^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * mm/percpu-vm.c - vmalloc area based chunk allocation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2010 SUSE Linux Products GmbH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright (C) 2010 Tejun Heo <tj@kernel.org>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Chunks are mapped into vmalloc areas and populated page by page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * This is the default chunk allocator.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) static struct page *pcpu_chunk_page(struct pcpu_chunk *chunk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) unsigned int cpu, int page_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) /* must not be used on pre-mapped chunk */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) WARN_ON(chunk->immutable);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) return vmalloc_to_page((void *)pcpu_chunk_addr(chunk, cpu, page_idx));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * pcpu_get_pages - get temp pages array
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * Returns pointer to array of pointers to struct page which can be indexed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * with pcpu_page_idx(). Note that there is only one array and accesses
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * should be serialized by pcpu_alloc_mutex.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * RETURNS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * Pointer to temp pages array on success.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) static struct page **pcpu_get_pages(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) static struct page **pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) size_t pages_size = pcpu_nr_units * pcpu_unit_pages * sizeof(pages[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) lockdep_assert_held(&pcpu_alloc_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) if (!pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) pages = pcpu_mem_zalloc(pages_size, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) return pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) * pcpu_free_pages - free pages which were allocated for @chunk
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) * @chunk: chunk pages were allocated for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) * @pages: array of pages to be freed, indexed by pcpu_page_idx()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) * @page_start: page index of the first page to be freed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) * @page_end: page index of the last page to be freed + 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) * Free pages [@page_start and @page_end) in @pages for all units.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) * The pages were allocated for @chunk.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) static void pcpu_free_pages(struct pcpu_chunk *chunk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) struct page **pages, int page_start, int page_end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) unsigned int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) for (i = page_start; i < page_end; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) struct page *page = pages[pcpu_page_idx(cpu, i)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) if (page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) __free_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) * pcpu_alloc_pages - allocates pages for @chunk
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) * @chunk: target chunk
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) * @pages: array to put the allocated pages into, indexed by pcpu_page_idx()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) * @page_start: page index of the first page to be allocated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) * @page_end: page index of the last page to be allocated + 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) * @gfp: allocation flags passed to the underlying allocator
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) * Allocate pages [@page_start,@page_end) into @pages for all units.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) * The allocation is for @chunk. Percpu core doesn't care about the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) * content of @pages and will pass it verbatim to pcpu_map_pages().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) static int pcpu_alloc_pages(struct pcpu_chunk *chunk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) struct page **pages, int page_start, int page_end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) gfp_t gfp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) unsigned int cpu, tcpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) gfp |= __GFP_HIGHMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) for (i = page_start; i < page_end; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) struct page **pagep = &pages[pcpu_page_idx(cpu, i)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) *pagep = alloc_pages_node(cpu_to_node(cpu), gfp, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) if (!*pagep)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) while (--i >= page_start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) __free_page(pages[pcpu_page_idx(cpu, i)]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) for_each_possible_cpu(tcpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) if (tcpu == cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) for (i = page_start; i < page_end; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) __free_page(pages[pcpu_page_idx(tcpu, i)]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) * pcpu_pre_unmap_flush - flush cache prior to unmapping
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) * @chunk: chunk the regions to be flushed belongs to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) * @page_start: page index of the first page to be flushed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) * @page_end: page index of the last page to be flushed + 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) * Pages in [@page_start,@page_end) of @chunk are about to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) * unmapped. Flush cache. As each flushing trial can be very
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) * expensive, issue flush on the whole region at once rather than
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) * doing it for each cpu. This could be an overkill but is more
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) * scalable.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) static void pcpu_pre_unmap_flush(struct pcpu_chunk *chunk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) int page_start, int page_end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) flush_cache_vunmap(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) pcpu_chunk_addr(chunk, pcpu_low_unit_cpu, page_start),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) pcpu_chunk_addr(chunk, pcpu_high_unit_cpu, page_end));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) static void __pcpu_unmap_pages(unsigned long addr, int nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) unmap_kernel_range_noflush(addr, nr_pages << PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) * pcpu_unmap_pages - unmap pages out of a pcpu_chunk
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) * @chunk: chunk of interest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) * @pages: pages array which can be used to pass information to free
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) * @page_start: page index of the first page to unmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) * @page_end: page index of the last page to unmap + 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) * For each cpu, unmap pages [@page_start,@page_end) out of @chunk.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) * Corresponding elements in @pages were cleared by the caller and can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) * be used to carry information to pcpu_free_pages() which will be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) * called after all unmaps are finished. The caller should call
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) * proper pre/post flush functions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) static void pcpu_unmap_pages(struct pcpu_chunk *chunk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) struct page **pages, int page_start, int page_end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) unsigned int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) for (i = page_start; i < page_end; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) page = pcpu_chunk_page(chunk, cpu, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) WARN_ON(!page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) pages[pcpu_page_idx(cpu, i)] = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) __pcpu_unmap_pages(pcpu_chunk_addr(chunk, cpu, page_start),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) page_end - page_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) * pcpu_post_unmap_tlb_flush - flush TLB after unmapping
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) * @chunk: pcpu_chunk the regions to be flushed belong to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) * @page_start: page index of the first page to be flushed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) * @page_end: page index of the last page to be flushed + 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) * Pages [@page_start,@page_end) of @chunk have been unmapped. Flush
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) * TLB for the regions. This can be skipped if the area is to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) * returned to vmalloc as vmalloc will handle TLB flushing lazily.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) * As with pcpu_pre_unmap_flush(), TLB flushing also is done at once
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) * for the whole region.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) static void pcpu_post_unmap_tlb_flush(struct pcpu_chunk *chunk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) int page_start, int page_end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) flush_tlb_kernel_range(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) pcpu_chunk_addr(chunk, pcpu_low_unit_cpu, page_start),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) pcpu_chunk_addr(chunk, pcpu_high_unit_cpu, page_end));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) static int __pcpu_map_pages(unsigned long addr, struct page **pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) int nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) return map_kernel_range_noflush(addr, nr_pages << PAGE_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) PAGE_KERNEL, pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) * pcpu_map_pages - map pages into a pcpu_chunk
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) * @chunk: chunk of interest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) * @pages: pages array containing pages to be mapped
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) * @page_start: page index of the first page to map
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) * @page_end: page index of the last page to map + 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) * For each cpu, map pages [@page_start,@page_end) into @chunk. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) * caller is responsible for calling pcpu_post_map_flush() after all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) * mappings are complete.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) * This function is responsible for setting up whatever is necessary for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) * reverse lookup (addr -> chunk).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) static int pcpu_map_pages(struct pcpu_chunk *chunk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) struct page **pages, int page_start, int page_end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) unsigned int cpu, tcpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) int i, err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) err = __pcpu_map_pages(pcpu_chunk_addr(chunk, cpu, page_start),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) &pages[pcpu_page_idx(cpu, page_start)],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) page_end - page_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) for (i = page_start; i < page_end; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) pcpu_set_page_chunk(pages[pcpu_page_idx(cpu, i)],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) chunk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) for_each_possible_cpu(tcpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) if (tcpu == cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) __pcpu_unmap_pages(pcpu_chunk_addr(chunk, tcpu, page_start),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) page_end - page_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) pcpu_post_unmap_tlb_flush(chunk, page_start, page_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) * pcpu_post_map_flush - flush cache after mapping
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) * @chunk: pcpu_chunk the regions to be flushed belong to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) * @page_start: page index of the first page to be flushed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) * @page_end: page index of the last page to be flushed + 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) * Pages [@page_start,@page_end) of @chunk have been mapped. Flush
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) * cache.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) * As with pcpu_pre_unmap_flush(), TLB flushing also is done at once
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) * for the whole region.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) static void pcpu_post_map_flush(struct pcpu_chunk *chunk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) int page_start, int page_end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) flush_cache_vmap(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) pcpu_chunk_addr(chunk, pcpu_low_unit_cpu, page_start),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) pcpu_chunk_addr(chunk, pcpu_high_unit_cpu, page_end));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) * pcpu_populate_chunk - populate and map an area of a pcpu_chunk
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) * @chunk: chunk of interest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) * @page_start: the start page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) * @page_end: the end page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) * @gfp: allocation flags passed to the underlying memory allocator
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) * For each cpu, populate and map pages [@page_start,@page_end) into
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) * @chunk.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) * CONTEXT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) * pcpu_alloc_mutex, does GFP_KERNEL allocation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) static int pcpu_populate_chunk(struct pcpu_chunk *chunk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) int page_start, int page_end, gfp_t gfp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) struct page **pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) pages = pcpu_get_pages();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) if (!pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) if (pcpu_alloc_pages(chunk, pages, page_start, page_end, gfp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) if (pcpu_map_pages(chunk, pages, page_start, page_end)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) pcpu_free_pages(chunk, pages, page_start, page_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) pcpu_post_map_flush(chunk, page_start, page_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) * pcpu_depopulate_chunk - depopulate and unmap an area of a pcpu_chunk
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) * @chunk: chunk to depopulate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) * @page_start: the start page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) * @page_end: the end page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) * For each cpu, depopulate and unmap pages [@page_start,@page_end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) * from @chunk.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) * CONTEXT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) * pcpu_alloc_mutex.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) int page_start, int page_end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) struct page **pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) * If control reaches here, there must have been at least one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) * successful population attempt so the temp pages array must
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) * be available now.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) pages = pcpu_get_pages();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) BUG_ON(!pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) /* unmap and free */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) pcpu_pre_unmap_flush(chunk, page_start, page_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) pcpu_unmap_pages(chunk, pages, page_start, page_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) /* no need to flush tlb, vmalloc will handle it lazily */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) pcpu_free_pages(chunk, pages, page_start, page_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) static struct pcpu_chunk *pcpu_create_chunk(enum pcpu_chunk_type type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) gfp_t gfp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) struct pcpu_chunk *chunk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) struct vm_struct **vms;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) chunk = pcpu_alloc_chunk(type, gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) if (!chunk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) vms = pcpu_get_vm_areas(pcpu_group_offsets, pcpu_group_sizes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) pcpu_nr_groups, pcpu_atom_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) if (!vms) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) pcpu_free_chunk(chunk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) chunk->data = vms;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) chunk->base_addr = vms[0]->addr - pcpu_group_offsets[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) pcpu_stats_chunk_alloc();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) trace_percpu_create_chunk(chunk->base_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) return chunk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) static void pcpu_destroy_chunk(struct pcpu_chunk *chunk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) if (!chunk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) pcpu_stats_chunk_dealloc();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) trace_percpu_destroy_chunk(chunk->base_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) if (chunk->data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) pcpu_free_vm_areas(chunk->data, pcpu_nr_groups);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) pcpu_free_chunk(chunk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) static struct page *pcpu_addr_to_page(void *addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) return vmalloc_to_page(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) static int __init pcpu_verify_alloc_info(const struct pcpu_alloc_info *ai)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) /* no extra restriction */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags