^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (C) Gao Xiang <xiang@kernel.org>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * For low-latency decompression algorithms (e.g. lz4), reserve consecutive
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * per-CPU virtual memory (in pages) in advance to store such inplace I/O
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * data if inplace decompression is failed (due to unmet inplace margin for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * example).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include "internal.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) struct erofs_pcpubuf {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) raw_spinlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) void *ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) struct page **pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) unsigned int nrpages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) static DEFINE_PER_CPU(struct erofs_pcpubuf, erofs_pcb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) void *erofs_get_pcpubuf(unsigned int requiredpages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) __acquires(pcb->lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) struct erofs_pcpubuf *pcb = &get_cpu_var(erofs_pcb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) raw_spin_lock(&pcb->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) /* check if the per-CPU buffer is too small */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) if (requiredpages > pcb->nrpages) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) raw_spin_unlock(&pcb->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) put_cpu_var(erofs_pcb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) /* (for sparse checker) pretend pcb->lock is still taken */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) __acquire(pcb->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) return pcb->ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) void erofs_put_pcpubuf(void *ptr) __releases(pcb->lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) DBG_BUGON(pcb->ptr != ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) raw_spin_unlock(&pcb->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) put_cpu_var(erofs_pcb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) /* the next step: support per-CPU page buffers hotplug */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) int erofs_pcpubuf_growsize(unsigned int nrpages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) static DEFINE_MUTEX(pcb_resize_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) static unsigned int pcb_nrpages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) LIST_HEAD(pagepool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) int delta, cpu, ret, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) mutex_lock(&pcb_resize_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) delta = nrpages - pcb_nrpages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) /* avoid shrinking pcpubuf, since no idea how many fses rely on */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) if (delta <= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) struct page **pages, **oldpages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) void *ptr, *old_ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) pages = kmalloc_array(nrpages, sizeof(*pages), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) if (!pages) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) for (i = 0; i < nrpages; ++i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) pages[i] = erofs_allocpage(&pagepool, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) if (!pages[i]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) oldpages = pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) goto free_pagearray;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) ptr = vmap(pages, nrpages, VM_MAP, PAGE_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) if (!ptr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) oldpages = pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) goto free_pagearray;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) raw_spin_lock(&pcb->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) old_ptr = pcb->ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) pcb->ptr = ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) oldpages = pcb->pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) pcb->pages = pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) i = pcb->nrpages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) pcb->nrpages = nrpages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) raw_spin_unlock(&pcb->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) if (!oldpages) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) DBG_BUGON(old_ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) if (old_ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) vunmap(old_ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) free_pagearray:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) while (i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) list_add(&oldpages[--i]->lru, &pagepool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) kfree(oldpages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) pcb_nrpages = nrpages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) put_pages_list(&pagepool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) mutex_unlock(&pcb_resize_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) void erofs_pcpubuf_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) raw_spin_lock_init(&pcb->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) void erofs_pcpubuf_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) int cpu, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) if (pcb->ptr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) vunmap(pcb->ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) pcb->ptr = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) if (!pcb->pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) for (i = 0; i < pcb->nrpages; ++i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) if (pcb->pages[i])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) put_page(pcb->pages[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) kfree(pcb->pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) pcb->pages = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags