^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * linux/mm/swap.c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * This file contains the default values for the operation of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * Linux VM subsystem. Fine-tuning documentation can be found in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * Documentation/admin-guide/sysctl/vm.rst.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * Started 18.12.91
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * Swap aging added 23.2.95, Stephen Tweedie.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * Buffermem limits added 12.3.98, Rik van Riel.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/kernel_stat.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/swap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/mman.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/pagemap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/pagevec.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/mm_inline.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/percpu_counter.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <linux/memremap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <linux/percpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <linux/notifier.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <linux/backing-dev.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <linux/memcontrol.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <linux/gfp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <linux/uio.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <linux/hugetlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include <linux/page_idle.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #include <linux/local_lock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #include <linux/buffer_head.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #include "internal.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #define CREATE_TRACE_POINTS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #include <trace/events/pagemap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) /* How many pages do we try to swap or page in/out together? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) int page_cluster;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) /* Protecting only lru_rotate.pvec which requires disabling interrupts */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) struct lru_rotate {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) local_lock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) struct pagevec pvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) static DEFINE_PER_CPU(struct lru_rotate, lru_rotate) = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) .lock = INIT_LOCAL_LOCK(lock),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) * The following struct pagevec are grouped together because they are protected
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) * by disabling preemption (and interrupts remain enabled).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) struct lru_pvecs {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) local_lock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) struct pagevec lru_add;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) struct pagevec lru_deactivate_file;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) struct pagevec lru_deactivate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) struct pagevec lru_lazyfree;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) struct pagevec lru_lazyfree_movetail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) #ifdef CONFIG_SMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) struct pagevec activate_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) static DEFINE_PER_CPU(struct lru_pvecs, lru_pvecs) = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) .lock = INIT_LOCAL_LOCK(lock),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) * This path almost never happens for VM activity - pages are normally
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) * freed via pagevecs. But it gets used by networking.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) static void __page_cache_release(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) if (PageLRU(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) pg_data_t *pgdat = page_pgdat(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) struct lruvec *lruvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) spin_lock_irqsave(&pgdat->lru_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) lruvec = mem_cgroup_page_lruvec(page, pgdat);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) VM_BUG_ON_PAGE(!PageLRU(page), page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) __ClearPageLRU(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) del_page_from_lru_list(page, lruvec, page_off_lru(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) spin_unlock_irqrestore(&pgdat->lru_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) __ClearPageWaiters(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) static void __put_single_page(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) __page_cache_release(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) mem_cgroup_uncharge(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) free_unref_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) static void __put_compound_page(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) * __page_cache_release() is supposed to be called for thp, not for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) * hugetlb. This is because hugetlb page does never have PageLRU set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) * (it's never listed to any LRU lists) and no memcg routines should
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) * be called for hugetlb (it has a separate hugetlb_cgroup.)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) if (!PageHuge(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) __page_cache_release(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) destroy_compound_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) void __put_page(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) if (is_zone_device_page(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) put_dev_pagemap(page->pgmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) * The page belongs to the device that created pgmap. Do
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) * not return it to page allocator.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) if (unlikely(PageCompound(page)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) __put_compound_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) __put_single_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) EXPORT_SYMBOL(__put_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) * put_pages_list() - release a list of pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) * @pages: list of pages threaded on page->lru
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) * Release a list of pages which are strung together on page.lru. Currently
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) * used by read_cache_pages() and related error recovery code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) void put_pages_list(struct list_head *pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) while (!list_empty(pages)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) struct page *victim;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) victim = lru_to_page(pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) list_del(&victim->lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) put_page(victim);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) EXPORT_SYMBOL(put_pages_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) * get_kernel_pages() - pin kernel pages in memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) * @kiov: An array of struct kvec structures
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) * @nr_segs: number of segments to pin
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) * @write: pinning for read/write, currently ignored
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) * @pages: array that receives pointers to the pages pinned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) * Should be at least nr_segs long.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) * Returns number of pages pinned. This may be fewer than the number
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) * requested. If nr_pages is 0 or negative, returns 0. If no pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) * were pinned, returns -errno. Each page returned must be released
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) * with a put_page() call when it is finished with.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) int get_kernel_pages(const struct kvec *kiov, int nr_segs, int write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) struct page **pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) int seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) for (seg = 0; seg < nr_segs; seg++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) if (WARN_ON(kiov[seg].iov_len != PAGE_SIZE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) return seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) pages[seg] = kmap_to_page(kiov[seg].iov_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) get_page(pages[seg]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) return seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) EXPORT_SYMBOL_GPL(get_kernel_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) * get_kernel_page() - pin a kernel page in memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) * @start: starting kernel address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) * @write: pinning for read/write, currently ignored
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) * @pages: array that receives pointer to the page pinned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) * Must be at least nr_segs long.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) * Returns 1 if page is pinned. If the page was not pinned, returns
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) * -errno. The page returned must be released with a put_page() call
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) * when it is finished with.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) int get_kernel_page(unsigned long start, int write, struct page **pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) const struct kvec kiov = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) .iov_base = (void *)start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) .iov_len = PAGE_SIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) return get_kernel_pages(&kiov, 1, write, pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) EXPORT_SYMBOL_GPL(get_kernel_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) static void pagevec_lru_move_fn(struct pagevec *pvec,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) void (*move_fn)(struct page *page, struct lruvec *lruvec, void *arg),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) struct pglist_data *pgdat = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) struct lruvec *lruvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) unsigned long flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) for (i = 0; i < pagevec_count(pvec); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) struct page *page = pvec->pages[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) struct pglist_data *pagepgdat = page_pgdat(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) if (pagepgdat != pgdat) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) if (pgdat)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) spin_unlock_irqrestore(&pgdat->lru_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) pgdat = pagepgdat;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) spin_lock_irqsave(&pgdat->lru_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) lruvec = mem_cgroup_page_lruvec(page, pgdat);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) (*move_fn)(page, lruvec, arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) if (pgdat)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) spin_unlock_irqrestore(&pgdat->lru_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) release_pages(pvec->pages, pvec->nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) pagevec_reinit(pvec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) static void pagevec_move_tail_fn(struct page *page, struct lruvec *lruvec,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) int *pgmoved = arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) if (PageLRU(page) && !PageUnevictable(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) del_page_from_lru_list(page, lruvec, page_lru(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) ClearPageActive(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) add_page_to_lru_list_tail(page, lruvec, page_lru(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) (*pgmoved) += thp_nr_pages(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) * pagevec_move_tail() must be called with IRQ disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) * Otherwise this may cause nasty races.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) static void pagevec_move_tail(struct pagevec *pvec)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) int pgmoved = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) pagevec_lru_move_fn(pvec, pagevec_move_tail_fn, &pgmoved);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) __count_vm_events(PGROTATED, pgmoved);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) /* return true if pagevec needs to drain */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) static bool pagevec_add_and_need_flush(struct pagevec *pvec, struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) bool ret = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) if (!pagevec_add(pvec, page) || PageCompound(page) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) lru_cache_disabled())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) ret = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) * Writeback is about to end against a page which has been marked for immediate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) * reclaim. If it still appears to be reclaimable, move it to the tail of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) * inactive list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) void rotate_reclaimable_page(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) if (!PageLocked(page) && !PageDirty(page) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) !PageUnevictable(page) && PageLRU(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) struct pagevec *pvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) get_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) local_lock_irqsave(&lru_rotate.lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) pvec = this_cpu_ptr(&lru_rotate.pvec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) if (pagevec_add_and_need_flush(pvec, page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) pagevec_move_tail(pvec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) local_unlock_irqrestore(&lru_rotate.lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) void lru_note_cost(struct lruvec *lruvec, bool file, unsigned int nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) unsigned long lrusize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) /* Record cost event */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) if (file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) lruvec->file_cost += nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) lruvec->anon_cost += nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) * Decay previous events
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) * Because workloads change over time (and to avoid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) * overflow) we keep these statistics as a floating
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) * average, which ends up weighing recent refaults
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) * more than old ones.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) lrusize = lruvec_page_state(lruvec, NR_INACTIVE_ANON) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) lruvec_page_state(lruvec, NR_ACTIVE_ANON) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) lruvec_page_state(lruvec, NR_INACTIVE_FILE) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) lruvec_page_state(lruvec, NR_ACTIVE_FILE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) if (lruvec->file_cost + lruvec->anon_cost > lrusize / 4) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) lruvec->file_cost /= 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) lruvec->anon_cost /= 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) } while ((lruvec = parent_lruvec(lruvec)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) void lru_note_cost_page(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) lru_note_cost(mem_cgroup_page_lruvec(page, page_pgdat(page)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) page_is_file_lru(page), thp_nr_pages(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) static void __activate_page(struct page *page, struct lruvec *lruvec,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) int lru = page_lru_base_type(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) int nr_pages = thp_nr_pages(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) del_page_from_lru_list(page, lruvec, lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) SetPageActive(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) lru += LRU_ACTIVE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) add_page_to_lru_list(page, lruvec, lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) trace_mm_lru_activate(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) __count_vm_events(PGACTIVATE, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) __count_memcg_events(lruvec_memcg(lruvec), PGACTIVATE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) #ifdef CONFIG_SMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) static void activate_page_drain(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) struct pagevec *pvec = &per_cpu(lru_pvecs.activate_page, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) if (pagevec_count(pvec))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) pagevec_lru_move_fn(pvec, __activate_page, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) static bool need_activate_page_drain(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) return pagevec_count(&per_cpu(lru_pvecs.activate_page, cpu)) != 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) static void activate_page(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) page = compound_head(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) struct pagevec *pvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) local_lock(&lru_pvecs.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) pvec = this_cpu_ptr(&lru_pvecs.activate_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) get_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) if (pagevec_add_and_need_flush(pvec, page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) pagevec_lru_move_fn(pvec, __activate_page, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) local_unlock(&lru_pvecs.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) static inline void activate_page_drain(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) static void activate_page(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) pg_data_t *pgdat = page_pgdat(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) page = compound_head(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) spin_lock_irq(&pgdat->lru_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) __activate_page(page, mem_cgroup_page_lruvec(page, pgdat), NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) spin_unlock_irq(&pgdat->lru_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) static void __lru_cache_activate_page(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) struct pagevec *pvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) local_lock(&lru_pvecs.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) pvec = this_cpu_ptr(&lru_pvecs.lru_add);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) * Search backwards on the optimistic assumption that the page being
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) * activated has just been added to this pagevec. Note that only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) * the local pagevec is examined as a !PageLRU page could be in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) * process of being released, reclaimed, migrated or on a remote
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) * pagevec that is currently being drained. Furthermore, marking
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) * a remote pagevec's page PageActive potentially hits a race where
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) * a page is marked PageActive just after it is added to the inactive
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) * list causing accounting errors and BUG_ON checks to trigger.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) for (i = pagevec_count(pvec) - 1; i >= 0; i--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) struct page *pagevec_page = pvec->pages[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) if (pagevec_page == page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) SetPageActive(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) local_unlock(&lru_pvecs.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) * Mark a page as having seen activity.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) * inactive,unreferenced -> inactive,referenced
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) * inactive,referenced -> active,unreferenced
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) * active,unreferenced -> active,referenced
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) * When a newly allocated page is not yet visible, so safe for non-atomic ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) * __SetPageReferenced(page) may be substituted for mark_page_accessed(page).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) void mark_page_accessed(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) page = compound_head(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) if (!PageReferenced(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) SetPageReferenced(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) } else if (PageUnevictable(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) * Unevictable pages are on the "LRU_UNEVICTABLE" list. But,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) * this list is never rotated or maintained, so marking an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) * evictable page accessed has no effect.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) } else if (!PageActive(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) * If the page is on the LRU, queue it for activation via
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) * lru_pvecs.activate_page. Otherwise, assume the page is on a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) * pagevec, mark it active and it'll be moved to the active
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) * LRU on the next drain.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) if (PageLRU(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) activate_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) __lru_cache_activate_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) ClearPageReferenced(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) workingset_activation(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) if (page_is_idle(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) clear_page_idle(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) EXPORT_SYMBOL(mark_page_accessed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) * lru_cache_add - add a page to a page list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) * @page: the page to be added to the LRU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) * Queue the page for addition to the LRU via pagevec. The decision on whether
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) * to add the page to the [in]active [file|anon] list is deferred until the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) * pagevec is drained. This gives a chance for the caller of lru_cache_add()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) * have the page added to the active list using mark_page_accessed().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) void lru_cache_add(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) struct pagevec *pvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) VM_BUG_ON_PAGE(PageActive(page) && PageUnevictable(page), page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) VM_BUG_ON_PAGE(PageLRU(page), page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) get_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) local_lock(&lru_pvecs.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) pvec = this_cpu_ptr(&lru_pvecs.lru_add);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) if (pagevec_add_and_need_flush(pvec, page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) __pagevec_lru_add(pvec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) local_unlock(&lru_pvecs.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) EXPORT_SYMBOL(lru_cache_add);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) * lru_cache_add_inactive_or_unevictable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) * @page: the page to be added to LRU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) * @vma: vma in which page is mapped for determining reclaimability
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) * Place @page on the inactive or unevictable LRU list, depending on its
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) * evictability.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) void __lru_cache_add_inactive_or_unevictable(struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) unsigned long vma_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) bool unevictable;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) VM_BUG_ON_PAGE(PageLRU(page), page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) unevictable = (vma_flags & (VM_LOCKED | VM_SPECIAL)) == VM_LOCKED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) if (unlikely(unevictable) && !TestSetPageMlocked(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) int nr_pages = thp_nr_pages(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) * We use the irq-unsafe __mod_zone_page_stat because this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) * counter is not modified from interrupt context, and the pte
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) * lock is held(spinlock), which implies preemption disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) __mod_zone_page_state(page_zone(page), NR_MLOCK, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) count_vm_events(UNEVICTABLE_PGMLOCKED, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) lru_cache_add(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) * If the page can not be invalidated, it is moved to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) * inactive list to speed up its reclaim. It is moved to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) * head of the list, rather than the tail, to give the flusher
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) * threads some time to write it out, as this is much more
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) * effective than the single-page writeout from reclaim.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) * If the page isn't page_mapped and dirty/writeback, the page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) * could reclaim asap using PG_reclaim.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) * 1. active, mapped page -> none
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) * 2. active, dirty/writeback page -> inactive, head, PG_reclaim
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) * 3. inactive, mapped page -> none
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) * 4. inactive, dirty/writeback page -> inactive, head, PG_reclaim
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) * 5. inactive, clean -> inactive, tail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) * 6. Others -> none
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) * In 4, why it moves inactive's head, the VM expects the page would
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) * be write it out by flusher threads as this is much more effective
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) * than the single-page writeout from reclaim.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) static void lru_deactivate_file_fn(struct page *page, struct lruvec *lruvec,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) int lru;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) bool active;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) int nr_pages = thp_nr_pages(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) if (!PageLRU(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) if (PageUnevictable(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) /* Some processes are using the page */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) if (page_mapped(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) active = PageActive(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) lru = page_lru_base_type(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) del_page_from_lru_list(page, lruvec, lru + active);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) ClearPageActive(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) ClearPageReferenced(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) if (PageWriteback(page) || PageDirty(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) * PG_reclaim could be raced with end_page_writeback
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) * It can make readahead confusing. But race window
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) * is _really_ small and it's non-critical problem.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) add_page_to_lru_list(page, lruvec, lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) SetPageReclaim(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) * The page's writeback ends up during pagevec
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) * We moves tha page into tail of inactive.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) add_page_to_lru_list_tail(page, lruvec, lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) __count_vm_events(PGROTATED, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) if (active) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) __count_vm_events(PGDEACTIVATE, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) __count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) static void lru_deactivate_fn(struct page *page, struct lruvec *lruvec,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) int lru = page_lru_base_type(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) int nr_pages = thp_nr_pages(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) del_page_from_lru_list(page, lruvec, lru + LRU_ACTIVE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) ClearPageActive(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) ClearPageReferenced(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) add_page_to_lru_list(page, lruvec, lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) __count_vm_events(PGDEACTIVATE, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) __count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) static void lru_lazyfree_fn(struct page *page, struct lruvec *lruvec,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) !PageSwapCache(page) && !PageUnevictable(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) bool active = PageActive(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) int nr_pages = thp_nr_pages(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) del_page_from_lru_list(page, lruvec,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) LRU_INACTIVE_ANON + active);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) ClearPageActive(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) ClearPageReferenced(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) * Lazyfree pages are clean anonymous pages. They have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) * PG_swapbacked flag cleared, to distinguish them from normal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) * anonymous pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) ClearPageSwapBacked(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) add_page_to_lru_list(page, lruvec, LRU_INACTIVE_FILE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) __count_vm_events(PGLAZYFREE, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) __count_memcg_events(lruvec_memcg(lruvec), PGLAZYFREE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) static void lru_lazyfree_movetail_fn(struct page *page, struct lruvec *lruvec,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) bool *add_to_tail = (bool *)arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) if (PageLRU(page) && !PageUnevictable(page) && PageSwapBacked(page) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) !PageSwapCache(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) bool active = PageActive(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) del_page_from_lru_list(page, lruvec,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) LRU_INACTIVE_ANON + active);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) ClearPageActive(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) ClearPageReferenced(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) if (add_to_tail && *add_to_tail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) add_page_to_lru_list_tail(page, lruvec, LRU_INACTIVE_FILE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) add_page_to_lru_list(page, lruvec, LRU_INACTIVE_FILE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) * Drain pages out of the cpu's pagevecs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) * Either "cpu" is the current CPU, and preemption has already been
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) * disabled; or "cpu" is being hot-unplugged, and is already dead.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) void lru_add_drain_cpu(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) struct pagevec *pvec = &per_cpu(lru_pvecs.lru_add, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) if (pagevec_count(pvec))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) __pagevec_lru_add(pvec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) pvec = &per_cpu(lru_rotate.pvec, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) /* Disabling interrupts below acts as a compiler barrier. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) if (data_race(pagevec_count(pvec))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) /* No harm done if a racing interrupt already did this */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) local_lock_irqsave(&lru_rotate.lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) pagevec_move_tail(pvec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) local_unlock_irqrestore(&lru_rotate.lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) pvec = &per_cpu(lru_pvecs.lru_deactivate_file, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) if (pagevec_count(pvec))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) pagevec_lru_move_fn(pvec, lru_deactivate_file_fn, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) pvec = &per_cpu(lru_pvecs.lru_deactivate, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) if (pagevec_count(pvec))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) pvec = &per_cpu(lru_pvecs.lru_lazyfree, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) if (pagevec_count(pvec))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) pagevec_lru_move_fn(pvec, lru_lazyfree_fn, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) pvec = &per_cpu(lru_pvecs.lru_lazyfree_movetail, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) if (pagevec_count(pvec))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) pagevec_lru_move_fn(pvec, lru_lazyfree_movetail_fn, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) activate_page_drain(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) * deactivate_file_page - forcefully deactivate a file page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) * @page: page to deactivate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) * This function hints the VM that @page is a good reclaim candidate,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) * for example if its invalidation fails due to the page being dirty
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) * or under writeback.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) void deactivate_file_page(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) * In a workload with many unevictable page such as mprotect,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) * unevictable page deactivation for accelerating reclaim is pointless.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) if (PageUnevictable(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) if (likely(get_page_unless_zero(page))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) struct pagevec *pvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) local_lock(&lru_pvecs.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) pvec = this_cpu_ptr(&lru_pvecs.lru_deactivate_file);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) if (pagevec_add_and_need_flush(pvec, page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) pagevec_lru_move_fn(pvec, lru_deactivate_file_fn, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) local_unlock(&lru_pvecs.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) * deactivate_page - deactivate a page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) * @page: page to deactivate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) * deactivate_page() moves @page to the inactive list if @page was on the active
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) * list and was not an unevictable page. This is done to accelerate the reclaim
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) * of @page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) void deactivate_page(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) struct pagevec *pvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) local_lock(&lru_pvecs.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) pvec = this_cpu_ptr(&lru_pvecs.lru_deactivate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) get_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) if (pagevec_add_and_need_flush(pvec, page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) local_unlock(&lru_pvecs.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) * mark_page_lazyfree - make an anon page lazyfree
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) * @page: page to deactivate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) * mark_page_lazyfree() moves @page to the inactive file list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) * This is done to accelerate the reclaim of @page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) void mark_page_lazyfree(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) !PageSwapCache(page) && !PageUnevictable(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) struct pagevec *pvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) local_lock(&lru_pvecs.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) pvec = this_cpu_ptr(&lru_pvecs.lru_lazyfree);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) get_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) if (pagevec_add_and_need_flush(pvec, page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) pagevec_lru_move_fn(pvec, lru_lazyfree_fn, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) local_unlock(&lru_pvecs.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) * mark_page_lazyfree_movetail - make a swapbacked page lazyfree
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) * @page: page to deactivate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) * mark_page_lazyfree_movetail() moves @page to the tail of inactive file list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) * This is done to accelerate the reclaim of @page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) void mark_page_lazyfree_movetail(struct page *page, bool tail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) if (PageLRU(page) && !PageUnevictable(page) && PageSwapBacked(page) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) !PageSwapCache(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) struct pagevec *pvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) local_lock(&lru_pvecs.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) pvec = this_cpu_ptr(&lru_pvecs.lru_lazyfree_movetail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) get_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) if (pagevec_add_and_need_flush(pvec, page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) pagevec_lru_move_fn(pvec,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) lru_lazyfree_movetail_fn, &tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) local_unlock(&lru_pvecs.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) void lru_add_drain(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) local_lock(&lru_pvecs.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) lru_add_drain_cpu(smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) local_unlock(&lru_pvecs.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) * It's called from per-cpu workqueue context in SMP case so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) * lru_add_drain_cpu and invalidate_bh_lrus_cpu should run on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) * the same cpu. It shouldn't be a problem in !SMP case since
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) * the core is only one and the locks will disable preemption.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) static void lru_add_and_bh_lrus_drain(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) local_lock(&lru_pvecs.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) lru_add_drain_cpu(smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) local_unlock(&lru_pvecs.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) invalidate_bh_lrus_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) void lru_add_drain_cpu_zone(struct zone *zone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) local_lock(&lru_pvecs.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) lru_add_drain_cpu(smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) drain_local_pages(zone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) local_unlock(&lru_pvecs.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) #ifdef CONFIG_SMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) static void lru_add_drain_per_cpu(struct work_struct *dummy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) lru_add_and_bh_lrus_drain();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) * Doesn't need any cpu hotplug locking because we do rely on per-cpu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) * kworkers being shut down before our page_alloc_cpu_dead callback is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) * executed on the offlined cpu.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) * Calling this function with cpu hotplug locks held can actually lead
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) * to obscure indirect dependencies via WQ context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) inline void __lru_add_drain_all(bool force_all_cpus)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) * lru_drain_gen - Global pages generation number
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) * (A) Definition: global lru_drain_gen = x implies that all generations
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) * 0 < n <= x are already *scheduled* for draining.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) * This is an optimization for the highly-contended use case where a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) * user space workload keeps constantly generating a flow of pages for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) * each CPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) static unsigned int lru_drain_gen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) static struct cpumask has_work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) static DEFINE_MUTEX(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) unsigned cpu, this_gen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) * Make sure nobody triggers this path before mm_percpu_wq is fully
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) * initialized.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) if (WARN_ON(!mm_percpu_wq))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) * Guarantee pagevec counter stores visible by this CPU are visible to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) * other CPUs before loading the current drain generation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) * (B) Locally cache global LRU draining generation number
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) * The read barrier ensures that the counter is loaded before the mutex
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) * is taken. It pairs with smp_mb() inside the mutex critical section
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) * at (D).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) this_gen = smp_load_acquire(&lru_drain_gen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) mutex_lock(&lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) * (C) Exit the draining operation if a newer generation, from another
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) * lru_add_drain_all(), was already scheduled for draining. Check (A).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) if (unlikely(this_gen != lru_drain_gen && !force_all_cpus))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) * (D) Increment global generation number
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) * Pairs with smp_load_acquire() at (B), outside of the critical
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) * section. Use a full memory barrier to guarantee that the new global
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) * drain generation number is stored before loading pagevec counters.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) * This pairing must be done here, before the for_each_online_cpu loop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) * below which drains the page vectors.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) * Let x, y, and z represent some system CPU numbers, where x < y < z.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) * Assume CPU #z is is in the middle of the for_each_online_cpu loop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) * below and has already reached CPU #y's per-cpu data. CPU #x comes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) * along, adds some pages to its per-cpu vectors, then calls
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) * lru_add_drain_all().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) * If the paired barrier is done at any later step, e.g. after the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) * loop, CPU #x will just exit at (C) and miss flushing out all of its
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) * added pages.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) WRITE_ONCE(lru_drain_gen, lru_drain_gen + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) cpumask_clear(&has_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) for_each_online_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) struct work_struct *work = &per_cpu(lru_add_drain_work, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) if (force_all_cpus ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) pagevec_count(&per_cpu(lru_pvecs.lru_add, cpu)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) data_race(pagevec_count(&per_cpu(lru_rotate.pvec, cpu))) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) pagevec_count(&per_cpu(lru_pvecs.lru_deactivate_file, cpu)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) pagevec_count(&per_cpu(lru_pvecs.lru_deactivate, cpu)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) pagevec_count(&per_cpu(lru_pvecs.lru_lazyfree, cpu)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) pagevec_count(&per_cpu(lru_pvecs.lru_lazyfree_movetail, cpu)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) need_activate_page_drain(cpu) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) has_bh_in_lru(cpu, NULL)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) INIT_WORK(work, lru_add_drain_per_cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) queue_work_on(cpu, mm_percpu_wq, work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) __cpumask_set_cpu(cpu, &has_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) for_each_cpu(cpu, &has_work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) flush_work(&per_cpu(lru_add_drain_work, cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) mutex_unlock(&lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) void lru_add_drain_all(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) __lru_add_drain_all(false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) void lru_add_drain_all(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) lru_add_drain();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) #endif /* CONFIG_SMP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) static atomic_t lru_disable_count = ATOMIC_INIT(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) bool lru_cache_disabled(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) return atomic_read(&lru_disable_count) != 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) void lru_cache_enable(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) atomic_dec(&lru_disable_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) EXPORT_SYMBOL_GPL(lru_cache_enable);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) * lru_cache_disable() needs to be called before we start compiling
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) * a list of pages to be migrated using isolate_lru_page().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) * It drains pages on LRU cache and then disable on all cpus until
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) * lru_cache_enable is called.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) * Must be paired with a call to lru_cache_enable().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) void lru_cache_disable(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) * If someone is already disabled lru_cache, just return with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) * increasing the lru_disable_count.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) if (atomic_inc_not_zero(&lru_disable_count))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) #ifdef CONFIG_SMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) * lru_add_drain_all in the force mode will schedule draining on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) * all online CPUs so any calls of lru_cache_disabled wrapped by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) * local_lock or preemption disabled would be ordered by that.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) * The atomic operation doesn't need to have stronger ordering
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) * requirements because that is enforeced by the scheduling
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) * guarantees.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) __lru_add_drain_all(true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) lru_add_and_bh_lrus_drain();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) atomic_inc(&lru_disable_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) EXPORT_SYMBOL_GPL(lru_cache_disable);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) * release_pages - batched put_page()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) * @pages: array of pages to release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) * @nr: number of pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) * Decrement the reference count on all the pages in @pages. If it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) * fell to zero, remove the page from the LRU and free it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) void release_pages(struct page **pages, int nr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) LIST_HEAD(pages_to_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) struct pglist_data *locked_pgdat = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) struct lruvec *lruvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) unsigned int lock_batch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) for (i = 0; i < nr; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) struct page *page = pages[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) * Make sure the IRQ-safe lock-holding time does not get
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) * excessive with a continuous string of pages from the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) * same pgdat. The lock is held only if pgdat != NULL.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) if (locked_pgdat && ++lock_batch == SWAP_CLUSTER_MAX) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) locked_pgdat = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) page = compound_head(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) if (is_huge_zero_page(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) if (is_zone_device_page(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) if (locked_pgdat) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) spin_unlock_irqrestore(&locked_pgdat->lru_lock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) locked_pgdat = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) * ZONE_DEVICE pages that return 'false' from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) * page_is_devmap_managed() do not require special
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) * processing, and instead, expect a call to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) * put_page_testzero().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) if (page_is_devmap_managed(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) put_devmap_managed_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) if (!put_page_testzero(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) if (PageCompound(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) if (locked_pgdat) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) locked_pgdat = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) __put_compound_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) if (PageLRU(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) struct pglist_data *pgdat = page_pgdat(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) if (pgdat != locked_pgdat) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) if (locked_pgdat)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) spin_unlock_irqrestore(&locked_pgdat->lru_lock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) lock_batch = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) locked_pgdat = pgdat;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) spin_lock_irqsave(&locked_pgdat->lru_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) lruvec = mem_cgroup_page_lruvec(page, locked_pgdat);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) VM_BUG_ON_PAGE(!PageLRU(page), page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) __ClearPageLRU(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) del_page_from_lru_list(page, lruvec, page_off_lru(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) __ClearPageWaiters(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) list_add(&page->lru, &pages_to_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) if (locked_pgdat)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) mem_cgroup_uncharge_list(&pages_to_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) free_unref_page_list(&pages_to_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) EXPORT_SYMBOL(release_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) * The pages which we're about to release may be in the deferred lru-addition
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) * queues. That would prevent them from really being freed right now. That's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) * OK from a correctness point of view but is inefficient - those pages may be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) * cache-warm and we want to give them back to the page allocator ASAP.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) * So __pagevec_release() will drain those queues here. __pagevec_lru_add()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) * and __pagevec_lru_add_active() call release_pages() directly to avoid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) * mutual recursion.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) void __pagevec_release(struct pagevec *pvec)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) if (!pvec->percpu_pvec_drained) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) lru_add_drain();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) pvec->percpu_pvec_drained = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) release_pages(pvec->pages, pagevec_count(pvec));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) pagevec_reinit(pvec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) EXPORT_SYMBOL(__pagevec_release);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) #ifdef CONFIG_TRANSPARENT_HUGEPAGE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) /* used by __split_huge_page_refcount() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) void lru_add_page_tail(struct page *page, struct page *page_tail,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) struct lruvec *lruvec, struct list_head *list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) VM_BUG_ON_PAGE(!PageHead(page), page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) VM_BUG_ON_PAGE(PageCompound(page_tail), page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) VM_BUG_ON_PAGE(PageLRU(page_tail), page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) lockdep_assert_held(&lruvec_pgdat(lruvec)->lru_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) if (!list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) SetPageLRU(page_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) if (likely(PageLRU(page)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) list_add_tail(&page_tail->lru, &page->lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) else if (list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) /* page reclaim is reclaiming a huge page */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) get_page(page_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) list_add_tail(&page_tail->lru, list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) * Head page has not yet been counted, as an hpage,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) * so we must account for each subpage individually.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) * Put page_tail on the list at the correct position
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) * so they all end up in order.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) add_page_to_lru_list_tail(page_tail, lruvec,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) page_lru(page_tail));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) enum lru_list lru;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) int was_unevictable = TestClearPageUnevictable(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) int nr_pages = thp_nr_pages(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) VM_BUG_ON_PAGE(PageLRU(page), page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) * Page becomes evictable in two ways:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) * 1) Within LRU lock [munlock_vma_page() and __munlock_pagevec()].
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) * 2) Before acquiring LRU lock to put the page to correct LRU and then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) * a) do PageLRU check with lock [check_move_unevictable_pages]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) * b) do PageLRU check before lock [clear_page_mlock]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) * (1) & (2a) are ok as LRU lock will serialize them. For (2b), we need
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) * following strict ordering:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) * #0: __pagevec_lru_add_fn #1: clear_page_mlock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) * SetPageLRU() TestClearPageMlocked()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) * smp_mb() // explicit ordering // above provides strict
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) * // ordering
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) * PageMlocked() PageLRU()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) * if '#1' does not observe setting of PG_lru by '#0' and fails
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) * isolation, the explicit barrier will make sure that page_evictable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) * check will put the page in correct LRU. Without smp_mb(), SetPageLRU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) * can be reordered after PageMlocked check and can make '#1' to fail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) * the isolation of the page whose Mlocked bit is cleared (#0 is also
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) * looking at the same page) and the evictable page will be stranded
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) * in an unevictable LRU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) SetPageLRU(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) smp_mb__after_atomic();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) if (page_evictable(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) lru = page_lru(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) if (was_unevictable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) __count_vm_events(UNEVICTABLE_PGRESCUED, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) lru = LRU_UNEVICTABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) ClearPageActive(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) SetPageUnevictable(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) if (!was_unevictable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) __count_vm_events(UNEVICTABLE_PGCULLED, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) add_page_to_lru_list(page, lruvec, lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) trace_mm_lru_insertion(page, lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) * Add the passed pages to the LRU, then drop the caller's refcount
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) * on them. Reinitialises the caller's pagevec.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) void __pagevec_lru_add(struct pagevec *pvec)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) pagevec_lru_move_fn(pvec, __pagevec_lru_add_fn, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) * pagevec_lookup_entries - gang pagecache lookup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) * @pvec: Where the resulting entries are placed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) * @mapping: The address_space to search
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) * @start: The starting entry index
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) * @nr_entries: The maximum number of pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) * @indices: The cache indices corresponding to the entries in @pvec
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) * pagevec_lookup_entries() will search for and return a group of up
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) * to @nr_pages pages and shadow entries in the mapping. All
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) * entries are placed in @pvec. pagevec_lookup_entries() takes a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) * reference against actual pages in @pvec.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) * The search returns a group of mapping-contiguous entries with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) * ascending indexes. There may be holes in the indices due to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) * not-present entries.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) * Only one subpage of a Transparent Huge Page is returned in one call:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) * allowing truncate_inode_pages_range() to evict the whole THP without
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) * cycling through a pagevec of extra references.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) * pagevec_lookup_entries() returns the number of entries which were
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) * found.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) unsigned pagevec_lookup_entries(struct pagevec *pvec,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) struct address_space *mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) pgoff_t start, unsigned nr_entries,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) pgoff_t *indices)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) pvec->nr = find_get_entries(mapping, start, nr_entries,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) pvec->pages, indices);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) return pagevec_count(pvec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) * pagevec_remove_exceptionals - pagevec exceptionals pruning
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) * @pvec: The pagevec to prune
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) * pagevec_lookup_entries() fills both pages and exceptional radix
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) * tree entries into the pagevec. This function prunes all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) * exceptionals from @pvec without leaving holes, so that it can be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) * passed on to page-only pagevec operations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) void pagevec_remove_exceptionals(struct pagevec *pvec)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) int i, j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) for (i = 0, j = 0; i < pagevec_count(pvec); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) struct page *page = pvec->pages[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) if (!xa_is_value(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) pvec->pages[j++] = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) pvec->nr = j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) * pagevec_lookup_range - gang pagecache lookup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) * @pvec: Where the resulting pages are placed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) * @mapping: The address_space to search
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) * @start: The starting page index
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) * @end: The final page index
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) * pagevec_lookup_range() will search for & return a group of up to PAGEVEC_SIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) * pages in the mapping starting from index @start and upto index @end
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) * (inclusive). The pages are placed in @pvec. pagevec_lookup() takes a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) * reference against the pages in @pvec.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) * The search returns a group of mapping-contiguous pages with ascending
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) * indexes. There may be holes in the indices due to not-present pages. We
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) * also update @start to index the next page for the traversal.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) * pagevec_lookup_range() returns the number of pages which were found. If this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) * number is smaller than PAGEVEC_SIZE, the end of specified range has been
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) * reached.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) unsigned pagevec_lookup_range(struct pagevec *pvec,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) struct address_space *mapping, pgoff_t *start, pgoff_t end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) pvec->nr = find_get_pages_range(mapping, start, end, PAGEVEC_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) pvec->pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) return pagevec_count(pvec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) EXPORT_SYMBOL(pagevec_lookup_range);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) unsigned pagevec_lookup_range_tag(struct pagevec *pvec,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) struct address_space *mapping, pgoff_t *index, pgoff_t end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) xa_mark_t tag)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) pvec->nr = find_get_pages_range_tag(mapping, index, end, tag,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) PAGEVEC_SIZE, pvec->pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) return pagevec_count(pvec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) EXPORT_SYMBOL(pagevec_lookup_range_tag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) unsigned pagevec_lookup_range_nr_tag(struct pagevec *pvec,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) struct address_space *mapping, pgoff_t *index, pgoff_t end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) xa_mark_t tag, unsigned max_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) pvec->nr = find_get_pages_range_tag(mapping, index, end, tag,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) min_t(unsigned int, max_pages, PAGEVEC_SIZE), pvec->pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) return pagevec_count(pvec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) EXPORT_SYMBOL(pagevec_lookup_range_nr_tag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) * Perform any setup for the swap system
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) void __init swap_setup(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) unsigned long megs = totalram_pages() >> (20 - PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) /* Use a smaller cluster for small-memory machines */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) if (megs < 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) page_cluster = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) page_cluster = 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) * Right now other parts of the system means that we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) * _really_ don't want to cluster much more
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) #ifdef CONFIG_DEV_PAGEMAP_OPS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) void put_devmap_managed_page(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) int count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) if (WARN_ON_ONCE(!page_is_devmap_managed(page)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) count = page_ref_dec_return(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) * devmap page refcounts are 1-based, rather than 0-based: if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) * refcount is 1, then the page is free and the refcount is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) * stable because nobody holds a reference on the page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) if (count == 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) free_devmap_managed_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) else if (!count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) __put_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) EXPORT_SYMBOL(put_devmap_managed_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) #endif
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags