^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-or-later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /* memcontrol.c - Memory Controller
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Copyright IBM Corporation, 2007
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Author Balbir Singh <balbir@linux.vnet.ibm.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Copyright 2007 OpenVZ SWsoft Inc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Author: Pavel Emelianov <xemul@openvz.org>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * Memory thresholds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * Copyright (C) 2009 Nokia Corporation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * Author: Kirill A. Shutemov
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * Kernel Memory Controller
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * Copyright (C) 2012 Parallels Inc. and Google Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * Authors: Glauber Costa and Suleiman Souhlal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * Native page reclaim
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * Charge lifetime sanitation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * Lockless page tracking & accounting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * Unified hierarchy configuration model
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * Copyright (C) 2015 Red Hat, Inc., Johannes Weiner
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/page_counter.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/memcontrol.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/cgroup.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <linux/pagewalk.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <linux/sched/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <linux/shmem_fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <linux/hugetlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <linux/pagemap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <linux/vm_event_item.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <linux/smp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <linux/page-flags.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <linux/backing-dev.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include <linux/bit_spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #include <linux/rcupdate.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #include <linux/limits.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #include <linux/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #include <linux/mutex.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #include <linux/rbtree.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #include <linux/swap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #include <linux/swapops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #include <linux/eventfd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #include <linux/poll.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #include <linux/sort.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #include <linux/fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #include <linux/seq_file.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #include <linux/vmpressure.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #include <linux/mm_inline.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #include <linux/swap_cgroup.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #include <linux/oom.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #include <linux/lockdep.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #include <linux/file.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #include <linux/tracehook.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #include <linux/psi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) #include <linux/seq_buf.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) #include "internal.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) #include <net/sock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) #include <net/ip.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) #include "slab.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) #include <trace/events/vmscan.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) #include <trace/hooks/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) struct cgroup_subsys memory_cgrp_subsys __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) EXPORT_SYMBOL(memory_cgrp_subsys);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) struct mem_cgroup *root_mem_cgroup __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) /* Active memory cgroup to use from an interrupt context */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) DEFINE_PER_CPU(struct mem_cgroup *, int_active_memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) /* Socket memory accounting disabled? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) static bool cgroup_memory_nosocket;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) /* Kernel memory accounting disabled? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) static bool cgroup_memory_nokmem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) /* Whether the swap controller is active */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) #ifdef CONFIG_MEMCG_SWAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) bool cgroup_memory_noswap __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) #define cgroup_memory_noswap 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) #ifdef CONFIG_CGROUP_WRITEBACK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) static DECLARE_WAIT_QUEUE_HEAD(memcg_cgwb_frn_waitq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) /* Whether legacy memory+swap accounting is active */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) static bool do_memsw_account(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) return !cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_noswap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) #define THRESHOLDS_EVENTS_TARGET 128
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) #define SOFTLIMIT_EVENTS_TARGET 1024
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) * Cgroups above their limits are maintained in a RB-Tree, independent of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) * their hierarchy representation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) struct mem_cgroup_tree_per_node {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) struct rb_root rb_root;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) struct rb_node *rb_rightmost;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) spinlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) struct mem_cgroup_tree {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) static struct mem_cgroup_tree soft_limit_tree __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) /* for OOM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) struct mem_cgroup_eventfd_list {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) struct list_head list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) struct eventfd_ctx *eventfd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) * cgroup_event represents events which userspace want to receive.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) struct mem_cgroup_event {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) * memcg which the event belongs to.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) struct mem_cgroup *memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) * eventfd to signal userspace about the event.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) struct eventfd_ctx *eventfd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) * Each of these stored in a list by the cgroup.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) struct list_head list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) * register_event() callback will be used to add new userspace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) * waiter for changes related to this event. Use eventfd_signal()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) * on eventfd to send notification to userspace.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) int (*register_event)(struct mem_cgroup *memcg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) struct eventfd_ctx *eventfd, const char *args);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) * unregister_event() callback will be called when userspace closes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) * the eventfd or on cgroup removing. This callback must be set,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) * if you want provide notification functionality.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) void (*unregister_event)(struct mem_cgroup *memcg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) struct eventfd_ctx *eventfd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) * All fields below needed to unregister event when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) * userspace closes eventfd.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) poll_table pt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) wait_queue_head_t *wqh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) wait_queue_entry_t wait;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) struct work_struct remove;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) static void mem_cgroup_threshold(struct mem_cgroup *memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) /* Stuffs for move charges at task migration. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) * Types of charges to be moved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) #define MOVE_ANON 0x1U
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) #define MOVE_FILE 0x2U
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) #define MOVE_MASK (MOVE_ANON | MOVE_FILE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) /* "mc" and its members are protected by cgroup_mutex */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) static struct move_charge_struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) spinlock_t lock; /* for from, to */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) struct mm_struct *mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) struct mem_cgroup *from;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) struct mem_cgroup *to;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) unsigned long precharge;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) unsigned long moved_charge;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) unsigned long moved_swap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) struct task_struct *moving_task; /* a task moving charges */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) wait_queue_head_t waitq; /* a waitq for other context */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) } mc = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) .lock = __SPIN_LOCK_UNLOCKED(mc.lock),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) * limit reclaim to prevent infinite loops, if they ever occur.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) #define MEM_CGROUP_MAX_RECLAIM_LOOPS 100
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) #define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) /* for encoding cft->private value on file */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) enum res_type {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) _MEM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) _MEMSWAP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) _OOM_TYPE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) _KMEM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) _TCP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) #define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) #define MEMFILE_TYPE(val) ((val) >> 16 & 0xffff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) #define MEMFILE_ATTR(val) ((val) & 0xffff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) /* Used for OOM nofiier */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) #define OOM_CONTROL (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) * Iteration constructs for visiting all cgroups (under a tree). If
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) * loops are exited prematurely (break), mem_cgroup_iter_break() must
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) * be used for reference counting.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) #define for_each_mem_cgroup_tree(iter, root) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) for (iter = mem_cgroup_iter(root, NULL, NULL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) iter != NULL; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) iter = mem_cgroup_iter(root, iter, NULL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) #define for_each_mem_cgroup(iter) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) for (iter = mem_cgroup_iter(NULL, NULL, NULL); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) iter != NULL; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) iter = mem_cgroup_iter(NULL, iter, NULL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) static inline bool task_is_dying(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) return tsk_is_oom_victim(current) || fatal_signal_pending(current) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) (current->flags & PF_EXITING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) /* Some nice accessors for the vmpressure. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) if (!memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) memcg = root_mem_cgroup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) return &memcg->vmpressure;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) #ifdef CONFIG_MEMCG_KMEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) static DEFINE_SPINLOCK(objcg_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) static void obj_cgroup_release(struct percpu_ref *ref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) struct obj_cgroup *objcg = container_of(ref, struct obj_cgroup, refcnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) struct mem_cgroup *memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) unsigned int nr_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) unsigned int nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) * At this point all allocated objects are freed, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) * objcg->nr_charged_bytes can't have an arbitrary byte value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) * However, it can be PAGE_SIZE or (x * PAGE_SIZE).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) * The following sequence can lead to it:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) * 1) CPU0: objcg == stock->cached_objcg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) * 2) CPU1: we do a small allocation (e.g. 92 bytes),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) * PAGE_SIZE bytes are charged
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) * 3) CPU1: a process from another memcg is allocating something,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) * the stock if flushed,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) * objcg->nr_charged_bytes = PAGE_SIZE - 92
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) * 5) CPU0: we do release this object,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) * 92 bytes are added to stock->nr_bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) * 6) CPU0: stock is flushed,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) * 92 bytes are added to objcg->nr_charged_bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) * In the result, nr_charged_bytes == PAGE_SIZE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) * This page will be uncharged in obj_cgroup_release().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) nr_bytes = atomic_read(&objcg->nr_charged_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) WARN_ON_ONCE(nr_bytes & (PAGE_SIZE - 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) nr_pages = nr_bytes >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) spin_lock_irqsave(&objcg_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) memcg = obj_cgroup_memcg(objcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) if (nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) __memcg_kmem_uncharge(memcg, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) list_del(&objcg->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) mem_cgroup_put(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) spin_unlock_irqrestore(&objcg_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) percpu_ref_exit(ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) kfree_rcu(objcg, rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) static struct obj_cgroup *obj_cgroup_alloc(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) struct obj_cgroup *objcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) objcg = kzalloc(sizeof(struct obj_cgroup), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) if (!objcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) ret = percpu_ref_init(&objcg->refcnt, obj_cgroup_release, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) kfree(objcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) INIT_LIST_HEAD(&objcg->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) return objcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) static void memcg_reparent_objcgs(struct mem_cgroup *memcg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) struct mem_cgroup *parent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) struct obj_cgroup *objcg, *iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) objcg = rcu_replace_pointer(memcg->objcg, NULL, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) spin_lock_irq(&objcg_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) /* Move active objcg to the parent's list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) xchg(&objcg->memcg, parent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) css_get(&parent->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) list_add(&objcg->list, &parent->objcg_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) /* Move already reparented objcgs to the parent's list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) list_for_each_entry(iter, &memcg->objcg_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) css_get(&parent->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) xchg(&iter->memcg, parent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) css_put(&memcg->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) list_splice(&memcg->objcg_list, &parent->objcg_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) spin_unlock_irq(&objcg_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) percpu_ref_kill(&objcg->refcnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) * This will be used as a shrinker list's index.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) * The main reason for not using cgroup id for this:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) * this works better in sparse environments, where we have a lot of memcgs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) * but only a few kmem-limited. Or also, if we have, for instance, 200
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) * memcgs, and none but the 200th is kmem-limited, we'd have to have a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) * 200 entry array for that.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) * The current size of the caches array is stored in memcg_nr_cache_ids. It
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) * will double each time we have to increase it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) static DEFINE_IDA(memcg_cache_ida);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) int memcg_nr_cache_ids;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) /* Protects memcg_nr_cache_ids */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) static DECLARE_RWSEM(memcg_cache_ids_sem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) void memcg_get_cache_ids(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) down_read(&memcg_cache_ids_sem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) void memcg_put_cache_ids(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) up_read(&memcg_cache_ids_sem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) * MIN_SIZE is different than 1, because we would like to avoid going through
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) * the alloc/free process all the time. In a small machine, 4 kmem-limited
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) * cgroups is a reasonable guess. In the future, it could be a parameter or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) * tunable, but that is strictly not necessary.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) * MAX_SIZE should be as large as the number of cgrp_ids. Ideally, we could get
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) * this constant directly from cgroup, but it is understandable that this is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) * better kept as an internal representation in cgroup.c. In any case, the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) * cgrp_id space is not getting any smaller, and we don't have to necessarily
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) * increase ours as well if it increases.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) #define MEMCG_CACHES_MIN_SIZE 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) #define MEMCG_CACHES_MAX_SIZE MEM_CGROUP_ID_MAX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) * A lot of the calls to the cache allocation functions are expected to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) * inlined by the compiler. Since the calls to memcg_slab_pre_alloc_hook() are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) * conditional to this static branch, we'll have to allow modules that does
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) * kmem_cache_alloc and the such to see this symbol as well
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) DEFINE_STATIC_KEY_FALSE(memcg_kmem_enabled_key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) EXPORT_SYMBOL(memcg_kmem_enabled_key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) static int memcg_shrinker_map_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) static DEFINE_MUTEX(memcg_shrinker_map_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) static void memcg_free_shrinker_map_rcu(struct rcu_head *head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) kvfree(container_of(head, struct memcg_shrinker_map, rcu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) static int memcg_expand_one_shrinker_map(struct mem_cgroup *memcg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) int size, int old_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) struct memcg_shrinker_map *new, *old;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) int nid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) lockdep_assert_held(&memcg_shrinker_map_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) for_each_node(nid) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) old = rcu_dereference_protected(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) mem_cgroup_nodeinfo(memcg, nid)->shrinker_map, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) /* Not yet online memcg */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) if (!old)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) new = kvmalloc_node(sizeof(*new) + size, GFP_KERNEL, nid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) if (!new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) /* Set all old bits, clear all new bits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) memset(new->map, (int)0xff, old_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) memset((void *)new->map + old_size, 0, size - old_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_map, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) call_rcu(&old->rcu, memcg_free_shrinker_map_rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) static void memcg_free_shrinker_maps(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) struct mem_cgroup_per_node *pn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) struct memcg_shrinker_map *map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) int nid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) if (mem_cgroup_is_root(memcg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) for_each_node(nid) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) pn = mem_cgroup_nodeinfo(memcg, nid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) map = rcu_dereference_protected(pn->shrinker_map, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) if (map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) kvfree(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) rcu_assign_pointer(pn->shrinker_map, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) static int memcg_alloc_shrinker_maps(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) struct memcg_shrinker_map *map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) int nid, size, ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) if (mem_cgroup_is_root(memcg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) mutex_lock(&memcg_shrinker_map_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) size = memcg_shrinker_map_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) for_each_node(nid) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) map = kvzalloc_node(sizeof(*map) + size, GFP_KERNEL, nid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) if (!map) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) memcg_free_shrinker_maps(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_map, map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) mutex_unlock(&memcg_shrinker_map_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) int memcg_expand_shrinker_maps(int new_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) int size, old_size, ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) struct mem_cgroup *memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) size = DIV_ROUND_UP(new_id + 1, BITS_PER_LONG) * sizeof(unsigned long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) old_size = memcg_shrinker_map_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) if (size <= old_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) mutex_lock(&memcg_shrinker_map_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) if (!root_mem_cgroup)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) for_each_mem_cgroup(memcg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) if (mem_cgroup_is_root(memcg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) ret = memcg_expand_one_shrinker_map(memcg, size, old_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) mem_cgroup_iter_break(NULL, memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) memcg_shrinker_map_size = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) mutex_unlock(&memcg_shrinker_map_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) void memcg_set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) if (shrinker_id >= 0 && memcg && !mem_cgroup_is_root(memcg)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) struct memcg_shrinker_map *map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) map = rcu_dereference(memcg->nodeinfo[nid]->shrinker_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) /* Pairs with smp mb in shrink_slab() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) smp_mb__before_atomic();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) set_bit(shrinker_id, map->map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) * mem_cgroup_css_from_page - css of the memcg associated with a page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) * @page: page of interest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) * If memcg is bound to the default hierarchy, css of the memcg associated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) * with @page is returned. The returned css remains associated with @page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) * until it is released.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) * is returned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) struct mem_cgroup *memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) memcg = page->mem_cgroup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) memcg = root_mem_cgroup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) return &memcg->css;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) * page_cgroup_ino - return inode number of the memcg a page is charged to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) * @page: the page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) * Look up the closest online ancestor of the memory cgroup @page is charged to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) * and return its inode number or 0 if @page is not charged to any cgroup. It
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) * is safe to call this function without holding a reference to @page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) * Note, this function is inherently racy, because there is nothing to prevent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) * the cgroup inode from getting torn down and potentially reallocated a moment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) * after page_cgroup_ino() returns, so it only should be used by callers that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) * do not care (such as procfs interfaces).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) ino_t page_cgroup_ino(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) struct mem_cgroup *memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) unsigned long ino = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) memcg = page->mem_cgroup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) * The lowest bit set means that memcg isn't a valid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) * memcg pointer, but a obj_cgroups pointer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) * In this case the page is shared and doesn't belong
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) * to any specific memory cgroup.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) if ((unsigned long) memcg & 0x1UL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) memcg = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) while (memcg && !(memcg->css.flags & CSS_ONLINE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) memcg = parent_mem_cgroup(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) if (memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) ino = cgroup_ino(memcg->css.cgroup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) return ino;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) static struct mem_cgroup_per_node *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) mem_cgroup_page_nodeinfo(struct mem_cgroup *memcg, struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) int nid = page_to_nid(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) return memcg->nodeinfo[nid];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) static struct mem_cgroup_tree_per_node *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) soft_limit_tree_node(int nid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) return soft_limit_tree.rb_tree_per_node[nid];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) static struct mem_cgroup_tree_per_node *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) soft_limit_tree_from_page(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) int nid = page_to_nid(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) return soft_limit_tree.rb_tree_per_node[nid];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_node *mz,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) struct mem_cgroup_tree_per_node *mctz,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) unsigned long new_usage_in_excess)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) struct rb_node **p = &mctz->rb_root.rb_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) struct rb_node *parent = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) struct mem_cgroup_per_node *mz_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) bool rightmost = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) if (mz->on_tree)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) mz->usage_in_excess = new_usage_in_excess;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) if (!mz->usage_in_excess)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) while (*p) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) parent = *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) mz_node = rb_entry(parent, struct mem_cgroup_per_node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) tree_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) if (mz->usage_in_excess < mz_node->usage_in_excess) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) p = &(*p)->rb_left;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) rightmost = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) * We can't avoid mem cgroups that are over their soft
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) * limit by the same amount
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) else if (mz->usage_in_excess >= mz_node->usage_in_excess)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) p = &(*p)->rb_right;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) if (rightmost)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) mctz->rb_rightmost = &mz->tree_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) rb_link_node(&mz->tree_node, parent, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) rb_insert_color(&mz->tree_node, &mctz->rb_root);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) mz->on_tree = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) struct mem_cgroup_tree_per_node *mctz)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) if (!mz->on_tree)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) if (&mz->tree_node == mctz->rb_rightmost)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) mctz->rb_rightmost = rb_prev(&mz->tree_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) rb_erase(&mz->tree_node, &mctz->rb_root);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) mz->on_tree = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) struct mem_cgroup_tree_per_node *mctz)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) spin_lock_irqsave(&mctz->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) __mem_cgroup_remove_exceeded(mz, mctz);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) spin_unlock_irqrestore(&mctz->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) static unsigned long soft_limit_excess(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) unsigned long nr_pages = page_counter_read(&memcg->memory);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) unsigned long soft_limit = READ_ONCE(memcg->soft_limit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) unsigned long excess = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) if (nr_pages > soft_limit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) excess = nr_pages - soft_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) return excess;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) unsigned long excess;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) struct mem_cgroup_per_node *mz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) struct mem_cgroup_tree_per_node *mctz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) mctz = soft_limit_tree_from_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) if (!mctz)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) * Necessary to update all ancestors when hierarchy is used.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) * because their event counter is not touched.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) for (; memcg; memcg = parent_mem_cgroup(memcg)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) mz = mem_cgroup_page_nodeinfo(memcg, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) excess = soft_limit_excess(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) * We have to update the tree if mz is on RB-tree or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) * mem is over its softlimit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) if (excess || mz->on_tree) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) spin_lock_irqsave(&mctz->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) /* if on-tree, remove it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) if (mz->on_tree)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) __mem_cgroup_remove_exceeded(mz, mctz);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) * Insert again. mz->usage_in_excess will be updated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) * If excess is 0, no tree ops.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) __mem_cgroup_insert_exceeded(mz, mctz, excess);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) spin_unlock_irqrestore(&mctz->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) struct mem_cgroup_tree_per_node *mctz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) struct mem_cgroup_per_node *mz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) int nid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) for_each_node(nid) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) mz = mem_cgroup_nodeinfo(memcg, nid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) mctz = soft_limit_tree_node(nid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) if (mctz)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) mem_cgroup_remove_exceeded(mz, mctz);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) static struct mem_cgroup_per_node *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) struct mem_cgroup_per_node *mz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) retry:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) mz = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) if (!mctz->rb_rightmost)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) goto done; /* Nothing to reclaim from */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) mz = rb_entry(mctz->rb_rightmost,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) struct mem_cgroup_per_node, tree_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) * Remove the node now but someone else can add it back,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) * we will to add it back at the end of reclaim to its correct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) * position in the tree.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) __mem_cgroup_remove_exceeded(mz, mctz);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) if (!soft_limit_excess(mz->memcg) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) !css_tryget(&mz->memcg->css))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) return mz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) static struct mem_cgroup_per_node *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) struct mem_cgroup_per_node *mz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) spin_lock_irq(&mctz->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) mz = __mem_cgroup_largest_soft_limit_node(mctz);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) spin_unlock_irq(&mctz->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) return mz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) * __mod_memcg_state - update cgroup memory statistics
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) * @memcg: the memory cgroup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) * @idx: the stat item - can be enum memcg_stat_item or enum node_stat_item
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) * @val: delta to add to the counter, can be negative
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) long x, threshold = MEMCG_CHARGE_BATCH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) if (mem_cgroup_disabled())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) if (memcg_stat_item_in_bytes(idx))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) threshold <<= PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) x = val + __this_cpu_read(memcg->vmstats_percpu->stat[idx]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) if (unlikely(abs(x) > threshold)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) struct mem_cgroup *mi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) * Batch local counters to keep them in sync with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) * the hierarchical ones.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) __this_cpu_add(memcg->vmstats_local->stat[idx], x);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) atomic_long_add(x, &mi->vmstats[idx]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) x = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) __this_cpu_write(memcg->vmstats_percpu->stat[idx], x);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) static struct mem_cgroup_per_node *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) parent_nodeinfo(struct mem_cgroup_per_node *pn, int nid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) struct mem_cgroup *parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) parent = parent_mem_cgroup(pn->memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) if (!parent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) return mem_cgroup_nodeinfo(parent, nid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) void __mod_memcg_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) int val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) struct mem_cgroup_per_node *pn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) struct mem_cgroup *memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) long x, threshold = MEMCG_CHARGE_BATCH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) memcg = pn->memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) /* Update memcg */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) __mod_memcg_state(memcg, idx, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) /* Update lruvec */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) __this_cpu_add(pn->lruvec_stat_local->count[idx], val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) if (vmstat_item_in_bytes(idx))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) threshold <<= PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) x = val + __this_cpu_read(pn->lruvec_stat_cpu->count[idx]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) if (unlikely(abs(x) > threshold)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) pg_data_t *pgdat = lruvec_pgdat(lruvec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) struct mem_cgroup_per_node *pi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) for (pi = pn; pi; pi = parent_nodeinfo(pi, pgdat->node_id))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) atomic_long_add(x, &pi->lruvec_stat[idx]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) x = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) __this_cpu_write(pn->lruvec_stat_cpu->count[idx], x);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) * __mod_lruvec_state - update lruvec memory statistics
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) * @lruvec: the lruvec
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) * @idx: the stat item
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) * @val: delta to add to the counter, can be negative
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) * The lruvec is the intersection of the NUMA node and a cgroup. This
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) * function updates the all three counters that are affected by a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) * change of state at this level: per-node, per-cgroup, per-lruvec.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) void __mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) int val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) /* Update node */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) __mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) /* Update memcg and lruvec */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) if (!mem_cgroup_disabled())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) __mod_memcg_lruvec_state(lruvec, idx, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) void __mod_lruvec_slab_state(void *p, enum node_stat_item idx, int val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) pg_data_t *pgdat = page_pgdat(virt_to_page(p));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) struct mem_cgroup *memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) struct lruvec *lruvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) memcg = mem_cgroup_from_obj(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) * Untracked pages have no memcg, no lruvec. Update only the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) * node. If we reparent the slab objects to the root memcg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) * when we free the slab object, we need to update the per-memcg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) * vmstats to keep it correct for the root memcg.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) if (!memcg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) __mod_node_page_state(pgdat, idx, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) lruvec = mem_cgroup_lruvec(memcg, pgdat);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) __mod_lruvec_state(lruvec, idx, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) void mod_memcg_obj_state(void *p, int idx, int val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) struct mem_cgroup *memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) memcg = mem_cgroup_from_obj(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) if (memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) mod_memcg_state(memcg, idx, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) * __count_memcg_events - account VM events in a cgroup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) * @memcg: the memory cgroup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) * @idx: the event item
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) * @count: the number of events that occured
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) unsigned long count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) unsigned long x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) if (mem_cgroup_disabled())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) x = count + __this_cpu_read(memcg->vmstats_percpu->events[idx]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) if (unlikely(x > MEMCG_CHARGE_BATCH)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) struct mem_cgroup *mi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) * Batch local counters to keep them in sync with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) * the hierarchical ones.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) __this_cpu_add(memcg->vmstats_local->events[idx], x);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) atomic_long_add(x, &mi->vmevents[idx]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) x = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) __this_cpu_write(memcg->vmstats_percpu->events[idx], x);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) static unsigned long memcg_events(struct mem_cgroup *memcg, int event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) return atomic_long_read(&memcg->vmevents[event]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) static unsigned long memcg_events_local(struct mem_cgroup *memcg, int event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) long x = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) for_each_possible_cpu(cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) x += per_cpu(memcg->vmstats_local->events[event], cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) return x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) int nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) /* pagein of a big page is an event. So, ignore page size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) if (nr_pages > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) __count_memcg_events(memcg, PGPGIN, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) __count_memcg_events(memcg, PGPGOUT, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) nr_pages = -nr_pages; /* for event */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) __this_cpu_add(memcg->vmstats_percpu->nr_page_events, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) enum mem_cgroup_events_target target)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) unsigned long val, next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) val = __this_cpu_read(memcg->vmstats_percpu->nr_page_events);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) next = __this_cpu_read(memcg->vmstats_percpu->targets[target]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) /* from time_after() in jiffies.h */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) if ((long)(next - val) < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) switch (target) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) case MEM_CGROUP_TARGET_THRESH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) next = val + THRESHOLDS_EVENTS_TARGET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) case MEM_CGROUP_TARGET_SOFTLIMIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) next = val + SOFTLIMIT_EVENTS_TARGET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) __this_cpu_write(memcg->vmstats_percpu->targets[target], next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) * Check events in order.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) /* threshold event is triggered in finer grain than soft limit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) if (unlikely(mem_cgroup_event_ratelimit(memcg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) MEM_CGROUP_TARGET_THRESH))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) bool do_softlimit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) do_softlimit = mem_cgroup_event_ratelimit(memcg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) MEM_CGROUP_TARGET_SOFTLIMIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) mem_cgroup_threshold(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) if (unlikely(do_softlimit))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) mem_cgroup_update_tree(memcg, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) * mm_update_next_owner() may clear mm->owner to NULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) * if it races with swapoff, page migration, etc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) * So this can be called with p == NULL.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) if (unlikely(!p))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) return mem_cgroup_from_css(task_css(p, memory_cgrp_id));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) EXPORT_SYMBOL(mem_cgroup_from_task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) * get_mem_cgroup_from_mm: Obtain a reference on given mm_struct's memcg.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) * @mm: mm from which memcg should be extracted. It can be NULL.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) * Obtain a reference on mm->memcg and returns it if successful. Otherwise
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) * root_mem_cgroup is returned. However if mem_cgroup is disabled, NULL is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) * returned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) struct mem_cgroup *memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) if (mem_cgroup_disabled())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) * Page cache insertions can happen withou an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) * actual mm context, e.g. during disk probing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) * on boot, loopback IO, acct() writes etc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) if (unlikely(!mm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) memcg = root_mem_cgroup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) if (unlikely(!memcg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) memcg = root_mem_cgroup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) } while (!css_tryget(&memcg->css));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) return memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) EXPORT_SYMBOL(get_mem_cgroup_from_mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) * get_mem_cgroup_from_page: Obtain a reference on given page's memcg.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) * @page: page from which memcg should be extracted.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) * Obtain a reference on page->memcg and returns it if successful. Otherwise
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) * root_mem_cgroup is returned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) struct mem_cgroup *get_mem_cgroup_from_page(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) struct mem_cgroup *memcg = page->mem_cgroup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) if (mem_cgroup_disabled())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) /* Page should not get uncharged and freed memcg under us. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) if (!memcg || WARN_ON_ONCE(!css_tryget(&memcg->css)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) memcg = root_mem_cgroup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) return memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) EXPORT_SYMBOL(get_mem_cgroup_from_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) static __always_inline struct mem_cgroup *active_memcg(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) if (in_interrupt())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) return this_cpu_read(int_active_memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) return current->active_memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) static __always_inline struct mem_cgroup *get_active_memcg(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) struct mem_cgroup *memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) memcg = active_memcg();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) /* remote memcg must hold a ref. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) if (memcg && WARN_ON_ONCE(!css_tryget(&memcg->css)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) memcg = root_mem_cgroup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) return memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) static __always_inline bool memcg_kmem_bypass(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) /* Allow remote memcg charging from any context. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) if (unlikely(active_memcg()))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) /* Memcg to charge can't be determined. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) if (in_interrupt() || !current->mm || (current->flags & PF_KTHREAD))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) * If active memcg is set, do not fallback to current->mm->memcg.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) static __always_inline struct mem_cgroup *get_mem_cgroup_from_current(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) if (memcg_kmem_bypass())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) if (unlikely(active_memcg()))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) return get_active_memcg();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) return get_mem_cgroup_from_mm(current->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) * mem_cgroup_iter - iterate over memory cgroup hierarchy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) * @root: hierarchy root
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) * @prev: previously returned memcg, NULL on first invocation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) * @reclaim: cookie for shared reclaim walks, NULL for full walks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) * Returns references to children of the hierarchy below @root, or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) * @root itself, or %NULL after a full round-trip.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) * Caller must pass the return value in @prev on subsequent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) * invocations for reference counting, or use mem_cgroup_iter_break()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) * to cancel a hierarchy walk before the round-trip is complete.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) * Reclaimers can specify a node in @reclaim to divide up the memcgs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) * in the hierarchy among all concurrent reclaimers operating on the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) * same node.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) struct mem_cgroup *prev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) struct mem_cgroup_reclaim_cookie *reclaim)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) struct mem_cgroup_reclaim_iter *iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) struct cgroup_subsys_state *css = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) struct mem_cgroup *memcg = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) struct mem_cgroup *pos = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) if (mem_cgroup_disabled())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) if (!root)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) root = root_mem_cgroup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) if (prev && !reclaim)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) pos = prev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) if (!root->use_hierarchy && root != root_mem_cgroup) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) if (prev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) return root;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) if (reclaim) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) struct mem_cgroup_per_node *mz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) mz = mem_cgroup_nodeinfo(root, reclaim->pgdat->node_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) iter = &mz->iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) if (prev && reclaim->generation != iter->generation)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) pos = READ_ONCE(iter->position);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) if (!pos || css_tryget(&pos->css))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) * css reference reached zero, so iter->position will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) * be cleared by ->css_released. However, we should not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) * rely on this happening soon, because ->css_released
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) * is called from a work queue, and by busy-waiting we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) * might block it. So we clear iter->position right
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) * away.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) (void)cmpxchg(&iter->position, pos, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) if (pos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) css = &pos->css;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) css = css_next_descendant_pre(css, &root->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) if (!css) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) * Reclaimers share the hierarchy walk, and a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) * new one might jump in right at the end of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) * the hierarchy - make sure they see at least
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) * one group and restart from the beginning.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) if (!prev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) * Verify the css and acquire a reference. The root
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) * is provided by the caller, so we know it's alive
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) * and kicking, and don't take an extra reference.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) memcg = mem_cgroup_from_css(css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) if (css == &root->css)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) if (css_tryget(css))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) memcg = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) if (reclaim) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) * The position could have already been updated by a competing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) * thread, so check that the value hasn't changed since we read
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) * it to avoid reclaiming from the same cgroup twice.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) (void)cmpxchg(&iter->position, pos, memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) if (pos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) css_put(&pos->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) if (!memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) iter->generation++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) else if (!prev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) reclaim->generation = iter->generation;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) out_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) if (prev && prev != root)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) css_put(&prev->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) return memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) * mem_cgroup_iter_break - abort a hierarchy walk prematurely
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) * @root: hierarchy root
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) void mem_cgroup_iter_break(struct mem_cgroup *root,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) struct mem_cgroup *prev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) if (!root)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) root = root_mem_cgroup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) if (prev && prev != root)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) css_put(&prev->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) static void __invalidate_reclaim_iterators(struct mem_cgroup *from,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) struct mem_cgroup *dead_memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) struct mem_cgroup_reclaim_iter *iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) struct mem_cgroup_per_node *mz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) int nid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) for_each_node(nid) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) mz = mem_cgroup_nodeinfo(from, nid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) iter = &mz->iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) cmpxchg(&iter->position, dead_memcg, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) struct mem_cgroup *memcg = dead_memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) struct mem_cgroup *last;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) __invalidate_reclaim_iterators(memcg, dead_memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) last = memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) } while ((memcg = parent_mem_cgroup(memcg)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) * When cgruop1 non-hierarchy mode is used,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) * parent_mem_cgroup() does not walk all the way up to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) * cgroup root (root_mem_cgroup). So we have to handle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) * dead_memcg from cgroup root separately.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) if (last != root_mem_cgroup)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) __invalidate_reclaim_iterators(root_mem_cgroup,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) dead_memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) * mem_cgroup_scan_tasks - iterate over tasks of a memory cgroup hierarchy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) * @memcg: hierarchy root
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) * @fn: function to call for each task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) * @arg: argument passed to @fn
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) * This function iterates over tasks attached to @memcg or to any of its
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) * descendants and calls @fn for each task. If @fn returns a non-zero
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) * value, the function breaks the iteration loop and returns the value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) * Otherwise, it will iterate over all tasks and return 0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) * This function must not be called for the root memory cgroup.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) int (*fn)(struct task_struct *, void *), void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) struct mem_cgroup *iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) BUG_ON(memcg == root_mem_cgroup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) for_each_mem_cgroup_tree(iter, memcg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) struct css_task_iter it;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) struct task_struct *task;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) css_task_iter_start(&iter->css, CSS_TASK_ITER_PROCS, &it);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) while (!ret && (task = css_task_iter_next(&it)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) ret = fn(task, arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) css_task_iter_end(&it);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) mem_cgroup_iter_break(memcg, iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) * mem_cgroup_page_lruvec - return lruvec for isolating/putting an LRU page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) * @page: the page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) * @pgdat: pgdat of the page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) * This function relies on page->mem_cgroup being stable - see the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) * access rules in commit_charge().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct pglist_data *pgdat)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) struct mem_cgroup_per_node *mz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) struct mem_cgroup *memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) struct lruvec *lruvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) if (mem_cgroup_disabled()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) lruvec = &pgdat->__lruvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) memcg = page->mem_cgroup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) * Swapcache readahead pages are added to the LRU - and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) * possibly migrated - before they are charged.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) if (!memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) memcg = root_mem_cgroup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) mz = mem_cgroup_page_nodeinfo(memcg, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) lruvec = &mz->lruvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) * Since a node can be onlined after the mem_cgroup was created,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) * we have to be prepared to initialize lruvec->zone here;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) * and if offlined then reonlined, we need to reinitialize it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) if (unlikely(lruvec->pgdat != pgdat))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) lruvec->pgdat = pgdat;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) return lruvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) * mem_cgroup_update_lru_size - account for adding or removing an lru page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) * @lruvec: mem_cgroup per zone lru vector
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) * @lru: index of lru list the page is sitting on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) * @zid: zone id of the accounted pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) * @nr_pages: positive when adding or negative when removing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) * This function must be called under lru_lock, just before a page is added
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) * to or just after a page is removed from an lru list (that ordering being
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) * so as to allow it to check that lru_size 0 is consistent with list_empty).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) int zid, int nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) struct mem_cgroup_per_node *mz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) unsigned long *lru_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) long size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) if (mem_cgroup_disabled())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) lru_size = &mz->lru_zone_size[zid][lru];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) if (nr_pages < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) *lru_size += nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) size = *lru_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) if (WARN_ONCE(size < 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) "%s(%p, %d, %d): lru_size %ld\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) __func__, lruvec, lru, nr_pages, size)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) VM_BUG_ON(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) *lru_size = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) if (nr_pages > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) *lru_size += nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) * mem_cgroup_margin - calculate chargeable space of a memory cgroup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) * @memcg: the memory cgroup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) * Returns the maximum amount of memory @mem can be charged with, in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) * pages.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) unsigned long margin = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) unsigned long count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) unsigned long limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) count = page_counter_read(&memcg->memory);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) limit = READ_ONCE(memcg->memory.max);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) if (count < limit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) margin = limit - count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) if (do_memsw_account()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) count = page_counter_read(&memcg->memsw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) limit = READ_ONCE(memcg->memsw.max);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) if (count < limit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) margin = min(margin, limit - count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) margin = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) return margin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) * A routine for checking "mem" is under move_account() or not.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) * Checking a cgroup is mc.from or mc.to or under hierarchy of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) * moving cgroups. This is for waiting at high-memory pressure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) * caused by "move".
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) struct mem_cgroup *from;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) struct mem_cgroup *to;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) bool ret = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) * Unlike task_move routines, we access mc.to, mc.from not under
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) spin_lock(&mc.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) from = mc.from;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) to = mc.to;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) if (!from)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) ret = mem_cgroup_is_descendant(from, memcg) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) mem_cgroup_is_descendant(to, memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) spin_unlock(&mc.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) if (mc.moving_task && current != mc.moving_task) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) if (mem_cgroup_under_move(memcg)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) DEFINE_WAIT(wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) /* moving charge context might have finished. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) if (mc.moving_task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) schedule();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) finish_wait(&mc.waitq, &wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) struct memory_stat {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) const char *name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) unsigned int ratio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) unsigned int idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) static struct memory_stat memory_stats[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) { "anon", PAGE_SIZE, NR_ANON_MAPPED },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) { "file", PAGE_SIZE, NR_FILE_PAGES },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) { "kernel_stack", 1024, NR_KERNEL_STACK_KB },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) { "percpu", 1, MEMCG_PERCPU_B },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) { "sock", PAGE_SIZE, MEMCG_SOCK },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) { "shmem", PAGE_SIZE, NR_SHMEM },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) { "file_mapped", PAGE_SIZE, NR_FILE_MAPPED },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) { "file_dirty", PAGE_SIZE, NR_FILE_DIRTY },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) { "file_writeback", PAGE_SIZE, NR_WRITEBACK },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) #ifdef CONFIG_TRANSPARENT_HUGEPAGE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) * The ratio will be initialized in memory_stats_init(). Because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) * on some architectures, the macro of HPAGE_PMD_SIZE is not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) * constant(e.g. powerpc).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) { "anon_thp", 0, NR_ANON_THPS },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) { "inactive_anon", PAGE_SIZE, NR_INACTIVE_ANON },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) { "active_anon", PAGE_SIZE, NR_ACTIVE_ANON },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) { "inactive_file", PAGE_SIZE, NR_INACTIVE_FILE },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) { "active_file", PAGE_SIZE, NR_ACTIVE_FILE },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) { "unevictable", PAGE_SIZE, NR_UNEVICTABLE },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) * Note: The slab_reclaimable and slab_unreclaimable must be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) * together and slab_reclaimable must be in front.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) { "slab_reclaimable", 1, NR_SLAB_RECLAIMABLE_B },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) { "slab_unreclaimable", 1, NR_SLAB_UNRECLAIMABLE_B },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) /* The memory events */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) { "workingset_refault_anon", 1, WORKINGSET_REFAULT_ANON },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) { "workingset_refault_file", 1, WORKINGSET_REFAULT_FILE },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) { "workingset_activate_anon", 1, WORKINGSET_ACTIVATE_ANON },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) { "workingset_activate_file", 1, WORKINGSET_ACTIVATE_FILE },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) { "workingset_restore_anon", 1, WORKINGSET_RESTORE_ANON },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) { "workingset_restore_file", 1, WORKINGSET_RESTORE_FILE },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) { "workingset_nodereclaim", 1, WORKINGSET_NODERECLAIM },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) static int __init memory_stats_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) for (i = 0; i < ARRAY_SIZE(memory_stats); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) #ifdef CONFIG_TRANSPARENT_HUGEPAGE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) if (memory_stats[i].idx == NR_ANON_THPS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) memory_stats[i].ratio = HPAGE_PMD_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) VM_BUG_ON(!memory_stats[i].ratio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) VM_BUG_ON(memory_stats[i].idx >= MEMCG_NR_STAT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) pure_initcall(memory_stats_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) static char *memory_stat_format(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) struct seq_buf s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) seq_buf_init(&s, kmalloc(PAGE_SIZE, GFP_KERNEL), PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) if (!s.buffer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) * Provide statistics on the state of the memory subsystem as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) * well as cumulative event counters that show past behavior.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) * This list is ordered following a combination of these gradients:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) * 1) generic big picture -> specifics and details
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) * 2) reflecting userspace activity -> reflecting kernel heuristics
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) * Current memory state:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) for (i = 0; i < ARRAY_SIZE(memory_stats); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) u64 size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) size = memcg_page_state(memcg, memory_stats[i].idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) size *= memory_stats[i].ratio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) seq_buf_printf(&s, "%s %llu\n", memory_stats[i].name, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) if (unlikely(memory_stats[i].idx == NR_SLAB_UNRECLAIMABLE_B)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) size = memcg_page_state(memcg, NR_SLAB_RECLAIMABLE_B) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) memcg_page_state(memcg, NR_SLAB_UNRECLAIMABLE_B);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) seq_buf_printf(&s, "slab %llu\n", size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) /* Accumulated memory events */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGFAULT),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) memcg_events(memcg, PGFAULT));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGMAJFAULT),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) memcg_events(memcg, PGMAJFAULT));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGREFILL),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) memcg_events(memcg, PGREFILL));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) seq_buf_printf(&s, "pgscan %lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) memcg_events(memcg, PGSCAN_KSWAPD) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) memcg_events(memcg, PGSCAN_DIRECT));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) seq_buf_printf(&s, "pgsteal %lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) memcg_events(memcg, PGSTEAL_KSWAPD) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) memcg_events(memcg, PGSTEAL_DIRECT));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGACTIVATE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) memcg_events(memcg, PGACTIVATE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGDEACTIVATE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) memcg_events(memcg, PGDEACTIVATE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGLAZYFREE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) memcg_events(memcg, PGLAZYFREE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGLAZYFREED),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) memcg_events(memcg, PGLAZYFREED));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) #ifdef CONFIG_TRANSPARENT_HUGEPAGE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) seq_buf_printf(&s, "%s %lu\n", vm_event_name(THP_FAULT_ALLOC),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) memcg_events(memcg, THP_FAULT_ALLOC));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) seq_buf_printf(&s, "%s %lu\n", vm_event_name(THP_COLLAPSE_ALLOC),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) memcg_events(memcg, THP_COLLAPSE_ALLOC));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) /* The above should easily fit into one page */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) WARN_ON_ONCE(seq_buf_has_overflowed(&s));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) return s.buffer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) #define K(x) ((x) << (PAGE_SHIFT-10))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) * mem_cgroup_print_oom_context: Print OOM information relevant to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) * memory controller.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) * @memcg: The memory cgroup that went over limit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) * @p: Task that is going to be killed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) * enabled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) void mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) if (memcg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) pr_cont(",oom_memcg=");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) pr_cont_cgroup_path(memcg->css.cgroup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) pr_cont(",global_oom");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) if (p) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) pr_cont(",task_memcg=");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) pr_cont_cgroup_path(task_cgroup(p, memory_cgrp_id));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) * mem_cgroup_print_oom_meminfo: Print OOM memory information relevant to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) * memory controller.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) * @memcg: The memory cgroup that went over limit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) char *buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) K((u64)page_counter_read(&memcg->memory)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) K((u64)READ_ONCE(memcg->memory.max)), memcg->memory.failcnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) pr_info("swap: usage %llukB, limit %llukB, failcnt %lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) K((u64)page_counter_read(&memcg->swap)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) K((u64)READ_ONCE(memcg->swap.max)), memcg->swap.failcnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) K((u64)page_counter_read(&memcg->memsw)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) K((u64)memcg->memsw.max), memcg->memsw.failcnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) pr_info("kmem: usage %llukB, limit %llukB, failcnt %lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) K((u64)page_counter_read(&memcg->kmem)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) K((u64)memcg->kmem.max), memcg->kmem.failcnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) pr_info("Memory cgroup stats for ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) pr_cont_cgroup_path(memcg->css.cgroup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) pr_cont(":");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) buf = memory_stat_format(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) if (!buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) pr_info("%s", buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) kfree(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) * Return the memory (and swap, if configured) limit for a memcg.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689) unsigned long max = READ_ONCE(memcg->memory.max);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) if (mem_cgroup_swappiness(memcg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) max += min(READ_ONCE(memcg->swap.max),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) (unsigned long)total_swap_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) } else { /* v1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) if (mem_cgroup_swappiness(memcg)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) /* Calculate swap excess capacity from memsw limit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) unsigned long swap = READ_ONCE(memcg->memsw.max) - max;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700) max += min(swap, (unsigned long)total_swap_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) return max;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) unsigned long mem_cgroup_size(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) return page_counter_read(&memcg->memory);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712) int order)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) struct oom_control oc = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) .zonelist = NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716) .nodemask = NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) .memcg = memcg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) .gfp_mask = gfp_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719) .order = order,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721) bool ret = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) if (mutex_lock_killable(&oom_lock))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) if (mem_cgroup_margin(memcg) >= (1 << order))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730) * A few threads which were not waiting at mutex_lock_killable() can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) * fail to bail out. Therefore, check again after holding oom_lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733) ret = task_is_dying() || out_of_memory(&oc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) mutex_unlock(&oom_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740) static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) pg_data_t *pgdat,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742) gfp_t gfp_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) unsigned long *total_scanned)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) struct mem_cgroup *victim = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746) int total = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747) int loop = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) unsigned long excess;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749) unsigned long nr_scanned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) struct mem_cgroup_reclaim_cookie reclaim = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751) .pgdat = pgdat,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754) excess = soft_limit_excess(root_memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757) victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758) if (!victim) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759) loop++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760) if (loop >= 2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762) * If we have not been able to reclaim
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) * anything, it might because there are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764) * no reclaimable pages under this hierarchy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766) if (!total)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) * We want to do more targeted reclaim.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770) * excess >> 2 is not to excessive so as to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) * reclaim too much, nor too less that we keep
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772) * coming back to reclaim from this cgroup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774) if (total >= (excess >> 2) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775) (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780) total += mem_cgroup_shrink_node(victim, gfp_mask, false,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781) pgdat, &nr_scanned);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782) *total_scanned += nr_scanned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783) if (!soft_limit_excess(root_memcg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786) mem_cgroup_iter_break(root_memcg, victim);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787) return total;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790) #ifdef CONFIG_LOCKDEP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791) static struct lockdep_map memcg_oom_lock_dep_map = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792) .name = "memcg_oom_lock",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796) static DEFINE_SPINLOCK(memcg_oom_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799) * Check OOM-Killer is already running under our hierarchy.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800) * If someone is running, return false.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802) static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804) struct mem_cgroup *iter, *failed = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806) spin_lock(&memcg_oom_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808) for_each_mem_cgroup_tree(iter, memcg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809) if (iter->oom_lock) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811) * this subtree of our hierarchy is already locked
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812) * so we cannot give a lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814) failed = iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815) mem_cgroup_iter_break(memcg, iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818) iter->oom_lock = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821) if (failed) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823) * OK, we failed to lock the whole subtree so we have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824) * to clean up what we set up to the failing subtree
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826) for_each_mem_cgroup_tree(iter, memcg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827) if (iter == failed) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828) mem_cgroup_iter_break(memcg, iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831) iter->oom_lock = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834) mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836) spin_unlock(&memcg_oom_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838) return !failed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841) static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843) struct mem_cgroup *iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845) spin_lock(&memcg_oom_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846) mutex_release(&memcg_oom_lock_dep_map, _RET_IP_);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847) for_each_mem_cgroup_tree(iter, memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848) iter->oom_lock = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849) spin_unlock(&memcg_oom_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852) static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854) struct mem_cgroup *iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856) spin_lock(&memcg_oom_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857) for_each_mem_cgroup_tree(iter, memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858) iter->under_oom++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1859) spin_unlock(&memcg_oom_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1860) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1861)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1862) static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1863) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1864) struct mem_cgroup *iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1865)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1866) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1867) * Be careful about under_oom underflows becase a child memcg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1868) * could have been added after mem_cgroup_mark_under_oom.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1869) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1870) spin_lock(&memcg_oom_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1871) for_each_mem_cgroup_tree(iter, memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1872) if (iter->under_oom > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1873) iter->under_oom--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1874) spin_unlock(&memcg_oom_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1875) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1876)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1877) static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1878)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1879) struct oom_wait_info {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1880) struct mem_cgroup *memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1881) wait_queue_entry_t wait;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1882) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1883)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1884) static int memcg_oom_wake_function(wait_queue_entry_t *wait,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1885) unsigned mode, int sync, void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1886) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1887) struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1888) struct mem_cgroup *oom_wait_memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1889) struct oom_wait_info *oom_wait_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1890)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1891) oom_wait_info = container_of(wait, struct oom_wait_info, wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1892) oom_wait_memcg = oom_wait_info->memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1893)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1894) if (!mem_cgroup_is_descendant(wake_memcg, oom_wait_memcg) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1895) !mem_cgroup_is_descendant(oom_wait_memcg, wake_memcg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1896) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1897) return autoremove_wake_function(wait, mode, sync, arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1898) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1899)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1900) static void memcg_oom_recover(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1901) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1902) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1903) * For the following lockless ->under_oom test, the only required
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1904) * guarantee is that it must see the state asserted by an OOM when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1905) * this function is called as a result of userland actions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1906) * triggered by the notification of the OOM. This is trivially
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1907) * achieved by invoking mem_cgroup_mark_under_oom() before
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1908) * triggering notification.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1909) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1910) if (memcg && memcg->under_oom)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1911) __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1912) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1913)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1914) enum oom_status {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1915) OOM_SUCCESS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1916) OOM_FAILED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1917) OOM_ASYNC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1918) OOM_SKIPPED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1919) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1920)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1921) static enum oom_status mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1922) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1923) enum oom_status ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1924) bool locked;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1925)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1926) if (order > PAGE_ALLOC_COSTLY_ORDER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1927) return OOM_SKIPPED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1928)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1929) memcg_memory_event(memcg, MEMCG_OOM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1930)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1931) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1932) * We are in the middle of the charge context here, so we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1933) * don't want to block when potentially sitting on a callstack
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1934) * that holds all kinds of filesystem and mm locks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1935) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1936) * cgroup1 allows disabling the OOM killer and waiting for outside
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1937) * handling until the charge can succeed; remember the context and put
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1938) * the task to sleep at the end of the page fault when all locks are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1939) * released.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1940) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1941) * On the other hand, in-kernel OOM killer allows for an async victim
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1942) * memory reclaim (oom_reaper) and that means that we are not solely
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1943) * relying on the oom victim to make a forward progress and we can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1944) * invoke the oom killer here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1945) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1946) * Please note that mem_cgroup_out_of_memory might fail to find a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1947) * victim and then we have to bail out from the charge path.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1948) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1949) if (memcg->oom_kill_disable) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1950) if (!current->in_user_fault)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1951) return OOM_SKIPPED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1952) css_get(&memcg->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1953) current->memcg_in_oom = memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1954) current->memcg_oom_gfp_mask = mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1955) current->memcg_oom_order = order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1956)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1957) return OOM_ASYNC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1958) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1959)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1960) mem_cgroup_mark_under_oom(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1961)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1962) locked = mem_cgroup_oom_trylock(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1963)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1964) if (locked)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1965) mem_cgroup_oom_notify(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1966)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1967) mem_cgroup_unmark_under_oom(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1968) if (mem_cgroup_out_of_memory(memcg, mask, order))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1969) ret = OOM_SUCCESS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1970) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1971) ret = OOM_FAILED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1972)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1973) if (locked)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1974) mem_cgroup_oom_unlock(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1975)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1976) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1977) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1978)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1979) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1980) * mem_cgroup_oom_synchronize - complete memcg OOM handling
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1981) * @handle: actually kill/wait or just clean up the OOM state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1982) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1983) * This has to be called at the end of a page fault if the memcg OOM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1984) * handler was enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1985) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1986) * Memcg supports userspace OOM handling where failed allocations must
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1987) * sleep on a waitqueue until the userspace task resolves the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1988) * situation. Sleeping directly in the charge context with all kinds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1989) * of locks held is not a good idea, instead we remember an OOM state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1990) * in the task and mem_cgroup_oom_synchronize() has to be called at
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1991) * the end of the page fault to complete the OOM handling.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1992) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1993) * Returns %true if an ongoing memcg OOM situation was detected and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1994) * completed, %false otherwise.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1995) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1996) bool mem_cgroup_oom_synchronize(bool handle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1997) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1998) struct mem_cgroup *memcg = current->memcg_in_oom;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1999) struct oom_wait_info owait;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2000) bool locked;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2001)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2002) /* OOM is global, do not handle */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2003) if (!memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2004) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2005)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2006) if (!handle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2007) goto cleanup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2008)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2009) owait.memcg = memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2010) owait.wait.flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2011) owait.wait.func = memcg_oom_wake_function;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2012) owait.wait.private = current;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2013) INIT_LIST_HEAD(&owait.wait.entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2014)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2015) prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2016) mem_cgroup_mark_under_oom(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2017)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2018) locked = mem_cgroup_oom_trylock(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2019)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2020) if (locked)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2021) mem_cgroup_oom_notify(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2022)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2023) if (locked && !memcg->oom_kill_disable) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2024) mem_cgroup_unmark_under_oom(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2025) finish_wait(&memcg_oom_waitq, &owait.wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2026) mem_cgroup_out_of_memory(memcg, current->memcg_oom_gfp_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2027) current->memcg_oom_order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2028) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2029) schedule();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2030) mem_cgroup_unmark_under_oom(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2031) finish_wait(&memcg_oom_waitq, &owait.wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2032) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2033)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2034) if (locked) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2035) mem_cgroup_oom_unlock(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2036) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2037) * There is no guarantee that an OOM-lock contender
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2038) * sees the wakeups triggered by the OOM kill
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2039) * uncharges. Wake any sleepers explicitely.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2040) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2041) memcg_oom_recover(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2042) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2043) cleanup:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2044) current->memcg_in_oom = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2045) css_put(&memcg->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2046) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2047) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2048)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2049) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2050) * mem_cgroup_get_oom_group - get a memory cgroup to clean up after OOM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2051) * @victim: task to be killed by the OOM killer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2052) * @oom_domain: memcg in case of memcg OOM, NULL in case of system-wide OOM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2053) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2054) * Returns a pointer to a memory cgroup, which has to be cleaned up
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2055) * by killing all belonging OOM-killable tasks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2056) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2057) * Caller has to call mem_cgroup_put() on the returned non-NULL memcg.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2058) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2059) struct mem_cgroup *mem_cgroup_get_oom_group(struct task_struct *victim,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2060) struct mem_cgroup *oom_domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2061) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2062) struct mem_cgroup *oom_group = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2063) struct mem_cgroup *memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2064)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2065) if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2066) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2067)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2068) if (!oom_domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2069) oom_domain = root_mem_cgroup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2070)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2071) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2072)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2073) memcg = mem_cgroup_from_task(victim);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2074) if (memcg == root_mem_cgroup)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2075) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2076)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2077) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2078) * If the victim task has been asynchronously moved to a different
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2079) * memory cgroup, we might end up killing tasks outside oom_domain.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2080) * In this case it's better to ignore memory.group.oom.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2081) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2082) if (unlikely(!mem_cgroup_is_descendant(memcg, oom_domain)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2083) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2084)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2085) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2086) * Traverse the memory cgroup hierarchy from the victim task's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2087) * cgroup up to the OOMing cgroup (or root) to find the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2088) * highest-level memory cgroup with oom.group set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2089) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2090) for (; memcg; memcg = parent_mem_cgroup(memcg)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2091) if (memcg->oom_group)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2092) oom_group = memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2093)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2094) if (memcg == oom_domain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2095) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2096) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2097)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2098) if (oom_group)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2099) css_get(&oom_group->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2100) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2101) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2103) return oom_group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2104) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2106) void mem_cgroup_print_oom_group(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2107) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2108) pr_info("Tasks in ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2109) pr_cont_cgroup_path(memcg->css.cgroup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2110) pr_cont(" are going to be killed due to memory.oom.group set\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2111) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2113) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2114) * lock_page_memcg - lock a page->mem_cgroup binding
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2115) * @page: the page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2116) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2117) * This function protects unlocked LRU pages from being moved to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2118) * another cgroup.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2119) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2120) * It ensures lifetime of the returned memcg. Caller is responsible
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2121) * for the lifetime of the page; __unlock_page_memcg() is available
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2122) * when @page might get freed inside the locked section.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2123) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2124) struct mem_cgroup *lock_page_memcg(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2125) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2126) struct page *head = compound_head(page); /* rmap on tail pages */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2127) struct mem_cgroup *memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2128) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2130) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2131) * The RCU lock is held throughout the transaction. The fast
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2132) * path can get away without acquiring the memcg->move_lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2133) * because page moving starts with an RCU grace period.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2134) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2135) * The RCU lock also protects the memcg from being freed when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2136) * the page state that is going to change is the only thing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2137) * preventing the page itself from being freed. E.g. writeback
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2138) * doesn't hold a page reference and relies on PG_writeback to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2139) * keep off truncation, migration and so forth.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2140) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2141) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2143) if (mem_cgroup_disabled())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2144) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2145) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2146) memcg = head->mem_cgroup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2147) if (unlikely(!memcg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2148) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2150) if (atomic_read(&memcg->moving_account) <= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2151) return memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2153) spin_lock_irqsave(&memcg->move_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2154) if (memcg != head->mem_cgroup) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2155) spin_unlock_irqrestore(&memcg->move_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2156) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2157) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2159) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2160) * When charge migration first begins, we can have locked and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2161) * unlocked page stat updates happening concurrently. Track
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2162) * the task who has the lock for unlock_page_memcg().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2163) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2164) memcg->move_lock_task = current;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2165) memcg->move_lock_flags = flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2167) return memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2168) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2169) EXPORT_SYMBOL(lock_page_memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2171) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2172) * __unlock_page_memcg - unlock and unpin a memcg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2173) * @memcg: the memcg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2174) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2175) * Unlock and unpin a memcg returned by lock_page_memcg().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2176) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2177) void __unlock_page_memcg(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2178) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2179) if (memcg && memcg->move_lock_task == current) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2180) unsigned long flags = memcg->move_lock_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2182) memcg->move_lock_task = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2183) memcg->move_lock_flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2185) spin_unlock_irqrestore(&memcg->move_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2186) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2188) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2189) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2191) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2192) * unlock_page_memcg - unlock a page->mem_cgroup binding
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2193) * @page: the page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2194) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2195) void unlock_page_memcg(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2196) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2197) struct page *head = compound_head(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2199) __unlock_page_memcg(head->mem_cgroup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2200) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2201) EXPORT_SYMBOL(unlock_page_memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2203) struct memcg_stock_pcp {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2204) struct mem_cgroup *cached; /* this never be root cgroup */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2205) unsigned int nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2207) #ifdef CONFIG_MEMCG_KMEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2208) struct obj_cgroup *cached_objcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2209) unsigned int nr_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2210) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2212) struct work_struct work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2213) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2214) #define FLUSHING_CACHED_CHARGE 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2215) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2216) static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2217) static DEFINE_MUTEX(percpu_charge_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2219) #ifdef CONFIG_MEMCG_KMEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2220) static void drain_obj_stock(struct memcg_stock_pcp *stock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2221) static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2222) struct mem_cgroup *root_memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2224) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2225) static inline void drain_obj_stock(struct memcg_stock_pcp *stock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2226) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2227) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2228) static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2229) struct mem_cgroup *root_memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2230) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2231) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2232) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2233) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2235) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2236) * consume_stock: Try to consume stocked charge on this cpu.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2237) * @memcg: memcg to consume from.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2238) * @nr_pages: how many pages to charge.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2239) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2240) * The charges will only happen if @memcg matches the current cpu's memcg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2241) * stock, and at least @nr_pages are available in that stock. Failure to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2242) * service an allocation will refill the stock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2243) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2244) * returns true if successful, false otherwise.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2245) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2246) static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2247) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2248) struct memcg_stock_pcp *stock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2249) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2250) bool ret = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2252) if (nr_pages > MEMCG_CHARGE_BATCH)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2253) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2255) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2257) stock = this_cpu_ptr(&memcg_stock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2258) if (memcg == stock->cached && stock->nr_pages >= nr_pages) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2259) stock->nr_pages -= nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2260) ret = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2261) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2263) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2265) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2266) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2268) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2269) * Returns stocks cached in percpu and reset cached information.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2270) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2271) static void drain_stock(struct memcg_stock_pcp *stock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2272) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2273) struct mem_cgroup *old = stock->cached;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2275) if (!old)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2276) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2278) if (stock->nr_pages) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2279) page_counter_uncharge(&old->memory, stock->nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2280) if (do_memsw_account())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2281) page_counter_uncharge(&old->memsw, stock->nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2282) stock->nr_pages = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2283) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2285) css_put(&old->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2286) stock->cached = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2287) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2289) static void drain_local_stock(struct work_struct *dummy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2290) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2291) struct memcg_stock_pcp *stock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2292) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2294) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2295) * The only protection from memory hotplug vs. drain_stock races is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2296) * that we always operate on local CPU stock here with IRQ disabled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2297) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2298) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2300) stock = this_cpu_ptr(&memcg_stock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2301) drain_obj_stock(stock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2302) drain_stock(stock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2303) clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2305) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2306) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2308) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2309) * Cache charges(val) to local per_cpu area.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2310) * This will be consumed by consume_stock() function, later.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2311) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2312) static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2313) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2314) struct memcg_stock_pcp *stock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2315) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2317) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2319) stock = this_cpu_ptr(&memcg_stock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2320) if (stock->cached != memcg) { /* reset if necessary */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2321) drain_stock(stock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2322) css_get(&memcg->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2323) stock->cached = memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2324) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2325) stock->nr_pages += nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2327) if (stock->nr_pages > MEMCG_CHARGE_BATCH)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2328) drain_stock(stock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2330) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2331) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2333) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2334) * Drains all per-CPU charge caches for given root_memcg resp. subtree
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2335) * of the hierarchy under it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2336) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2337) static void drain_all_stock(struct mem_cgroup *root_memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2338) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2339) int cpu, curcpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2341) /* If someone's already draining, avoid adding running more workers. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2342) if (!mutex_trylock(&percpu_charge_mutex))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2343) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2344) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2345) * Notify other cpus that system-wide "drain" is running
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2346) * We do not care about races with the cpu hotplug because cpu down
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2347) * as well as workers from this path always operate on the local
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2348) * per-cpu data. CPU up doesn't touch memcg_stock at all.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2349) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2350) curcpu = get_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2351) for_each_online_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2352) struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2353) struct mem_cgroup *memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2354) bool flush = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2356) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2357) memcg = stock->cached;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2358) if (memcg && stock->nr_pages &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2359) mem_cgroup_is_descendant(memcg, root_memcg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2360) flush = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2361) if (obj_stock_flush_required(stock, root_memcg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2362) flush = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2363) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2364)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2365) if (flush &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2366) !test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2367) if (cpu == curcpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2368) drain_local_stock(&stock->work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2369) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2370) schedule_work_on(cpu, &stock->work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2371) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2372) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2373) put_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2374) mutex_unlock(&percpu_charge_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2375) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2377) static int memcg_hotplug_cpu_dead(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2378) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2379) struct memcg_stock_pcp *stock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2380) struct mem_cgroup *memcg, *mi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2382) stock = &per_cpu(memcg_stock, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2383) drain_stock(stock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2385) for_each_mem_cgroup(memcg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2386) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2388) for (i = 0; i < MEMCG_NR_STAT; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2389) int nid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2390) long x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2392) x = this_cpu_xchg(memcg->vmstats_percpu->stat[i], 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2393) if (x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2394) for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2395) atomic_long_add(x, &memcg->vmstats[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2397) if (i >= NR_VM_NODE_STAT_ITEMS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2398) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2400) for_each_node(nid) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2401) struct mem_cgroup_per_node *pn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2403) pn = mem_cgroup_nodeinfo(memcg, nid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2404) x = this_cpu_xchg(pn->lruvec_stat_cpu->count[i], 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2405) if (x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2406) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2407) atomic_long_add(x, &pn->lruvec_stat[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2408) } while ((pn = parent_nodeinfo(pn, nid)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2409) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2410) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2412) for (i = 0; i < NR_VM_EVENT_ITEMS; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2413) long x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2415) x = this_cpu_xchg(memcg->vmstats_percpu->events[i], 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2416) if (x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2417) for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2418) atomic_long_add(x, &memcg->vmevents[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2419) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2420) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2422) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2423) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2425) static unsigned long reclaim_high(struct mem_cgroup *memcg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2426) unsigned int nr_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2427) gfp_t gfp_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2428) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2429) unsigned long nr_reclaimed = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2431) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2432) unsigned long pflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2434) if (page_counter_read(&memcg->memory) <=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2435) READ_ONCE(memcg->memory.high))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2436) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2438) memcg_memory_event(memcg, MEMCG_HIGH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2440) psi_memstall_enter(&pflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2441) nr_reclaimed += try_to_free_mem_cgroup_pages(memcg, nr_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2442) gfp_mask, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2443) psi_memstall_leave(&pflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2444) } while ((memcg = parent_mem_cgroup(memcg)) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2445) !mem_cgroup_is_root(memcg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2447) return nr_reclaimed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2448) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2450) static void high_work_func(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2451) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2452) struct mem_cgroup *memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2454) memcg = container_of(work, struct mem_cgroup, high_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2455) reclaim_high(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2456) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2458) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2459) * Clamp the maximum sleep time per allocation batch to 2 seconds. This is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2460) * enough to still cause a significant slowdown in most cases, while still
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2461) * allowing diagnostics and tracing to proceed without becoming stuck.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2462) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2463) #define MEMCG_MAX_HIGH_DELAY_JIFFIES (2UL*HZ)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2465) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2466) * When calculating the delay, we use these either side of the exponentiation to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2467) * maintain precision and scale to a reasonable number of jiffies (see the table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2468) * below.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2469) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2470) * - MEMCG_DELAY_PRECISION_SHIFT: Extra precision bits while translating the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2471) * overage ratio to a delay.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2472) * - MEMCG_DELAY_SCALING_SHIFT: The number of bits to scale down the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2473) * proposed penalty in order to reduce to a reasonable number of jiffies, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2474) * to produce a reasonable delay curve.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2475) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2476) * MEMCG_DELAY_SCALING_SHIFT just happens to be a number that produces a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2477) * reasonable delay curve compared to precision-adjusted overage, not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2478) * penalising heavily at first, but still making sure that growth beyond the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2479) * limit penalises misbehaviour cgroups by slowing them down exponentially. For
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2480) * example, with a high of 100 megabytes:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2481) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2482) * +-------+------------------------+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2483) * | usage | time to allocate in ms |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2484) * +-------+------------------------+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2485) * | 100M | 0 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2486) * | 101M | 6 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2487) * | 102M | 25 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2488) * | 103M | 57 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2489) * | 104M | 102 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2490) * | 105M | 159 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2491) * | 106M | 230 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2492) * | 107M | 313 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2493) * | 108M | 409 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2494) * | 109M | 518 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2495) * | 110M | 639 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2496) * | 111M | 774 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2497) * | 112M | 921 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2498) * | 113M | 1081 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2499) * | 114M | 1254 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2500) * | 115M | 1439 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2501) * | 116M | 1638 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2502) * | 117M | 1849 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2503) * | 118M | 2000 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2504) * | 119M | 2000 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2505) * | 120M | 2000 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2506) * +-------+------------------------+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2507) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2508) #define MEMCG_DELAY_PRECISION_SHIFT 20
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2509) #define MEMCG_DELAY_SCALING_SHIFT 14
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2511) static u64 calculate_overage(unsigned long usage, unsigned long high)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2512) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2513) u64 overage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2515) if (usage <= high)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2516) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2518) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2519) * Prevent division by 0 in overage calculation by acting as if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2520) * it was a threshold of 1 page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2521) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2522) high = max(high, 1UL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2524) overage = usage - high;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2525) overage <<= MEMCG_DELAY_PRECISION_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2526) return div64_u64(overage, high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2527) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2529) static u64 mem_find_max_overage(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2530) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2531) u64 overage, max_overage = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2533) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2534) overage = calculate_overage(page_counter_read(&memcg->memory),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2535) READ_ONCE(memcg->memory.high));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2536) max_overage = max(overage, max_overage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2537) } while ((memcg = parent_mem_cgroup(memcg)) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2538) !mem_cgroup_is_root(memcg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2540) return max_overage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2541) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2543) static u64 swap_find_max_overage(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2544) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2545) u64 overage, max_overage = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2547) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2548) overage = calculate_overage(page_counter_read(&memcg->swap),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2549) READ_ONCE(memcg->swap.high));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2550) if (overage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2551) memcg_memory_event(memcg, MEMCG_SWAP_HIGH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2552) max_overage = max(overage, max_overage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2553) } while ((memcg = parent_mem_cgroup(memcg)) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2554) !mem_cgroup_is_root(memcg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2555)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2556) return max_overage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2557) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2559) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2560) * Get the number of jiffies that we should penalise a mischievous cgroup which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2561) * is exceeding its memory.high by checking both it and its ancestors.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2562) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2563) static unsigned long calculate_high_delay(struct mem_cgroup *memcg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2564) unsigned int nr_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2565) u64 max_overage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2566) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2567) unsigned long penalty_jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2568)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2569) if (!max_overage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2570) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2572) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2573) * We use overage compared to memory.high to calculate the number of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2574) * jiffies to sleep (penalty_jiffies). Ideally this value should be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2575) * fairly lenient on small overages, and increasingly harsh when the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2576) * memcg in question makes it clear that it has no intention of stopping
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2577) * its crazy behaviour, so we exponentially increase the delay based on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2578) * overage amount.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2579) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2580) penalty_jiffies = max_overage * max_overage * HZ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2581) penalty_jiffies >>= MEMCG_DELAY_PRECISION_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2582) penalty_jiffies >>= MEMCG_DELAY_SCALING_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2584) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2585) * Factor in the task's own contribution to the overage, such that four
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2586) * N-sized allocations are throttled approximately the same as one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2587) * 4N-sized allocation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2588) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2589) * MEMCG_CHARGE_BATCH pages is nominal, so work out how much smaller or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2590) * larger the current charge patch is than that.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2591) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2592) return penalty_jiffies * nr_pages / MEMCG_CHARGE_BATCH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2593) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2595) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2596) * Scheduled by try_charge() to be executed from the userland return path
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2597) * and reclaims memory over the high limit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2598) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2599) void mem_cgroup_handle_over_high(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2600) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2601) unsigned long penalty_jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2602) unsigned long pflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2603) unsigned long nr_reclaimed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2604) unsigned int nr_pages = current->memcg_nr_pages_over_high;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2605) int nr_retries = MAX_RECLAIM_RETRIES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2606) struct mem_cgroup *memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2607) bool in_retry = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2609) if (likely(!nr_pages))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2610) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2612) memcg = get_mem_cgroup_from_mm(current->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2613) current->memcg_nr_pages_over_high = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2614)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2615) retry_reclaim:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2616) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2617) * The allocating task should reclaim at least the batch size, but for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2618) * subsequent retries we only want to do what's necessary to prevent oom
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2619) * or breaching resource isolation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2620) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2621) * This is distinct from memory.max or page allocator behaviour because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2622) * memory.high is currently batched, whereas memory.max and the page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2623) * allocator run every time an allocation is made.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2624) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2625) nr_reclaimed = reclaim_high(memcg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2626) in_retry ? SWAP_CLUSTER_MAX : nr_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2627) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2628)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2629) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2630) * memory.high is breached and reclaim is unable to keep up. Throttle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2631) * allocators proactively to slow down excessive growth.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2632) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2633) penalty_jiffies = calculate_high_delay(memcg, nr_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2634) mem_find_max_overage(memcg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2635)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2636) penalty_jiffies += calculate_high_delay(memcg, nr_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2637) swap_find_max_overage(memcg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2639) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2640) * Clamp the max delay per usermode return so as to still keep the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2641) * application moving forwards and also permit diagnostics, albeit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2642) * extremely slowly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2643) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2644) penalty_jiffies = min(penalty_jiffies, MEMCG_MAX_HIGH_DELAY_JIFFIES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2646) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2647) * Don't sleep if the amount of jiffies this memcg owes us is so low
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2648) * that it's not even worth doing, in an attempt to be nice to those who
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2649) * go only a small amount over their memory.high value and maybe haven't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2650) * been aggressively reclaimed enough yet.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2651) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2652) if (penalty_jiffies <= HZ / 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2653) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2654)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2655) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2656) * If reclaim is making forward progress but we're still over
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2657) * memory.high, we want to encourage that rather than doing allocator
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2658) * throttling.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2659) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2660) if (nr_reclaimed || nr_retries--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2661) in_retry = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2662) goto retry_reclaim;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2663) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2664)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2665) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2666) * If we exit early, we're guaranteed to die (since
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2667) * schedule_timeout_killable sets TASK_KILLABLE). This means we don't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2668) * need to account for any ill-begotten jiffies to pay them off later.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2669) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2670) psi_memstall_enter(&pflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2671) schedule_timeout_killable(penalty_jiffies);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2672) psi_memstall_leave(&pflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2673)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2674) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2675) css_put(&memcg->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2676) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2677)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2678) static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2679) unsigned int nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2680) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2681) unsigned int batch = max(MEMCG_CHARGE_BATCH, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2682) int nr_retries = MAX_RECLAIM_RETRIES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2683) struct mem_cgroup *mem_over_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2684) struct page_counter *counter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2685) enum oom_status oom_status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2686) unsigned long nr_reclaimed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2687) bool passed_oom = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2688) bool may_swap = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2689) bool drained = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2690) unsigned long pflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2692) if (mem_cgroup_is_root(memcg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2693) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2694) retry:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2695) if (consume_stock(memcg, nr_pages))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2696) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2697)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2698) if (!do_memsw_account() ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2699) page_counter_try_charge(&memcg->memsw, batch, &counter)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2700) if (page_counter_try_charge(&memcg->memory, batch, &counter))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2701) goto done_restock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2702) if (do_memsw_account())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2703) page_counter_uncharge(&memcg->memsw, batch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2704) mem_over_limit = mem_cgroup_from_counter(counter, memory);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2705) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2706) mem_over_limit = mem_cgroup_from_counter(counter, memsw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2707) may_swap = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2708) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2709)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2710) if (batch > nr_pages) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2711) batch = nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2712) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2713) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2715) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2716) * Memcg doesn't have a dedicated reserve for atomic
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2717) * allocations. But like the global atomic pool, we need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2718) * put the burden of reclaim on regular allocation requests
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2719) * and let these go through as privileged allocations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2720) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2721) if (gfp_mask & __GFP_ATOMIC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2722) goto force;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2723)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2724) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2725) * Prevent unbounded recursion when reclaim operations need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2726) * allocate memory. This might exceed the limits temporarily,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2727) * but we prefer facilitating memory reclaim and getting back
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2728) * under the limit over triggering OOM kills in these cases.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2729) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2730) if (unlikely(current->flags & PF_MEMALLOC))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2731) goto force;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2732)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2733) if (unlikely(task_in_memcg_oom(current)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2734) goto nomem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2735)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2736) if (!gfpflags_allow_blocking(gfp_mask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2737) goto nomem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2738)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2739) memcg_memory_event(mem_over_limit, MEMCG_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2741) psi_memstall_enter(&pflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2742) nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2743) gfp_mask, may_swap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2744) psi_memstall_leave(&pflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2745)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2746) if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2747) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2749) if (!drained) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2750) drain_all_stock(mem_over_limit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2751) drained = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2752) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2753) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2754)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2755) if (gfp_mask & __GFP_NORETRY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2756) goto nomem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2757) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2758) * Even though the limit is exceeded at this point, reclaim
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2759) * may have been able to free some pages. Retry the charge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2760) * before killing the task.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2761) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2762) * Only for regular pages, though: huge pages are rather
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2763) * unlikely to succeed so close to the limit, and we fall back
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2764) * to regular pages anyway in case of failure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2765) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2766) if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2767) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2768) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2769) * At task move, charge accounts can be doubly counted. So, it's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2770) * better to wait until the end of task_move if something is going on.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2771) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2772) if (mem_cgroup_wait_acct_move(mem_over_limit))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2773) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2774)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2775) if (nr_retries--)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2776) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2778) if (gfp_mask & __GFP_RETRY_MAYFAIL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2779) goto nomem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2781) if (gfp_mask & __GFP_NOFAIL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2782) goto force;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2783)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2784) /* Avoid endless loop for tasks bypassed by the oom killer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2785) if (passed_oom && task_is_dying())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2786) goto nomem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2787)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2788) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2789) * keep retrying as long as the memcg oom killer is able to make
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2790) * a forward progress or bypass the charge if the oom killer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2791) * couldn't make any progress.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2792) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2793) oom_status = mem_cgroup_oom(mem_over_limit, gfp_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2794) get_order(nr_pages * PAGE_SIZE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2795) if (oom_status == OOM_SUCCESS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2796) passed_oom = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2797) nr_retries = MAX_RECLAIM_RETRIES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2798) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2799) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2800) nomem:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2801) if (!(gfp_mask & __GFP_NOFAIL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2802) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2803) force:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2804) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2805) * The allocation either can't fail or will lead to more memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2806) * being freed very soon. Allow memory usage go over the limit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2807) * temporarily by force charging it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2808) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2809) page_counter_charge(&memcg->memory, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2810) if (do_memsw_account())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2811) page_counter_charge(&memcg->memsw, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2812)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2813) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2814)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2815) done_restock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2816) if (batch > nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2817) refill_stock(memcg, batch - nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2818)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2819) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2820) * If the hierarchy is above the normal consumption range, schedule
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2821) * reclaim on returning to userland. We can perform reclaim here
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2822) * if __GFP_RECLAIM but let's always punt for simplicity and so that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2823) * GFP_KERNEL can consistently be used during reclaim. @memcg is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2824) * not recorded as it most likely matches current's and won't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2825) * change in the meantime. As high limit is checked again before
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2826) * reclaim, the cost of mismatch is negligible.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2827) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2828) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2829) bool mem_high, swap_high;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2830)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2831) mem_high = page_counter_read(&memcg->memory) >
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2832) READ_ONCE(memcg->memory.high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2833) swap_high = page_counter_read(&memcg->swap) >
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2834) READ_ONCE(memcg->swap.high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2835)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2836) /* Don't bother a random interrupted task */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2837) if (in_interrupt()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2838) if (mem_high) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2839) schedule_work(&memcg->high_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2840) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2841) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2842) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2843) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2844)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2845) if (mem_high || swap_high) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2846) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2847) * The allocating tasks in this cgroup will need to do
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2848) * reclaim or be throttled to prevent further growth
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2849) * of the memory or swap footprints.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2850) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2851) * Target some best-effort fairness between the tasks,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2852) * and distribute reclaim work and delay penalties
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2853) * based on how much each task is actually allocating.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2854) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2855) current->memcg_nr_pages_over_high += batch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2856) set_notify_resume(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2857) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2858) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2859) } while ((memcg = parent_mem_cgroup(memcg)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2860)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2861) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2862) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2863)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2864) #if defined(CONFIG_MEMCG_KMEM) || defined(CONFIG_MMU)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2865) static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2866) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2867) if (mem_cgroup_is_root(memcg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2868) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2869)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2870) page_counter_uncharge(&memcg->memory, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2871) if (do_memsw_account())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2872) page_counter_uncharge(&memcg->memsw, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2873) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2874) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2875)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2876) static void commit_charge(struct page *page, struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2877) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2878) VM_BUG_ON_PAGE(page->mem_cgroup, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2879) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2880) * Any of the following ensures page->mem_cgroup stability:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2881) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2882) * - the page lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2883) * - LRU isolation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2884) * - lock_page_memcg()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2885) * - exclusive reference
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2886) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2887) page->mem_cgroup = memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2888) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2889)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2890) #ifdef CONFIG_MEMCG_KMEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2891) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2892) * The allocated objcg pointers array is not accounted directly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2893) * Moreover, it should not come from DMA buffer and is not readily
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2894) * reclaimable. So those GFP bits should be masked off.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2895) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2896) #define OBJCGS_CLEAR_MASK (__GFP_DMA | __GFP_RECLAIMABLE | __GFP_ACCOUNT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2897)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2898) int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2899) gfp_t gfp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2900) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2901) unsigned int objects = objs_per_slab_page(s, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2902) void *vec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2904) gfp &= ~OBJCGS_CLEAR_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2905) vec = kcalloc_node(objects, sizeof(struct obj_cgroup *), gfp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2906) page_to_nid(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2907) if (!vec)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2908) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2909)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2910) if (cmpxchg(&page->obj_cgroups, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2911) (struct obj_cgroup **) ((unsigned long)vec | 0x1UL)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2912) kfree(vec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2913) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2914) kmemleak_not_leak(vec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2915)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2916) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2917) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2918)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2919) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2920) * Returns a pointer to the memory cgroup to which the kernel object is charged.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2921) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2922) * The caller must ensure the memcg lifetime, e.g. by taking rcu_read_lock(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2923) * cgroup_mutex, etc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2924) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2925) struct mem_cgroup *mem_cgroup_from_obj(void *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2926) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2927) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2928)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2929) if (mem_cgroup_disabled())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2930) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2931)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2932) page = virt_to_head_page(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2933)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2934) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2935) * If page->mem_cgroup is set, it's either a simple mem_cgroup pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2936) * or a pointer to obj_cgroup vector. In the latter case the lowest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2937) * bit of the pointer is set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2938) * The page->mem_cgroup pointer can be asynchronously changed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2939) * from NULL to (obj_cgroup_vec | 0x1UL), but can't be changed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2940) * from a valid memcg pointer to objcg vector or back.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2941) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2942) if (!page->mem_cgroup)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2943) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2944)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2945) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2946) * Slab objects are accounted individually, not per-page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2947) * Memcg membership data for each individual object is saved in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2948) * the page->obj_cgroups.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2949) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2950) if (page_has_obj_cgroups(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2951) struct obj_cgroup *objcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2952) unsigned int off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2953)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2954) off = obj_to_index(page->slab_cache, page, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2955) objcg = page_obj_cgroups(page)[off];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2956) if (objcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2957) return obj_cgroup_memcg(objcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2958)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2959) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2960) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2961)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2962) /* All other pages use page->mem_cgroup */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2963) return page->mem_cgroup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2964) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2965)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2966) __always_inline struct obj_cgroup *get_obj_cgroup_from_current(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2967) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2968) struct obj_cgroup *objcg = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2969) struct mem_cgroup *memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2970)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2971) if (memcg_kmem_bypass())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2972) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2973)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2974) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2975) if (unlikely(active_memcg()))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2976) memcg = active_memcg();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2977) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2978) memcg = mem_cgroup_from_task(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2979)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2980) for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2981) objcg = rcu_dereference(memcg->objcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2982) if (objcg && obj_cgroup_tryget(objcg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2983) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2984) objcg = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2985) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2986) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2987)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2988) return objcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2989) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2990)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2991) static int memcg_alloc_cache_id(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2992) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2993) int id, size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2994) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2995)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2996) id = ida_simple_get(&memcg_cache_ida,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2997) 0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2998) if (id < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2999) return id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3001) if (id < memcg_nr_cache_ids)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3002) return id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3003)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3004) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3005) * There's no space for the new id in memcg_caches arrays,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3006) * so we have to grow them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3007) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3008) down_write(&memcg_cache_ids_sem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3009)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3010) size = 2 * (id + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3011) if (size < MEMCG_CACHES_MIN_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3012) size = MEMCG_CACHES_MIN_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3013) else if (size > MEMCG_CACHES_MAX_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3014) size = MEMCG_CACHES_MAX_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3016) err = memcg_update_all_list_lrus(size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3017) if (!err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3018) memcg_nr_cache_ids = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3019)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3020) up_write(&memcg_cache_ids_sem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3021)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3022) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3023) ida_simple_remove(&memcg_cache_ida, id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3024) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3025) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3026) return id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3027) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3028)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3029) static void memcg_free_cache_id(int id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3030) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3031) ida_simple_remove(&memcg_cache_ida, id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3032) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3033)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3034) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3035) * __memcg_kmem_charge: charge a number of kernel pages to a memcg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3036) * @memcg: memory cgroup to charge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3037) * @gfp: reclaim mode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3038) * @nr_pages: number of pages to charge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3039) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3040) * Returns 0 on success, an error code on failure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3041) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3042) int __memcg_kmem_charge(struct mem_cgroup *memcg, gfp_t gfp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3043) unsigned int nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3044) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3045) struct page_counter *counter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3046) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3047)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3048) ret = try_charge(memcg, gfp, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3049) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3050) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3051)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3052) if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3053) !page_counter_try_charge(&memcg->kmem, nr_pages, &counter)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3054)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3055) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3056) * Enforce __GFP_NOFAIL allocation because callers are not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3057) * prepared to see failures and likely do not have any failure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3058) * handling code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3059) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3060) if (gfp & __GFP_NOFAIL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3061) page_counter_charge(&memcg->kmem, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3062) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3063) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3064) cancel_charge(memcg, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3065) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3066) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3067) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3068) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3069)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3070) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3071) * __memcg_kmem_uncharge: uncharge a number of kernel pages from a memcg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3072) * @memcg: memcg to uncharge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3073) * @nr_pages: number of pages to uncharge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3074) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3075) void __memcg_kmem_uncharge(struct mem_cgroup *memcg, unsigned int nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3076) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3077) if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3078) page_counter_uncharge(&memcg->kmem, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3079)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3080) refill_stock(memcg, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3081) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3082)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3083) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3084) * __memcg_kmem_charge_page: charge a kmem page to the current memory cgroup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3085) * @page: page to charge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3086) * @gfp: reclaim mode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3087) * @order: allocation order
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3088) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3089) * Returns 0 on success, an error code on failure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3090) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3091) int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3092) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3093) struct mem_cgroup *memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3094) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3095)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3096) memcg = get_mem_cgroup_from_current();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3097) if (memcg && !mem_cgroup_is_root(memcg)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3098) ret = __memcg_kmem_charge(memcg, gfp, 1 << order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3099) if (!ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3100) page->mem_cgroup = memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3101) __SetPageKmemcg(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3102) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3103) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3104) css_put(&memcg->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3105) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3106) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3107) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3109) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3110) * __memcg_kmem_uncharge_page: uncharge a kmem page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3111) * @page: page to uncharge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3112) * @order: allocation order
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3113) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3114) void __memcg_kmem_uncharge_page(struct page *page, int order)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3115) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3116) struct mem_cgroup *memcg = page->mem_cgroup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3117) unsigned int nr_pages = 1 << order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3119) if (!memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3120) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3122) VM_BUG_ON_PAGE(mem_cgroup_is_root(memcg), page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3123) __memcg_kmem_uncharge(memcg, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3124) page->mem_cgroup = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3125) css_put(&memcg->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3127) /* slab pages do not have PageKmemcg flag set */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3128) if (PageKmemcg(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3129) __ClearPageKmemcg(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3130) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3132) static bool consume_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3133) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3134) struct memcg_stock_pcp *stock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3135) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3136) bool ret = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3138) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3140) stock = this_cpu_ptr(&memcg_stock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3141) if (objcg == stock->cached_objcg && stock->nr_bytes >= nr_bytes) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3142) stock->nr_bytes -= nr_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3143) ret = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3144) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3146) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3148) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3149) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3151) static void drain_obj_stock(struct memcg_stock_pcp *stock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3152) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3153) struct obj_cgroup *old = stock->cached_objcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3155) if (!old)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3156) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3158) if (stock->nr_bytes) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3159) unsigned int nr_pages = stock->nr_bytes >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3160) unsigned int nr_bytes = stock->nr_bytes & (PAGE_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3162) if (nr_pages) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3163) struct mem_cgroup *memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3165) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3166) retry:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3167) memcg = obj_cgroup_memcg(old);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3168) if (unlikely(!css_tryget(&memcg->css)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3169) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3170) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3172) __memcg_kmem_uncharge(memcg, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3173) css_put(&memcg->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3174) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3176) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3177) * The leftover is flushed to the centralized per-memcg value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3178) * On the next attempt to refill obj stock it will be moved
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3179) * to a per-cpu stock (probably, on an other CPU), see
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3180) * refill_obj_stock().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3181) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3182) * How often it's flushed is a trade-off between the memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3183) * limit enforcement accuracy and potential CPU contention,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3184) * so it might be changed in the future.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3185) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3186) atomic_add(nr_bytes, &old->nr_charged_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3187) stock->nr_bytes = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3188) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3190) obj_cgroup_put(old);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3191) stock->cached_objcg = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3192) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3194) static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3195) struct mem_cgroup *root_memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3196) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3197) struct mem_cgroup *memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3199) if (stock->cached_objcg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3200) memcg = obj_cgroup_memcg(stock->cached_objcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3201) if (memcg && mem_cgroup_is_descendant(memcg, root_memcg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3202) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3203) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3205) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3206) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3208) static void refill_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3209) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3210) struct memcg_stock_pcp *stock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3211) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3213) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3215) stock = this_cpu_ptr(&memcg_stock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3216) if (stock->cached_objcg != objcg) { /* reset if necessary */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3217) drain_obj_stock(stock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3218) obj_cgroup_get(objcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3219) stock->cached_objcg = objcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3220) stock->nr_bytes = atomic_xchg(&objcg->nr_charged_bytes, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3221) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3222) stock->nr_bytes += nr_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3224) if (stock->nr_bytes > PAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3225) drain_obj_stock(stock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3227) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3228) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3230) int obj_cgroup_charge(struct obj_cgroup *objcg, gfp_t gfp, size_t size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3231) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3232) struct mem_cgroup *memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3233) unsigned int nr_pages, nr_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3234) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3236) if (consume_obj_stock(objcg, size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3237) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3239) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3240) * In theory, memcg->nr_charged_bytes can have enough
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3241) * pre-charged bytes to satisfy the allocation. However,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3242) * flushing memcg->nr_charged_bytes requires two atomic
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3243) * operations, and memcg->nr_charged_bytes can't be big,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3244) * so it's better to ignore it and try grab some new pages.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3245) * memcg->nr_charged_bytes will be flushed in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3246) * refill_obj_stock(), called from this function or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3247) * independently later.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3248) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3249) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3250) retry:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3251) memcg = obj_cgroup_memcg(objcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3252) if (unlikely(!css_tryget(&memcg->css)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3253) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3254) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3256) nr_pages = size >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3257) nr_bytes = size & (PAGE_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3259) if (nr_bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3260) nr_pages += 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3262) ret = __memcg_kmem_charge(memcg, gfp, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3263) if (!ret && nr_bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3264) refill_obj_stock(objcg, PAGE_SIZE - nr_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3266) css_put(&memcg->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3267) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3268) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3270) void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3271) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3272) refill_obj_stock(objcg, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3273) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3275) #endif /* CONFIG_MEMCG_KMEM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3277) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3278) * Because head->mem_cgroup is not set on tails, set it now.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3279) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3280) void split_page_memcg(struct page *head, unsigned int nr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3281) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3282) struct mem_cgroup *memcg = head->mem_cgroup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3283) int kmemcg = PageKmemcg(head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3284) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3286) if (mem_cgroup_disabled() || !memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3287) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3289) for (i = 1; i < nr; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3290) head[i].mem_cgroup = memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3291) if (kmemcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3292) __SetPageKmemcg(head + i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3293) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3294) css_get_many(&memcg->css, nr - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3295) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3297) #ifdef CONFIG_MEMCG_SWAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3298) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3299) * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3300) * @entry: swap entry to be moved
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3301) * @from: mem_cgroup which the entry is moved from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3302) * @to: mem_cgroup which the entry is moved to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3303) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3304) * It succeeds only when the swap_cgroup's record for this entry is the same
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3305) * as the mem_cgroup's id of @from.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3306) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3307) * Returns 0 on success, -EINVAL on failure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3308) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3309) * The caller must have charged to @to, IOW, called page_counter_charge() about
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3310) * both res and memsw, and called css_get().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3311) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3312) static int mem_cgroup_move_swap_account(swp_entry_t entry,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3313) struct mem_cgroup *from, struct mem_cgroup *to)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3314) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3315) unsigned short old_id, new_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3317) old_id = mem_cgroup_id(from);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3318) new_id = mem_cgroup_id(to);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3320) if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3321) mod_memcg_state(from, MEMCG_SWAP, -1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3322) mod_memcg_state(to, MEMCG_SWAP, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3323) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3324) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3325) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3326) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3327) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3328) static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3329) struct mem_cgroup *from, struct mem_cgroup *to)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3330) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3331) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3332) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3333) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3335) static DEFINE_MUTEX(memcg_max_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3337) static int mem_cgroup_resize_max(struct mem_cgroup *memcg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3338) unsigned long max, bool memsw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3339) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3340) bool enlarge = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3341) bool drained = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3342) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3343) bool limits_invariant;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3344) struct page_counter *counter = memsw ? &memcg->memsw : &memcg->memory;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3346) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3347) if (signal_pending(current)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3348) ret = -EINTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3349) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3350) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3352) mutex_lock(&memcg_max_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3353) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3354) * Make sure that the new limit (memsw or memory limit) doesn't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3355) * break our basic invariant rule memory.max <= memsw.max.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3356) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3357) limits_invariant = memsw ? max >= READ_ONCE(memcg->memory.max) :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3358) max <= memcg->memsw.max;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3359) if (!limits_invariant) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3360) mutex_unlock(&memcg_max_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3361) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3362) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3363) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3364) if (max > counter->max)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3365) enlarge = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3366) ret = page_counter_set_max(counter, max);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3367) mutex_unlock(&memcg_max_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3369) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3370) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3372) if (!drained) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3373) drain_all_stock(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3374) drained = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3375) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3376) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3378) if (!try_to_free_mem_cgroup_pages(memcg, 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3379) GFP_KERNEL, !memsw)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3380) ret = -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3381) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3382) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3383) } while (true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3385) if (!ret && enlarge)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3386) memcg_oom_recover(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3388) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3389) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3391) unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3392) gfp_t gfp_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3393) unsigned long *total_scanned)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3394) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3395) unsigned long nr_reclaimed = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3396) struct mem_cgroup_per_node *mz, *next_mz = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3397) unsigned long reclaimed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3398) int loop = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3399) struct mem_cgroup_tree_per_node *mctz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3400) unsigned long excess;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3401) unsigned long nr_scanned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3403) if (order > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3404) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3406) mctz = soft_limit_tree_node(pgdat->node_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3408) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3409) * Do not even bother to check the largest node if the root
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3410) * is empty. Do it lockless to prevent lock bouncing. Races
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3411) * are acceptable as soft limit is best effort anyway.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3412) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3413) if (!mctz || RB_EMPTY_ROOT(&mctz->rb_root))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3414) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3416) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3417) * This loop can run a while, specially if mem_cgroup's continuously
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3418) * keep exceeding their soft limit and putting the system under
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3419) * pressure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3420) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3421) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3422) if (next_mz)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3423) mz = next_mz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3424) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3425) mz = mem_cgroup_largest_soft_limit_node(mctz);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3426) if (!mz)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3427) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3428)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3429) nr_scanned = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3430) reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3431) gfp_mask, &nr_scanned);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3432) nr_reclaimed += reclaimed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3433) *total_scanned += nr_scanned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3434) spin_lock_irq(&mctz->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3435) __mem_cgroup_remove_exceeded(mz, mctz);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3436)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3437) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3438) * If we failed to reclaim anything from this memory cgroup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3439) * it is time to move on to the next cgroup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3440) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3441) next_mz = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3442) if (!reclaimed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3443) next_mz = __mem_cgroup_largest_soft_limit_node(mctz);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3445) excess = soft_limit_excess(mz->memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3446) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3447) * One school of thought says that we should not add
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3448) * back the node to the tree if reclaim returns 0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3449) * But our reclaim could return 0, simply because due
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3450) * to priority we are exposing a smaller subset of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3451) * memory to reclaim from. Consider this as a longer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3452) * term TODO.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3453) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3454) /* If excess == 0, no tree ops */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3455) __mem_cgroup_insert_exceeded(mz, mctz, excess);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3456) spin_unlock_irq(&mctz->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3457) css_put(&mz->memcg->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3458) loop++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3459) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3460) * Could not reclaim anything and there are no more
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3461) * mem cgroups to try or we seem to be looping without
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3462) * reclaiming anything.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3463) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3464) if (!nr_reclaimed &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3465) (next_mz == NULL ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3466) loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3467) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3468) } while (!nr_reclaimed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3469) if (next_mz)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3470) css_put(&next_mz->memcg->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3471) return nr_reclaimed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3472) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3474) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3475) * Test whether @memcg has children, dead or alive. Note that this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3476) * function doesn't care whether @memcg has use_hierarchy enabled and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3477) * returns %true if there are child csses according to the cgroup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3478) * hierarchy. Testing use_hierarchy is the caller's responsibility.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3479) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3480) static inline bool memcg_has_children(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3481) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3482) bool ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3484) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3485) ret = css_next_child(NULL, &memcg->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3486) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3487) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3488) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3490) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3491) * Reclaims as many pages from the given memcg as possible.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3492) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3493) * Caller is responsible for holding css reference for memcg.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3494) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3495) static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3496) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3497) int nr_retries = MAX_RECLAIM_RETRIES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3499) /* we call try-to-free pages for make this cgroup empty */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3500) lru_add_drain_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3502) drain_all_stock(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3504) /* try to free all pages in this cgroup */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3505) while (nr_retries && page_counter_read(&memcg->memory)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3506) int progress;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3508) if (signal_pending(current))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3509) return -EINTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3511) progress = try_to_free_mem_cgroup_pages(memcg, 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3512) GFP_KERNEL, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3513) if (!progress) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3514) nr_retries--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3515) /* maybe some writeback is necessary */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3516) congestion_wait(BLK_RW_ASYNC, HZ/10);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3517) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3519) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3521) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3522) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3524) static ssize_t mem_cgroup_force_empty_write(struct kernfs_open_file *of,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3525) char *buf, size_t nbytes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3526) loff_t off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3527) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3528) struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3530) if (mem_cgroup_is_root(memcg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3531) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3532) return mem_cgroup_force_empty(memcg) ?: nbytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3533) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3535) static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3536) struct cftype *cft)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3537) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3538) return mem_cgroup_from_css(css)->use_hierarchy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3539) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3541) static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3542) struct cftype *cft, u64 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3543) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3544) int retval = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3545) struct mem_cgroup *memcg = mem_cgroup_from_css(css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3546) struct mem_cgroup *parent_memcg = mem_cgroup_from_css(memcg->css.parent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3548) if (memcg->use_hierarchy == val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3549) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3551) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3552) * If parent's use_hierarchy is set, we can't make any modifications
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3553) * in the child subtrees. If it is unset, then the change can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3554) * occur, provided the current cgroup has no children.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3555) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3556) * For the root cgroup, parent_mem is NULL, we allow value to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3557) * set if there are no children.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3558) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3559) if ((!parent_memcg || !parent_memcg->use_hierarchy) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3560) (val == 1 || val == 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3561) if (!memcg_has_children(memcg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3562) memcg->use_hierarchy = val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3563) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3564) retval = -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3565) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3566) retval = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3568) return retval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3569) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3571) static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3572) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3573) unsigned long val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3575) if (mem_cgroup_is_root(memcg)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3576) val = memcg_page_state(memcg, NR_FILE_PAGES) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3577) memcg_page_state(memcg, NR_ANON_MAPPED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3578) if (swap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3579) val += memcg_page_state(memcg, MEMCG_SWAP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3580) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3581) if (!swap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3582) val = page_counter_read(&memcg->memory);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3583) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3584) val = page_counter_read(&memcg->memsw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3585) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3586) return val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3587) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3589) enum {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3590) RES_USAGE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3591) RES_LIMIT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3592) RES_MAX_USAGE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3593) RES_FAILCNT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3594) RES_SOFT_LIMIT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3595) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3597) static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3598) struct cftype *cft)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3599) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3600) struct mem_cgroup *memcg = mem_cgroup_from_css(css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3601) struct page_counter *counter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3603) switch (MEMFILE_TYPE(cft->private)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3604) case _MEM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3605) counter = &memcg->memory;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3606) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3607) case _MEMSWAP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3608) counter = &memcg->memsw;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3609) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3610) case _KMEM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3611) counter = &memcg->kmem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3612) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3613) case _TCP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3614) counter = &memcg->tcpmem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3615) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3616) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3617) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3618) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3620) switch (MEMFILE_ATTR(cft->private)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3621) case RES_USAGE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3622) if (counter == &memcg->memory)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3623) return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3624) if (counter == &memcg->memsw)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3625) return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3626) return (u64)page_counter_read(counter) * PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3627) case RES_LIMIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3628) return (u64)counter->max * PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3629) case RES_MAX_USAGE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3630) return (u64)counter->watermark * PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3631) case RES_FAILCNT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3632) return counter->failcnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3633) case RES_SOFT_LIMIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3634) return (u64)memcg->soft_limit * PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3635) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3636) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3637) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3638) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3639)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3640) static void memcg_flush_percpu_vmstats(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3641) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3642) unsigned long stat[MEMCG_NR_STAT] = {0};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3643) struct mem_cgroup *mi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3644) int node, cpu, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3646) for_each_online_cpu(cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3647) for (i = 0; i < MEMCG_NR_STAT; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3648) stat[i] += per_cpu(memcg->vmstats_percpu->stat[i], cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3650) for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3651) for (i = 0; i < MEMCG_NR_STAT; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3652) atomic_long_add(stat[i], &mi->vmstats[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3654) for_each_node(node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3655) struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3656) struct mem_cgroup_per_node *pi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3657)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3658) for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3659) stat[i] = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3660)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3661) for_each_online_cpu(cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3662) for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3663) stat[i] += per_cpu(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3664) pn->lruvec_stat_cpu->count[i], cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3666) for (pi = pn; pi; pi = parent_nodeinfo(pi, node))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3667) for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3668) atomic_long_add(stat[i], &pi->lruvec_stat[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3669) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3670) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3672) static void memcg_flush_percpu_vmevents(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3673) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3674) unsigned long events[NR_VM_EVENT_ITEMS];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3675) struct mem_cgroup *mi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3676) int cpu, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3677)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3678) for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3679) events[i] = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3680)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3681) for_each_online_cpu(cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3682) for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3683) events[i] += per_cpu(memcg->vmstats_percpu->events[i],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3684) cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3685)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3686) for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3687) for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3688) atomic_long_add(events[i], &mi->vmevents[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3689) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3691) #ifdef CONFIG_MEMCG_KMEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3692) static int memcg_online_kmem(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3693) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3694) struct obj_cgroup *objcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3695) int memcg_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3696)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3697) if (cgroup_memory_nokmem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3698) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3700) BUG_ON(memcg->kmemcg_id >= 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3701) BUG_ON(memcg->kmem_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3703) memcg_id = memcg_alloc_cache_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3704) if (memcg_id < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3705) return memcg_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3706)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3707) objcg = obj_cgroup_alloc();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3708) if (!objcg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3709) memcg_free_cache_id(memcg_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3710) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3711) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3712) objcg->memcg = memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3713) rcu_assign_pointer(memcg->objcg, objcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3715) static_branch_enable(&memcg_kmem_enabled_key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3717) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3718) * A memory cgroup is considered kmem-online as soon as it gets
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3719) * kmemcg_id. Setting the id after enabling static branching will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3720) * guarantee no one starts accounting before all call sites are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3721) * patched.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3722) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3723) memcg->kmemcg_id = memcg_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3724) memcg->kmem_state = KMEM_ONLINE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3726) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3727) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3728)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3729) static void memcg_offline_kmem(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3730) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3731) struct cgroup_subsys_state *css;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3732) struct mem_cgroup *parent, *child;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3733) int kmemcg_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3734)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3735) if (memcg->kmem_state != KMEM_ONLINE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3736) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3738) memcg->kmem_state = KMEM_ALLOCATED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3739)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3740) parent = parent_mem_cgroup(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3741) if (!parent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3742) parent = root_mem_cgroup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3743)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3744) memcg_reparent_objcgs(memcg, parent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3745)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3746) kmemcg_id = memcg->kmemcg_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3747) BUG_ON(kmemcg_id < 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3749) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3750) * Change kmemcg_id of this cgroup and all its descendants to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3751) * parent's id, and then move all entries from this cgroup's list_lrus
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3752) * to ones of the parent. After we have finished, all list_lrus
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3753) * corresponding to this cgroup are guaranteed to remain empty. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3754) * ordering is imposed by list_lru_node->lock taken by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3755) * memcg_drain_all_list_lrus().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3756) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3757) rcu_read_lock(); /* can be called from css_free w/o cgroup_mutex */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3758) css_for_each_descendant_pre(css, &memcg->css) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3759) child = mem_cgroup_from_css(css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3760) BUG_ON(child->kmemcg_id != kmemcg_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3761) child->kmemcg_id = parent->kmemcg_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3762) if (!memcg->use_hierarchy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3763) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3764) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3765) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3767) memcg_drain_all_list_lrus(kmemcg_id, parent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3769) memcg_free_cache_id(kmemcg_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3770) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3771)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3772) static void memcg_free_kmem(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3773) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3774) /* css_alloc() failed, offlining didn't happen */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3775) if (unlikely(memcg->kmem_state == KMEM_ONLINE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3776) memcg_offline_kmem(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3777) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3778) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3779) static int memcg_online_kmem(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3780) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3781) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3782) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3783) static void memcg_offline_kmem(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3784) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3785) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3786) static void memcg_free_kmem(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3787) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3788) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3789) #endif /* CONFIG_MEMCG_KMEM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3790)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3791) static int memcg_update_kmem_max(struct mem_cgroup *memcg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3792) unsigned long max)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3793) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3794) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3795)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3796) mutex_lock(&memcg_max_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3797) ret = page_counter_set_max(&memcg->kmem, max);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3798) mutex_unlock(&memcg_max_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3799) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3800) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3801)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3802) static int memcg_update_tcp_max(struct mem_cgroup *memcg, unsigned long max)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3803) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3804) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3805)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3806) mutex_lock(&memcg_max_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3807)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3808) ret = page_counter_set_max(&memcg->tcpmem, max);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3809) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3810) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3812) if (!memcg->tcpmem_active) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3813) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3814) * The active flag needs to be written after the static_key
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3815) * update. This is what guarantees that the socket activation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3816) * function is the last one to run. See mem_cgroup_sk_alloc()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3817) * for details, and note that we don't mark any socket as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3818) * belonging to this memcg until that flag is up.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3819) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3820) * We need to do this, because static_keys will span multiple
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3821) * sites, but we can't control their order. If we mark a socket
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3822) * as accounted, but the accounting functions are not patched in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3823) * yet, we'll lose accounting.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3824) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3825) * We never race with the readers in mem_cgroup_sk_alloc(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3826) * because when this value change, the code to process it is not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3827) * patched in yet.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3828) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3829) static_branch_inc(&memcg_sockets_enabled_key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3830) memcg->tcpmem_active = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3831) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3832) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3833) mutex_unlock(&memcg_max_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3834) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3835) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3836)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3837) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3838) * The user of this function is...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3839) * RES_LIMIT.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3840) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3841) static ssize_t mem_cgroup_write(struct kernfs_open_file *of,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3842) char *buf, size_t nbytes, loff_t off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3843) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3844) struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3845) unsigned long nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3846) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3847)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3848) buf = strstrip(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3849) ret = page_counter_memparse(buf, "-1", &nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3850) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3851) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3852)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3853) switch (MEMFILE_ATTR(of_cft(of)->private)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3854) case RES_LIMIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3855) if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3856) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3857) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3858) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3859) switch (MEMFILE_TYPE(of_cft(of)->private)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3860) case _MEM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3861) ret = mem_cgroup_resize_max(memcg, nr_pages, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3862) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3863) case _MEMSWAP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3864) ret = mem_cgroup_resize_max(memcg, nr_pages, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3865) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3866) case _KMEM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3867) pr_warn_once("kmem.limit_in_bytes is deprecated and will be removed. "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3868) "Please report your usecase to linux-mm@kvack.org if you "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3869) "depend on this functionality.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3870) ret = memcg_update_kmem_max(memcg, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3871) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3872) case _TCP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3873) ret = memcg_update_tcp_max(memcg, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3874) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3875) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3876) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3877) case RES_SOFT_LIMIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3878) memcg->soft_limit = nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3879) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3880) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3881) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3882) return ret ?: nbytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3883) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3884)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3885) static ssize_t mem_cgroup_reset(struct kernfs_open_file *of, char *buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3886) size_t nbytes, loff_t off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3887) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3888) struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3889) struct page_counter *counter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3890)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3891) switch (MEMFILE_TYPE(of_cft(of)->private)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3892) case _MEM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3893) counter = &memcg->memory;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3894) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3895) case _MEMSWAP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3896) counter = &memcg->memsw;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3897) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3898) case _KMEM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3899) counter = &memcg->kmem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3900) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3901) case _TCP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3902) counter = &memcg->tcpmem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3903) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3904) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3905) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3906) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3907)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3908) switch (MEMFILE_ATTR(of_cft(of)->private)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3909) case RES_MAX_USAGE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3910) page_counter_reset_watermark(counter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3911) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3912) case RES_FAILCNT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3913) counter->failcnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3914) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3915) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3916) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3917) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3918)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3919) return nbytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3920) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3921)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3922) static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3923) struct cftype *cft)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3924) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3925) return mem_cgroup_from_css(css)->move_charge_at_immigrate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3926) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3927)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3928) #ifdef CONFIG_MMU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3929) static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3930) struct cftype *cft, u64 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3931) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3932) struct mem_cgroup *memcg = mem_cgroup_from_css(css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3933)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3934) if (val & ~MOVE_MASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3935) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3936)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3937) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3938) * No kind of locking is needed in here, because ->can_attach() will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3939) * check this value once in the beginning of the process, and then carry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3940) * on with stale data. This means that changes to this value will only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3941) * affect task migrations starting after the change.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3942) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3943) memcg->move_charge_at_immigrate = val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3944) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3945) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3946) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3947) static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3948) struct cftype *cft, u64 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3949) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3950) return -ENOSYS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3951) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3952) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3953)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3954) #ifdef CONFIG_NUMA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3955)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3956) #define LRU_ALL_FILE (BIT(LRU_INACTIVE_FILE) | BIT(LRU_ACTIVE_FILE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3957) #define LRU_ALL_ANON (BIT(LRU_INACTIVE_ANON) | BIT(LRU_ACTIVE_ANON))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3958) #define LRU_ALL ((1 << NR_LRU_LISTS) - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3959)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3960) static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3961) int nid, unsigned int lru_mask, bool tree)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3962) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3963) struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3964) unsigned long nr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3965) enum lru_list lru;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3966)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3967) VM_BUG_ON((unsigned)nid >= nr_node_ids);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3968)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3969) for_each_lru(lru) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3970) if (!(BIT(lru) & lru_mask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3971) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3972) if (tree)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3973) nr += lruvec_page_state(lruvec, NR_LRU_BASE + lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3974) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3975) nr += lruvec_page_state_local(lruvec, NR_LRU_BASE + lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3976) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3977) return nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3978) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3979)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3980) static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3981) unsigned int lru_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3982) bool tree)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3983) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3984) unsigned long nr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3985) enum lru_list lru;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3986)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3987) for_each_lru(lru) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3988) if (!(BIT(lru) & lru_mask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3989) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3990) if (tree)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3991) nr += memcg_page_state(memcg, NR_LRU_BASE + lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3992) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3993) nr += memcg_page_state_local(memcg, NR_LRU_BASE + lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3994) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3995) return nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3996) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3997)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3998) static int memcg_numa_stat_show(struct seq_file *m, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3999) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4000) struct numa_stat {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4001) const char *name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4002) unsigned int lru_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4003) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4004)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4005) static const struct numa_stat stats[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4006) { "total", LRU_ALL },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4007) { "file", LRU_ALL_FILE },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4008) { "anon", LRU_ALL_ANON },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4009) { "unevictable", BIT(LRU_UNEVICTABLE) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4010) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4011) const struct numa_stat *stat;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4012) int nid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4013) struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4014)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4015) for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4016) seq_printf(m, "%s=%lu", stat->name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4017) mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4018) false));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4019) for_each_node_state(nid, N_MEMORY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4020) seq_printf(m, " N%d=%lu", nid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4021) mem_cgroup_node_nr_lru_pages(memcg, nid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4022) stat->lru_mask, false));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4023) seq_putc(m, '\n');
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4024) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4025)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4026) for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4027)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4028) seq_printf(m, "hierarchical_%s=%lu", stat->name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4029) mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4030) true));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4031) for_each_node_state(nid, N_MEMORY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4032) seq_printf(m, " N%d=%lu", nid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4033) mem_cgroup_node_nr_lru_pages(memcg, nid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4034) stat->lru_mask, true));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4035) seq_putc(m, '\n');
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4036) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4037)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4038) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4039) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4040) #endif /* CONFIG_NUMA */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4041)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4042) static const unsigned int memcg1_stats[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4043) NR_FILE_PAGES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4044) NR_ANON_MAPPED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4045) #ifdef CONFIG_TRANSPARENT_HUGEPAGE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4046) NR_ANON_THPS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4047) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4048) NR_SHMEM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4049) NR_FILE_MAPPED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4050) NR_FILE_DIRTY,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4051) NR_WRITEBACK,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4052) MEMCG_SWAP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4053) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4054)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4055) static const char *const memcg1_stat_names[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4056) "cache",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4057) "rss",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4058) #ifdef CONFIG_TRANSPARENT_HUGEPAGE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4059) "rss_huge",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4060) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4061) "shmem",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4062) "mapped_file",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4063) "dirty",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4064) "writeback",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4065) "swap",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4066) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4067)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4068) /* Universal VM events cgroup1 shows, original sort order */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4069) static const unsigned int memcg1_events[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4070) PGPGIN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4071) PGPGOUT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4072) PGFAULT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4073) PGMAJFAULT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4074) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4075)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4076) static int memcg_stat_show(struct seq_file *m, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4077) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4078) struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4079) unsigned long memory, memsw;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4080) struct mem_cgroup *mi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4081) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4082)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4083) BUILD_BUG_ON(ARRAY_SIZE(memcg1_stat_names) != ARRAY_SIZE(memcg1_stats));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4084)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4085) for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4086) unsigned long nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4087)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4088) if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4089) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4090) nr = memcg_page_state_local(memcg, memcg1_stats[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4091) #ifdef CONFIG_TRANSPARENT_HUGEPAGE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4092) if (memcg1_stats[i] == NR_ANON_THPS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4093) nr *= HPAGE_PMD_NR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4094) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4095) seq_printf(m, "%s %lu\n", memcg1_stat_names[i], nr * PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4096) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4097)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4098) for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4099) seq_printf(m, "%s %lu\n", vm_event_name(memcg1_events[i]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4100) memcg_events_local(memcg, memcg1_events[i]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4102) for (i = 0; i < NR_LRU_LISTS; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4103) seq_printf(m, "%s %lu\n", lru_list_name(i),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4104) memcg_page_state_local(memcg, NR_LRU_BASE + i) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4105) PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4107) /* Hierarchical information */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4108) memory = memsw = PAGE_COUNTER_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4109) for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4110) memory = min(memory, READ_ONCE(mi->memory.max));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4111) memsw = min(memsw, READ_ONCE(mi->memsw.max));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4112) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4113) seq_printf(m, "hierarchical_memory_limit %llu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4114) (u64)memory * PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4115) if (do_memsw_account())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4116) seq_printf(m, "hierarchical_memsw_limit %llu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4117) (u64)memsw * PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4119) for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4120) unsigned long nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4122) if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4123) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4124) nr = memcg_page_state(memcg, memcg1_stats[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4125) #ifdef CONFIG_TRANSPARENT_HUGEPAGE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4126) if (memcg1_stats[i] == NR_ANON_THPS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4127) nr *= HPAGE_PMD_NR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4128) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4129) seq_printf(m, "total_%s %llu\n", memcg1_stat_names[i],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4130) (u64)nr * PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4131) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4133) for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4134) seq_printf(m, "total_%s %llu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4135) vm_event_name(memcg1_events[i]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4136) (u64)memcg_events(memcg, memcg1_events[i]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4138) for (i = 0; i < NR_LRU_LISTS; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4139) seq_printf(m, "total_%s %llu\n", lru_list_name(i),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4140) (u64)memcg_page_state(memcg, NR_LRU_BASE + i) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4141) PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4143) #ifdef CONFIG_DEBUG_VM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4144) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4145) pg_data_t *pgdat;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4146) struct mem_cgroup_per_node *mz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4147) unsigned long anon_cost = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4148) unsigned long file_cost = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4150) for_each_online_pgdat(pgdat) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4151) mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4153) anon_cost += mz->lruvec.anon_cost;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4154) file_cost += mz->lruvec.file_cost;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4155) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4156) seq_printf(m, "anon_cost %lu\n", anon_cost);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4157) seq_printf(m, "file_cost %lu\n", file_cost);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4158) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4159) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4161) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4162) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4164) static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4165) struct cftype *cft)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4166) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4167) struct mem_cgroup *memcg = mem_cgroup_from_css(css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4169) return mem_cgroup_swappiness(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4170) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4172) static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4173) struct cftype *cft, u64 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4174) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4175) struct mem_cgroup *memcg = mem_cgroup_from_css(css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4177) if (val > 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4178) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4180) if (css->parent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4181) memcg->swappiness = val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4182) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4183) vm_swappiness = val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4185) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4186) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4188) static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4189) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4190) struct mem_cgroup_threshold_ary *t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4191) unsigned long usage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4192) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4194) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4195) if (!swap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4196) t = rcu_dereference(memcg->thresholds.primary);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4197) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4198) t = rcu_dereference(memcg->memsw_thresholds.primary);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4200) if (!t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4201) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4203) usage = mem_cgroup_usage(memcg, swap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4205) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4206) * current_threshold points to threshold just below or equal to usage.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4207) * If it's not true, a threshold was crossed after last
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4208) * call of __mem_cgroup_threshold().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4209) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4210) i = t->current_threshold;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4212) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4213) * Iterate backward over array of thresholds starting from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4214) * current_threshold and check if a threshold is crossed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4215) * If none of thresholds below usage is crossed, we read
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4216) * only one element of the array here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4217) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4218) for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4219) eventfd_signal(t->entries[i].eventfd, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4221) /* i = current_threshold + 1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4222) i++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4224) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4225) * Iterate forward over array of thresholds starting from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4226) * current_threshold+1 and check if a threshold is crossed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4227) * If none of thresholds above usage is crossed, we read
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4228) * only one element of the array here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4229) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4230) for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4231) eventfd_signal(t->entries[i].eventfd, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4233) /* Update current_threshold */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4234) t->current_threshold = i - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4235) unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4236) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4237) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4239) static void mem_cgroup_threshold(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4240) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4241) while (memcg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4242) __mem_cgroup_threshold(memcg, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4243) if (do_memsw_account())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4244) __mem_cgroup_threshold(memcg, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4246) memcg = parent_mem_cgroup(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4247) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4248) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4250) static int compare_thresholds(const void *a, const void *b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4251) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4252) const struct mem_cgroup_threshold *_a = a;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4253) const struct mem_cgroup_threshold *_b = b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4255) if (_a->threshold > _b->threshold)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4256) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4258) if (_a->threshold < _b->threshold)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4259) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4261) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4262) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4264) static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4265) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4266) struct mem_cgroup_eventfd_list *ev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4268) spin_lock(&memcg_oom_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4270) list_for_each_entry(ev, &memcg->oom_notify, list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4271) eventfd_signal(ev->eventfd, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4273) spin_unlock(&memcg_oom_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4274) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4275) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4277) static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4278) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4279) struct mem_cgroup *iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4281) for_each_mem_cgroup_tree(iter, memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4282) mem_cgroup_oom_notify_cb(iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4283) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4285) static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4286) struct eventfd_ctx *eventfd, const char *args, enum res_type type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4287) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4288) struct mem_cgroup_thresholds *thresholds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4289) struct mem_cgroup_threshold_ary *new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4290) unsigned long threshold;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4291) unsigned long usage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4292) int i, size, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4294) ret = page_counter_memparse(args, "-1", &threshold);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4295) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4296) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4298) mutex_lock(&memcg->thresholds_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4300) if (type == _MEM) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4301) thresholds = &memcg->thresholds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4302) usage = mem_cgroup_usage(memcg, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4303) } else if (type == _MEMSWAP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4304) thresholds = &memcg->memsw_thresholds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4305) usage = mem_cgroup_usage(memcg, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4306) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4307) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4309) /* Check if a threshold crossed before adding a new one */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4310) if (thresholds->primary)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4311) __mem_cgroup_threshold(memcg, type == _MEMSWAP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4312)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4313) size = thresholds->primary ? thresholds->primary->size + 1 : 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4315) /* Allocate memory for new array of thresholds */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4316) new = kmalloc(struct_size(new, entries, size), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4317) if (!new) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4318) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4319) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4320) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4321) new->size = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4323) /* Copy thresholds (if any) to new array */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4324) if (thresholds->primary)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4325) memcpy(new->entries, thresholds->primary->entries,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4326) flex_array_size(new, entries, size - 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4328) /* Add new threshold */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4329) new->entries[size - 1].eventfd = eventfd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4330) new->entries[size - 1].threshold = threshold;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4332) /* Sort thresholds. Registering of new threshold isn't time-critical */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4333) sort(new->entries, size, sizeof(*new->entries),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4334) compare_thresholds, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4336) /* Find current threshold */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4337) new->current_threshold = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4338) for (i = 0; i < size; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4339) if (new->entries[i].threshold <= usage) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4340) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4341) * new->current_threshold will not be used until
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4342) * rcu_assign_pointer(), so it's safe to increment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4343) * it here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4344) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4345) ++new->current_threshold;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4346) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4347) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4348) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4350) /* Free old spare buffer and save old primary buffer as spare */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4351) kfree(thresholds->spare);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4352) thresholds->spare = thresholds->primary;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4354) rcu_assign_pointer(thresholds->primary, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4356) /* To be sure that nobody uses thresholds */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4357) synchronize_rcu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4358)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4359) unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4360) mutex_unlock(&memcg->thresholds_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4362) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4363) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4364)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4365) static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4366) struct eventfd_ctx *eventfd, const char *args)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4367) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4368) return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4369) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4371) static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4372) struct eventfd_ctx *eventfd, const char *args)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4373) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4374) return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4375) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4377) static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4378) struct eventfd_ctx *eventfd, enum res_type type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4379) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4380) struct mem_cgroup_thresholds *thresholds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4381) struct mem_cgroup_threshold_ary *new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4382) unsigned long usage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4383) int i, j, size, entries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4385) mutex_lock(&memcg->thresholds_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4387) if (type == _MEM) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4388) thresholds = &memcg->thresholds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4389) usage = mem_cgroup_usage(memcg, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4390) } else if (type == _MEMSWAP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4391) thresholds = &memcg->memsw_thresholds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4392) usage = mem_cgroup_usage(memcg, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4393) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4394) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4396) if (!thresholds->primary)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4397) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4399) /* Check if a threshold crossed before removing */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4400) __mem_cgroup_threshold(memcg, type == _MEMSWAP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4402) /* Calculate new number of threshold */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4403) size = entries = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4404) for (i = 0; i < thresholds->primary->size; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4405) if (thresholds->primary->entries[i].eventfd != eventfd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4406) size++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4407) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4408) entries++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4409) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4411) new = thresholds->spare;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4413) /* If no items related to eventfd have been cleared, nothing to do */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4414) if (!entries)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4415) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4417) /* Set thresholds array to NULL if we don't have thresholds */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4418) if (!size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4419) kfree(new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4420) new = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4421) goto swap_buffers;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4422) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4424) new->size = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4426) /* Copy thresholds and find current threshold */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4427) new->current_threshold = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4428) for (i = 0, j = 0; i < thresholds->primary->size; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4429) if (thresholds->primary->entries[i].eventfd == eventfd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4430) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4432) new->entries[j] = thresholds->primary->entries[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4433) if (new->entries[j].threshold <= usage) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4434) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4435) * new->current_threshold will not be used
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4436) * until rcu_assign_pointer(), so it's safe to increment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4437) * it here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4438) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4439) ++new->current_threshold;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4440) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4441) j++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4442) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4444) swap_buffers:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4445) /* Swap primary and spare array */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4446) thresholds->spare = thresholds->primary;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4448) rcu_assign_pointer(thresholds->primary, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4450) /* To be sure that nobody uses thresholds */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4451) synchronize_rcu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4453) /* If all events are unregistered, free the spare array */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4454) if (!new) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4455) kfree(thresholds->spare);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4456) thresholds->spare = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4457) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4458) unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4459) mutex_unlock(&memcg->thresholds_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4460) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4462) static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4463) struct eventfd_ctx *eventfd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4464) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4465) return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4466) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4468) static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4469) struct eventfd_ctx *eventfd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4470) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4471) return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4472) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4474) static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4475) struct eventfd_ctx *eventfd, const char *args)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4476) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4477) struct mem_cgroup_eventfd_list *event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4479) event = kmalloc(sizeof(*event), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4480) if (!event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4481) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4483) spin_lock(&memcg_oom_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4485) event->eventfd = eventfd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4486) list_add(&event->list, &memcg->oom_notify);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4488) /* already in OOM ? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4489) if (memcg->under_oom)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4490) eventfd_signal(eventfd, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4491) spin_unlock(&memcg_oom_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4493) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4494) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4495)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4496) static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4497) struct eventfd_ctx *eventfd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4498) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4499) struct mem_cgroup_eventfd_list *ev, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4501) spin_lock(&memcg_oom_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4502)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4503) list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4504) if (ev->eventfd == eventfd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4505) list_del(&ev->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4506) kfree(ev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4507) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4508) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4510) spin_unlock(&memcg_oom_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4511) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4513) static int mem_cgroup_oom_control_read(struct seq_file *sf, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4514) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4515) struct mem_cgroup *memcg = mem_cgroup_from_seq(sf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4517) seq_printf(sf, "oom_kill_disable %d\n", memcg->oom_kill_disable);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4518) seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4519) seq_printf(sf, "oom_kill %lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4520) atomic_long_read(&memcg->memory_events[MEMCG_OOM_KILL]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4521) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4522) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4524) static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4525) struct cftype *cft, u64 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4526) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4527) struct mem_cgroup *memcg = mem_cgroup_from_css(css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4529) /* cannot set to root cgroup and only 0 and 1 are allowed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4530) if (!css->parent || !((val == 0) || (val == 1)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4531) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4533) memcg->oom_kill_disable = val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4534) if (!val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4535) memcg_oom_recover(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4537) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4538) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4540) #ifdef CONFIG_CGROUP_WRITEBACK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4542) #include <trace/events/writeback.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4544) static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4545) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4546) return wb_domain_init(&memcg->cgwb_domain, gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4547) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4549) static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4550) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4551) wb_domain_exit(&memcg->cgwb_domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4552) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4554) static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4555) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4556) wb_domain_size_changed(&memcg->cgwb_domain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4557) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4559) struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4560) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4561) struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4563) if (!memcg->css.parent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4564) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4566) return &memcg->cgwb_domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4567) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4568)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4569) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4570) * idx can be of type enum memcg_stat_item or node_stat_item.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4571) * Keep in sync with memcg_exact_page().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4572) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4573) static unsigned long memcg_exact_page_state(struct mem_cgroup *memcg, int idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4574) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4575) long x = atomic_long_read(&memcg->vmstats[idx]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4576) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4578) for_each_online_cpu(cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4579) x += per_cpu_ptr(memcg->vmstats_percpu, cpu)->stat[idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4580) if (x < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4581) x = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4582) return x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4583) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4585) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4586) * mem_cgroup_wb_stats - retrieve writeback related stats from its memcg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4587) * @wb: bdi_writeback in question
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4588) * @pfilepages: out parameter for number of file pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4589) * @pheadroom: out parameter for number of allocatable pages according to memcg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4590) * @pdirty: out parameter for number of dirty pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4591) * @pwriteback: out parameter for number of pages under writeback
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4592) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4593) * Determine the numbers of file, headroom, dirty, and writeback pages in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4594) * @wb's memcg. File, dirty and writeback are self-explanatory. Headroom
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4595) * is a bit more involved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4596) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4597) * A memcg's headroom is "min(max, high) - used". In the hierarchy, the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4598) * headroom is calculated as the lowest headroom of itself and the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4599) * ancestors. Note that this doesn't consider the actual amount of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4600) * available memory in the system. The caller should further cap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4601) * *@pheadroom accordingly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4602) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4603) void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4604) unsigned long *pheadroom, unsigned long *pdirty,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4605) unsigned long *pwriteback)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4606) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4607) struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4608) struct mem_cgroup *parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4610) *pdirty = memcg_exact_page_state(memcg, NR_FILE_DIRTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4612) *pwriteback = memcg_exact_page_state(memcg, NR_WRITEBACK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4613) *pfilepages = memcg_exact_page_state(memcg, NR_INACTIVE_FILE) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4614) memcg_exact_page_state(memcg, NR_ACTIVE_FILE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4615) *pheadroom = PAGE_COUNTER_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4617) while ((parent = parent_mem_cgroup(memcg))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4618) unsigned long ceiling = min(READ_ONCE(memcg->memory.max),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4619) READ_ONCE(memcg->memory.high));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4620) unsigned long used = page_counter_read(&memcg->memory);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4621)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4622) *pheadroom = min(*pheadroom, ceiling - min(ceiling, used));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4623) memcg = parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4624) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4625) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4627) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4628) * Foreign dirty flushing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4629) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4630) * There's an inherent mismatch between memcg and writeback. The former
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4631) * trackes ownership per-page while the latter per-inode. This was a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4632) * deliberate design decision because honoring per-page ownership in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4633) * writeback path is complicated, may lead to higher CPU and IO overheads
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4634) * and deemed unnecessary given that write-sharing an inode across
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4635) * different cgroups isn't a common use-case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4636) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4637) * Combined with inode majority-writer ownership switching, this works well
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4638) * enough in most cases but there are some pathological cases. For
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4639) * example, let's say there are two cgroups A and B which keep writing to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4640) * different but confined parts of the same inode. B owns the inode and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4641) * A's memory is limited far below B's. A's dirty ratio can rise enough to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4642) * trigger balance_dirty_pages() sleeps but B's can be low enough to avoid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4643) * triggering background writeback. A will be slowed down without a way to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4644) * make writeback of the dirty pages happen.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4645) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4646) * Conditions like the above can lead to a cgroup getting repatedly and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4647) * severely throttled after making some progress after each
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4648) * dirty_expire_interval while the underyling IO device is almost
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4649) * completely idle.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4650) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4651) * Solving this problem completely requires matching the ownership tracking
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4652) * granularities between memcg and writeback in either direction. However,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4653) * the more egregious behaviors can be avoided by simply remembering the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4654) * most recent foreign dirtying events and initiating remote flushes on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4655) * them when local writeback isn't enough to keep the memory clean enough.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4656) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4657) * The following two functions implement such mechanism. When a foreign
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4658) * page - a page whose memcg and writeback ownerships don't match - is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4659) * dirtied, mem_cgroup_track_foreign_dirty() records the inode owning
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4660) * bdi_writeback on the page owning memcg. When balance_dirty_pages()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4661) * decides that the memcg needs to sleep due to high dirty ratio, it calls
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4662) * mem_cgroup_flush_foreign() which queues writeback on the recorded
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4663) * foreign bdi_writebacks which haven't expired. Both the numbers of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4664) * recorded bdi_writebacks and concurrent in-flight foreign writebacks are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4665) * limited to MEMCG_CGWB_FRN_CNT.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4666) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4667) * The mechanism only remembers IDs and doesn't hold any object references.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4668) * As being wrong occasionally doesn't matter, updates and accesses to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4669) * records are lockless and racy.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4670) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4671) void mem_cgroup_track_foreign_dirty_slowpath(struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4672) struct bdi_writeback *wb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4673) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4674) struct mem_cgroup *memcg = page->mem_cgroup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4675) struct memcg_cgwb_frn *frn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4676) u64 now = get_jiffies_64();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4677) u64 oldest_at = now;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4678) int oldest = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4679) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4680)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4681) trace_track_foreign_dirty(page, wb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4682)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4683) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4684) * Pick the slot to use. If there is already a slot for @wb, keep
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4685) * using it. If not replace the oldest one which isn't being
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4686) * written out.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4687) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4688) for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4689) frn = &memcg->cgwb_frn[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4690) if (frn->bdi_id == wb->bdi->id &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4691) frn->memcg_id == wb->memcg_css->id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4692) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4693) if (time_before64(frn->at, oldest_at) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4694) atomic_read(&frn->done.cnt) == 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4695) oldest = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4696) oldest_at = frn->at;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4697) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4698) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4700) if (i < MEMCG_CGWB_FRN_CNT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4701) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4702) * Re-using an existing one. Update timestamp lazily to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4703) * avoid making the cacheline hot. We want them to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4704) * reasonably up-to-date and significantly shorter than
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4705) * dirty_expire_interval as that's what expires the record.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4706) * Use the shorter of 1s and dirty_expire_interval / 8.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4707) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4708) unsigned long update_intv =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4709) min_t(unsigned long, HZ,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4710) msecs_to_jiffies(dirty_expire_interval * 10) / 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4711)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4712) if (time_before64(frn->at, now - update_intv))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4713) frn->at = now;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4714) } else if (oldest >= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4715) /* replace the oldest free one */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4716) frn = &memcg->cgwb_frn[oldest];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4717) frn->bdi_id = wb->bdi->id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4718) frn->memcg_id = wb->memcg_css->id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4719) frn->at = now;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4720) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4721) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4723) /* issue foreign writeback flushes for recorded foreign dirtying events */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4724) void mem_cgroup_flush_foreign(struct bdi_writeback *wb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4725) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4726) struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4727) unsigned long intv = msecs_to_jiffies(dirty_expire_interval * 10);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4728) u64 now = jiffies_64;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4729) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4730)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4731) for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4732) struct memcg_cgwb_frn *frn = &memcg->cgwb_frn[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4734) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4735) * If the record is older than dirty_expire_interval,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4736) * writeback on it has already started. No need to kick it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4737) * off again. Also, don't start a new one if there's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4738) * already one in flight.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4739) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4740) if (time_after64(frn->at, now - intv) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4741) atomic_read(&frn->done.cnt) == 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4742) frn->at = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4743) trace_flush_foreign(wb, frn->bdi_id, frn->memcg_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4744) cgroup_writeback_by_id(frn->bdi_id, frn->memcg_id, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4745) WB_REASON_FOREIGN_FLUSH,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4746) &frn->done);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4747) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4748) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4749) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4750)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4751) #else /* CONFIG_CGROUP_WRITEBACK */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4753) static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4754) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4755) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4756) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4758) static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4759) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4760) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4761)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4762) static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4763) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4764) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4765)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4766) #endif /* CONFIG_CGROUP_WRITEBACK */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4767)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4768) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4769) * DO NOT USE IN NEW FILES.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4770) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4771) * "cgroup.event_control" implementation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4772) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4773) * This is way over-engineered. It tries to support fully configurable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4774) * events for each user. Such level of flexibility is completely
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4775) * unnecessary especially in the light of the planned unified hierarchy.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4776) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4777) * Please deprecate this and replace with something simpler if at all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4778) * possible.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4779) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4781) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4782) * Unregister event and free resources.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4783) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4784) * Gets called from workqueue.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4785) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4786) static void memcg_event_remove(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4787) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4788) struct mem_cgroup_event *event =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4789) container_of(work, struct mem_cgroup_event, remove);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4790) struct mem_cgroup *memcg = event->memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4791)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4792) remove_wait_queue(event->wqh, &event->wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4793)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4794) event->unregister_event(memcg, event->eventfd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4795)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4796) /* Notify userspace the event is going away. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4797) eventfd_signal(event->eventfd, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4798)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4799) eventfd_ctx_put(event->eventfd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4800) kfree(event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4801) css_put(&memcg->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4802) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4804) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4805) * Gets called on EPOLLHUP on eventfd when user closes it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4806) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4807) * Called with wqh->lock held and interrupts disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4808) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4809) static int memcg_event_wake(wait_queue_entry_t *wait, unsigned mode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4810) int sync, void *key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4811) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4812) struct mem_cgroup_event *event =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4813) container_of(wait, struct mem_cgroup_event, wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4814) struct mem_cgroup *memcg = event->memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4815) __poll_t flags = key_to_poll(key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4816)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4817) if (flags & EPOLLHUP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4818) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4819) * If the event has been detached at cgroup removal, we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4820) * can simply return knowing the other side will cleanup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4821) * for us.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4822) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4823) * We can't race against event freeing since the other
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4824) * side will require wqh->lock via remove_wait_queue(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4825) * which we hold.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4826) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4827) spin_lock(&memcg->event_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4828) if (!list_empty(&event->list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4829) list_del_init(&event->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4830) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4831) * We are in atomic context, but cgroup_event_remove()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4832) * may sleep, so we have to call it in workqueue.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4833) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4834) schedule_work(&event->remove);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4835) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4836) spin_unlock(&memcg->event_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4837) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4838)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4839) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4840) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4841)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4842) static void memcg_event_ptable_queue_proc(struct file *file,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4843) wait_queue_head_t *wqh, poll_table *pt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4844) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4845) struct mem_cgroup_event *event =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4846) container_of(pt, struct mem_cgroup_event, pt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4847)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4848) event->wqh = wqh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4849) add_wait_queue(wqh, &event->wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4850) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4852) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4853) * DO NOT USE IN NEW FILES.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4854) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4855) * Parse input and register new cgroup event handler.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4856) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4857) * Input must be in format '<event_fd> <control_fd> <args>'.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4858) * Interpretation of args is defined by control file implementation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4859) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4860) static ssize_t memcg_write_event_control(struct kernfs_open_file *of,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4861) char *buf, size_t nbytes, loff_t off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4862) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4863) struct cgroup_subsys_state *css = of_css(of);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4864) struct mem_cgroup *memcg = mem_cgroup_from_css(css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4865) struct mem_cgroup_event *event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4866) struct cgroup_subsys_state *cfile_css;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4867) unsigned int efd, cfd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4868) struct fd efile;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4869) struct fd cfile;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4870) const char *name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4871) char *endp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4872) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4873)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4874) buf = strstrip(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4875)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4876) efd = simple_strtoul(buf, &endp, 10);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4877) if (*endp != ' ')
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4878) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4879) buf = endp + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4880)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4881) cfd = simple_strtoul(buf, &endp, 10);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4882) if ((*endp != ' ') && (*endp != '\0'))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4883) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4884) buf = endp + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4885)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4886) event = kzalloc(sizeof(*event), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4887) if (!event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4888) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4889)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4890) event->memcg = memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4891) INIT_LIST_HEAD(&event->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4892) init_poll_funcptr(&event->pt, memcg_event_ptable_queue_proc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4893) init_waitqueue_func_entry(&event->wait, memcg_event_wake);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4894) INIT_WORK(&event->remove, memcg_event_remove);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4895)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4896) efile = fdget(efd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4897) if (!efile.file) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4898) ret = -EBADF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4899) goto out_kfree;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4900) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4901)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4902) event->eventfd = eventfd_ctx_fileget(efile.file);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4903) if (IS_ERR(event->eventfd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4904) ret = PTR_ERR(event->eventfd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4905) goto out_put_efile;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4906) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4907)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4908) cfile = fdget(cfd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4909) if (!cfile.file) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4910) ret = -EBADF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4911) goto out_put_eventfd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4912) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4913)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4914) /* the process need read permission on control file */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4915) /* AV: shouldn't we check that it's been opened for read instead? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4916) ret = inode_permission(file_inode(cfile.file), MAY_READ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4917) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4918) goto out_put_cfile;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4919)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4920) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4921) * Determine the event callbacks and set them in @event. This used
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4922) * to be done via struct cftype but cgroup core no longer knows
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4923) * about these events. The following is crude but the whole thing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4924) * is for compatibility anyway.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4925) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4926) * DO NOT ADD NEW FILES.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4927) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4928) name = cfile.file->f_path.dentry->d_name.name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4929)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4930) if (!strcmp(name, "memory.usage_in_bytes")) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4931) event->register_event = mem_cgroup_usage_register_event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4932) event->unregister_event = mem_cgroup_usage_unregister_event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4933) } else if (!strcmp(name, "memory.oom_control")) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4934) event->register_event = mem_cgroup_oom_register_event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4935) event->unregister_event = mem_cgroup_oom_unregister_event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4936) } else if (!strcmp(name, "memory.pressure_level")) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4937) event->register_event = vmpressure_register_event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4938) event->unregister_event = vmpressure_unregister_event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4939) } else if (!strcmp(name, "memory.memsw.usage_in_bytes")) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4940) event->register_event = memsw_cgroup_usage_register_event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4941) event->unregister_event = memsw_cgroup_usage_unregister_event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4942) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4943) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4944) goto out_put_cfile;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4945) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4946)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4947) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4948) * Verify @cfile should belong to @css. Also, remaining events are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4949) * automatically removed on cgroup destruction but the removal is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4950) * asynchronous, so take an extra ref on @css.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4951) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4952) cfile_css = css_tryget_online_from_dir(cfile.file->f_path.dentry->d_parent,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4953) &memory_cgrp_subsys);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4954) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4955) if (IS_ERR(cfile_css))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4956) goto out_put_cfile;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4957) if (cfile_css != css) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4958) css_put(cfile_css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4959) goto out_put_cfile;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4960) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4961)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4962) ret = event->register_event(memcg, event->eventfd, buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4963) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4964) goto out_put_css;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4965)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4966) vfs_poll(efile.file, &event->pt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4967)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4968) spin_lock(&memcg->event_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4969) list_add(&event->list, &memcg->event_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4970) spin_unlock(&memcg->event_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4971)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4972) fdput(cfile);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4973) fdput(efile);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4974)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4975) return nbytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4976)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4977) out_put_css:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4978) css_put(css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4979) out_put_cfile:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4980) fdput(cfile);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4981) out_put_eventfd:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4982) eventfd_ctx_put(event->eventfd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4983) out_put_efile:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4984) fdput(efile);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4985) out_kfree:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4986) kfree(event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4987)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4988) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4989) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4990)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4991) static struct cftype mem_cgroup_legacy_files[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4992) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4993) .name = "usage_in_bytes",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4994) .private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4995) .read_u64 = mem_cgroup_read_u64,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4996) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4997) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4998) .name = "max_usage_in_bytes",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4999) .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5000) .write = mem_cgroup_reset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5001) .read_u64 = mem_cgroup_read_u64,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5002) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5003) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5004) .name = "limit_in_bytes",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5005) .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5006) .write = mem_cgroup_write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5007) .read_u64 = mem_cgroup_read_u64,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5008) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5009) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5010) .name = "soft_limit_in_bytes",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5011) .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5012) .write = mem_cgroup_write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5013) .read_u64 = mem_cgroup_read_u64,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5014) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5015) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5016) .name = "failcnt",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5017) .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5018) .write = mem_cgroup_reset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5019) .read_u64 = mem_cgroup_read_u64,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5020) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5021) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5022) .name = "stat",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5023) .seq_show = memcg_stat_show,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5024) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5025) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5026) .name = "force_empty",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5027) .write = mem_cgroup_force_empty_write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5028) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5029) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5030) .name = "use_hierarchy",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5031) .write_u64 = mem_cgroup_hierarchy_write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5032) .read_u64 = mem_cgroup_hierarchy_read,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5033) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5034) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5035) .name = "cgroup.event_control", /* XXX: for compat */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5036) .write = memcg_write_event_control,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5037) .flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5038) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5039) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5040) .name = "swappiness",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5041) .read_u64 = mem_cgroup_swappiness_read,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5042) .write_u64 = mem_cgroup_swappiness_write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5043) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5044) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5045) .name = "move_charge_at_immigrate",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5046) .read_u64 = mem_cgroup_move_charge_read,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5047) .write_u64 = mem_cgroup_move_charge_write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5048) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5049) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5050) .name = "oom_control",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5051) .seq_show = mem_cgroup_oom_control_read,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5052) .write_u64 = mem_cgroup_oom_control_write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5053) .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5054) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5055) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5056) .name = "pressure_level",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5057) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5058) #ifdef CONFIG_NUMA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5059) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5060) .name = "numa_stat",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5061) .seq_show = memcg_numa_stat_show,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5062) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5063) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5064) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5065) .name = "kmem.limit_in_bytes",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5066) .private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5067) .write = mem_cgroup_write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5068) .read_u64 = mem_cgroup_read_u64,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5069) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5070) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5071) .name = "kmem.usage_in_bytes",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5072) .private = MEMFILE_PRIVATE(_KMEM, RES_USAGE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5073) .read_u64 = mem_cgroup_read_u64,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5074) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5075) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5076) .name = "kmem.failcnt",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5077) .private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5078) .write = mem_cgroup_reset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5079) .read_u64 = mem_cgroup_read_u64,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5080) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5081) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5082) .name = "kmem.max_usage_in_bytes",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5083) .private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5084) .write = mem_cgroup_reset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5085) .read_u64 = mem_cgroup_read_u64,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5086) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5087) #if defined(CONFIG_MEMCG_KMEM) && \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5088) (defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5089) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5090) .name = "kmem.slabinfo",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5091) .seq_show = memcg_slab_show,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5092) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5093) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5094) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5095) .name = "kmem.tcp.limit_in_bytes",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5096) .private = MEMFILE_PRIVATE(_TCP, RES_LIMIT),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5097) .write = mem_cgroup_write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5098) .read_u64 = mem_cgroup_read_u64,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5099) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5100) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5101) .name = "kmem.tcp.usage_in_bytes",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5102) .private = MEMFILE_PRIVATE(_TCP, RES_USAGE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5103) .read_u64 = mem_cgroup_read_u64,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5104) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5105) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5106) .name = "kmem.tcp.failcnt",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5107) .private = MEMFILE_PRIVATE(_TCP, RES_FAILCNT),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5108) .write = mem_cgroup_reset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5109) .read_u64 = mem_cgroup_read_u64,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5110) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5111) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5112) .name = "kmem.tcp.max_usage_in_bytes",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5113) .private = MEMFILE_PRIVATE(_TCP, RES_MAX_USAGE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5114) .write = mem_cgroup_reset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5115) .read_u64 = mem_cgroup_read_u64,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5116) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5117) { }, /* terminate */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5118) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5120) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5121) * Private memory cgroup IDR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5122) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5123) * Swap-out records and page cache shadow entries need to store memcg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5124) * references in constrained space, so we maintain an ID space that is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5125) * limited to 16 bit (MEM_CGROUP_ID_MAX), limiting the total number of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5126) * memory-controlled cgroups to 64k.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5127) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5128) * However, there usually are many references to the offline CSS after
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5129) * the cgroup has been destroyed, such as page cache or reclaimable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5130) * slab objects, that don't need to hang on to the ID. We want to keep
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5131) * those dead CSS from occupying IDs, or we might quickly exhaust the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5132) * relatively small ID space and prevent the creation of new cgroups
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5133) * even when there are much fewer than 64k cgroups - possibly none.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5134) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5135) * Maintain a private 16-bit ID space for memcg, and allow the ID to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5136) * be freed and recycled when it's no longer needed, which is usually
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5137) * when the CSS is offlined.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5138) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5139) * The only exception to that are records of swapped out tmpfs/shmem
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5140) * pages that need to be attributed to live ancestors on swapin. But
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5141) * those references are manageable from userspace.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5142) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5144) static DEFINE_IDR(mem_cgroup_idr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5146) static void mem_cgroup_id_remove(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5147) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5148) if (memcg->id.id > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5149) trace_android_vh_mem_cgroup_id_remove(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5150) idr_remove(&mem_cgroup_idr, memcg->id.id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5151) memcg->id.id = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5153) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5155) static void __maybe_unused mem_cgroup_id_get_many(struct mem_cgroup *memcg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5156) unsigned int n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5157) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5158) refcount_add(n, &memcg->id.ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5159) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5161) static void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5162) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5163) if (refcount_sub_and_test(n, &memcg->id.ref)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5164) mem_cgroup_id_remove(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5166) /* Memcg ID pins CSS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5167) css_put(&memcg->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5168) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5169) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5171) static inline void mem_cgroup_id_put(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5172) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5173) mem_cgroup_id_put_many(memcg, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5174) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5176) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5177) * mem_cgroup_from_id - look up a memcg from a memcg id
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5178) * @id: the memcg id to look up
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5179) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5180) * Caller must hold rcu_read_lock().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5181) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5182) struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5183) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5184) WARN_ON_ONCE(!rcu_read_lock_held());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5185) return idr_find(&mem_cgroup_idr, id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5186) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5187) EXPORT_SYMBOL_GPL(mem_cgroup_from_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5189) static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5190) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5191) struct mem_cgroup_per_node *pn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5192) int tmp = node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5193) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5194) * This routine is called against possible nodes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5195) * But it's BUG to call kmalloc() against offline node.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5196) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5197) * TODO: this routine can waste much memory for nodes which will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5198) * never be onlined. It's better to use memory hotplug callback
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5199) * function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5200) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5201) if (!node_state(node, N_NORMAL_MEMORY))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5202) tmp = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5203) pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5204) if (!pn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5205) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5207) pn->lruvec_stat_local = alloc_percpu_gfp(struct lruvec_stat,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5208) GFP_KERNEL_ACCOUNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5209) if (!pn->lruvec_stat_local) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5210) kfree(pn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5211) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5212) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5214) pn->lruvec_stat_cpu = alloc_percpu_gfp(struct lruvec_stat,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5215) GFP_KERNEL_ACCOUNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5216) if (!pn->lruvec_stat_cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5217) free_percpu(pn->lruvec_stat_local);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5218) kfree(pn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5219) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5220) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5222) lruvec_init(&pn->lruvec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5223) pn->usage_in_excess = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5224) pn->on_tree = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5225) pn->memcg = memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5227) memcg->nodeinfo[node] = pn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5228) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5229) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5231) static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5232) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5233) struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5235) if (!pn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5236) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5238) free_percpu(pn->lruvec_stat_cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5239) free_percpu(pn->lruvec_stat_local);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5240) kfree(pn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5243) static void __mem_cgroup_free(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5244) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5245) int node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5247) trace_android_vh_mem_cgroup_free(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5248) for_each_node(node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5249) free_mem_cgroup_per_node_info(memcg, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5250) free_percpu(memcg->vmstats_percpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5251) free_percpu(memcg->vmstats_local);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5252) kfree(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5253) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5255) static void mem_cgroup_free(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5256) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5257) memcg_wb_domain_exit(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5258) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5259) * Flush percpu vmstats and vmevents to guarantee the value correctness
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5260) * on parent's and all ancestor levels.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5261) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5262) memcg_flush_percpu_vmstats(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5263) memcg_flush_percpu_vmevents(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5264) __mem_cgroup_free(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5265) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5267) static struct mem_cgroup *mem_cgroup_alloc(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5268) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5269) struct mem_cgroup *memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5270) unsigned int size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5271) int node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5272) int __maybe_unused i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5273) long error = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5275) size = sizeof(struct mem_cgroup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5276) size += nr_node_ids * sizeof(struct mem_cgroup_per_node *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5278) memcg = kzalloc(size, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5279) if (!memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5280) return ERR_PTR(error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5282) memcg->id.id = idr_alloc(&mem_cgroup_idr, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5283) 1, MEM_CGROUP_ID_MAX,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5284) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5285) if (memcg->id.id < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5286) error = memcg->id.id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5287) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5288) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5290) memcg->vmstats_local = alloc_percpu_gfp(struct memcg_vmstats_percpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5291) GFP_KERNEL_ACCOUNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5292) if (!memcg->vmstats_local)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5293) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5295) memcg->vmstats_percpu = alloc_percpu_gfp(struct memcg_vmstats_percpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5296) GFP_KERNEL_ACCOUNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5297) if (!memcg->vmstats_percpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5298) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5300) for_each_node(node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5301) if (alloc_mem_cgroup_per_node_info(memcg, node))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5302) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5304) if (memcg_wb_domain_init(memcg, GFP_KERNEL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5305) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5307) INIT_WORK(&memcg->high_work, high_work_func);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5308) INIT_LIST_HEAD(&memcg->oom_notify);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5309) mutex_init(&memcg->thresholds_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5310) spin_lock_init(&memcg->move_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5311) vmpressure_init(&memcg->vmpressure);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5312) INIT_LIST_HEAD(&memcg->event_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5313) spin_lock_init(&memcg->event_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5314) memcg->socket_pressure = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5315) #ifdef CONFIG_MEMCG_KMEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5316) memcg->kmemcg_id = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5317) INIT_LIST_HEAD(&memcg->objcg_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5318) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5319) #ifdef CONFIG_CGROUP_WRITEBACK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5320) INIT_LIST_HEAD(&memcg->cgwb_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5321) for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5322) memcg->cgwb_frn[i].done =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5323) __WB_COMPLETION_INIT(&memcg_cgwb_frn_waitq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5324) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5325) #ifdef CONFIG_TRANSPARENT_HUGEPAGE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5326) spin_lock_init(&memcg->deferred_split_queue.split_queue_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5327) INIT_LIST_HEAD(&memcg->deferred_split_queue.split_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5328) memcg->deferred_split_queue.split_queue_len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5329) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5330) idr_replace(&mem_cgroup_idr, memcg, memcg->id.id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5331) trace_android_vh_mem_cgroup_alloc(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5332) return memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5333) fail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5334) mem_cgroup_id_remove(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5335) __mem_cgroup_free(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5336) return ERR_PTR(error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5337) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5339) static struct cgroup_subsys_state * __ref
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5340) mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5341) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5342) struct mem_cgroup *parent = mem_cgroup_from_css(parent_css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5343) struct mem_cgroup *memcg, *old_memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5344) long error = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5346) old_memcg = set_active_memcg(parent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5347) memcg = mem_cgroup_alloc();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5348) set_active_memcg(old_memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5349) if (IS_ERR(memcg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5350) return ERR_CAST(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5352) page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5353) memcg->soft_limit = PAGE_COUNTER_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5354) page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5355) if (parent) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5356) memcg->swappiness = mem_cgroup_swappiness(parent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5357) memcg->oom_kill_disable = parent->oom_kill_disable;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5358) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5359) if (!parent) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5360) page_counter_init(&memcg->memory, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5361) page_counter_init(&memcg->swap, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5362) page_counter_init(&memcg->kmem, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5363) page_counter_init(&memcg->tcpmem, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5364) } else if (parent->use_hierarchy) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5365) memcg->use_hierarchy = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5366) page_counter_init(&memcg->memory, &parent->memory);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5367) page_counter_init(&memcg->swap, &parent->swap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5368) page_counter_init(&memcg->kmem, &parent->kmem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5369) page_counter_init(&memcg->tcpmem, &parent->tcpmem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5370) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5371) page_counter_init(&memcg->memory, &root_mem_cgroup->memory);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5372) page_counter_init(&memcg->swap, &root_mem_cgroup->swap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5373) page_counter_init(&memcg->kmem, &root_mem_cgroup->kmem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5374) page_counter_init(&memcg->tcpmem, &root_mem_cgroup->tcpmem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5375) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5376) * Deeper hierachy with use_hierarchy == false doesn't make
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5377) * much sense so let cgroup subsystem know about this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5378) * unfortunate state in our controller.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5379) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5380) if (parent != root_mem_cgroup)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5381) memory_cgrp_subsys.broken_hierarchy = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5382) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5384) /* The following stuff does not apply to the root */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5385) if (!parent) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5386) root_mem_cgroup = memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5387) return &memcg->css;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5388) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5390) error = memcg_online_kmem(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5391) if (error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5392) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5394) if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5395) static_branch_inc(&memcg_sockets_enabled_key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5397) return &memcg->css;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5398) fail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5399) mem_cgroup_id_remove(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5400) mem_cgroup_free(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5401) return ERR_PTR(error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5402) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5404) static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5405) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5406) struct mem_cgroup *memcg = mem_cgroup_from_css(css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5408) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5409) * A memcg must be visible for memcg_expand_shrinker_maps()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5410) * by the time the maps are allocated. So, we allocate maps
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5411) * here, when for_each_mem_cgroup() can't skip it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5412) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5413) if (memcg_alloc_shrinker_maps(memcg)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5414) mem_cgroup_id_remove(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5415) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5416) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5418) /* Online state pins memcg ID, memcg ID pins CSS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5419) refcount_set(&memcg->id.ref, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5420) css_get(css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5421) trace_android_vh_mem_cgroup_css_online(css, memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5422) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5423) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5425) static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5426) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5427) struct mem_cgroup *memcg = mem_cgroup_from_css(css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5428) struct mem_cgroup_event *event, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5430) trace_android_vh_mem_cgroup_css_offline(css, memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5431) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5432) * Unregister events and notify userspace.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5433) * Notify userspace about cgroup removing only after rmdir of cgroup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5434) * directory to avoid race between userspace and kernelspace.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5435) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5436) spin_lock(&memcg->event_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5437) list_for_each_entry_safe(event, tmp, &memcg->event_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5438) list_del_init(&event->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5439) schedule_work(&event->remove);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5440) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5441) spin_unlock(&memcg->event_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5442)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5443) page_counter_set_min(&memcg->memory, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5444) page_counter_set_low(&memcg->memory, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5446) memcg_offline_kmem(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5447) wb_memcg_offline(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5449) drain_all_stock(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5451) mem_cgroup_id_put(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5452) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5454) static void mem_cgroup_css_released(struct cgroup_subsys_state *css)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5455) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5456) struct mem_cgroup *memcg = mem_cgroup_from_css(css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5458) invalidate_reclaim_iterators(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5459) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5461) static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5462) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5463) struct mem_cgroup *memcg = mem_cgroup_from_css(css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5464) int __maybe_unused i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5466) #ifdef CONFIG_CGROUP_WRITEBACK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5467) for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5468) wb_wait_for_completion(&memcg->cgwb_frn[i].done);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5469) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5470) if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5471) static_branch_dec(&memcg_sockets_enabled_key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5473) if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5474) static_branch_dec(&memcg_sockets_enabled_key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5476) vmpressure_cleanup(&memcg->vmpressure);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5477) cancel_work_sync(&memcg->high_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5478) mem_cgroup_remove_from_trees(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5479) memcg_free_shrinker_maps(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5480) memcg_free_kmem(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5481) mem_cgroup_free(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5482) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5484) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5485) * mem_cgroup_css_reset - reset the states of a mem_cgroup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5486) * @css: the target css
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5487) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5488) * Reset the states of the mem_cgroup associated with @css. This is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5489) * invoked when the userland requests disabling on the default hierarchy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5490) * but the memcg is pinned through dependency. The memcg should stop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5491) * applying policies and should revert to the vanilla state as it may be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5492) * made visible again.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5493) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5494) * The current implementation only resets the essential configurations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5495) * This needs to be expanded to cover all the visible parts.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5496) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5497) static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5498) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5499) struct mem_cgroup *memcg = mem_cgroup_from_css(css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5501) page_counter_set_max(&memcg->memory, PAGE_COUNTER_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5502) page_counter_set_max(&memcg->swap, PAGE_COUNTER_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5503) page_counter_set_max(&memcg->kmem, PAGE_COUNTER_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5504) page_counter_set_max(&memcg->tcpmem, PAGE_COUNTER_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5505) page_counter_set_min(&memcg->memory, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5506) page_counter_set_low(&memcg->memory, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5507) page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5508) memcg->soft_limit = PAGE_COUNTER_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5509) page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5510) memcg_wb_domain_size_changed(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5511) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5513) #ifdef CONFIG_MMU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5514) /* Handlers for move charge at task migration. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5515) static int mem_cgroup_do_precharge(unsigned long count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5516) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5517) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5519) /* Try a single bulk charge without reclaim first, kswapd may wake */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5520) ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_DIRECT_RECLAIM, count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5521) if (!ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5522) mc.precharge += count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5523) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5524) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5526) /* Try charges one by one with reclaim, but do not retry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5527) while (count--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5528) ret = try_charge(mc.to, GFP_KERNEL | __GFP_NORETRY, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5529) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5530) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5531) mc.precharge++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5532) cond_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5533) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5534) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5535) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5537) union mc_target {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5538) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5539) swp_entry_t ent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5540) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5542) enum mc_target_type {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5543) MC_TARGET_NONE = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5544) MC_TARGET_PAGE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5545) MC_TARGET_SWAP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5546) MC_TARGET_DEVICE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5547) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5549) static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5550) unsigned long addr, pte_t ptent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5551) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5552) struct page *page = vm_normal_page(vma, addr, ptent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5554) if (!page || !page_mapped(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5555) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5556) if (PageAnon(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5557) if (!(mc.flags & MOVE_ANON))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5558) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5559) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5560) if (!(mc.flags & MOVE_FILE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5561) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5562) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5563) if (!get_page_unless_zero(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5564) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5566) return page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5567) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5568)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5569) #if defined(CONFIG_SWAP) || defined(CONFIG_DEVICE_PRIVATE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5570) static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5571) pte_t ptent, swp_entry_t *entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5572) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5573) struct page *page = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5574) swp_entry_t ent = pte_to_swp_entry(ptent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5576) if (!(mc.flags & MOVE_ANON))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5577) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5579) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5580) * Handle MEMORY_DEVICE_PRIVATE which are ZONE_DEVICE page belonging to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5581) * a device and because they are not accessible by CPU they are store
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5582) * as special swap entry in the CPU page table.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5583) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5584) if (is_device_private_entry(ent)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5585) page = device_private_entry_to_page(ent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5586) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5587) * MEMORY_DEVICE_PRIVATE means ZONE_DEVICE page and which have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5588) * a refcount of 1 when free (unlike normal page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5589) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5590) if (!page_ref_add_unless(page, 1, 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5591) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5592) return page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5593) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5595) if (non_swap_entry(ent))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5596) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5597)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5598) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5599) * Because lookup_swap_cache() updates some statistics counter,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5600) * we call find_get_page() with swapper_space directly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5601) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5602) page = find_get_page(swap_address_space(ent), swp_offset(ent));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5603) entry->val = ent.val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5605) return page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5606) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5607) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5608) static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5609) pte_t ptent, swp_entry_t *entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5610) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5611) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5612) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5613) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5614)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5615) static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5616) unsigned long addr, pte_t ptent, swp_entry_t *entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5617) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5618) if (!vma->vm_file) /* anonymous vma */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5619) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5620) if (!(mc.flags & MOVE_FILE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5621) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5623) /* page is moved even if it's not RSS of this task(page-faulted). */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5624) /* shmem/tmpfs may report page out on swap: account for that too. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5625) return find_get_incore_page(vma->vm_file->f_mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5626) linear_page_index(vma, addr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5627) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5628)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5629) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5630) * mem_cgroup_move_account - move account of the page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5631) * @page: the page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5632) * @compound: charge the page as compound or small page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5633) * @from: mem_cgroup which the page is moved from.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5634) * @to: mem_cgroup which the page is moved to. @from != @to.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5635) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5636) * The caller must make sure the page is not on LRU (isolate_page() is useful.)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5637) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5638) * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5639) * from old cgroup.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5640) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5641) static int mem_cgroup_move_account(struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5642) bool compound,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5643) struct mem_cgroup *from,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5644) struct mem_cgroup *to)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5645) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5646) struct lruvec *from_vec, *to_vec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5647) struct pglist_data *pgdat;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5648) unsigned int nr_pages = compound ? thp_nr_pages(page) : 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5649) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5651) VM_BUG_ON(from == to);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5652) VM_BUG_ON_PAGE(PageLRU(page), page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5653) VM_BUG_ON(compound && !PageTransHuge(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5654)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5655) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5656) * Prevent mem_cgroup_migrate() from looking at
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5657) * page->mem_cgroup of its source page while we change it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5658) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5659) ret = -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5660) if (!trylock_page(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5661) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5663) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5664) if (page->mem_cgroup != from)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5665) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5667) pgdat = page_pgdat(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5668) from_vec = mem_cgroup_lruvec(from, pgdat);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5669) to_vec = mem_cgroup_lruvec(to, pgdat);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5670)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5671) lock_page_memcg(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5672)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5673) if (PageAnon(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5674) if (page_mapped(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5675) __mod_lruvec_state(from_vec, NR_ANON_MAPPED, -nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5676) __mod_lruvec_state(to_vec, NR_ANON_MAPPED, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5677) if (PageTransHuge(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5678) __dec_lruvec_state(from_vec, NR_ANON_THPS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5679) __inc_lruvec_state(to_vec, NR_ANON_THPS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5680) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5682) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5683) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5684) __mod_lruvec_state(from_vec, NR_FILE_PAGES, -nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5685) __mod_lruvec_state(to_vec, NR_FILE_PAGES, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5686)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5687) if (PageSwapBacked(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5688) __mod_lruvec_state(from_vec, NR_SHMEM, -nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5689) __mod_lruvec_state(to_vec, NR_SHMEM, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5690) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5692) if (page_mapped(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5693) __mod_lruvec_state(from_vec, NR_FILE_MAPPED, -nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5694) __mod_lruvec_state(to_vec, NR_FILE_MAPPED, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5695) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5696)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5697) if (PageDirty(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5698) struct address_space *mapping = page_mapping(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5700) if (mapping_can_writeback(mapping)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5701) __mod_lruvec_state(from_vec, NR_FILE_DIRTY,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5702) -nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5703) __mod_lruvec_state(to_vec, NR_FILE_DIRTY,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5704) nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5705) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5706) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5707) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5708)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5709) if (PageWriteback(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5710) __mod_lruvec_state(from_vec, NR_WRITEBACK, -nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5711) __mod_lruvec_state(to_vec, NR_WRITEBACK, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5712) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5713)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5714) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5715) * All state has been migrated, let's switch to the new memcg.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5716) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5717) * It is safe to change page->mem_cgroup here because the page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5718) * is referenced, charged, isolated, and locked: we can't race
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5719) * with (un)charging, migration, LRU putback, or anything else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5720) * that would rely on a stable page->mem_cgroup.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5721) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5722) * Note that lock_page_memcg is a memcg lock, not a page lock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5723) * to save space. As soon as we switch page->mem_cgroup to a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5724) * new memcg that isn't locked, the above state can change
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5725) * concurrently again. Make sure we're truly done with it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5726) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5727) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5728)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5729) css_get(&to->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5730) css_put(&from->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5731)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5732) page->mem_cgroup = to;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5734) __unlock_page_memcg(from);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5735)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5736) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5738) local_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5739) mem_cgroup_charge_statistics(to, page, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5740) memcg_check_events(to, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5741) mem_cgroup_charge_statistics(from, page, -nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5742) memcg_check_events(from, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5743) local_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5744) out_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5745) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5746) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5747) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5748) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5749)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5750) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5751) * get_mctgt_type - get target type of moving charge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5752) * @vma: the vma the pte to be checked belongs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5753) * @addr: the address corresponding to the pte to be checked
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5754) * @ptent: the pte to be checked
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5755) * @target: the pointer the target page or swap ent will be stored(can be NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5756) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5757) * Returns
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5758) * 0(MC_TARGET_NONE): if the pte is not a target for move charge.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5759) * 1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5760) * move charge. if @target is not NULL, the page is stored in target->page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5761) * with extra refcnt got(Callers should handle it).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5762) * 2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5763) * target for charge migration. if @target is not NULL, the entry is stored
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5764) * in target->ent.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5765) * 3(MC_TARGET_DEVICE): like MC_TARGET_PAGE but page is MEMORY_DEVICE_PRIVATE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5766) * (so ZONE_DEVICE page and thus not on the lru).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5767) * For now we such page is charge like a regular page would be as for all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5768) * intent and purposes it is just special memory taking the place of a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5769) * regular page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5770) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5771) * See Documentations/vm/hmm.txt and include/linux/hmm.h
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5772) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5773) * Called with pte lock held.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5774) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5775)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5776) static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5777) unsigned long addr, pte_t ptent, union mc_target *target)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5778) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5779) struct page *page = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5780) enum mc_target_type ret = MC_TARGET_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5781) swp_entry_t ent = { .val = 0 };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5782)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5783) if (pte_present(ptent))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5784) page = mc_handle_present_pte(vma, addr, ptent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5785) else if (is_swap_pte(ptent))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5786) page = mc_handle_swap_pte(vma, ptent, &ent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5787) else if (pte_none(ptent))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5788) page = mc_handle_file_pte(vma, addr, ptent, &ent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5789)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5790) if (!page && !ent.val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5791) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5792) if (page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5793) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5794) * Do only loose check w/o serialization.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5795) * mem_cgroup_move_account() checks the page is valid or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5796) * not under LRU exclusion.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5797) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5798) if (page->mem_cgroup == mc.from) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5799) ret = MC_TARGET_PAGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5800) if (is_device_private_page(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5801) ret = MC_TARGET_DEVICE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5802) if (target)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5803) target->page = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5804) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5805) if (!ret || !target)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5806) put_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5807) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5808) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5809) * There is a swap entry and a page doesn't exist or isn't charged.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5810) * But we cannot move a tail-page in a THP.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5811) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5812) if (ent.val && !ret && (!page || !PageTransCompound(page)) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5813) mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5814) ret = MC_TARGET_SWAP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5815) if (target)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5816) target->ent = ent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5817) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5818) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5819) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5820)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5821) #ifdef CONFIG_TRANSPARENT_HUGEPAGE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5822) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5823) * We don't consider PMD mapped swapping or file mapped pages because THP does
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5824) * not support them for now.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5825) * Caller should make sure that pmd_trans_huge(pmd) is true.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5826) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5827) static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5828) unsigned long addr, pmd_t pmd, union mc_target *target)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5829) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5830) struct page *page = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5831) enum mc_target_type ret = MC_TARGET_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5832)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5833) if (unlikely(is_swap_pmd(pmd))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5834) VM_BUG_ON(thp_migration_supported() &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5835) !is_pmd_migration_entry(pmd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5836) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5837) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5838) page = pmd_page(pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5839) VM_BUG_ON_PAGE(!page || !PageHead(page), page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5840) if (!(mc.flags & MOVE_ANON))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5841) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5842) if (page->mem_cgroup == mc.from) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5843) ret = MC_TARGET_PAGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5844) if (target) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5845) get_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5846) target->page = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5847) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5848) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5849) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5850) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5851) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5852) static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5853) unsigned long addr, pmd_t pmd, union mc_target *target)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5854) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5855) return MC_TARGET_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5856) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5857) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5858)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5859) static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5860) unsigned long addr, unsigned long end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5861) struct mm_walk *walk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5862) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5863) struct vm_area_struct *vma = walk->vma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5864) pte_t *pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5865) spinlock_t *ptl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5866)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5867) ptl = pmd_trans_huge_lock(pmd, vma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5868) if (ptl) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5869) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5870) * Note their can not be MC_TARGET_DEVICE for now as we do not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5871) * support transparent huge page with MEMORY_DEVICE_PRIVATE but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5872) * this might change.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5873) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5874) if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5875) mc.precharge += HPAGE_PMD_NR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5876) spin_unlock(ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5877) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5878) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5879)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5880) if (pmd_trans_unstable(pmd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5881) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5882) pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5883) for (; addr != end; pte++, addr += PAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5884) if (get_mctgt_type(vma, addr, *pte, NULL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5885) mc.precharge++; /* increment precharge temporarily */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5886) pte_unmap_unlock(pte - 1, ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5887) cond_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5889) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5890) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5891)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5892) static const struct mm_walk_ops precharge_walk_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5893) .pmd_entry = mem_cgroup_count_precharge_pte_range,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5894) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5895)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5896) static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5897) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5898) unsigned long precharge;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5899)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5900) mmap_read_lock(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5901) walk_page_range(mm, 0, mm->highest_vm_end, &precharge_walk_ops, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5902) mmap_read_unlock(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5904) precharge = mc.precharge;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5905) mc.precharge = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5906)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5907) return precharge;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5908) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5909)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5910) static int mem_cgroup_precharge_mc(struct mm_struct *mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5911) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5912) unsigned long precharge = mem_cgroup_count_precharge(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5913)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5914) VM_BUG_ON(mc.moving_task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5915) mc.moving_task = current;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5916) return mem_cgroup_do_precharge(precharge);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5917) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5918)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5919) /* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5920) static void __mem_cgroup_clear_mc(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5921) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5922) struct mem_cgroup *from = mc.from;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5923) struct mem_cgroup *to = mc.to;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5925) /* we must uncharge all the leftover precharges from mc.to */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5926) if (mc.precharge) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5927) cancel_charge(mc.to, mc.precharge);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5928) mc.precharge = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5929) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5930) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5931) * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5932) * we must uncharge here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5933) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5934) if (mc.moved_charge) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5935) cancel_charge(mc.from, mc.moved_charge);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5936) mc.moved_charge = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5937) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5938) /* we must fixup refcnts and charges */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5939) if (mc.moved_swap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5940) /* uncharge swap account from the old cgroup */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5941) if (!mem_cgroup_is_root(mc.from))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5942) page_counter_uncharge(&mc.from->memsw, mc.moved_swap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5943)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5944) mem_cgroup_id_put_many(mc.from, mc.moved_swap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5945)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5946) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5947) * we charged both to->memory and to->memsw, so we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5948) * should uncharge to->memory.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5949) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5950) if (!mem_cgroup_is_root(mc.to))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5951) page_counter_uncharge(&mc.to->memory, mc.moved_swap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5952)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5953) mc.moved_swap = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5954) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5955) memcg_oom_recover(from);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5956) memcg_oom_recover(to);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5957) wake_up_all(&mc.waitq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5958) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5959)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5960) static void mem_cgroup_clear_mc(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5961) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5962) struct mm_struct *mm = mc.mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5963)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5964) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5965) * we must clear moving_task before waking up waiters at the end of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5966) * task migration.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5967) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5968) mc.moving_task = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5969) __mem_cgroup_clear_mc();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5970) spin_lock(&mc.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5971) mc.from = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5972) mc.to = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5973) mc.mm = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5974) spin_unlock(&mc.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5975)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5976) mmput(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5977) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5978)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5979) static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5980) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5981) struct cgroup_subsys_state *css;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5982) struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5983) struct mem_cgroup *from;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5984) struct task_struct *leader, *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5985) struct mm_struct *mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5986) unsigned long move_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5987) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5988)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5989) /* charge immigration isn't supported on the default hierarchy */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5990) if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5991) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5992)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5993) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5994) * Multi-process migrations only happen on the default hierarchy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5995) * where charge immigration is not used. Perform charge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5996) * immigration if @tset contains a leader and whine if there are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5997) * multiple.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5998) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5999) p = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6000) cgroup_taskset_for_each_leader(leader, css, tset) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6001) WARN_ON_ONCE(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6002) p = leader;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6003) memcg = mem_cgroup_from_css(css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6004) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6005) if (!p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6006) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6007)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6008) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6009) * We are now commited to this value whatever it is. Changes in this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6010) * tunable will only affect upcoming migrations, not the current one.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6011) * So we need to save it, and keep it going.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6012) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6013) move_flags = READ_ONCE(memcg->move_charge_at_immigrate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6014) if (!move_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6015) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6016)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6017) from = mem_cgroup_from_task(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6018)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6019) VM_BUG_ON(from == memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6020)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6021) mm = get_task_mm(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6022) if (!mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6023) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6024) /* We move charges only when we move a owner of the mm */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6025) if (mm->owner == p) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6026) VM_BUG_ON(mc.from);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6027) VM_BUG_ON(mc.to);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6028) VM_BUG_ON(mc.precharge);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6029) VM_BUG_ON(mc.moved_charge);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6030) VM_BUG_ON(mc.moved_swap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6031)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6032) spin_lock(&mc.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6033) mc.mm = mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6034) mc.from = from;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6035) mc.to = memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6036) mc.flags = move_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6037) spin_unlock(&mc.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6038) /* We set mc.moving_task later */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6039)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6040) ret = mem_cgroup_precharge_mc(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6041) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6042) mem_cgroup_clear_mc();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6043) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6044) mmput(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6045) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6046) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6047) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6048)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6049) static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6050) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6051) if (mc.to)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6052) mem_cgroup_clear_mc();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6053) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6054)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6055) static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6056) unsigned long addr, unsigned long end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6057) struct mm_walk *walk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6058) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6059) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6060) struct vm_area_struct *vma = walk->vma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6061) pte_t *pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6062) spinlock_t *ptl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6063) enum mc_target_type target_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6064) union mc_target target;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6065) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6066)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6067) ptl = pmd_trans_huge_lock(pmd, vma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6068) if (ptl) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6069) if (mc.precharge < HPAGE_PMD_NR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6070) spin_unlock(ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6071) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6072) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6073) target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6074) if (target_type == MC_TARGET_PAGE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6075) page = target.page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6076) if (!isolate_lru_page(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6077) if (!mem_cgroup_move_account(page, true,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6078) mc.from, mc.to)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6079) mc.precharge -= HPAGE_PMD_NR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6080) mc.moved_charge += HPAGE_PMD_NR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6081) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6082) putback_lru_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6083) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6084) put_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6085) } else if (target_type == MC_TARGET_DEVICE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6086) page = target.page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6087) if (!mem_cgroup_move_account(page, true,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6088) mc.from, mc.to)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6089) mc.precharge -= HPAGE_PMD_NR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6090) mc.moved_charge += HPAGE_PMD_NR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6091) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6092) put_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6093) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6094) spin_unlock(ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6095) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6096) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6097)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6098) if (pmd_trans_unstable(pmd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6099) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6100) retry:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6101) pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6102) for (; addr != end; addr += PAGE_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6103) pte_t ptent = *(pte++);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6104) bool device = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6105) swp_entry_t ent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6107) if (!mc.precharge)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6108) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6110) switch (get_mctgt_type(vma, addr, ptent, &target)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6111) case MC_TARGET_DEVICE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6112) device = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6113) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6114) case MC_TARGET_PAGE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6115) page = target.page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6116) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6117) * We can have a part of the split pmd here. Moving it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6118) * can be done but it would be too convoluted so simply
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6119) * ignore such a partial THP and keep it in original
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6120) * memcg. There should be somebody mapping the head.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6121) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6122) if (PageTransCompound(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6123) goto put;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6124) if (!device && isolate_lru_page(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6125) goto put;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6126) if (!mem_cgroup_move_account(page, false,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6127) mc.from, mc.to)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6128) mc.precharge--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6129) /* we uncharge from mc.from later. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6130) mc.moved_charge++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6131) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6132) if (!device)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6133) putback_lru_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6134) put: /* get_mctgt_type() gets the page */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6135) put_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6136) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6137) case MC_TARGET_SWAP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6138) ent = target.ent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6139) if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6140) mc.precharge--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6141) mem_cgroup_id_get_many(mc.to, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6142) /* we fixup other refcnts and charges later. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6143) mc.moved_swap++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6144) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6145) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6146) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6147) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6148) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6149) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6150) pte_unmap_unlock(pte - 1, ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6151) cond_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6153) if (addr != end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6154) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6155) * We have consumed all precharges we got in can_attach().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6156) * We try charge one by one, but don't do any additional
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6157) * charges to mc.to if we have failed in charge once in attach()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6158) * phase.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6159) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6160) ret = mem_cgroup_do_precharge(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6161) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6162) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6163) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6165) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6166) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6168) static const struct mm_walk_ops charge_walk_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6169) .pmd_entry = mem_cgroup_move_charge_pte_range,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6170) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6172) static void mem_cgroup_move_charge(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6173) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6174) lru_add_drain_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6175) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6176) * Signal lock_page_memcg() to take the memcg's move_lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6177) * while we're moving its pages to another memcg. Then wait
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6178) * for already started RCU-only updates to finish.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6179) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6180) atomic_inc(&mc.from->moving_account);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6181) synchronize_rcu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6182) retry:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6183) if (unlikely(!mmap_read_trylock(mc.mm))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6184) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6185) * Someone who are holding the mmap_lock might be waiting in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6186) * waitq. So we cancel all extra charges, wake up all waiters,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6187) * and retry. Because we cancel precharges, we might not be able
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6188) * to move enough charges, but moving charge is a best-effort
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6189) * feature anyway, so it wouldn't be a big problem.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6190) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6191) __mem_cgroup_clear_mc();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6192) cond_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6193) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6194) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6195) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6196) * When we have consumed all precharges and failed in doing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6197) * additional charge, the page walk just aborts.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6198) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6199) walk_page_range(mc.mm, 0, mc.mm->highest_vm_end, &charge_walk_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6200) NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6202) mmap_read_unlock(mc.mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6203) atomic_dec(&mc.from->moving_account);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6204) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6206) static void mem_cgroup_move_task(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6207) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6208) if (mc.to) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6209) mem_cgroup_move_charge();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6210) mem_cgroup_clear_mc();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6211) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6212) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6213) #else /* !CONFIG_MMU */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6214) static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6215) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6216) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6217) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6218) static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6219) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6220) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6221) static void mem_cgroup_move_task(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6222) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6223) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6224) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6226) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6227) * Cgroup retains root cgroups across [un]mount cycles making it necessary
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6228) * to verify whether we're attached to the default hierarchy on each mount
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6229) * attempt.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6230) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6231) static void mem_cgroup_bind(struct cgroup_subsys_state *root_css)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6232) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6233) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6234) * use_hierarchy is forced on the default hierarchy. cgroup core
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6235) * guarantees that @root doesn't have any children, so turning it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6236) * on for the root memcg is enough.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6237) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6238) if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6239) root_mem_cgroup->use_hierarchy = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6240) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6241) root_mem_cgroup->use_hierarchy = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6242) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6244) static int seq_puts_memcg_tunable(struct seq_file *m, unsigned long value)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6245) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6246) if (value == PAGE_COUNTER_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6247) seq_puts(m, "max\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6248) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6249) seq_printf(m, "%llu\n", (u64)value * PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6251) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6252) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6254) static u64 memory_current_read(struct cgroup_subsys_state *css,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6255) struct cftype *cft)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6256) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6257) struct mem_cgroup *memcg = mem_cgroup_from_css(css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6259) return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6260) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6262) static int memory_min_show(struct seq_file *m, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6263) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6264) return seq_puts_memcg_tunable(m,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6265) READ_ONCE(mem_cgroup_from_seq(m)->memory.min));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6266) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6268) static ssize_t memory_min_write(struct kernfs_open_file *of,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6269) char *buf, size_t nbytes, loff_t off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6270) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6271) struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6272) unsigned long min;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6273) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6275) buf = strstrip(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6276) err = page_counter_memparse(buf, "max", &min);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6277) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6278) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6280) page_counter_set_min(&memcg->memory, min);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6282) return nbytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6283) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6285) static int memory_low_show(struct seq_file *m, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6286) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6287) return seq_puts_memcg_tunable(m,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6288) READ_ONCE(mem_cgroup_from_seq(m)->memory.low));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6289) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6291) static ssize_t memory_low_write(struct kernfs_open_file *of,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6292) char *buf, size_t nbytes, loff_t off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6293) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6294) struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6295) unsigned long low;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6296) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6298) buf = strstrip(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6299) err = page_counter_memparse(buf, "max", &low);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6300) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6301) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6303) page_counter_set_low(&memcg->memory, low);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6305) return nbytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6306) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6308) static int memory_high_show(struct seq_file *m, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6309) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6310) return seq_puts_memcg_tunable(m,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6311) READ_ONCE(mem_cgroup_from_seq(m)->memory.high));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6312) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6314) static ssize_t memory_high_write(struct kernfs_open_file *of,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6315) char *buf, size_t nbytes, loff_t off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6316) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6317) struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6318) unsigned int nr_retries = MAX_RECLAIM_RETRIES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6319) bool drained = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6320) unsigned long high;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6321) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6323) buf = strstrip(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6324) err = page_counter_memparse(buf, "max", &high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6325) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6326) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6328) page_counter_set_high(&memcg->memory, high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6330) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6331) unsigned long nr_pages = page_counter_read(&memcg->memory);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6332) unsigned long reclaimed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6334) if (nr_pages <= high)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6335) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6337) if (signal_pending(current))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6338) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6340) if (!drained) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6341) drain_all_stock(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6342) drained = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6343) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6344) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6346) reclaimed = try_to_free_mem_cgroup_pages(memcg, nr_pages - high,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6347) GFP_KERNEL, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6349) if (!reclaimed && !nr_retries--)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6350) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6351) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6353) memcg_wb_domain_size_changed(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6354) return nbytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6355) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6357) static int memory_max_show(struct seq_file *m, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6358) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6359) return seq_puts_memcg_tunable(m,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6360) READ_ONCE(mem_cgroup_from_seq(m)->memory.max));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6361) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6363) static ssize_t memory_max_write(struct kernfs_open_file *of,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6364) char *buf, size_t nbytes, loff_t off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6365) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6366) struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6367) unsigned int nr_reclaims = MAX_RECLAIM_RETRIES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6368) bool drained = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6369) unsigned long max;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6370) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6372) buf = strstrip(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6373) err = page_counter_memparse(buf, "max", &max);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6374) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6375) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6377) xchg(&memcg->memory.max, max);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6379) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6380) unsigned long nr_pages = page_counter_read(&memcg->memory);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6382) if (nr_pages <= max)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6383) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6385) if (signal_pending(current))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6386) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6388) if (!drained) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6389) drain_all_stock(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6390) drained = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6391) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6392) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6394) if (nr_reclaims) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6395) if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6396) GFP_KERNEL, true))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6397) nr_reclaims--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6398) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6399) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6401) memcg_memory_event(memcg, MEMCG_OOM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6402) if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6403) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6404) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6406) memcg_wb_domain_size_changed(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6407) return nbytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6408) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6410) static void __memory_events_show(struct seq_file *m, atomic_long_t *events)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6411) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6412) seq_printf(m, "low %lu\n", atomic_long_read(&events[MEMCG_LOW]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6413) seq_printf(m, "high %lu\n", atomic_long_read(&events[MEMCG_HIGH]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6414) seq_printf(m, "max %lu\n", atomic_long_read(&events[MEMCG_MAX]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6415) seq_printf(m, "oom %lu\n", atomic_long_read(&events[MEMCG_OOM]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6416) seq_printf(m, "oom_kill %lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6417) atomic_long_read(&events[MEMCG_OOM_KILL]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6418) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6420) static int memory_events_show(struct seq_file *m, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6421) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6422) struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6424) __memory_events_show(m, memcg->memory_events);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6425) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6426) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6428) static int memory_events_local_show(struct seq_file *m, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6429) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6430) struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6432) __memory_events_show(m, memcg->memory_events_local);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6433) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6434) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6436) static int memory_stat_show(struct seq_file *m, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6437) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6438) struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6439) char *buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6441) buf = memory_stat_format(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6442) if (!buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6443) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6444) seq_puts(m, buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6445) kfree(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6446) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6447) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6449) #ifdef CONFIG_NUMA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6450) static int memory_numa_stat_show(struct seq_file *m, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6451) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6452) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6453) struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6455) for (i = 0; i < ARRAY_SIZE(memory_stats); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6456) int nid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6458) if (memory_stats[i].idx >= NR_VM_NODE_STAT_ITEMS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6459) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6461) seq_printf(m, "%s", memory_stats[i].name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6462) for_each_node_state(nid, N_MEMORY) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6463) u64 size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6464) struct lruvec *lruvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6466) lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6467) size = lruvec_page_state(lruvec, memory_stats[i].idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6468) size *= memory_stats[i].ratio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6469) seq_printf(m, " N%d=%llu", nid, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6470) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6471) seq_putc(m, '\n');
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6472) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6474) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6475) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6476) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6478) static int memory_oom_group_show(struct seq_file *m, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6479) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6480) struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6482) seq_printf(m, "%d\n", memcg->oom_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6484) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6485) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6487) static ssize_t memory_oom_group_write(struct kernfs_open_file *of,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6488) char *buf, size_t nbytes, loff_t off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6489) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6490) struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6491) int ret, oom_group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6493) buf = strstrip(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6494) if (!buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6495) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6496)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6497) ret = kstrtoint(buf, 0, &oom_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6498) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6499) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6501) if (oom_group != 0 && oom_group != 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6502) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6504) memcg->oom_group = oom_group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6506) return nbytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6507) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6509) static struct cftype memory_files[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6510) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6511) .name = "current",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6512) .flags = CFTYPE_NOT_ON_ROOT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6513) .read_u64 = memory_current_read,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6514) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6515) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6516) .name = "min",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6517) .flags = CFTYPE_NOT_ON_ROOT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6518) .seq_show = memory_min_show,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6519) .write = memory_min_write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6520) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6521) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6522) .name = "low",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6523) .flags = CFTYPE_NOT_ON_ROOT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6524) .seq_show = memory_low_show,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6525) .write = memory_low_write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6526) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6527) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6528) .name = "high",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6529) .flags = CFTYPE_NOT_ON_ROOT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6530) .seq_show = memory_high_show,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6531) .write = memory_high_write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6532) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6533) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6534) .name = "max",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6535) .flags = CFTYPE_NOT_ON_ROOT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6536) .seq_show = memory_max_show,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6537) .write = memory_max_write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6538) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6539) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6540) .name = "events",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6541) .flags = CFTYPE_NOT_ON_ROOT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6542) .file_offset = offsetof(struct mem_cgroup, events_file),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6543) .seq_show = memory_events_show,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6544) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6545) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6546) .name = "events.local",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6547) .flags = CFTYPE_NOT_ON_ROOT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6548) .file_offset = offsetof(struct mem_cgroup, events_local_file),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6549) .seq_show = memory_events_local_show,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6550) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6551) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6552) .name = "stat",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6553) .seq_show = memory_stat_show,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6554) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6555) #ifdef CONFIG_NUMA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6556) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6557) .name = "numa_stat",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6558) .seq_show = memory_numa_stat_show,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6559) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6560) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6561) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6562) .name = "oom.group",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6563) .flags = CFTYPE_NOT_ON_ROOT | CFTYPE_NS_DELEGATABLE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6564) .seq_show = memory_oom_group_show,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6565) .write = memory_oom_group_write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6566) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6567) { } /* terminate */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6568) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6570) struct cgroup_subsys memory_cgrp_subsys = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6571) .css_alloc = mem_cgroup_css_alloc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6572) .css_online = mem_cgroup_css_online,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6573) .css_offline = mem_cgroup_css_offline,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6574) .css_released = mem_cgroup_css_released,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6575) .css_free = mem_cgroup_css_free,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6576) .css_reset = mem_cgroup_css_reset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6577) .can_attach = mem_cgroup_can_attach,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6578) .cancel_attach = mem_cgroup_cancel_attach,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6579) .post_attach = mem_cgroup_move_task,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6580) .bind = mem_cgroup_bind,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6581) .dfl_cftypes = memory_files,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6582) .legacy_cftypes = mem_cgroup_legacy_files,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6583) .early_init = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6584) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6586) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6587) * This function calculates an individual cgroup's effective
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6588) * protection which is derived from its own memory.min/low, its
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6589) * parent's and siblings' settings, as well as the actual memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6590) * distribution in the tree.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6591) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6592) * The following rules apply to the effective protection values:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6593) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6594) * 1. At the first level of reclaim, effective protection is equal to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6595) * the declared protection in memory.min and memory.low.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6596) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6597) * 2. To enable safe delegation of the protection configuration, at
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6598) * subsequent levels the effective protection is capped to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6599) * parent's effective protection.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6600) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6601) * 3. To make complex and dynamic subtrees easier to configure, the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6602) * user is allowed to overcommit the declared protection at a given
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6603) * level. If that is the case, the parent's effective protection is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6604) * distributed to the children in proportion to how much protection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6605) * they have declared and how much of it they are utilizing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6606) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6607) * This makes distribution proportional, but also work-conserving:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6608) * if one cgroup claims much more protection than it uses memory,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6609) * the unused remainder is available to its siblings.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6610) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6611) * 4. Conversely, when the declared protection is undercommitted at a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6612) * given level, the distribution of the larger parental protection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6613) * budget is NOT proportional. A cgroup's protection from a sibling
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6614) * is capped to its own memory.min/low setting.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6615) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6616) * 5. However, to allow protecting recursive subtrees from each other
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6617) * without having to declare each individual cgroup's fixed share
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6618) * of the ancestor's claim to protection, any unutilized -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6619) * "floating" - protection from up the tree is distributed in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6620) * proportion to each cgroup's *usage*. This makes the protection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6621) * neutral wrt sibling cgroups and lets them compete freely over
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6622) * the shared parental protection budget, but it protects the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6623) * subtree as a whole from neighboring subtrees.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6624) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6625) * Note that 4. and 5. are not in conflict: 4. is about protecting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6626) * against immediate siblings whereas 5. is about protecting against
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6627) * neighboring subtrees.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6628) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6629) static unsigned long effective_protection(unsigned long usage,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6630) unsigned long parent_usage,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6631) unsigned long setting,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6632) unsigned long parent_effective,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6633) unsigned long siblings_protected)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6634) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6635) unsigned long protected;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6636) unsigned long ep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6638) protected = min(usage, setting);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6639) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6640) * If all cgroups at this level combined claim and use more
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6641) * protection then what the parent affords them, distribute
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6642) * shares in proportion to utilization.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6643) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6644) * We are using actual utilization rather than the statically
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6645) * claimed protection in order to be work-conserving: claimed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6646) * but unused protection is available to siblings that would
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6647) * otherwise get a smaller chunk than what they claimed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6648) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6649) if (siblings_protected > parent_effective)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6650) return protected * parent_effective / siblings_protected;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6651)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6652) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6653) * Ok, utilized protection of all children is within what the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6654) * parent affords them, so we know whatever this child claims
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6655) * and utilizes is effectively protected.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6656) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6657) * If there is unprotected usage beyond this value, reclaim
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6658) * will apply pressure in proportion to that amount.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6659) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6660) * If there is unutilized protection, the cgroup will be fully
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6661) * shielded from reclaim, but we do return a smaller value for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6662) * protection than what the group could enjoy in theory. This
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6663) * is okay. With the overcommit distribution above, effective
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6664) * protection is always dependent on how memory is actually
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6665) * consumed among the siblings anyway.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6666) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6667) ep = protected;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6669) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6670) * If the children aren't claiming (all of) the protection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6671) * afforded to them by the parent, distribute the remainder in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6672) * proportion to the (unprotected) memory of each cgroup. That
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6673) * way, cgroups that aren't explicitly prioritized wrt each
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6674) * other compete freely over the allowance, but they are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6675) * collectively protected from neighboring trees.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6676) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6677) * We're using unprotected memory for the weight so that if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6678) * some cgroups DO claim explicit protection, we don't protect
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6679) * the same bytes twice.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6680) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6681) * Check both usage and parent_usage against the respective
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6682) * protected values. One should imply the other, but they
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6683) * aren't read atomically - make sure the division is sane.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6684) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6685) if (!(cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_RECURSIVE_PROT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6686) return ep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6687) if (parent_effective > siblings_protected &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6688) parent_usage > siblings_protected &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6689) usage > protected) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6690) unsigned long unclaimed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6692) unclaimed = parent_effective - siblings_protected;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6693) unclaimed *= usage - protected;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6694) unclaimed /= parent_usage - siblings_protected;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6696) ep += unclaimed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6697) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6699) return ep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6700) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6701)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6702) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6703) * mem_cgroup_protected - check if memory consumption is in the normal range
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6704) * @root: the top ancestor of the sub-tree being checked
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6705) * @memcg: the memory cgroup to check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6706) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6707) * WARNING: This function is not stateless! It can only be used as part
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6708) * of a top-down tree iteration, not for isolated queries.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6709) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6710) void mem_cgroup_calculate_protection(struct mem_cgroup *root,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6711) struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6712) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6713) unsigned long usage, parent_usage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6714) struct mem_cgroup *parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6715)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6716) if (mem_cgroup_disabled())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6717) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6718)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6719) if (!root)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6720) root = root_mem_cgroup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6721)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6722) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6723) * Effective values of the reclaim targets are ignored so they
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6724) * can be stale. Have a look at mem_cgroup_protection for more
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6725) * details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6726) * TODO: calculation should be more robust so that we do not need
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6727) * that special casing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6728) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6729) if (memcg == root)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6730) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6731)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6732) usage = page_counter_read(&memcg->memory);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6733) if (!usage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6734) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6735)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6736) parent = parent_mem_cgroup(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6737) /* No parent means a non-hierarchical mode on v1 memcg */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6738) if (!parent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6739) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6741) if (parent == root) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6742) memcg->memory.emin = READ_ONCE(memcg->memory.min);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6743) memcg->memory.elow = READ_ONCE(memcg->memory.low);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6744) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6745) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6747) parent_usage = page_counter_read(&parent->memory);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6749) WRITE_ONCE(memcg->memory.emin, effective_protection(usage, parent_usage,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6750) READ_ONCE(memcg->memory.min),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6751) READ_ONCE(parent->memory.emin),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6752) atomic_long_read(&parent->memory.children_min_usage)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6753)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6754) WRITE_ONCE(memcg->memory.elow, effective_protection(usage, parent_usage,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6755) READ_ONCE(memcg->memory.low),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6756) READ_ONCE(parent->memory.elow),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6757) atomic_long_read(&parent->memory.children_low_usage)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6758) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6759)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6760) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6761) * __mem_cgroup_charge - charge a newly allocated page to a cgroup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6762) * @page: page to charge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6763) * @mm: mm context of the victim
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6764) * @gfp_mask: reclaim mode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6765) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6766) * Try to charge @page to the memcg that @mm belongs to, reclaiming
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6767) * pages according to @gfp_mask if necessary.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6768) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6769) * Returns 0 on success. Otherwise, an error code is returned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6770) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6771) int __mem_cgroup_charge(struct page *page, struct mm_struct *mm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6772) gfp_t gfp_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6773) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6774) unsigned int nr_pages = thp_nr_pages(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6775) struct mem_cgroup *memcg = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6776) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6778) if (PageSwapCache(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6779) swp_entry_t ent = { .val = page_private(page), };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6780) unsigned short id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6781)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6782) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6783) * Every swap fault against a single page tries to charge the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6784) * page, bail as early as possible. shmem_unuse() encounters
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6785) * already charged pages, too. page->mem_cgroup is protected
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6786) * by the page lock, which serializes swap cache removal, which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6787) * in turn serializes uncharging.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6788) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6789) VM_BUG_ON_PAGE(!PageLocked(page), page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6790) if (compound_head(page)->mem_cgroup)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6791) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6792)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6793) id = lookup_swap_cgroup_id(ent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6794) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6795) memcg = mem_cgroup_from_id(id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6796) if (memcg && !css_tryget_online(&memcg->css))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6797) memcg = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6798) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6799) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6800)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6801) if (!memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6802) memcg = get_mem_cgroup_from_mm(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6804) ret = try_charge(memcg, gfp_mask, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6805) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6806) goto out_put;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6807)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6808) css_get(&memcg->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6809) commit_charge(page, memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6810)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6811) local_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6812) mem_cgroup_charge_statistics(memcg, page, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6813) memcg_check_events(memcg, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6814) local_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6815)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6816) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6817) * Cgroup1's unified memory+swap counter has been charged with the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6818) * new swapcache page, finish the transfer by uncharging the swap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6819) * slot. The swap slot would also get uncharged when it dies, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6820) * it can stick around indefinitely and we'd count the page twice
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6821) * the entire time.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6822) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6823) * Cgroup2 has separate resource counters for memory and swap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6824) * so this is a non-issue here. Memory and swap charge lifetimes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6825) * correspond 1:1 to page and swap slot lifetimes: we charge the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6826) * page to memory here, and uncharge swap when the slot is freed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6827) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6828) if (do_memsw_account() && PageSwapCache(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6829) swp_entry_t entry = { .val = page_private(page) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6830) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6831) * The swap entry might not get freed for a long time,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6832) * let's not wait for it. The page already received a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6833) * memory+swap charge, drop the swap entry duplicate.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6834) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6835) mem_cgroup_uncharge_swap(entry, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6836) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6837)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6838) out_put:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6839) css_put(&memcg->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6840) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6841) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6842) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6843)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6844) struct uncharge_gather {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6845) struct mem_cgroup *memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6846) unsigned long nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6847) unsigned long pgpgout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6848) unsigned long nr_kmem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6849) struct page *dummy_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6850) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6852) static inline void uncharge_gather_clear(struct uncharge_gather *ug)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6853) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6854) memset(ug, 0, sizeof(*ug));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6855) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6856)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6857) static void uncharge_batch(const struct uncharge_gather *ug)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6858) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6859) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6860)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6861) if (!mem_cgroup_is_root(ug->memcg)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6862) page_counter_uncharge(&ug->memcg->memory, ug->nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6863) if (do_memsw_account())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6864) page_counter_uncharge(&ug->memcg->memsw, ug->nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6865) if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && ug->nr_kmem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6866) page_counter_uncharge(&ug->memcg->kmem, ug->nr_kmem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6867) memcg_oom_recover(ug->memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6868) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6869)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6870) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6871) __count_memcg_events(ug->memcg, PGPGOUT, ug->pgpgout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6872) __this_cpu_add(ug->memcg->vmstats_percpu->nr_page_events, ug->nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6873) memcg_check_events(ug->memcg, ug->dummy_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6874) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6875)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6876) /* drop reference from uncharge_page */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6877) css_put(&ug->memcg->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6878) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6879)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6880) static void uncharge_page(struct page *page, struct uncharge_gather *ug)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6881) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6882) unsigned long nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6883)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6884) VM_BUG_ON_PAGE(PageLRU(page), page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6885)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6886) if (!page->mem_cgroup)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6887) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6889) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6890) * Nobody should be changing or seriously looking at
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6891) * page->mem_cgroup at this point, we have fully
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6892) * exclusive access to the page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6893) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6894)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6895) if (ug->memcg != page->mem_cgroup) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6896) if (ug->memcg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6897) uncharge_batch(ug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6898) uncharge_gather_clear(ug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6899) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6900) ug->memcg = page->mem_cgroup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6901)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6902) /* pairs with css_put in uncharge_batch */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6903) css_get(&ug->memcg->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6904) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6905)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6906) nr_pages = compound_nr(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6907) ug->nr_pages += nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6908)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6909) if (!PageKmemcg(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6910) ug->pgpgout++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6911) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6912) ug->nr_kmem += nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6913) __ClearPageKmemcg(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6914) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6915)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6916) ug->dummy_page = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6917) page->mem_cgroup = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6918) css_put(&ug->memcg->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6919) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6920)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6921) static void uncharge_list(struct list_head *page_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6922) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6923) struct uncharge_gather ug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6924) struct list_head *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6925)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6926) uncharge_gather_clear(&ug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6927)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6928) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6929) * Note that the list can be a single page->lru; hence the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6930) * do-while loop instead of a simple list_for_each_entry().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6931) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6932) next = page_list->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6933) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6934) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6935)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6936) page = list_entry(next, struct page, lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6937) next = page->lru.next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6938)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6939) uncharge_page(page, &ug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6940) } while (next != page_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6941)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6942) if (ug.memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6943) uncharge_batch(&ug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6944) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6945)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6946) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6947) * __mem_cgroup_uncharge - uncharge a page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6948) * @page: page to uncharge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6949) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6950) * Uncharge a page previously charged with __mem_cgroup_charge().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6951) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6952) void __mem_cgroup_uncharge(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6953) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6954) struct uncharge_gather ug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6955)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6956) /* Don't touch page->lru of any random page, pre-check: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6957) if (!page->mem_cgroup)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6958) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6959)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6960) uncharge_gather_clear(&ug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6961) uncharge_page(page, &ug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6962) uncharge_batch(&ug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6963) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6964)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6965) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6966) * __mem_cgroup_uncharge_list - uncharge a list of page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6967) * @page_list: list of pages to uncharge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6968) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6969) * Uncharge a list of pages previously charged with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6970) * __mem_cgroup_charge().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6971) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6972) void __mem_cgroup_uncharge_list(struct list_head *page_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6973) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6974) if (!list_empty(page_list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6975) uncharge_list(page_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6976) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6977)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6978) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6979) * mem_cgroup_migrate - charge a page's replacement
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6980) * @oldpage: currently circulating page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6981) * @newpage: replacement page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6982) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6983) * Charge @newpage as a replacement page for @oldpage. @oldpage will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6984) * be uncharged upon free.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6985) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6986) * Both pages must be locked, @newpage->mapping must be set up.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6987) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6988) void mem_cgroup_migrate(struct page *oldpage, struct page *newpage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6989) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6990) struct mem_cgroup *memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6991) unsigned int nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6992) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6993)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6994) VM_BUG_ON_PAGE(!PageLocked(oldpage), oldpage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6995) VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6996) VM_BUG_ON_PAGE(PageAnon(oldpage) != PageAnon(newpage), newpage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6997) VM_BUG_ON_PAGE(PageTransHuge(oldpage) != PageTransHuge(newpage),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6998) newpage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6999)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7000) if (mem_cgroup_disabled())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7001) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7002)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7003) /* Page cache replacement: new page already charged? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7004) if (newpage->mem_cgroup)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7005) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7006)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7007) /* Swapcache readahead pages can get replaced before being charged */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7008) memcg = oldpage->mem_cgroup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7009) if (!memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7010) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7011)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7012) /* Force-charge the new page. The old one will be freed soon */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7013) nr_pages = thp_nr_pages(newpage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7014)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7015) page_counter_charge(&memcg->memory, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7016) if (do_memsw_account())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7017) page_counter_charge(&memcg->memsw, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7018)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7019) css_get(&memcg->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7020) commit_charge(newpage, memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7021)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7022) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7023) mem_cgroup_charge_statistics(memcg, newpage, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7024) memcg_check_events(memcg, newpage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7025) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7026) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7027)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7028) DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7029) EXPORT_SYMBOL(memcg_sockets_enabled_key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7030)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7031) void mem_cgroup_sk_alloc(struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7032) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7033) struct mem_cgroup *memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7034)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7035) if (!mem_cgroup_sockets_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7036) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7037)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7038) /* Do not associate the sock with unrelated interrupted task's memcg. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7039) if (in_interrupt())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7040) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7041)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7042) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7043) memcg = mem_cgroup_from_task(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7044) if (memcg == root_mem_cgroup)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7045) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7046) if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7047) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7048) if (css_tryget(&memcg->css))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7049) sk->sk_memcg = memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7050) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7051) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7052) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7053)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7054) void mem_cgroup_sk_free(struct sock *sk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7055) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7056) if (sk->sk_memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7057) css_put(&sk->sk_memcg->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7058) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7060) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7061) * mem_cgroup_charge_skmem - charge socket memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7062) * @memcg: memcg to charge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7063) * @nr_pages: number of pages to charge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7064) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7065) * Charges @nr_pages to @memcg. Returns %true if the charge fit within
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7066) * @memcg's configured limit, %false if the charge had to be forced.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7067) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7068) bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7069) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7070) gfp_t gfp_mask = GFP_KERNEL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7072) if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7073) struct page_counter *fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7074)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7075) if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7076) memcg->tcpmem_pressure = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7077) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7078) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7079) page_counter_charge(&memcg->tcpmem, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7080) memcg->tcpmem_pressure = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7081) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7082) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7083)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7084) /* Don't block in the packet receive path */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7085) if (in_softirq())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7086) gfp_mask = GFP_NOWAIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7087)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7088) mod_memcg_state(memcg, MEMCG_SOCK, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7089)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7090) if (try_charge(memcg, gfp_mask, nr_pages) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7091) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7092)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7093) try_charge(memcg, gfp_mask|__GFP_NOFAIL, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7094) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7095) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7096)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7097) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7098) * mem_cgroup_uncharge_skmem - uncharge socket memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7099) * @memcg: memcg to uncharge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7100) * @nr_pages: number of pages to uncharge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7101) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7102) void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7103) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7104) if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7105) page_counter_uncharge(&memcg->tcpmem, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7106) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7107) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7109) mod_memcg_state(memcg, MEMCG_SOCK, -nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7111) refill_stock(memcg, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7112) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7114) static int __init cgroup_memory(char *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7115) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7116) char *token;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7118) while ((token = strsep(&s, ",")) != NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7119) if (!*token)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7120) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7121) if (!strcmp(token, "nosocket"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7122) cgroup_memory_nosocket = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7123) if (!strcmp(token, "nokmem"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7124) cgroup_memory_nokmem = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7125) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7126) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7127) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7128) __setup("cgroup.memory=", cgroup_memory);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7130) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7131) * subsys_initcall() for memory controller.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7132) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7133) * Some parts like memcg_hotplug_cpu_dead() have to be initialized from this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7134) * context because of lock dependencies (cgroup_lock -> cpu hotplug) but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7135) * basically everything that doesn't depend on a specific mem_cgroup structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7136) * should be initialized from here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7137) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7138) static int __init mem_cgroup_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7139) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7140) int cpu, node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7142) cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7143) memcg_hotplug_cpu_dead);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7145) for_each_possible_cpu(cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7146) INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7147) drain_local_stock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7149) for_each_node(node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7150) struct mem_cgroup_tree_per_node *rtpn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7152) rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7153) node_online(node) ? node : NUMA_NO_NODE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7155) rtpn->rb_root = RB_ROOT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7156) rtpn->rb_rightmost = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7157) spin_lock_init(&rtpn->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7158) soft_limit_tree.rb_tree_per_node[node] = rtpn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7159) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7161) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7162) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7163) subsys_initcall(mem_cgroup_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7165) #ifdef CONFIG_MEMCG_SWAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7166) static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7167) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7168) while (!refcount_inc_not_zero(&memcg->id.ref)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7169) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7170) * The root cgroup cannot be destroyed, so it's refcount must
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7171) * always be >= 1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7172) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7173) if (WARN_ON_ONCE(memcg == root_mem_cgroup)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7174) VM_BUG_ON(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7175) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7176) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7177) memcg = parent_mem_cgroup(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7178) if (!memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7179) memcg = root_mem_cgroup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7180) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7181) return memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7182) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7184) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7185) * mem_cgroup_swapout - transfer a memsw charge to swap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7186) * @page: page whose memsw charge to transfer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7187) * @entry: swap entry to move the charge to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7188) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7189) * Transfer the memsw charge of @page to @entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7190) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7191) void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7192) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7193) struct mem_cgroup *memcg, *swap_memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7194) unsigned int nr_entries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7195) unsigned short oldid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7197) VM_BUG_ON_PAGE(PageLRU(page), page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7198) VM_BUG_ON_PAGE(page_count(page), page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7200) if (mem_cgroup_disabled())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7201) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7203) if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7204) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7206) memcg = page->mem_cgroup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7208) /* Readahead page, never charged */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7209) if (!memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7210) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7212) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7213) * In case the memcg owning these pages has been offlined and doesn't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7214) * have an ID allocated to it anymore, charge the closest online
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7215) * ancestor for the swap instead and transfer the memory+swap charge.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7216) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7217) swap_memcg = mem_cgroup_id_get_online(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7218) nr_entries = thp_nr_pages(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7219) /* Get references for the tail pages, too */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7220) if (nr_entries > 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7221) mem_cgroup_id_get_many(swap_memcg, nr_entries - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7222) oldid = swap_cgroup_record(entry, mem_cgroup_id(swap_memcg),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7223) nr_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7224) VM_BUG_ON_PAGE(oldid, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7225) mod_memcg_state(swap_memcg, MEMCG_SWAP, nr_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7227) page->mem_cgroup = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7229) if (!mem_cgroup_is_root(memcg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7230) page_counter_uncharge(&memcg->memory, nr_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7232) if (!cgroup_memory_noswap && memcg != swap_memcg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7233) if (!mem_cgroup_is_root(swap_memcg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7234) page_counter_charge(&swap_memcg->memsw, nr_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7235) page_counter_uncharge(&memcg->memsw, nr_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7236) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7238) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7239) * Interrupts should be disabled here because the caller holds the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7240) * i_pages lock which is taken with interrupts-off. It is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7241) * important here to have the interrupts disabled because it is the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7242) * only synchronisation we have for updating the per-CPU variables.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7243) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7244) VM_BUG_ON(!irqs_disabled());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7245) mem_cgroup_charge_statistics(memcg, page, -nr_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7246) memcg_check_events(memcg, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7248) css_put(&memcg->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7249) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7251) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7252) * __mem_cgroup_try_charge_swap - try charging swap space for a page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7253) * @page: page being added to swap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7254) * @entry: swap entry to charge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7255) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7256) * Try to charge @page's memcg for the swap space at @entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7257) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7258) * Returns 0 on success, -ENOMEM on failure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7259) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7260) int __mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7261) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7262) unsigned int nr_pages = thp_nr_pages(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7263) struct page_counter *counter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7264) struct mem_cgroup *memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7265) unsigned short oldid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7267) if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7268) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7270) memcg = page->mem_cgroup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7272) /* Readahead page, never charged */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7273) if (!memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7274) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7276) if (!entry.val) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7277) memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7278) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7279) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7281) memcg = mem_cgroup_id_get_online(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7283) if (!cgroup_memory_noswap && !mem_cgroup_is_root(memcg) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7284) !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7285) memcg_memory_event(memcg, MEMCG_SWAP_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7286) memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7287) mem_cgroup_id_put(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7288) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7289) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7291) /* Get references for the tail pages, too */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7292) if (nr_pages > 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7293) mem_cgroup_id_get_many(memcg, nr_pages - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7294) oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg), nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7295) VM_BUG_ON_PAGE(oldid, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7296) mod_memcg_state(memcg, MEMCG_SWAP, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7298) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7299) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7301) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7302) * __mem_cgroup_uncharge_swap - uncharge swap space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7303) * @entry: swap entry to uncharge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7304) * @nr_pages: the amount of swap space to uncharge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7305) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7306) void __mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7307) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7308) struct mem_cgroup *memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7309) unsigned short id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7311) id = swap_cgroup_record(entry, 0, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7312) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7313) memcg = mem_cgroup_from_id(id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7314) if (memcg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7315) if (!cgroup_memory_noswap && !mem_cgroup_is_root(memcg)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7316) if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7317) page_counter_uncharge(&memcg->swap, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7318) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7319) page_counter_uncharge(&memcg->memsw, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7320) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7321) mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7322) mem_cgroup_id_put_many(memcg, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7323) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7324) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7325) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7327) long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7328) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7329) long nr_swap_pages = get_nr_swap_pages();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7331) if (cgroup_memory_noswap || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7332) return nr_swap_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7333) for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7334) nr_swap_pages = min_t(long, nr_swap_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7335) READ_ONCE(memcg->swap.max) -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7336) page_counter_read(&memcg->swap));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7337) return nr_swap_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7338) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7340) bool mem_cgroup_swap_full(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7341) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7342) struct mem_cgroup *memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7344) VM_BUG_ON_PAGE(!PageLocked(page), page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7346) if (vm_swap_full())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7347) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7348) if (cgroup_memory_noswap || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7349) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7351) memcg = page->mem_cgroup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7352) if (!memcg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7353) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7355) for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7356) unsigned long usage = page_counter_read(&memcg->swap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7358) if (usage * 2 >= READ_ONCE(memcg->swap.high) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7359) usage * 2 >= READ_ONCE(memcg->swap.max))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7360) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7361) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7363) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7364) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7366) static int __init setup_swap_account(char *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7367) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7368) if (!strcmp(s, "1"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7369) cgroup_memory_noswap = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7370) else if (!strcmp(s, "0"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7371) cgroup_memory_noswap = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7372) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7373) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7374) __setup("swapaccount=", setup_swap_account);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7376) static u64 swap_current_read(struct cgroup_subsys_state *css,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7377) struct cftype *cft)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7378) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7379) struct mem_cgroup *memcg = mem_cgroup_from_css(css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7381) return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7382) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7384) static int swap_high_show(struct seq_file *m, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7385) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7386) return seq_puts_memcg_tunable(m,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7387) READ_ONCE(mem_cgroup_from_seq(m)->swap.high));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7388) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7390) static ssize_t swap_high_write(struct kernfs_open_file *of,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7391) char *buf, size_t nbytes, loff_t off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7392) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7393) struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7394) unsigned long high;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7395) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7397) buf = strstrip(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7398) err = page_counter_memparse(buf, "max", &high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7399) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7400) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7402) page_counter_set_high(&memcg->swap, high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7404) return nbytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7405) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7407) static int swap_max_show(struct seq_file *m, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7408) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7409) return seq_puts_memcg_tunable(m,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7410) READ_ONCE(mem_cgroup_from_seq(m)->swap.max));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7411) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7413) static ssize_t swap_max_write(struct kernfs_open_file *of,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7414) char *buf, size_t nbytes, loff_t off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7415) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7416) struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7417) unsigned long max;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7418) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7420) buf = strstrip(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7421) err = page_counter_memparse(buf, "max", &max);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7422) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7423) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7425) xchg(&memcg->swap.max, max);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7427) return nbytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7428) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7430) static int swap_events_show(struct seq_file *m, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7431) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7432) struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7434) seq_printf(m, "high %lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7435) atomic_long_read(&memcg->memory_events[MEMCG_SWAP_HIGH]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7436) seq_printf(m, "max %lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7437) atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7438) seq_printf(m, "fail %lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7439) atomic_long_read(&memcg->memory_events[MEMCG_SWAP_FAIL]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7441) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7442) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7444) static struct cftype swap_files[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7445) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7446) .name = "swap.current",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7447) .flags = CFTYPE_NOT_ON_ROOT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7448) .read_u64 = swap_current_read,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7449) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7450) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7451) .name = "swap.high",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7452) .flags = CFTYPE_NOT_ON_ROOT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7453) .seq_show = swap_high_show,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7454) .write = swap_high_write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7455) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7456) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7457) .name = "swap.max",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7458) .flags = CFTYPE_NOT_ON_ROOT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7459) .seq_show = swap_max_show,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7460) .write = swap_max_write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7461) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7462) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7463) .name = "swap.events",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7464) .flags = CFTYPE_NOT_ON_ROOT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7465) .file_offset = offsetof(struct mem_cgroup, swap_events_file),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7466) .seq_show = swap_events_show,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7467) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7468) { } /* terminate */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7469) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7471) static struct cftype memsw_files[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7472) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7473) .name = "memsw.usage_in_bytes",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7474) .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7475) .read_u64 = mem_cgroup_read_u64,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7476) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7477) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7478) .name = "memsw.max_usage_in_bytes",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7479) .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7480) .write = mem_cgroup_reset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7481) .read_u64 = mem_cgroup_read_u64,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7482) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7483) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7484) .name = "memsw.limit_in_bytes",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7485) .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7486) .write = mem_cgroup_write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7487) .read_u64 = mem_cgroup_read_u64,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7488) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7489) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7490) .name = "memsw.failcnt",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7491) .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7492) .write = mem_cgroup_reset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7493) .read_u64 = mem_cgroup_read_u64,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7494) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7495) { }, /* terminate */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7496) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7498) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7499) * If mem_cgroup_swap_init() is implemented as a subsys_initcall()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7500) * instead of a core_initcall(), this could mean cgroup_memory_noswap still
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7501) * remains set to false even when memcg is disabled via "cgroup_disable=memory"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7502) * boot parameter. This may result in premature OOPS inside
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7503) * mem_cgroup_get_nr_swap_pages() function in corner cases.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7504) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7505) static int __init mem_cgroup_swap_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7506) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7507) /* No memory control -> no swap control */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7508) if (mem_cgroup_disabled())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7509) cgroup_memory_noswap = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7511) if (cgroup_memory_noswap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7512) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7514) WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys, swap_files));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7515) WARN_ON(cgroup_add_legacy_cftypes(&memory_cgrp_subsys, memsw_files));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7517) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7518) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7519) core_initcall(mem_cgroup_swap_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7521) #endif /* CONFIG_MEMCG_SWAP */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags