^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright IBM Corporation, 2012
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Author Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Cgroup v2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Copyright (C) 2019 Red Hat, Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Author: Giuseppe Scrivano <gscrivan@redhat.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * This program is free software; you can redistribute it and/or modify it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * under the terms of version 2.1 of the GNU Lesser General Public License
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * as published by the Free Software Foundation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * This program is distributed in the hope that it would be useful, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * WITHOUT ANY WARRANTY; without even the implied warranty of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/cgroup.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/page_counter.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/hugetlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/hugetlb_cgroup.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #define MEMFILE_PRIVATE(x, val) (((x) << 16) | (val))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #define MEMFILE_IDX(val) (((val) >> 16) & 0xffff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #define MEMFILE_ATTR(val) ((val) & 0xffff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #define hugetlb_cgroup_from_counter(counter, idx) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) container_of(counter, struct hugetlb_cgroup, hugepage[idx])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) static struct hugetlb_cgroup *root_h_cgroup __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) static inline struct page_counter *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) __hugetlb_cgroup_counter_from_cgroup(struct hugetlb_cgroup *h_cg, int idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) bool rsvd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) if (rsvd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) return &h_cg->rsvd_hugepage[idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) return &h_cg->hugepage[idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) static inline struct page_counter *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) hugetlb_cgroup_counter_from_cgroup(struct hugetlb_cgroup *h_cg, int idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) return __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) static inline struct page_counter *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) hugetlb_cgroup_counter_from_cgroup_rsvd(struct hugetlb_cgroup *h_cg, int idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) return __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) static inline
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) struct hugetlb_cgroup *hugetlb_cgroup_from_css(struct cgroup_subsys_state *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) return s ? container_of(s, struct hugetlb_cgroup, css) : NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) static inline
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) struct hugetlb_cgroup *hugetlb_cgroup_from_task(struct task_struct *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) return hugetlb_cgroup_from_css(task_css(task, hugetlb_cgrp_id));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) static inline bool hugetlb_cgroup_is_root(struct hugetlb_cgroup *h_cg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) return (h_cg == root_h_cgroup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) static inline struct hugetlb_cgroup *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) parent_hugetlb_cgroup(struct hugetlb_cgroup *h_cg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) return hugetlb_cgroup_from_css(h_cg->css.parent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) static inline bool hugetlb_cgroup_have_usage(struct hugetlb_cgroup *h_cg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) int idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) for (idx = 0; idx < hugetlb_max_hstate; idx++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) if (page_counter_read(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) hugetlb_cgroup_counter_from_cgroup(h_cg, idx)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) static void hugetlb_cgroup_init(struct hugetlb_cgroup *h_cgroup,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) struct hugetlb_cgroup *parent_h_cgroup)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) int idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) for (idx = 0; idx < HUGE_MAX_HSTATE; idx++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) struct page_counter *fault_parent = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) struct page_counter *rsvd_parent = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) unsigned long limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) if (parent_h_cgroup) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) fault_parent = hugetlb_cgroup_counter_from_cgroup(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) parent_h_cgroup, idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) rsvd_parent = hugetlb_cgroup_counter_from_cgroup_rsvd(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) parent_h_cgroup, idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) page_counter_init(hugetlb_cgroup_counter_from_cgroup(h_cgroup,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) idx),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) fault_parent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) page_counter_init(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) hugetlb_cgroup_counter_from_cgroup_rsvd(h_cgroup, idx),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) rsvd_parent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) limit = round_down(PAGE_COUNTER_MAX,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 1 << huge_page_order(&hstates[idx]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) ret = page_counter_set_max(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) hugetlb_cgroup_counter_from_cgroup(h_cgroup, idx),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) limit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) VM_BUG_ON(ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) ret = page_counter_set_max(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) hugetlb_cgroup_counter_from_cgroup_rsvd(h_cgroup, idx),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) limit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) VM_BUG_ON(ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) static struct cgroup_subsys_state *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) hugetlb_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) struct hugetlb_cgroup *parent_h_cgroup = hugetlb_cgroup_from_css(parent_css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) struct hugetlb_cgroup *h_cgroup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) h_cgroup = kzalloc(sizeof(*h_cgroup), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) if (!h_cgroup)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) if (!parent_h_cgroup)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) root_h_cgroup = h_cgroup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) hugetlb_cgroup_init(h_cgroup, parent_h_cgroup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) return &h_cgroup->css;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) static void hugetlb_cgroup_css_free(struct cgroup_subsys_state *css)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) struct hugetlb_cgroup *h_cgroup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) h_cgroup = hugetlb_cgroup_from_css(css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) kfree(h_cgroup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) * Should be called with hugetlb_lock held.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) * Since we are holding hugetlb_lock, pages cannot get moved from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) * active list or uncharged from the cgroup, So no need to get
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) * page reference and test for page active here. This function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) * cannot fail.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) static void hugetlb_cgroup_move_parent(int idx, struct hugetlb_cgroup *h_cg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) unsigned int nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) struct page_counter *counter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) struct hugetlb_cgroup *page_hcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) struct hugetlb_cgroup *parent = parent_hugetlb_cgroup(h_cg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) page_hcg = hugetlb_cgroup_from_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) * We can have pages in active list without any cgroup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) * ie, hugepage with less than 3 pages. We can safely
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) * ignore those pages.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) if (!page_hcg || page_hcg != h_cg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) nr_pages = compound_nr(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) if (!parent) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) parent = root_h_cgroup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) /* root has no limit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) page_counter_charge(&parent->hugepage[idx], nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) counter = &h_cg->hugepage[idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) /* Take the pages off the local counter */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) page_counter_cancel(counter, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) set_hugetlb_cgroup(page, parent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) * Force the hugetlb cgroup to empty the hugetlb resources by moving them to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) * the parent cgroup.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) static void hugetlb_cgroup_css_offline(struct cgroup_subsys_state *css)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) struct hstate *h;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) int idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) idx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) for_each_hstate(h) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) spin_lock(&hugetlb_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) list_for_each_entry(page, &h->hugepage_activelist, lru)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) hugetlb_cgroup_move_parent(idx, h_cg, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) spin_unlock(&hugetlb_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) idx++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) cond_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) } while (hugetlb_cgroup_have_usage(h_cg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) static inline void hugetlb_event(struct hugetlb_cgroup *hugetlb, int idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) enum hugetlb_memory_event event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) atomic_long_inc(&hugetlb->events_local[idx][event]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) cgroup_file_notify(&hugetlb->events_local_file[idx]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) atomic_long_inc(&hugetlb->events[idx][event]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) cgroup_file_notify(&hugetlb->events_file[idx]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) } while ((hugetlb = parent_hugetlb_cgroup(hugetlb)) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) !hugetlb_cgroup_is_root(hugetlb));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) static int __hugetlb_cgroup_charge_cgroup(int idx, unsigned long nr_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) struct hugetlb_cgroup **ptr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) bool rsvd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) struct page_counter *counter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) struct hugetlb_cgroup *h_cg = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) if (hugetlb_cgroup_disabled())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) * We don't charge any cgroup if the compound page have less
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) * than 3 pages.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) if (huge_page_order(&hstates[idx]) < HUGETLB_CGROUP_MIN_ORDER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) h_cg = hugetlb_cgroup_from_task(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) if (!css_tryget(&h_cg->css)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) if (!page_counter_try_charge(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, rsvd),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) nr_pages, &counter)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) hugetlb_event(h_cg, idx, HUGETLB_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) css_put(&h_cg->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) /* Reservations take a reference to the css because they do not get
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) * reparented.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) if (!rsvd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) css_put(&h_cg->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) *ptr = h_cg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) int hugetlb_cgroup_charge_cgroup(int idx, unsigned long nr_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) struct hugetlb_cgroup **ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) return __hugetlb_cgroup_charge_cgroup(idx, nr_pages, ptr, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) int hugetlb_cgroup_charge_cgroup_rsvd(int idx, unsigned long nr_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) struct hugetlb_cgroup **ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) return __hugetlb_cgroup_charge_cgroup(idx, nr_pages, ptr, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) /* Should be called with hugetlb_lock held */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) static void __hugetlb_cgroup_commit_charge(int idx, unsigned long nr_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) struct hugetlb_cgroup *h_cg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) struct page *page, bool rsvd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) if (hugetlb_cgroup_disabled() || !h_cg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) __set_hugetlb_cgroup(page, h_cg, rsvd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) void hugetlb_cgroup_commit_charge(int idx, unsigned long nr_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) struct hugetlb_cgroup *h_cg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) __hugetlb_cgroup_commit_charge(idx, nr_pages, h_cg, page, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) void hugetlb_cgroup_commit_charge_rsvd(int idx, unsigned long nr_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) struct hugetlb_cgroup *h_cg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) __hugetlb_cgroup_commit_charge(idx, nr_pages, h_cg, page, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) * Should be called with hugetlb_lock held
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) static void __hugetlb_cgroup_uncharge_page(int idx, unsigned long nr_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) struct page *page, bool rsvd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) struct hugetlb_cgroup *h_cg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) if (hugetlb_cgroup_disabled())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) lockdep_assert_held(&hugetlb_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) h_cg = __hugetlb_cgroup_from_page(page, rsvd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) if (unlikely(!h_cg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) __set_hugetlb_cgroup(page, NULL, rsvd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) page_counter_uncharge(__hugetlb_cgroup_counter_from_cgroup(h_cg, idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) rsvd),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) if (rsvd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) css_put(&h_cg->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) void hugetlb_cgroup_uncharge_page(int idx, unsigned long nr_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) __hugetlb_cgroup_uncharge_page(idx, nr_pages, page, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) void hugetlb_cgroup_uncharge_page_rsvd(int idx, unsigned long nr_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) __hugetlb_cgroup_uncharge_page(idx, nr_pages, page, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) static void __hugetlb_cgroup_uncharge_cgroup(int idx, unsigned long nr_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) struct hugetlb_cgroup *h_cg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) bool rsvd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) if (hugetlb_cgroup_disabled() || !h_cg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) if (huge_page_order(&hstates[idx]) < HUGETLB_CGROUP_MIN_ORDER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) page_counter_uncharge(__hugetlb_cgroup_counter_from_cgroup(h_cg, idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) rsvd),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) if (rsvd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) css_put(&h_cg->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) void hugetlb_cgroup_uncharge_cgroup(int idx, unsigned long nr_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) struct hugetlb_cgroup *h_cg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) __hugetlb_cgroup_uncharge_cgroup(idx, nr_pages, h_cg, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) void hugetlb_cgroup_uncharge_cgroup_rsvd(int idx, unsigned long nr_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) struct hugetlb_cgroup *h_cg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) __hugetlb_cgroup_uncharge_cgroup(idx, nr_pages, h_cg, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) void hugetlb_cgroup_uncharge_counter(struct resv_map *resv, unsigned long start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) unsigned long end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) if (hugetlb_cgroup_disabled() || !resv || !resv->reservation_counter ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) !resv->css)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) page_counter_uncharge(resv->reservation_counter,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) (end - start) * resv->pages_per_hpage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) css_put(resv->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) void hugetlb_cgroup_uncharge_file_region(struct resv_map *resv,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) struct file_region *rg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) unsigned long nr_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) bool region_del)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) if (hugetlb_cgroup_disabled() || !resv || !rg || !nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) if (rg->reservation_counter && resv->pages_per_hpage && nr_pages > 0 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) !resv->reservation_counter) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) page_counter_uncharge(rg->reservation_counter,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) nr_pages * resv->pages_per_hpage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) * Only do css_put(rg->css) when we delete the entire region
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) * because one file_region must hold exactly one css reference.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) if (region_del)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) css_put(rg->css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) enum {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) RES_USAGE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) RES_RSVD_USAGE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) RES_LIMIT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) RES_RSVD_LIMIT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) RES_MAX_USAGE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) RES_RSVD_MAX_USAGE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) RES_FAILCNT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) RES_RSVD_FAILCNT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) static u64 hugetlb_cgroup_read_u64(struct cgroup_subsys_state *css,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) struct cftype *cft)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) struct page_counter *counter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) struct page_counter *rsvd_counter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(css);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) counter = &h_cg->hugepage[MEMFILE_IDX(cft->private)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) rsvd_counter = &h_cg->rsvd_hugepage[MEMFILE_IDX(cft->private)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) switch (MEMFILE_ATTR(cft->private)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) case RES_USAGE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) return (u64)page_counter_read(counter) * PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) case RES_RSVD_USAGE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) return (u64)page_counter_read(rsvd_counter) * PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) case RES_LIMIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) return (u64)counter->max * PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) case RES_RSVD_LIMIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) return (u64)rsvd_counter->max * PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) case RES_MAX_USAGE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) return (u64)counter->watermark * PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) case RES_RSVD_MAX_USAGE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) return (u64)rsvd_counter->watermark * PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) case RES_FAILCNT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) return counter->failcnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) case RES_RSVD_FAILCNT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) return rsvd_counter->failcnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) static int hugetlb_cgroup_read_u64_max(struct seq_file *seq, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) int idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) u64 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) struct cftype *cft = seq_cft(seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) unsigned long limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) struct page_counter *counter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(seq_css(seq));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) idx = MEMFILE_IDX(cft->private);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) counter = &h_cg->hugepage[idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) limit = round_down(PAGE_COUNTER_MAX,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) 1 << huge_page_order(&hstates[idx]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) switch (MEMFILE_ATTR(cft->private)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) case RES_RSVD_USAGE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) counter = &h_cg->rsvd_hugepage[idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) case RES_USAGE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) val = (u64)page_counter_read(counter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) seq_printf(seq, "%llu\n", val * PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) case RES_RSVD_LIMIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) counter = &h_cg->rsvd_hugepage[idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) case RES_LIMIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) val = (u64)counter->max;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) if (val == limit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) seq_puts(seq, "max\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) seq_printf(seq, "%llu\n", val * PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) static DEFINE_MUTEX(hugetlb_limit_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) static ssize_t hugetlb_cgroup_write(struct kernfs_open_file *of,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) char *buf, size_t nbytes, loff_t off,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) const char *max)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) int ret, idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) unsigned long nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(of_css(of));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) bool rsvd = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) if (hugetlb_cgroup_is_root(h_cg)) /* Can't set limit on root */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) buf = strstrip(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) ret = page_counter_memparse(buf, max, &nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) idx = MEMFILE_IDX(of_cft(of)->private);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) nr_pages = round_down(nr_pages, 1 << huge_page_order(&hstates[idx]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) switch (MEMFILE_ATTR(of_cft(of)->private)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) case RES_RSVD_LIMIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) rsvd = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) case RES_LIMIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) mutex_lock(&hugetlb_limit_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) ret = page_counter_set_max(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, rsvd),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) mutex_unlock(&hugetlb_limit_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) return ret ?: nbytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) static ssize_t hugetlb_cgroup_write_legacy(struct kernfs_open_file *of,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) char *buf, size_t nbytes, loff_t off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) return hugetlb_cgroup_write(of, buf, nbytes, off, "-1");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) static ssize_t hugetlb_cgroup_write_dfl(struct kernfs_open_file *of,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) char *buf, size_t nbytes, loff_t off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) return hugetlb_cgroup_write(of, buf, nbytes, off, "max");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) static ssize_t hugetlb_cgroup_reset(struct kernfs_open_file *of,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) char *buf, size_t nbytes, loff_t off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) struct page_counter *counter, *rsvd_counter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(of_css(of));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) counter = &h_cg->hugepage[MEMFILE_IDX(of_cft(of)->private)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) rsvd_counter = &h_cg->rsvd_hugepage[MEMFILE_IDX(of_cft(of)->private)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) switch (MEMFILE_ATTR(of_cft(of)->private)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) case RES_MAX_USAGE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) page_counter_reset_watermark(counter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) case RES_RSVD_MAX_USAGE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) page_counter_reset_watermark(rsvd_counter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) case RES_FAILCNT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) counter->failcnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) case RES_RSVD_FAILCNT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) rsvd_counter->failcnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) return ret ?: nbytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) static char *mem_fmt(char *buf, int size, unsigned long hsize)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) if (hsize >= (1UL << 30))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) snprintf(buf, size, "%luGB", hsize >> 30);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) else if (hsize >= (1UL << 20))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) snprintf(buf, size, "%luMB", hsize >> 20);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) snprintf(buf, size, "%luKB", hsize >> 10);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) return buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) static int __hugetlb_events_show(struct seq_file *seq, bool local)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) int idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) long max;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) struct cftype *cft = seq_cft(seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(seq_css(seq));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) idx = MEMFILE_IDX(cft->private);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) if (local)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) max = atomic_long_read(&h_cg->events_local[idx][HUGETLB_MAX]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) max = atomic_long_read(&h_cg->events[idx][HUGETLB_MAX]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) seq_printf(seq, "max %lu\n", max);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) static int hugetlb_events_show(struct seq_file *seq, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) return __hugetlb_events_show(seq, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) static int hugetlb_events_local_show(struct seq_file *seq, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) return __hugetlb_events_show(seq, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) static void __init __hugetlb_cgroup_file_dfl_init(int idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) char buf[32];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) struct cftype *cft;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) struct hstate *h = &hstates[idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) /* format the size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) mem_fmt(buf, sizeof(buf), huge_page_size(h));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) /* Add the limit file */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) cft = &h->cgroup_files_dfl[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) snprintf(cft->name, MAX_CFTYPE_NAME, "%s.max", buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) cft->private = MEMFILE_PRIVATE(idx, RES_LIMIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) cft->seq_show = hugetlb_cgroup_read_u64_max;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) cft->write = hugetlb_cgroup_write_dfl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) cft->flags = CFTYPE_NOT_ON_ROOT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) /* Add the reservation limit file */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) cft = &h->cgroup_files_dfl[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) snprintf(cft->name, MAX_CFTYPE_NAME, "%s.rsvd.max", buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) cft->private = MEMFILE_PRIVATE(idx, RES_RSVD_LIMIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) cft->seq_show = hugetlb_cgroup_read_u64_max;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) cft->write = hugetlb_cgroup_write_dfl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) cft->flags = CFTYPE_NOT_ON_ROOT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) /* Add the current usage file */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) cft = &h->cgroup_files_dfl[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) snprintf(cft->name, MAX_CFTYPE_NAME, "%s.current", buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) cft->private = MEMFILE_PRIVATE(idx, RES_USAGE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) cft->seq_show = hugetlb_cgroup_read_u64_max;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) cft->flags = CFTYPE_NOT_ON_ROOT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) /* Add the current reservation usage file */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) cft = &h->cgroup_files_dfl[3];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) snprintf(cft->name, MAX_CFTYPE_NAME, "%s.rsvd.current", buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) cft->private = MEMFILE_PRIVATE(idx, RES_RSVD_USAGE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) cft->seq_show = hugetlb_cgroup_read_u64_max;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) cft->flags = CFTYPE_NOT_ON_ROOT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) /* Add the events file */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) cft = &h->cgroup_files_dfl[4];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) snprintf(cft->name, MAX_CFTYPE_NAME, "%s.events", buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) cft->private = MEMFILE_PRIVATE(idx, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) cft->seq_show = hugetlb_events_show;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) cft->file_offset = offsetof(struct hugetlb_cgroup, events_file[idx]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) cft->flags = CFTYPE_NOT_ON_ROOT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) /* Add the events.local file */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) cft = &h->cgroup_files_dfl[5];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) snprintf(cft->name, MAX_CFTYPE_NAME, "%s.events.local", buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) cft->private = MEMFILE_PRIVATE(idx, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) cft->seq_show = hugetlb_events_local_show;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) cft->file_offset = offsetof(struct hugetlb_cgroup,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) events_local_file[idx]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) cft->flags = CFTYPE_NOT_ON_ROOT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) /* NULL terminate the last cft */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) cft = &h->cgroup_files_dfl[6];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) memset(cft, 0, sizeof(*cft));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) WARN_ON(cgroup_add_dfl_cftypes(&hugetlb_cgrp_subsys,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) h->cgroup_files_dfl));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) static void __init __hugetlb_cgroup_file_legacy_init(int idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) char buf[32];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) struct cftype *cft;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) struct hstate *h = &hstates[idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) /* format the size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) mem_fmt(buf, sizeof(buf), huge_page_size(h));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) /* Add the limit file */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) cft = &h->cgroup_files_legacy[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) snprintf(cft->name, MAX_CFTYPE_NAME, "%s.limit_in_bytes", buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) cft->private = MEMFILE_PRIVATE(idx, RES_LIMIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) cft->read_u64 = hugetlb_cgroup_read_u64;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) cft->write = hugetlb_cgroup_write_legacy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) /* Add the reservation limit file */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) cft = &h->cgroup_files_legacy[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) snprintf(cft->name, MAX_CFTYPE_NAME, "%s.rsvd.limit_in_bytes", buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) cft->private = MEMFILE_PRIVATE(idx, RES_RSVD_LIMIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) cft->read_u64 = hugetlb_cgroup_read_u64;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) cft->write = hugetlb_cgroup_write_legacy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) /* Add the usage file */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) cft = &h->cgroup_files_legacy[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) snprintf(cft->name, MAX_CFTYPE_NAME, "%s.usage_in_bytes", buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) cft->private = MEMFILE_PRIVATE(idx, RES_USAGE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) cft->read_u64 = hugetlb_cgroup_read_u64;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) /* Add the reservation usage file */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) cft = &h->cgroup_files_legacy[3];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) snprintf(cft->name, MAX_CFTYPE_NAME, "%s.rsvd.usage_in_bytes", buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) cft->private = MEMFILE_PRIVATE(idx, RES_RSVD_USAGE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) cft->read_u64 = hugetlb_cgroup_read_u64;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) /* Add the MAX usage file */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) cft = &h->cgroup_files_legacy[4];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) snprintf(cft->name, MAX_CFTYPE_NAME, "%s.max_usage_in_bytes", buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) cft->private = MEMFILE_PRIVATE(idx, RES_MAX_USAGE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) cft->write = hugetlb_cgroup_reset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) cft->read_u64 = hugetlb_cgroup_read_u64;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) /* Add the MAX reservation usage file */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) cft = &h->cgroup_files_legacy[5];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) snprintf(cft->name, MAX_CFTYPE_NAME, "%s.rsvd.max_usage_in_bytes", buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) cft->private = MEMFILE_PRIVATE(idx, RES_RSVD_MAX_USAGE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) cft->write = hugetlb_cgroup_reset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) cft->read_u64 = hugetlb_cgroup_read_u64;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) /* Add the failcntfile */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) cft = &h->cgroup_files_legacy[6];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) snprintf(cft->name, MAX_CFTYPE_NAME, "%s.failcnt", buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) cft->private = MEMFILE_PRIVATE(idx, RES_FAILCNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) cft->write = hugetlb_cgroup_reset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) cft->read_u64 = hugetlb_cgroup_read_u64;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) /* Add the reservation failcntfile */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) cft = &h->cgroup_files_legacy[7];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) snprintf(cft->name, MAX_CFTYPE_NAME, "%s.rsvd.failcnt", buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) cft->private = MEMFILE_PRIVATE(idx, RES_RSVD_FAILCNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) cft->write = hugetlb_cgroup_reset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) cft->read_u64 = hugetlb_cgroup_read_u64;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) /* NULL terminate the last cft */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) cft = &h->cgroup_files_legacy[8];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) memset(cft, 0, sizeof(*cft));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) WARN_ON(cgroup_add_legacy_cftypes(&hugetlb_cgrp_subsys,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) h->cgroup_files_legacy));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) static void __init __hugetlb_cgroup_file_init(int idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) __hugetlb_cgroup_file_dfl_init(idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) __hugetlb_cgroup_file_legacy_init(idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) void __init hugetlb_cgroup_file_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) struct hstate *h;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) for_each_hstate(h) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) * Add cgroup control files only if the huge page consists
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) * of more than two normal pages. This is because we use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) * page[2].private for storing cgroup details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) if (huge_page_order(h) >= HUGETLB_CGROUP_MIN_ORDER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) __hugetlb_cgroup_file_init(hstate_index(h));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) * hugetlb_lock will make sure a parallel cgroup rmdir won't happen
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) * when we migrate hugepages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) void hugetlb_cgroup_migrate(struct page *oldhpage, struct page *newhpage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) struct hugetlb_cgroup *h_cg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) struct hugetlb_cgroup *h_cg_rsvd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) struct hstate *h = page_hstate(oldhpage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) if (hugetlb_cgroup_disabled())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) VM_BUG_ON_PAGE(!PageHuge(oldhpage), oldhpage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) spin_lock(&hugetlb_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) h_cg = hugetlb_cgroup_from_page(oldhpage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) h_cg_rsvd = hugetlb_cgroup_from_page_rsvd(oldhpage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) set_hugetlb_cgroup(oldhpage, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) set_hugetlb_cgroup_rsvd(oldhpage, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) /* move the h_cg details to new cgroup */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) set_hugetlb_cgroup(newhpage, h_cg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) set_hugetlb_cgroup_rsvd(newhpage, h_cg_rsvd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) list_move(&newhpage->lru, &h->hugepage_activelist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) spin_unlock(&hugetlb_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) static struct cftype hugetlb_files[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) {} /* terminate */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) struct cgroup_subsys hugetlb_cgrp_subsys = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) .css_alloc = hugetlb_cgroup_css_alloc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) .css_offline = hugetlb_cgroup_css_offline,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) .css_free = hugetlb_cgroup_css_free,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) .dfl_cftypes = hugetlb_files,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) .legacy_cftypes = hugetlb_files,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) };
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags