^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Basic Node interface support
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/memory.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/vmstat.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/notifier.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/node.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/hugetlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/compaction.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/cpumask.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/topology.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/nodemask.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/device.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/pm_runtime.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/swap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) static struct bus_type node_subsys = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) .name = "node",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) .dev_name = "node",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) static ssize_t node_read_cpumap(struct device *dev, bool list, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) ssize_t n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) cpumask_var_t mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) struct node *node_dev = to_node(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) /* 2008/04/07: buf currently PAGE_SIZE, need 9 chars per 32 bits. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) BUILD_BUG_ON((NR_CPUS/32 * 9) > (PAGE_SIZE-1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) if (!alloc_cpumask_var(&mask, GFP_KERNEL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) cpumask_and(mask, cpumask_of_node(node_dev->dev.id), cpu_online_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) n = cpumap_print_to_pagebuf(list, buf, mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) free_cpumask_var(mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) return n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) static inline ssize_t cpumap_show(struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) struct device_attribute *attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) return node_read_cpumap(dev, false, buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) static DEVICE_ATTR_RO(cpumap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) static inline ssize_t cpulist_show(struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) struct device_attribute *attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) return node_read_cpumap(dev, true, buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) static DEVICE_ATTR_RO(cpulist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) * struct node_access_nodes - Access class device to hold user visible
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) * relationships to other nodes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) * @dev: Device for this memory access class
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) * @list_node: List element in the node's access list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) * @access: The access class rank
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) * @hmem_attrs: Heterogeneous memory performance attributes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) struct node_access_nodes {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) struct device dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) struct list_head list_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) unsigned access;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) #ifdef CONFIG_HMEM_REPORTING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) struct node_hmem_attrs hmem_attrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) #define to_access_nodes(dev) container_of(dev, struct node_access_nodes, dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) static struct attribute *node_init_access_node_attrs[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) static struct attribute *node_targ_access_node_attrs[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) static const struct attribute_group initiators = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) .name = "initiators",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) .attrs = node_init_access_node_attrs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) static const struct attribute_group targets = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) .name = "targets",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) .attrs = node_targ_access_node_attrs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) static const struct attribute_group *node_access_node_groups[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) &initiators,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) &targets,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) static void node_remove_accesses(struct node *node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) struct node_access_nodes *c, *cnext;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) list_for_each_entry_safe(c, cnext, &node->access_list, list_node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) list_del(&c->list_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) device_unregister(&c->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) static void node_access_release(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) kfree(to_access_nodes(dev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) static struct node_access_nodes *node_init_node_access(struct node *node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) unsigned access)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) struct node_access_nodes *access_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) struct device *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) list_for_each_entry(access_node, &node->access_list, list_node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) if (access_node->access == access)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) return access_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) access_node = kzalloc(sizeof(*access_node), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) if (!access_node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) access_node->access = access;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) dev = &access_node->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) dev->parent = &node->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) dev->release = node_access_release;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) dev->groups = node_access_node_groups;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) if (dev_set_name(dev, "access%u", access))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) goto free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) if (device_register(dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) goto free_name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) pm_runtime_no_callbacks(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) list_add_tail(&access_node->list_node, &node->access_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) return access_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) free_name:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) kfree_const(dev->kobj.name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) free:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) kfree(access_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) #ifdef CONFIG_HMEM_REPORTING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) #define ACCESS_ATTR(name) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) static ssize_t name##_show(struct device *dev, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) struct device_attribute *attr, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) char *buf) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) return sysfs_emit(buf, "%u\n", \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) to_access_nodes(dev)->hmem_attrs.name); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) } \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) static DEVICE_ATTR_RO(name)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) ACCESS_ATTR(read_bandwidth);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) ACCESS_ATTR(read_latency);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) ACCESS_ATTR(write_bandwidth);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) ACCESS_ATTR(write_latency);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) static struct attribute *access_attrs[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) &dev_attr_read_bandwidth.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) &dev_attr_read_latency.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) &dev_attr_write_bandwidth.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) &dev_attr_write_latency.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) * node_set_perf_attrs - Set the performance values for given access class
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) * @nid: Node identifier to be set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) * @hmem_attrs: Heterogeneous memory performance attributes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) * @access: The access class the for the given attributes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) void node_set_perf_attrs(unsigned int nid, struct node_hmem_attrs *hmem_attrs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) unsigned access)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) struct node_access_nodes *c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) struct node *node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) if (WARN_ON_ONCE(!node_online(nid)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) node = node_devices[nid];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) c = node_init_node_access(node, access);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) if (!c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) c->hmem_attrs = *hmem_attrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) for (i = 0; access_attrs[i] != NULL; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) if (sysfs_add_file_to_group(&c->dev.kobj, access_attrs[i],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) "initiators")) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) pr_info("failed to add performance attribute to node %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) nid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) * struct node_cache_info - Internal tracking for memory node caches
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) * @dev: Device represeting the cache level
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) * @node: List element for tracking in the node
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) * @cache_attrs:Attributes for this cache level
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) struct node_cache_info {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) struct device dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) struct list_head node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) struct node_cache_attrs cache_attrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) #define to_cache_info(device) container_of(device, struct node_cache_info, dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) #define CACHE_ATTR(name, fmt) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) static ssize_t name##_show(struct device *dev, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) struct device_attribute *attr, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) char *buf) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) return sysfs_emit(buf, fmt "\n", \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) to_cache_info(dev)->cache_attrs.name); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) } \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) DEVICE_ATTR_RO(name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) CACHE_ATTR(size, "%llu")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) CACHE_ATTR(line_size, "%u")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) CACHE_ATTR(indexing, "%u")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) CACHE_ATTR(write_policy, "%u")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) static struct attribute *cache_attrs[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) &dev_attr_indexing.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) &dev_attr_size.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) &dev_attr_line_size.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) &dev_attr_write_policy.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) ATTRIBUTE_GROUPS(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) static void node_cache_release(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) kfree(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) static void node_cacheinfo_release(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) struct node_cache_info *info = to_cache_info(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) kfree(info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) static void node_init_cache_dev(struct node *node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) struct device *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) dev = kzalloc(sizeof(*dev), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) if (!dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) device_initialize(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) dev->parent = &node->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) dev->release = node_cache_release;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) if (dev_set_name(dev, "memory_side_cache"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) goto put_device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) if (device_add(dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) goto put_device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) pm_runtime_no_callbacks(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) node->cache_dev = dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) put_device:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) put_device(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) * node_add_cache() - add cache attribute to a memory node
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) * @nid: Node identifier that has new cache attributes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) * @cache_attrs: Attributes for the cache being added
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) void node_add_cache(unsigned int nid, struct node_cache_attrs *cache_attrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) struct node_cache_info *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) struct device *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) struct node *node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) if (!node_online(nid) || !node_devices[nid])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) node = node_devices[nid];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) list_for_each_entry(info, &node->cache_attrs, node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) if (info->cache_attrs.level == cache_attrs->level) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) dev_warn(&node->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) "attempt to add duplicate cache level:%d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) cache_attrs->level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) if (!node->cache_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) node_init_cache_dev(node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) if (!node->cache_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) info = kzalloc(sizeof(*info), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) if (!info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) dev = &info->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) device_initialize(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) dev->parent = node->cache_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) dev->release = node_cacheinfo_release;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) dev->groups = cache_groups;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) if (dev_set_name(dev, "index%d", cache_attrs->level))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) goto put_device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) info->cache_attrs = *cache_attrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) if (device_add(dev)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) dev_warn(&node->dev, "failed to add cache level:%d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) cache_attrs->level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) goto put_device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) pm_runtime_no_callbacks(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) list_add_tail(&info->node, &node->cache_attrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) put_device:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) put_device(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) static void node_remove_caches(struct node *node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) struct node_cache_info *info, *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) if (!node->cache_dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) list_for_each_entry_safe(info, next, &node->cache_attrs, node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) list_del(&info->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) device_unregister(&info->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) device_unregister(node->cache_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) static void node_init_caches(unsigned int nid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) INIT_LIST_HEAD(&node_devices[nid]->cache_attrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) static void node_init_caches(unsigned int nid) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) static void node_remove_caches(struct node *node) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) #define K(x) ((x) << (PAGE_SHIFT - 10))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) static ssize_t node_read_meminfo(struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) struct device_attribute *attr, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) int len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) int nid = dev->id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) struct pglist_data *pgdat = NODE_DATA(nid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) struct sysinfo i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) unsigned long sreclaimable, sunreclaimable;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) si_meminfo_node(&i, nid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) sreclaimable = node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) sunreclaimable = node_page_state_pages(pgdat, NR_SLAB_UNRECLAIMABLE_B);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) len = sysfs_emit_at(buf, len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) "Node %d MemTotal: %8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) "Node %d MemFree: %8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) "Node %d MemUsed: %8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) "Node %d Active: %8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) "Node %d Inactive: %8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) "Node %d Active(anon): %8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) "Node %d Inactive(anon): %8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) "Node %d Active(file): %8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) "Node %d Inactive(file): %8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) "Node %d Unevictable: %8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) "Node %d Mlocked: %8lu kB\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) nid, K(i.totalram),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) nid, K(i.freeram),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) nid, K(i.totalram - i.freeram),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) nid, K(node_page_state(pgdat, NR_ACTIVE_ANON) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) node_page_state(pgdat, NR_ACTIVE_FILE)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) nid, K(node_page_state(pgdat, NR_INACTIVE_ANON) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) node_page_state(pgdat, NR_INACTIVE_FILE)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) nid, K(node_page_state(pgdat, NR_ACTIVE_ANON)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) nid, K(node_page_state(pgdat, NR_INACTIVE_ANON)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) nid, K(node_page_state(pgdat, NR_ACTIVE_FILE)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) nid, K(node_page_state(pgdat, NR_INACTIVE_FILE)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) nid, K(node_page_state(pgdat, NR_UNEVICTABLE)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) nid, K(sum_zone_node_page_state(nid, NR_MLOCK)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) #ifdef CONFIG_HIGHMEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) len += sysfs_emit_at(buf, len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) "Node %d HighTotal: %8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) "Node %d HighFree: %8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) "Node %d LowTotal: %8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) "Node %d LowFree: %8lu kB\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) nid, K(i.totalhigh),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) nid, K(i.freehigh),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) nid, K(i.totalram - i.totalhigh),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) nid, K(i.freeram - i.freehigh));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) len += sysfs_emit_at(buf, len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) "Node %d Dirty: %8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) "Node %d Writeback: %8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) "Node %d FilePages: %8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) "Node %d Mapped: %8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) "Node %d AnonPages: %8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) "Node %d Shmem: %8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) "Node %d KernelStack: %8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) #ifdef CONFIG_SHADOW_CALL_STACK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) "Node %d ShadowCallStack:%8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) "Node %d PageTables: %8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) "Node %d NFS_Unstable: %8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) "Node %d Bounce: %8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) "Node %d WritebackTmp: %8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) "Node %d KReclaimable: %8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) "Node %d Slab: %8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) "Node %d SReclaimable: %8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) "Node %d SUnreclaim: %8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) #ifdef CONFIG_TRANSPARENT_HUGEPAGE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) "Node %d AnonHugePages: %8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) "Node %d ShmemHugePages: %8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) "Node %d ShmemPmdMapped: %8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) "Node %d FileHugePages: %8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) "Node %d FilePmdMapped: %8lu kB\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) ,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) nid, K(node_page_state(pgdat, NR_FILE_DIRTY)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) nid, K(node_page_state(pgdat, NR_WRITEBACK)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) nid, K(node_page_state(pgdat, NR_FILE_PAGES)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) nid, K(node_page_state(pgdat, NR_FILE_MAPPED)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) nid, K(node_page_state(pgdat, NR_ANON_MAPPED)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) nid, K(i.sharedram),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) nid, node_page_state(pgdat, NR_KERNEL_STACK_KB),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) #ifdef CONFIG_SHADOW_CALL_STACK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) nid, node_page_state(pgdat, NR_KERNEL_SCS_KB),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) nid, K(sum_zone_node_page_state(nid, NR_PAGETABLE)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) nid, 0UL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) nid, K(sum_zone_node_page_state(nid, NR_BOUNCE)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) nid, K(node_page_state(pgdat, NR_WRITEBACK_TEMP)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) nid, K(sreclaimable +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) node_page_state(pgdat, NR_KERNEL_MISC_RECLAIMABLE)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) nid, K(sreclaimable + sunreclaimable),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) nid, K(sreclaimable),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) nid, K(sunreclaimable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) #ifdef CONFIG_TRANSPARENT_HUGEPAGE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) ,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) nid, K(node_page_state(pgdat, NR_ANON_THPS) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) HPAGE_PMD_NR),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) nid, K(node_page_state(pgdat, NR_SHMEM_THPS) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) HPAGE_PMD_NR),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) nid, K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) HPAGE_PMD_NR),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) nid, K(node_page_state(pgdat, NR_FILE_THPS) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) HPAGE_PMD_NR),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) nid, K(node_page_state(pgdat, NR_FILE_PMDMAPPED) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) HPAGE_PMD_NR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) len += hugetlb_report_node_meminfo(buf, len, nid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) return len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) #undef K
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) static DEVICE_ATTR(meminfo, 0444, node_read_meminfo, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) static ssize_t node_read_numastat(struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) struct device_attribute *attr, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) return sysfs_emit(buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) "numa_hit %lu\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) "numa_miss %lu\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) "numa_foreign %lu\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) "interleave_hit %lu\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) "local_node %lu\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) "other_node %lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) sum_zone_numa_state(dev->id, NUMA_HIT),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) sum_zone_numa_state(dev->id, NUMA_MISS),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) sum_zone_numa_state(dev->id, NUMA_FOREIGN),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) sum_zone_numa_state(dev->id, NUMA_INTERLEAVE_HIT),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) sum_zone_numa_state(dev->id, NUMA_LOCAL),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) sum_zone_numa_state(dev->id, NUMA_OTHER));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) static DEVICE_ATTR(numastat, 0444, node_read_numastat, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) static ssize_t node_read_vmstat(struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) struct device_attribute *attr, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) int nid = dev->id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) struct pglist_data *pgdat = NODE_DATA(nid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) int len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) len += sysfs_emit_at(buf, len, "%s %lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) zone_stat_name(i),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) sum_zone_node_page_state(nid, i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) #ifdef CONFIG_NUMA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) for (i = 0; i < NR_VM_NUMA_STAT_ITEMS; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) len += sysfs_emit_at(buf, len, "%s %lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) numa_stat_name(i),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) sum_zone_numa_state(nid, i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) len += sysfs_emit_at(buf, len, "%s %lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) node_stat_name(i),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) node_page_state_pages(pgdat, i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) return len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) static DEVICE_ATTR(vmstat, 0444, node_read_vmstat, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) static ssize_t node_read_distance(struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) struct device_attribute *attr, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) int nid = dev->id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) int len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) * buf is currently PAGE_SIZE in length and each node needs 4 chars
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) * at the most (distance + space or newline).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) BUILD_BUG_ON(MAX_NUMNODES * 4 > PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) for_each_online_node(i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) len += sysfs_emit_at(buf, len, "%s%d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) i ? " " : "", node_distance(nid, i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) len += sysfs_emit_at(buf, len, "\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) return len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) static DEVICE_ATTR(distance, 0444, node_read_distance, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) static struct attribute *node_dev_attrs[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) &dev_attr_cpumap.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) &dev_attr_cpulist.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) &dev_attr_meminfo.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) &dev_attr_numastat.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) &dev_attr_distance.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) &dev_attr_vmstat.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) NULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) ATTRIBUTE_GROUPS(node_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) #ifdef CONFIG_HUGETLBFS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) * hugetlbfs per node attributes registration interface:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) * When/if hugetlb[fs] subsystem initializes [sometime after this module],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) * it will register its per node attributes for all online nodes with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) * memory. It will also call register_hugetlbfs_with_node(), below, to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) * register its attribute registration functions with this node driver.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) * Once these hooks have been initialized, the node driver will call into
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) * the hugetlb module to [un]register attributes for hot-plugged nodes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) static node_registration_func_t __hugetlb_register_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) static node_registration_func_t __hugetlb_unregister_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) static inline bool hugetlb_register_node(struct node *node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) if (__hugetlb_register_node &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) node_state(node->dev.id, N_MEMORY)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) __hugetlb_register_node(node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) static inline void hugetlb_unregister_node(struct node *node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) if (__hugetlb_unregister_node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) __hugetlb_unregister_node(node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) void register_hugetlbfs_with_node(node_registration_func_t doregister,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) node_registration_func_t unregister)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) __hugetlb_register_node = doregister;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) __hugetlb_unregister_node = unregister;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) static inline void hugetlb_register_node(struct node *node) {}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) static inline void hugetlb_unregister_node(struct node *node) {}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) static void node_device_release(struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) struct node *node = to_node(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) #if defined(CONFIG_MEMORY_HOTPLUG_SPARSE) && defined(CONFIG_HUGETLBFS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) * We schedule the work only when a memory section is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) * onlined/offlined on this node. When we come here,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) * all the memory on this node has been offlined,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) * so we won't enqueue new work to this work.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) * The work is using node->node_work, so we should
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) * flush work before freeing the memory.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) flush_work(&node->node_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) kfree(node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) * register_node - Setup a sysfs device for a node.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) * @num - Node number to use when creating the device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) * Initialize and register the node device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) static int register_node(struct node *node, int num)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) int error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) node->dev.id = num;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) node->dev.bus = &node_subsys;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) node->dev.release = node_device_release;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) node->dev.groups = node_dev_groups;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) error = device_register(&node->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) if (error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) put_device(&node->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) hugetlb_register_node(node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) compaction_register_node(node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) return error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) * unregister_node - unregister a node device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) * @node: node going away
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) * Unregisters a node device @node. All the devices on the node must be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) * unregistered before calling this function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) void unregister_node(struct node *node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) hugetlb_unregister_node(node); /* no-op, if memoryless node */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) node_remove_accesses(node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) node_remove_caches(node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) device_unregister(&node->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) struct node *node_devices[MAX_NUMNODES];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) * register cpu under node
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) int register_cpu_under_node(unsigned int cpu, unsigned int nid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) struct device *obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) if (!node_online(nid))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) obj = get_cpu_device(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) if (!obj)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) ret = sysfs_create_link(&node_devices[nid]->dev.kobj,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) &obj->kobj,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) kobject_name(&obj->kobj));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) return sysfs_create_link(&obj->kobj,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) &node_devices[nid]->dev.kobj,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) kobject_name(&node_devices[nid]->dev.kobj));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) * register_memory_node_under_compute_node - link memory node to its compute
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) * node for a given access class.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) * @mem_nid: Memory node number
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) * @cpu_nid: Cpu node number
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) * @access: Access class to register
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) * For use with platforms that may have separate memory and compute nodes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) * This function will export node relationships linking which memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) * initiator nodes can access memory targets at a given ranked access
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) * class.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) int register_memory_node_under_compute_node(unsigned int mem_nid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) unsigned int cpu_nid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) unsigned access)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) struct node *init_node, *targ_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) struct node_access_nodes *initiator, *target;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) if (!node_online(cpu_nid) || !node_online(mem_nid))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) init_node = node_devices[cpu_nid];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) targ_node = node_devices[mem_nid];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) initiator = node_init_node_access(init_node, access);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) target = node_init_node_access(targ_node, access);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) if (!initiator || !target)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) ret = sysfs_add_link_to_group(&initiator->dev.kobj, "targets",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) &targ_node->dev.kobj,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) dev_name(&targ_node->dev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) ret = sysfs_add_link_to_group(&target->dev.kobj, "initiators",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) &init_node->dev.kobj,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) dev_name(&init_node->dev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) sysfs_remove_link_from_group(&initiator->dev.kobj, "targets",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) dev_name(&targ_node->dev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) int unregister_cpu_under_node(unsigned int cpu, unsigned int nid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) struct device *obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) if (!node_online(nid))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) obj = get_cpu_device(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) if (!obj)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) sysfs_remove_link(&node_devices[nid]->dev.kobj,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) kobject_name(&obj->kobj));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) sysfs_remove_link(&obj->kobj,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) kobject_name(&node_devices[nid]->dev.kobj));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) static int __ref get_nid_for_pfn(unsigned long pfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) if (!pfn_valid_within(pfn))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) if (system_state < SYSTEM_RUNNING)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) return early_pfn_to_nid(pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) return pfn_to_nid(pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) static void do_register_memory_block_under_node(int nid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) struct memory_block *mem_blk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) * If this memory block spans multiple nodes, we only indicate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) * the last processed node.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) mem_blk->nid = nid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) ret = sysfs_create_link_nowarn(&node_devices[nid]->dev.kobj,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) &mem_blk->dev.kobj,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) kobject_name(&mem_blk->dev.kobj));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) if (ret && ret != -EEXIST)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) dev_err_ratelimited(&node_devices[nid]->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) "can't create link to %s in sysfs (%d)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) kobject_name(&mem_blk->dev.kobj), ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) ret = sysfs_create_link_nowarn(&mem_blk->dev.kobj,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) &node_devices[nid]->dev.kobj,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) kobject_name(&node_devices[nid]->dev.kobj));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) if (ret && ret != -EEXIST)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) dev_err_ratelimited(&mem_blk->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) "can't create link to %s in sysfs (%d)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) kobject_name(&node_devices[nid]->dev.kobj),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) /* register memory section under specified node if it spans that node */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) static int register_mem_block_under_node_early(struct memory_block *mem_blk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) unsigned long memory_block_pfns = memory_block_size_bytes() / PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) unsigned long start_pfn = section_nr_to_pfn(mem_blk->start_section_nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) unsigned long end_pfn = start_pfn + memory_block_pfns - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) int nid = *(int *)arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) unsigned long pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) for (pfn = start_pfn; pfn <= end_pfn; pfn++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) int page_nid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) * memory block could have several absent sections from start.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) * skip pfn range from absent section
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) if (!pfn_in_present_section(pfn)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) pfn = round_down(pfn + PAGES_PER_SECTION,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) PAGES_PER_SECTION) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) * We need to check if page belongs to nid only at the boot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) * case because node's ranges can be interleaved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) page_nid = get_nid_for_pfn(pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) if (page_nid < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) if (page_nid != nid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) do_register_memory_block_under_node(nid, mem_blk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) /* mem section does not span the specified node */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) * During hotplug we know that all pages in the memory block belong to the same
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) * node.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) static int register_mem_block_under_node_hotplug(struct memory_block *mem_blk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) int nid = *(int *)arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) do_register_memory_block_under_node(nid, mem_blk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) * Unregister a memory block device under the node it spans. Memory blocks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) * with multiple nodes cannot be offlined and therefore also never be removed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) void unregister_memory_block_under_nodes(struct memory_block *mem_blk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) if (mem_blk->nid == NUMA_NO_NODE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) sysfs_remove_link(&node_devices[mem_blk->nid]->dev.kobj,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) kobject_name(&mem_blk->dev.kobj));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) sysfs_remove_link(&mem_blk->dev.kobj,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) kobject_name(&node_devices[mem_blk->nid]->dev.kobj));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) void link_mem_sections(int nid, unsigned long start_pfn, unsigned long end_pfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) enum meminit_context context)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) walk_memory_blocks_func_t func;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) if (context == MEMINIT_HOTPLUG)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) func = register_mem_block_under_node_hotplug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) func = register_mem_block_under_node_early;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) walk_memory_blocks(PFN_PHYS(start_pfn), PFN_PHYS(end_pfn - start_pfn),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) (void *)&nid, func);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) #ifdef CONFIG_HUGETLBFS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) * Handle per node hstate attribute [un]registration on transistions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) * to/from memoryless state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) static void node_hugetlb_work(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) struct node *node = container_of(work, struct node, node_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) * We only get here when a node transitions to/from memoryless state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) * We can detect which transition occurred by examining whether the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) * node has memory now. hugetlb_register_node() already check this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) * so we try to register the attributes. If that fails, then the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) * node has transitioned to memoryless, try to unregister the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) * attributes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) if (!hugetlb_register_node(node))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) hugetlb_unregister_node(node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) static void init_node_hugetlb_work(int nid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) INIT_WORK(&node_devices[nid]->node_work, node_hugetlb_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) static int node_memory_callback(struct notifier_block *self,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) unsigned long action, void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) struct memory_notify *mnb = arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) int nid = mnb->status_change_nid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) switch (action) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) case MEM_ONLINE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) case MEM_OFFLINE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) * offload per node hstate [un]registration to a work thread
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) * when transitioning to/from memoryless state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) if (nid != NUMA_NO_NODE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) schedule_work(&node_devices[nid]->node_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) case MEM_GOING_ONLINE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) case MEM_GOING_OFFLINE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) case MEM_CANCEL_ONLINE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) case MEM_CANCEL_OFFLINE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) return NOTIFY_OK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) #endif /* CONFIG_HUGETLBFS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) #endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) #if !defined(CONFIG_MEMORY_HOTPLUG_SPARSE) || \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) !defined(CONFIG_HUGETLBFS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) static inline int node_memory_callback(struct notifier_block *self,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) unsigned long action, void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) return NOTIFY_OK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) static void init_node_hugetlb_work(int nid) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) int __register_one_node(int nid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) int error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) node_devices[nid] = kzalloc(sizeof(struct node), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) if (!node_devices[nid])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) error = register_node(node_devices[nid], nid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) /* link cpu under this node */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) for_each_present_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) if (cpu_to_node(cpu) == nid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) register_cpu_under_node(cpu, nid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) INIT_LIST_HEAD(&node_devices[nid]->access_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) /* initialize work queue for memory hot plug */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) init_node_hugetlb_work(nid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) node_init_caches(nid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) return error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) void unregister_one_node(int nid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) if (!node_devices[nid])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) unregister_node(node_devices[nid]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) node_devices[nid] = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) * node states attributes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) struct node_attr {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) struct device_attribute attr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) enum node_states state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) static ssize_t show_node_state(struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) struct device_attribute *attr, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) struct node_attr *na = container_of(attr, struct node_attr, attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) return sysfs_emit(buf, "%*pbl\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) nodemask_pr_args(&node_states[na->state]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) #define _NODE_ATTR(name, state) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) { __ATTR(name, 0444, show_node_state, NULL), state }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) static struct node_attr node_state_attr[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) [N_POSSIBLE] = _NODE_ATTR(possible, N_POSSIBLE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) [N_ONLINE] = _NODE_ATTR(online, N_ONLINE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) [N_NORMAL_MEMORY] = _NODE_ATTR(has_normal_memory, N_NORMAL_MEMORY),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) #ifdef CONFIG_HIGHMEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) [N_HIGH_MEMORY] = _NODE_ATTR(has_high_memory, N_HIGH_MEMORY),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) [N_MEMORY] = _NODE_ATTR(has_memory, N_MEMORY),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) [N_CPU] = _NODE_ATTR(has_cpu, N_CPU),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) [N_GENERIC_INITIATOR] = _NODE_ATTR(has_generic_initiator,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) N_GENERIC_INITIATOR),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) static struct attribute *node_state_attrs[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) &node_state_attr[N_POSSIBLE].attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) &node_state_attr[N_ONLINE].attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) &node_state_attr[N_NORMAL_MEMORY].attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) #ifdef CONFIG_HIGHMEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) &node_state_attr[N_HIGH_MEMORY].attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) &node_state_attr[N_MEMORY].attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) &node_state_attr[N_CPU].attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) &node_state_attr[N_GENERIC_INITIATOR].attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) NULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) static struct attribute_group memory_root_attr_group = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) .attrs = node_state_attrs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) static const struct attribute_group *cpu_root_attr_groups[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) &memory_root_attr_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) #define NODE_CALLBACK_PRI 2 /* lower than SLAB */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) static int __init register_node_type(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) BUILD_BUG_ON(ARRAY_SIZE(node_state_attr) != NR_NODE_STATES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) BUILD_BUG_ON(ARRAY_SIZE(node_state_attrs)-1 != NR_NODE_STATES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) ret = subsys_system_register(&node_subsys, cpu_root_attr_groups);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) if (!ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) static struct notifier_block node_memory_callback_nb = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) .notifier_call = node_memory_callback,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) .priority = NODE_CALLBACK_PRI,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) register_hotmemory_notifier(&node_memory_callback_nb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) * Note: we're not going to unregister the node class if we fail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) * to register the node state class attribute files.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) postcore_initcall(register_node_type);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags