^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * DAMON-based page reclamation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Author: SeongJae Park <sj@kernel.org>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #define pr_fmt(fmt) "damon-reclaim: " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/damon.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/ioport.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/workqueue.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #ifdef MODULE_PARAM_PREFIX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #undef MODULE_PARAM_PREFIX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #define MODULE_PARAM_PREFIX "damon_reclaim."
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * Enable or disable DAMON_RECLAIM.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * You can enable DAMON_RCLAIM by setting the value of this parameter as ``Y``.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * Setting it as ``N`` disables DAMON_RECLAIM. Note that DAMON_RECLAIM could
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * do no real monitoring and reclamation due to the watermarks-based activation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * condition. Refer to below descriptions for the watermarks parameter for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * this.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) static bool enabled __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * Time threshold for cold memory regions identification in microseconds.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * If a memory region is not accessed for this or longer time, DAMON_RECLAIM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * identifies the region as cold, and reclaims. 120 seconds by default.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) static unsigned long min_age __read_mostly = 120000000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) module_param(min_age, ulong, 0600);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) * Limit of time for trying the reclamation in milliseconds.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) * DAMON_RECLAIM tries to use only up to this time within a time window
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) * (quota_reset_interval_ms) for trying reclamation of cold pages. This can be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) * used for limiting CPU consumption of DAMON_RECLAIM. If the value is zero,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) * the limit is disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) * 10 ms by default.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) static unsigned long quota_ms __read_mostly = 10;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) module_param(quota_ms, ulong, 0600);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) * Limit of size of memory for the reclamation in bytes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) * DAMON_RECLAIM charges amount of memory which it tried to reclaim within a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) * time window (quota_reset_interval_ms) and makes no more than this limit is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) * tried. This can be used for limiting consumption of CPU and IO. If this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) * value is zero, the limit is disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) * 128 MiB by default.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) static unsigned long quota_sz __read_mostly = 128 * 1024 * 1024;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) module_param(quota_sz, ulong, 0600);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) * The time/size quota charge reset interval in milliseconds.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) * The charge reset interval for the quota of time (quota_ms) and size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) * (quota_sz). That is, DAMON_RECLAIM does not try reclamation for more than
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) * quota_ms milliseconds or quota_sz bytes within quota_reset_interval_ms
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) * milliseconds.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) * 1 second by default.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) static unsigned long quota_reset_interval_ms __read_mostly = 1000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) module_param(quota_reset_interval_ms, ulong, 0600);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) * The watermarks check time interval in microseconds.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) * Minimal time to wait before checking the watermarks, when DAMON_RECLAIM is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) * enabled but inactive due to its watermarks rule. 5 seconds by default.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) static unsigned long wmarks_interval __read_mostly = 5000000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) module_param(wmarks_interval, ulong, 0600);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) * Free memory rate (per thousand) for the high watermark.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) * If free memory of the system in bytes per thousand bytes is higher than
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) * this, DAMON_RECLAIM becomes inactive, so it does nothing but periodically
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) * checks the watermarks. 500 (50%) by default.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) static unsigned long wmarks_high __read_mostly = 500;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) module_param(wmarks_high, ulong, 0600);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) * Free memory rate (per thousand) for the middle watermark.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) * If free memory of the system in bytes per thousand bytes is between this and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) * the low watermark, DAMON_RECLAIM becomes active, so starts the monitoring
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) * and the reclaiming. 400 (40%) by default.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) static unsigned long wmarks_mid __read_mostly = 400;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) module_param(wmarks_mid, ulong, 0600);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) * Free memory rate (per thousand) for the low watermark.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) * If free memory of the system in bytes per thousand bytes is lower than this,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) * DAMON_RECLAIM becomes inactive, so it does nothing but periodically checks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) * the watermarks. In the case, the system falls back to the LRU-based page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) * granularity reclamation logic. 200 (20%) by default.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) static unsigned long wmarks_low __read_mostly = 200;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) module_param(wmarks_low, ulong, 0600);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) * Sampling interval for the monitoring in microseconds.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) * The sampling interval of DAMON for the cold memory monitoring. Please refer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) * to the DAMON documentation for more detail. 5 ms by default.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) static unsigned long sample_interval __read_mostly = 5000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) module_param(sample_interval, ulong, 0600);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) * Aggregation interval for the monitoring in microseconds.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) * The aggregation interval of DAMON for the cold memory monitoring. Please
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) * refer to the DAMON documentation for more detail. 100 ms by default.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) static unsigned long aggr_interval __read_mostly = 100000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) module_param(aggr_interval, ulong, 0600);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) * Minimum number of monitoring regions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) * The minimal number of monitoring regions of DAMON for the cold memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) * monitoring. This can be used to set lower-bound of the monitoring quality.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) * But, setting this too high could result in increased monitoring overhead.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) * Please refer to the DAMON documentation for more detail. 10 by default.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) static unsigned long min_nr_regions __read_mostly = 10;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) module_param(min_nr_regions, ulong, 0600);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) * Maximum number of monitoring regions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) * The maximum number of monitoring regions of DAMON for the cold memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) * monitoring. This can be used to set upper-bound of the monitoring overhead.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) * However, setting this too low could result in bad monitoring quality.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) * Please refer to the DAMON documentation for more detail. 1000 by default.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) static unsigned long max_nr_regions __read_mostly = 1000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) module_param(max_nr_regions, ulong, 0600);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) * Start of the target memory region in physical address.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) * The start physical address of memory region that DAMON_RECLAIM will do work
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) * against. By default, biggest System RAM is used as the region.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) static unsigned long monitor_region_start __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) module_param(monitor_region_start, ulong, 0600);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) * End of the target memory region in physical address.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) * The end physical address of memory region that DAMON_RECLAIM will do work
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) * against. By default, biggest System RAM is used as the region.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) static unsigned long monitor_region_end __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) module_param(monitor_region_end, ulong, 0600);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) * PID of the DAMON thread
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) * If DAMON_RECLAIM is enabled, this becomes the PID of the worker thread.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) * Else, -1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) static int kdamond_pid __read_mostly = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) module_param(kdamond_pid, int, 0400);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) * Number of memory regions that tried to be reclaimed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) static unsigned long nr_reclaim_tried_regions __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) module_param(nr_reclaim_tried_regions, ulong, 0400);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) * Total bytes of memory regions that tried to be reclaimed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) static unsigned long bytes_reclaim_tried_regions __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) module_param(bytes_reclaim_tried_regions, ulong, 0400);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) * Number of memory regions that successfully be reclaimed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) static unsigned long nr_reclaimed_regions __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) module_param(nr_reclaimed_regions, ulong, 0400);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) * Total bytes of memory regions that successfully be reclaimed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) static unsigned long bytes_reclaimed_regions __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) module_param(bytes_reclaimed_regions, ulong, 0400);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) * Number of times that the time/space quota limits have exceeded
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) static unsigned long nr_quota_exceeds __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) module_param(nr_quota_exceeds, ulong, 0400);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) static struct damon_ctx *ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) static struct damon_target *target;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) struct damon_reclaim_ram_walk_arg {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) unsigned long start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) unsigned long end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) static int walk_system_ram(struct resource *res, void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) struct damon_reclaim_ram_walk_arg *a = arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) if (a->end - a->start < res->end - res->start) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) a->start = res->start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) a->end = res->end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) * Find biggest 'System RAM' resource and store its start and end address in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) * @start and @end, respectively. If no System RAM is found, returns false.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) static bool get_monitoring_region(unsigned long *start, unsigned long *end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) struct damon_reclaim_ram_walk_arg arg = {};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) walk_system_ram_res(0, ULONG_MAX, &arg, walk_system_ram);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) if (arg.end <= arg.start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) *start = arg.start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) *end = arg.end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) static struct damos *damon_reclaim_new_scheme(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) struct damos_watermarks wmarks = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) .metric = DAMOS_WMARK_FREE_MEM_RATE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) .interval = wmarks_interval,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) .high = wmarks_high,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) .mid = wmarks_mid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) .low = wmarks_low,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) struct damos_quota quota = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) * Do not try reclamation for more than quota_ms milliseconds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) * or quota_sz bytes within quota_reset_interval_ms.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) .ms = quota_ms,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) .sz = quota_sz,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) .reset_interval = quota_reset_interval_ms,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) /* Within the quota, page out older regions first. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) .weight_sz = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) .weight_nr_accesses = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) .weight_age = 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) struct damos *scheme = damon_new_scheme(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) /* Find regions having PAGE_SIZE or larger size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) PAGE_SIZE, ULONG_MAX,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) /* and not accessed at all */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) 0, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) /* for min_age or more micro-seconds, and */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) min_age / aggr_interval, UINT_MAX,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) /* page out those, as soon as found */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) DAMOS_PAGEOUT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) /* under the quota. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) "a,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) /* (De)activate this according to the watermarks. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) &wmarks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) return scheme;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) static int damon_reclaim_turn(bool on)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) struct damon_region *region;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) struct damos *scheme;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) if (!on) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) err = damon_stop(&ctx, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) if (!err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) kdamond_pid = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) err = damon_set_attrs(ctx, sample_interval, aggr_interval, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) min_nr_regions, max_nr_regions);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) if (monitor_region_start > monitor_region_end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) if (!monitor_region_start && !monitor_region_end &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) !get_monitoring_region(&monitor_region_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) &monitor_region_end))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) /* DAMON will free this on its own when finish monitoring */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) region = damon_new_region(monitor_region_start, monitor_region_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) if (!region)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) damon_add_region(region, target);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) /* Will be freed by 'damon_set_schemes()' below */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) scheme = damon_reclaim_new_scheme();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) if (!scheme) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) err = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) goto free_region_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) err = damon_set_schemes(ctx, &scheme, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) goto free_scheme_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) err = damon_start(&ctx, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) if (!err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) kdamond_pid = ctx->kdamond->pid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) free_scheme_out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) damon_destroy_scheme(scheme);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) free_region_out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) damon_destroy_region(region, target);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) #define ENABLE_CHECK_INTERVAL_MS 1000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) static struct delayed_work damon_reclaim_timer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) static void damon_reclaim_timer_fn(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) static bool last_enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) bool now_enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) now_enabled = enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) if (last_enabled != now_enabled) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) if (!damon_reclaim_turn(now_enabled))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) last_enabled = now_enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) enabled = last_enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) if (enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) schedule_delayed_work(&damon_reclaim_timer,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) msecs_to_jiffies(ENABLE_CHECK_INTERVAL_MS));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) static DECLARE_DELAYED_WORK(damon_reclaim_timer, damon_reclaim_timer_fn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) static int enabled_store(const char *val,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) const struct kernel_param *kp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) int rc = param_set_bool(val, kp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) if (rc < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) if (enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) schedule_delayed_work(&damon_reclaim_timer, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) static const struct kernel_param_ops enabled_param_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) .set = enabled_store,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) .get = param_get_bool,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) module_param_cb(enabled, &enabled_param_ops, &enabled, 0600);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) MODULE_PARM_DESC(enabled,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) "Enable or disable DAMON_RECLAIM (default: disabled)");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) static int damon_reclaim_after_aggregation(struct damon_ctx *c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) struct damos *s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) /* update the stats parameter */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) damon_for_each_scheme(s, c) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) nr_reclaim_tried_regions = s->stat.nr_tried;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) bytes_reclaim_tried_regions = s->stat.sz_tried;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) nr_reclaimed_regions = s->stat.nr_applied;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) bytes_reclaimed_regions = s->stat.sz_applied;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) nr_quota_exceeds = s->stat.qt_exceeds;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) static int __init damon_reclaim_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) ctx = damon_new_ctx();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) if (!ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) damon_pa_set_primitives(ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) ctx->callback.after_aggregation = damon_reclaim_after_aggregation;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) /* 4242 means nothing but fun */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) target = damon_new_target(4242);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) if (!target) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) damon_destroy_ctx(ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) damon_add_target(ctx, target);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) schedule_delayed_work(&damon_reclaim_timer, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) module_init(damon_reclaim_init);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags