^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) #include <linux/jiffies.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include <linux/ktime.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/list.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/math64.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/sizes.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/workqueue.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include "ctree.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include "block-group.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include "discard.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include "free-space-cache.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * This contains the logic to handle async discard.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * Async discard manages trimming of free space outside of transaction commit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * Discarding is done by managing the block_groups on a LRU list based on free
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * space recency. Two passes are used to first prioritize discarding extents
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * and then allow for trimming in the bitmap the best opportunity to coalesce.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * The block_groups are maintained on multiple lists to allow for multiple
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) * passes with different discard filter requirements. A delayed work item is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * used to manage discarding with timeout determined by a max of the delay
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * incurred by the iops rate limit, the byte rate limit, and the max delay of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * BTRFS_DISCARD_MAX_DELAY.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * Note, this only keeps track of block_groups that are explicitly for data.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * Mixed block_groups are not supported.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * The first list is special to manage discarding of fully free block groups.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * This is necessary because we issue a final trim for a full free block group
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * after forgetting it. When a block group becomes unused, instead of directly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * being added to the unused_bgs list, we add it to this first list. Then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * from there, if it becomes fully discarded, we place it onto the unused_bgs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * The in-memory free space cache serves as the backing state for discard.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) * Consequently this means there is no persistence. We opt to load all the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) * block groups in as not discarded, so the mount case degenerates to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) * crashing case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) * As the free space cache uses bitmaps, there exists a tradeoff between
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) * ease/efficiency for find_free_extent() and the accuracy of discard state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) * Here we opt to let untrimmed regions merge with everything while only letting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) * trimmed regions merge with other trimmed regions. This can cause
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) * overtrimming, but the coalescing benefit seems to be worth it. Additionally,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) * bitmap state is tracked as a whole. If we're able to fully trim a bitmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) * the trimmed flag is set on the bitmap. Otherwise, if an allocation comes in,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) * this resets the state and we will retry trimming the whole bitmap. This is a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) * tradeoff between discard state accuracy and the cost of accounting.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) /* This is an initial delay to give some chance for block reuse */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #define BTRFS_DISCARD_DELAY (120ULL * NSEC_PER_SEC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #define BTRFS_DISCARD_UNUSED_DELAY (10ULL * NSEC_PER_SEC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) /* Target completion latency of discarding all discardable extents */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #define BTRFS_DISCARD_TARGET_MSEC (6 * 60 * 60UL * MSEC_PER_SEC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #define BTRFS_DISCARD_MIN_DELAY_MSEC (1UL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) #define BTRFS_DISCARD_MAX_DELAY_MSEC (1000UL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) #define BTRFS_DISCARD_MAX_IOPS (10U)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) /* Montonically decreasing minimum length filters after index 0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) static int discard_minlen[BTRFS_NR_DISCARD_LISTS] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) BTRFS_ASYNC_DISCARD_MAX_FILTER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) BTRFS_ASYNC_DISCARD_MIN_FILTER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) static struct list_head *get_discard_list(struct btrfs_discard_ctl *discard_ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) struct btrfs_block_group *block_group)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) return &discard_ctl->discard_list[block_group->discard_index];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) static void __add_to_discard_list(struct btrfs_discard_ctl *discard_ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) struct btrfs_block_group *block_group)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) if (!btrfs_run_discard_work(discard_ctl))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) if (list_empty(&block_group->discard_list) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) block_group->discard_index == BTRFS_DISCARD_INDEX_UNUSED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) if (block_group->discard_index == BTRFS_DISCARD_INDEX_UNUSED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) block_group->discard_index = BTRFS_DISCARD_INDEX_START;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) block_group->discard_eligible_time = (ktime_get_ns() +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) BTRFS_DISCARD_DELAY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) block_group->discard_state = BTRFS_DISCARD_RESET_CURSOR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) list_move_tail(&block_group->discard_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) get_discard_list(discard_ctl, block_group));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) static void add_to_discard_list(struct btrfs_discard_ctl *discard_ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) struct btrfs_block_group *block_group)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) if (!btrfs_is_block_group_data_only(block_group))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) spin_lock(&discard_ctl->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) __add_to_discard_list(discard_ctl, block_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) spin_unlock(&discard_ctl->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) static void add_to_discard_unused_list(struct btrfs_discard_ctl *discard_ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) struct btrfs_block_group *block_group)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) spin_lock(&discard_ctl->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) if (!btrfs_run_discard_work(discard_ctl)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) spin_unlock(&discard_ctl->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) list_del_init(&block_group->discard_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) block_group->discard_index = BTRFS_DISCARD_INDEX_UNUSED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) block_group->discard_eligible_time = (ktime_get_ns() +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) BTRFS_DISCARD_UNUSED_DELAY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) block_group->discard_state = BTRFS_DISCARD_RESET_CURSOR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) list_add_tail(&block_group->discard_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) &discard_ctl->discard_list[BTRFS_DISCARD_INDEX_UNUSED]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) spin_unlock(&discard_ctl->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) static bool remove_from_discard_list(struct btrfs_discard_ctl *discard_ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) struct btrfs_block_group *block_group)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) bool running = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) spin_lock(&discard_ctl->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) if (block_group == discard_ctl->block_group) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) running = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) discard_ctl->block_group = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) block_group->discard_eligible_time = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) list_del_init(&block_group->discard_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) spin_unlock(&discard_ctl->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) return running;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) * find_next_block_group - find block_group that's up next for discarding
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) * @discard_ctl: discard control
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) * @now: current time
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) * Iterate over the discard lists to find the next block_group up for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) * discarding checking the discard_eligible_time of block_group.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) static struct btrfs_block_group *find_next_block_group(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) struct btrfs_discard_ctl *discard_ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) u64 now)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) struct btrfs_block_group *ret_block_group = NULL, *block_group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) for (i = 0; i < BTRFS_NR_DISCARD_LISTS; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) struct list_head *discard_list = &discard_ctl->discard_list[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) if (!list_empty(discard_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) block_group = list_first_entry(discard_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) struct btrfs_block_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) discard_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) if (!ret_block_group)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) ret_block_group = block_group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) if (ret_block_group->discard_eligible_time < now)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) if (ret_block_group->discard_eligible_time >
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) block_group->discard_eligible_time)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) ret_block_group = block_group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) return ret_block_group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) * peek_discard_list - wrap find_next_block_group()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) * @discard_ctl: discard control
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) * @discard_state: the discard_state of the block_group after state management
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) * @discard_index: the discard_index of the block_group after state management
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) * This wraps find_next_block_group() and sets the block_group to be in use.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) * discard_state's control flow is managed here. Variables related to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) * discard_state are reset here as needed (eg discard_cursor). @discard_state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) * and @discard_index are remembered as it may change while we're discarding,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) * but we want the discard to execute in the context determined here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) static struct btrfs_block_group *peek_discard_list(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) struct btrfs_discard_ctl *discard_ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) enum btrfs_discard_state *discard_state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) int *discard_index, u64 now)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) struct btrfs_block_group *block_group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) spin_lock(&discard_ctl->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) block_group = find_next_block_group(discard_ctl, now);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) if (block_group && now >= block_group->discard_eligible_time) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) if (block_group->discard_index == BTRFS_DISCARD_INDEX_UNUSED &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) block_group->used != 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) if (btrfs_is_block_group_data_only(block_group))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) __add_to_discard_list(discard_ctl, block_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) list_del_init(&block_group->discard_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) if (block_group->discard_state == BTRFS_DISCARD_RESET_CURSOR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) block_group->discard_cursor = block_group->start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) block_group->discard_state = BTRFS_DISCARD_EXTENTS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) discard_ctl->block_group = block_group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) if (block_group) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) *discard_state = block_group->discard_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) *discard_index = block_group->discard_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) spin_unlock(&discard_ctl->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) return block_group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) * btrfs_discard_check_filter - updates a block groups filters
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) * @block_group: block group of interest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) * @bytes: recently freed region size after coalescing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) * Async discard maintains multiple lists with progressively smaller filters
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) * to prioritize discarding based on size. Should a free space that matches
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) * a larger filter be returned to the free_space_cache, prioritize that discard
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) * by moving @block_group to the proper filter.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) void btrfs_discard_check_filter(struct btrfs_block_group *block_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) u64 bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) struct btrfs_discard_ctl *discard_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) if (!block_group ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) !btrfs_test_opt(block_group->fs_info, DISCARD_ASYNC))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) discard_ctl = &block_group->fs_info->discard_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) if (block_group->discard_index > BTRFS_DISCARD_INDEX_START &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) bytes >= discard_minlen[block_group->discard_index - 1]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) remove_from_discard_list(discard_ctl, block_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) for (i = BTRFS_DISCARD_INDEX_START; i < BTRFS_NR_DISCARD_LISTS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) if (bytes >= discard_minlen[i]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) block_group->discard_index = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) add_to_discard_list(discard_ctl, block_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) * btrfs_update_discard_index - moves a block group along the discard lists
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) * @discard_ctl: discard control
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) * @block_group: block_group of interest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) * Increment @block_group's discard_index. If it falls of the list, let it be.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) * Otherwise add it back to the appropriate list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) static void btrfs_update_discard_index(struct btrfs_discard_ctl *discard_ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) struct btrfs_block_group *block_group)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) block_group->discard_index++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) if (block_group->discard_index == BTRFS_NR_DISCARD_LISTS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) block_group->discard_index = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) add_to_discard_list(discard_ctl, block_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) * btrfs_discard_cancel_work - remove a block_group from the discard lists
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) * @discard_ctl: discard control
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) * @block_group: block_group of interest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) * This removes @block_group from the discard lists. If necessary, it waits on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) * the current work and then reschedules the delayed work.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) void btrfs_discard_cancel_work(struct btrfs_discard_ctl *discard_ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) struct btrfs_block_group *block_group)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) if (remove_from_discard_list(discard_ctl, block_group)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) cancel_delayed_work_sync(&discard_ctl->work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) btrfs_discard_schedule_work(discard_ctl, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) * btrfs_discard_queue_work - handles queuing the block_groups
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) * @discard_ctl: discard control
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) * @block_group: block_group of interest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) * This maintains the LRU order of the discard lists.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) void btrfs_discard_queue_work(struct btrfs_discard_ctl *discard_ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) struct btrfs_block_group *block_group)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) if (!block_group || !btrfs_test_opt(block_group->fs_info, DISCARD_ASYNC))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) if (block_group->used == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) add_to_discard_unused_list(discard_ctl, block_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) add_to_discard_list(discard_ctl, block_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) if (!delayed_work_pending(&discard_ctl->work))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) btrfs_discard_schedule_work(discard_ctl, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) static void __btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) u64 now, bool override)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) struct btrfs_block_group *block_group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) if (!btrfs_run_discard_work(discard_ctl))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) if (!override && delayed_work_pending(&discard_ctl->work))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) block_group = find_next_block_group(discard_ctl, now);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) if (block_group) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) unsigned long delay = discard_ctl->delay;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) u32 kbps_limit = READ_ONCE(discard_ctl->kbps_limit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) * A single delayed workqueue item is responsible for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) * discarding, so we can manage the bytes rate limit by keeping
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) * track of the previous discard.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) if (kbps_limit && discard_ctl->prev_discard) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) u64 bps_limit = ((u64)kbps_limit) * SZ_1K;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) u64 bps_delay = div64_u64(discard_ctl->prev_discard *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) MSEC_PER_SEC, bps_limit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) delay = max(delay, msecs_to_jiffies(bps_delay));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) * This timeout is to hopefully prevent immediate discarding
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) * in a recently allocated block group.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) if (now < block_group->discard_eligible_time) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) u64 bg_timeout = block_group->discard_eligible_time - now;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) delay = max(delay, nsecs_to_jiffies(bg_timeout));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) mod_delayed_work(discard_ctl->discard_workers,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) &discard_ctl->work, delay);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) * btrfs_discard_schedule_work - responsible for scheduling the discard work
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) * @discard_ctl: discard control
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) * @override: override the current timer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) * Discards are issued by a delayed workqueue item. @override is used to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) * update the current delay as the baseline delay interval is reevaluated on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) * transaction commit. This is also maxed with any other rate limit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) void btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) bool override)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) const u64 now = ktime_get_ns();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) spin_lock(&discard_ctl->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) __btrfs_discard_schedule_work(discard_ctl, now, override);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) spin_unlock(&discard_ctl->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) * btrfs_finish_discard_pass - determine next step of a block_group
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) * @discard_ctl: discard control
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) * @block_group: block_group of interest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) * This determines the next step for a block group after it's finished going
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) * through a pass on a discard list. If it is unused and fully trimmed, we can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) * mark it unused and send it to the unused_bgs path. Otherwise, pass it onto
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) * the appropriate filter list or let it fall off.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) static void btrfs_finish_discard_pass(struct btrfs_discard_ctl *discard_ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) struct btrfs_block_group *block_group)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) remove_from_discard_list(discard_ctl, block_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) if (block_group->used == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) if (btrfs_is_free_space_trimmed(block_group))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) btrfs_mark_bg_unused(block_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) add_to_discard_unused_list(discard_ctl, block_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) btrfs_update_discard_index(discard_ctl, block_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) * btrfs_discard_workfn - discard work function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) * @work: work
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) * This finds the next block_group to start discarding and then discards a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) * single region. It does this in a two-pass fashion: first extents and second
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) * bitmaps. Completely discarded block groups are sent to the unused_bgs path.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) static void btrfs_discard_workfn(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) struct btrfs_discard_ctl *discard_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) struct btrfs_block_group *block_group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) enum btrfs_discard_state discard_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) int discard_index = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) u64 trimmed = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) u64 minlen = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) u64 now = ktime_get_ns();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) discard_ctl = container_of(work, struct btrfs_discard_ctl, work.work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) block_group = peek_discard_list(discard_ctl, &discard_state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) &discard_index, now);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) if (!block_group || !btrfs_run_discard_work(discard_ctl))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) if (now < block_group->discard_eligible_time) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) btrfs_discard_schedule_work(discard_ctl, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) /* Perform discarding */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) minlen = discard_minlen[discard_index];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) if (discard_state == BTRFS_DISCARD_BITMAPS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) u64 maxlen = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) * Use the previous levels minimum discard length as the max
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) * length filter. In the case something is added to make a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) * region go beyond the max filter, the entire bitmap is set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) * back to BTRFS_TRIM_STATE_UNTRIMMED.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) if (discard_index != BTRFS_DISCARD_INDEX_UNUSED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) maxlen = discard_minlen[discard_index - 1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) btrfs_trim_block_group_bitmaps(block_group, &trimmed,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) block_group->discard_cursor,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) btrfs_block_group_end(block_group),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) minlen, maxlen, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) discard_ctl->discard_bitmap_bytes += trimmed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) btrfs_trim_block_group_extents(block_group, &trimmed,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) block_group->discard_cursor,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) btrfs_block_group_end(block_group),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) minlen, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) discard_ctl->discard_extent_bytes += trimmed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) discard_ctl->prev_discard = trimmed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) /* Determine next steps for a block_group */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) if (block_group->discard_cursor >= btrfs_block_group_end(block_group)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) if (discard_state == BTRFS_DISCARD_BITMAPS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) btrfs_finish_discard_pass(discard_ctl, block_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) block_group->discard_cursor = block_group->start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) spin_lock(&discard_ctl->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) if (block_group->discard_state !=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) BTRFS_DISCARD_RESET_CURSOR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) block_group->discard_state =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) BTRFS_DISCARD_BITMAPS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) spin_unlock(&discard_ctl->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) spin_lock(&discard_ctl->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) discard_ctl->block_group = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) __btrfs_discard_schedule_work(discard_ctl, now, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) spin_unlock(&discard_ctl->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) * btrfs_run_discard_work - determines if async discard should be running
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) * @discard_ctl: discard control
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) * Checks if the file system is writeable and BTRFS_FS_DISCARD_RUNNING is set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) bool btrfs_run_discard_work(struct btrfs_discard_ctl *discard_ctl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) struct btrfs_fs_info *fs_info = container_of(discard_ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) struct btrfs_fs_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) discard_ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) return (!(fs_info->sb->s_flags & SB_RDONLY) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) test_bit(BTRFS_FS_DISCARD_RUNNING, &fs_info->flags));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) * btrfs_discard_calc_delay - recalculate the base delay
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) * @discard_ctl: discard control
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) * Recalculate the base delay which is based off the total number of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) * discardable_extents. Clamp this between the lower_limit (iops_limit or 1ms)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) * and the upper_limit (BTRFS_DISCARD_MAX_DELAY_MSEC).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) void btrfs_discard_calc_delay(struct btrfs_discard_ctl *discard_ctl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) s32 discardable_extents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) s64 discardable_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) u32 iops_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) unsigned long delay;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) unsigned long lower_limit = BTRFS_DISCARD_MIN_DELAY_MSEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) discardable_extents = atomic_read(&discard_ctl->discardable_extents);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) if (!discardable_extents)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) spin_lock(&discard_ctl->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) * The following is to fix a potential -1 discrepenancy that we're not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) * sure how to reproduce. But given that this is the only place that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) * utilizes these numbers and this is only called by from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) * btrfs_finish_extent_commit() which is synchronized, we can correct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) * here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) if (discardable_extents < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) atomic_add(-discardable_extents,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) &discard_ctl->discardable_extents);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) discardable_bytes = atomic64_read(&discard_ctl->discardable_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) if (discardable_bytes < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) atomic64_add(-discardable_bytes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) &discard_ctl->discardable_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) if (discardable_extents <= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) spin_unlock(&discard_ctl->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) iops_limit = READ_ONCE(discard_ctl->iops_limit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) if (iops_limit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) lower_limit = max_t(unsigned long, lower_limit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) MSEC_PER_SEC / iops_limit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) delay = BTRFS_DISCARD_TARGET_MSEC / discardable_extents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) delay = clamp(delay, lower_limit, BTRFS_DISCARD_MAX_DELAY_MSEC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) discard_ctl->delay = msecs_to_jiffies(delay);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) spin_unlock(&discard_ctl->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) * btrfs_discard_update_discardable - propagate discard counters
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) * @block_group: block_group of interest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) * @ctl: free_space_ctl of @block_group
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) * This propagates deltas of counters up to the discard_ctl. It maintains a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) * current counter and a previous counter passing the delta up to the global
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) * stat. Then the current counter value becomes the previous counter value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) void btrfs_discard_update_discardable(struct btrfs_block_group *block_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) struct btrfs_free_space_ctl *ctl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) struct btrfs_discard_ctl *discard_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) s32 extents_delta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) s64 bytes_delta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) if (!block_group ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) !btrfs_test_opt(block_group->fs_info, DISCARD_ASYNC) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) !btrfs_is_block_group_data_only(block_group))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) discard_ctl = &block_group->fs_info->discard_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) extents_delta = ctl->discardable_extents[BTRFS_STAT_CURR] -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) ctl->discardable_extents[BTRFS_STAT_PREV];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) if (extents_delta) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) atomic_add(extents_delta, &discard_ctl->discardable_extents);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) ctl->discardable_extents[BTRFS_STAT_PREV] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) ctl->discardable_extents[BTRFS_STAT_CURR];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) bytes_delta = ctl->discardable_bytes[BTRFS_STAT_CURR] -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) ctl->discardable_bytes[BTRFS_STAT_PREV];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) if (bytes_delta) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) atomic64_add(bytes_delta, &discard_ctl->discardable_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) ctl->discardable_bytes[BTRFS_STAT_PREV] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) ctl->discardable_bytes[BTRFS_STAT_CURR];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) * btrfs_discard_punt_unused_bgs_list - punt unused_bgs list to discard lists
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) * @fs_info: fs_info of interest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) * The unused_bgs list needs to be punted to the discard lists because the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) * order of operations is changed. In the normal sychronous discard path, the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) * block groups are trimmed via a single large trim in transaction commit. This
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) * is ultimately what we are trying to avoid with asynchronous discard. Thus,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) * it must be done before going down the unused_bgs path.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) void btrfs_discard_punt_unused_bgs_list(struct btrfs_fs_info *fs_info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) struct btrfs_block_group *block_group, *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) spin_lock(&fs_info->unused_bgs_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) /* We enabled async discard, so punt all to the queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) list_for_each_entry_safe(block_group, next, &fs_info->unused_bgs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) bg_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) list_del_init(&block_group->bg_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) btrfs_put_block_group(block_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) btrfs_discard_queue_work(&fs_info->discard_ctl, block_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) spin_unlock(&fs_info->unused_bgs_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) * btrfs_discard_purge_list - purge discard lists
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) * @discard_ctl: discard control
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) * If we are disabling async discard, we may have intercepted block groups that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) * are completely free and ready for the unused_bgs path. As discarding will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) * now happen in transaction commit or not at all, we can safely mark the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) * corresponding block groups as unused and they will be sent on their merry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) * way to the unused_bgs list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) static void btrfs_discard_purge_list(struct btrfs_discard_ctl *discard_ctl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) struct btrfs_block_group *block_group, *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) spin_lock(&discard_ctl->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) for (i = 0; i < BTRFS_NR_DISCARD_LISTS; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) list_for_each_entry_safe(block_group, next,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) &discard_ctl->discard_list[i],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) discard_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) list_del_init(&block_group->discard_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) spin_unlock(&discard_ctl->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) if (block_group->used == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) btrfs_mark_bg_unused(block_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) spin_lock(&discard_ctl->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) spin_unlock(&discard_ctl->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) void btrfs_discard_resume(struct btrfs_fs_info *fs_info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) if (!btrfs_test_opt(fs_info, DISCARD_ASYNC)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) btrfs_discard_cleanup(fs_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) btrfs_discard_punt_unused_bgs_list(fs_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) set_bit(BTRFS_FS_DISCARD_RUNNING, &fs_info->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) void btrfs_discard_stop(struct btrfs_fs_info *fs_info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) clear_bit(BTRFS_FS_DISCARD_RUNNING, &fs_info->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) void btrfs_discard_init(struct btrfs_fs_info *fs_info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) struct btrfs_discard_ctl *discard_ctl = &fs_info->discard_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) spin_lock_init(&discard_ctl->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) INIT_DELAYED_WORK(&discard_ctl->work, btrfs_discard_workfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) for (i = 0; i < BTRFS_NR_DISCARD_LISTS; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) INIT_LIST_HEAD(&discard_ctl->discard_list[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) discard_ctl->prev_discard = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) atomic_set(&discard_ctl->discardable_extents, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) atomic64_set(&discard_ctl->discardable_bytes, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) discard_ctl->max_discard_size = BTRFS_ASYNC_DISCARD_DEFAULT_MAX_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) discard_ctl->delay = BTRFS_DISCARD_MAX_DELAY_MSEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) discard_ctl->iops_limit = BTRFS_DISCARD_MAX_IOPS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) discard_ctl->kbps_limit = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) discard_ctl->discard_extent_bytes = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) discard_ctl->discard_bitmap_bytes = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) atomic64_set(&discard_ctl->discard_bytes_saved, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) void btrfs_discard_cleanup(struct btrfs_fs_info *fs_info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) btrfs_discard_stop(fs_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) cancel_delayed_work_sync(&fs_info->discard_ctl.work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) btrfs_discard_purge_list(&fs_info->discard_ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags