^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (C) 2008 Red Hat. All rights reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/pagemap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/sched/signal.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/math64.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/ratelimit.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/error-injection.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/sched/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include "ctree.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include "free-space-cache.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include "transaction.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include "disk-io.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include "extent_io.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include "inode-map.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include "volumes.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include "space-info.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include "delalloc-space.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include "block-group.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include "discard.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #define BITS_PER_BITMAP (PAGE_SIZE * 8UL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #define MAX_CACHE_BYTES_PER_GIG SZ_64K
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #define FORCE_EXTENT_THRESHOLD SZ_1M
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) struct btrfs_trim_range {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) u64 start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) u64 bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) struct list_head list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) static int count_bitmap_extents(struct btrfs_free_space_ctl *ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) struct btrfs_free_space *bitmap_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) static int link_free_space(struct btrfs_free_space_ctl *ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) struct btrfs_free_space *info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) static void unlink_free_space(struct btrfs_free_space_ctl *ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) struct btrfs_free_space *info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) static int btrfs_wait_cache_io_root(struct btrfs_root *root,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) struct btrfs_trans_handle *trans,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) struct btrfs_io_ctl *io_ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) struct btrfs_path *path);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) static struct inode *__lookup_free_space_inode(struct btrfs_root *root,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) struct btrfs_path *path,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) u64 offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) struct btrfs_fs_info *fs_info = root->fs_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) struct btrfs_key key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) struct btrfs_key location;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) struct btrfs_disk_key disk_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) struct btrfs_free_space_header *header;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) struct extent_buffer *leaf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) struct inode *inode = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) unsigned nofs_flag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) key.objectid = BTRFS_FREE_SPACE_OBJECTID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) key.offset = offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) key.type = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) return ERR_PTR(ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) if (ret > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) btrfs_release_path(path);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) return ERR_PTR(-ENOENT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) leaf = path->nodes[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) header = btrfs_item_ptr(leaf, path->slots[0],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) struct btrfs_free_space_header);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) btrfs_free_space_key(leaf, header, &disk_key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) btrfs_disk_key_to_cpu(&location, &disk_key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) btrfs_release_path(path);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) * We are often under a trans handle at this point, so we need to make
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) * sure NOFS is set to keep us from deadlocking.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) nofs_flag = memalloc_nofs_save();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) inode = btrfs_iget_path(fs_info->sb, location.objectid, root, path);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) btrfs_release_path(path);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) memalloc_nofs_restore(nofs_flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) if (IS_ERR(inode))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) return inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) mapping_set_gfp_mask(inode->i_mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) mapping_gfp_constraint(inode->i_mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) ~(__GFP_FS | __GFP_HIGHMEM)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) return inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) struct inode *lookup_free_space_inode(struct btrfs_block_group *block_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) struct btrfs_path *path)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) struct btrfs_fs_info *fs_info = block_group->fs_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) struct inode *inode = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) u32 flags = BTRFS_INODE_NODATASUM | BTRFS_INODE_NODATACOW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) spin_lock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) if (block_group->inode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) inode = igrab(block_group->inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) spin_unlock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) if (inode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) return inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) inode = __lookup_free_space_inode(fs_info->tree_root, path,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) block_group->start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) if (IS_ERR(inode))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) return inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) spin_lock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) if (!((BTRFS_I(inode)->flags & flags) == flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) btrfs_info(fs_info, "Old style space inode found, converting.");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) BTRFS_INODE_NODATACOW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) block_group->disk_cache_state = BTRFS_DC_CLEAR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) if (!block_group->iref) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) block_group->inode = igrab(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) block_group->iref = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) spin_unlock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) return inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) static int __create_free_space_inode(struct btrfs_root *root,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) struct btrfs_trans_handle *trans,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) struct btrfs_path *path,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) u64 ino, u64 offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) struct btrfs_key key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) struct btrfs_disk_key disk_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) struct btrfs_free_space_header *header;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) struct btrfs_inode_item *inode_item;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) struct extent_buffer *leaf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) u64 flags = BTRFS_INODE_NOCOMPRESS | BTRFS_INODE_PREALLOC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) ret = btrfs_insert_empty_inode(trans, root, path, ino);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) /* We inline crc's for the free disk space cache */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) if (ino != BTRFS_FREE_INO_OBJECTID)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) flags |= BTRFS_INODE_NODATASUM | BTRFS_INODE_NODATACOW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) leaf = path->nodes[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) inode_item = btrfs_item_ptr(leaf, path->slots[0],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) struct btrfs_inode_item);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) btrfs_item_key(leaf, &disk_key, path->slots[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) memzero_extent_buffer(leaf, (unsigned long)inode_item,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) sizeof(*inode_item));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) btrfs_set_inode_generation(leaf, inode_item, trans->transid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) btrfs_set_inode_size(leaf, inode_item, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) btrfs_set_inode_nbytes(leaf, inode_item, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) btrfs_set_inode_uid(leaf, inode_item, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) btrfs_set_inode_gid(leaf, inode_item, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) btrfs_set_inode_mode(leaf, inode_item, S_IFREG | 0600);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) btrfs_set_inode_flags(leaf, inode_item, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) btrfs_set_inode_nlink(leaf, inode_item, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) btrfs_set_inode_transid(leaf, inode_item, trans->transid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) btrfs_set_inode_block_group(leaf, inode_item, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) btrfs_mark_buffer_dirty(leaf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) btrfs_release_path(path);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) key.objectid = BTRFS_FREE_SPACE_OBJECTID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) key.offset = offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) key.type = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) ret = btrfs_insert_empty_item(trans, root, path, &key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) sizeof(struct btrfs_free_space_header));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) btrfs_release_path(path);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) leaf = path->nodes[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) header = btrfs_item_ptr(leaf, path->slots[0],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) struct btrfs_free_space_header);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) memzero_extent_buffer(leaf, (unsigned long)header, sizeof(*header));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) btrfs_set_free_space_key(leaf, header, &disk_key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) btrfs_mark_buffer_dirty(leaf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) btrfs_release_path(path);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) int create_free_space_inode(struct btrfs_trans_handle *trans,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) struct btrfs_block_group *block_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) struct btrfs_path *path)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) u64 ino;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) ret = btrfs_find_free_objectid(trans->fs_info->tree_root, &ino);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) return __create_free_space_inode(trans->fs_info->tree_root, trans, path,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) ino, block_group->start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) int btrfs_check_trunc_cache_free_space(struct btrfs_fs_info *fs_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) struct btrfs_block_rsv *rsv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) u64 needed_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) /* 1 for slack space, 1 for updating the inode */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) needed_bytes = btrfs_calc_insert_metadata_size(fs_info, 1) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) btrfs_calc_metadata_size(fs_info, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) spin_lock(&rsv->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) if (rsv->reserved < needed_bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) ret = -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) spin_unlock(&rsv->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) int btrfs_truncate_free_space_cache(struct btrfs_trans_handle *trans,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) struct btrfs_block_group *block_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) struct inode *inode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) struct btrfs_root *root = BTRFS_I(inode)->root;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) bool locked = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) if (block_group) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) struct btrfs_path *path = btrfs_alloc_path();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) if (!path) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) locked = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) mutex_lock(&trans->transaction->cache_write_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) if (!list_empty(&block_group->io_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) list_del_init(&block_group->io_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) btrfs_wait_cache_io(trans, block_group, path);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) btrfs_put_block_group(block_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) * now that we've truncated the cache away, its no longer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) * setup or written
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) spin_lock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) block_group->disk_cache_state = BTRFS_DC_CLEAR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) spin_unlock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) btrfs_free_path(path);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) btrfs_i_size_write(BTRFS_I(inode), 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) truncate_pagecache(inode, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) * We skip the throttling logic for free space cache inodes, so we don't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) * need to check for -EAGAIN.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) ret = btrfs_truncate_inode_items(trans, root, inode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 0, BTRFS_EXTENT_DATA_KEY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) ret = btrfs_update_inode(trans, root, inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) fail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) if (locked)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) mutex_unlock(&trans->transaction->cache_write_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) btrfs_abort_transaction(trans, ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) static void readahead_cache(struct inode *inode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) struct file_ra_state *ra;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) unsigned long last_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) ra = kzalloc(sizeof(*ra), GFP_NOFS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) if (!ra)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) file_ra_state_init(ra, inode->i_mapping);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) last_index = (i_size_read(inode) - 1) >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) page_cache_sync_readahead(inode->i_mapping, ra, NULL, 0, last_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) kfree(ra);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) static int io_ctl_init(struct btrfs_io_ctl *io_ctl, struct inode *inode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) int write)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) int num_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) int check_crcs = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) num_pages = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) if (btrfs_ino(BTRFS_I(inode)) != BTRFS_FREE_INO_OBJECTID)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) check_crcs = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) /* Make sure we can fit our crcs and generation into the first page */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) if (write && check_crcs &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) (num_pages * sizeof(u32) + sizeof(u64)) > PAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) memset(io_ctl, 0, sizeof(struct btrfs_io_ctl));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) io_ctl->pages = kcalloc(num_pages, sizeof(struct page *), GFP_NOFS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) if (!io_ctl->pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) io_ctl->num_pages = num_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) io_ctl->fs_info = btrfs_sb(inode->i_sb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) io_ctl->check_crcs = check_crcs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) io_ctl->inode = inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) ALLOW_ERROR_INJECTION(io_ctl_init, ERRNO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) static void io_ctl_free(struct btrfs_io_ctl *io_ctl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) kfree(io_ctl->pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) io_ctl->pages = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) static void io_ctl_unmap_page(struct btrfs_io_ctl *io_ctl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) if (io_ctl->cur) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) io_ctl->cur = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) io_ctl->orig = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) static void io_ctl_map_page(struct btrfs_io_ctl *io_ctl, int clear)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) ASSERT(io_ctl->index < io_ctl->num_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) io_ctl->page = io_ctl->pages[io_ctl->index++];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) io_ctl->cur = page_address(io_ctl->page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) io_ctl->orig = io_ctl->cur;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) io_ctl->size = PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) if (clear)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) clear_page(io_ctl->cur);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) static void io_ctl_drop_pages(struct btrfs_io_ctl *io_ctl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) io_ctl_unmap_page(io_ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) for (i = 0; i < io_ctl->num_pages; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) if (io_ctl->pages[i]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) ClearPageChecked(io_ctl->pages[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) unlock_page(io_ctl->pages[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) put_page(io_ctl->pages[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) static int io_ctl_prepare_pages(struct btrfs_io_ctl *io_ctl, bool uptodate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) struct inode *inode = io_ctl->inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) gfp_t mask = btrfs_alloc_write_mask(inode->i_mapping);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) for (i = 0; i < io_ctl->num_pages; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) page = find_or_create_page(inode->i_mapping, i, mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) if (!page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) io_ctl_drop_pages(io_ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) io_ctl->pages[i] = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) if (uptodate && !PageUptodate(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) btrfs_readpage(NULL, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) lock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) if (page->mapping != inode->i_mapping) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) btrfs_err(BTRFS_I(inode)->root->fs_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) "free space cache page truncated");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) io_ctl_drop_pages(io_ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) if (!PageUptodate(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) btrfs_err(BTRFS_I(inode)->root->fs_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) "error reading free space cache");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) io_ctl_drop_pages(io_ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) for (i = 0; i < io_ctl->num_pages; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) clear_page_dirty_for_io(io_ctl->pages[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) set_page_extent_mapped(io_ctl->pages[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) static void io_ctl_set_generation(struct btrfs_io_ctl *io_ctl, u64 generation)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) io_ctl_map_page(io_ctl, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) * Skip the csum areas. If we don't check crcs then we just have a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) * 64bit chunk at the front of the first page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) if (io_ctl->check_crcs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) io_ctl->cur += (sizeof(u32) * io_ctl->num_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) io_ctl->size -= sizeof(u64) + (sizeof(u32) * io_ctl->num_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) io_ctl->cur += sizeof(u64);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) io_ctl->size -= sizeof(u64) * 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) put_unaligned_le64(generation, io_ctl->cur);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) io_ctl->cur += sizeof(u64);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) static int io_ctl_check_generation(struct btrfs_io_ctl *io_ctl, u64 generation)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) u64 cache_gen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) * Skip the crc area. If we don't check crcs then we just have a 64bit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) * chunk at the front of the first page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) if (io_ctl->check_crcs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) io_ctl->cur += sizeof(u32) * io_ctl->num_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) io_ctl->size -= sizeof(u64) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) (sizeof(u32) * io_ctl->num_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) io_ctl->cur += sizeof(u64);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) io_ctl->size -= sizeof(u64) * 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) cache_gen = get_unaligned_le64(io_ctl->cur);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) if (cache_gen != generation) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) btrfs_err_rl(io_ctl->fs_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) "space cache generation (%llu) does not match inode (%llu)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) cache_gen, generation);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) io_ctl_unmap_page(io_ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) io_ctl->cur += sizeof(u64);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) static void io_ctl_set_crc(struct btrfs_io_ctl *io_ctl, int index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) u32 *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) u32 crc = ~(u32)0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) unsigned offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) if (!io_ctl->check_crcs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) io_ctl_unmap_page(io_ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) if (index == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) offset = sizeof(u32) * io_ctl->num_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) crc = btrfs_crc32c(crc, io_ctl->orig + offset, PAGE_SIZE - offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) btrfs_crc32c_final(crc, (u8 *)&crc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) io_ctl_unmap_page(io_ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) tmp = page_address(io_ctl->pages[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) tmp += index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) *tmp = crc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) static int io_ctl_check_crc(struct btrfs_io_ctl *io_ctl, int index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) u32 *tmp, val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) u32 crc = ~(u32)0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) unsigned offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) if (!io_ctl->check_crcs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) io_ctl_map_page(io_ctl, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) if (index == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) offset = sizeof(u32) * io_ctl->num_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) tmp = page_address(io_ctl->pages[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) tmp += index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) val = *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) io_ctl_map_page(io_ctl, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) crc = btrfs_crc32c(crc, io_ctl->orig + offset, PAGE_SIZE - offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) btrfs_crc32c_final(crc, (u8 *)&crc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) if (val != crc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) btrfs_err_rl(io_ctl->fs_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) "csum mismatch on free space cache");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) io_ctl_unmap_page(io_ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) static int io_ctl_add_entry(struct btrfs_io_ctl *io_ctl, u64 offset, u64 bytes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) void *bitmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) struct btrfs_free_space_entry *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) if (!io_ctl->cur)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) entry = io_ctl->cur;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) put_unaligned_le64(offset, &entry->offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) put_unaligned_le64(bytes, &entry->bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) entry->type = (bitmap) ? BTRFS_FREE_SPACE_BITMAP :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) BTRFS_FREE_SPACE_EXTENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) io_ctl->cur += sizeof(struct btrfs_free_space_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) io_ctl->size -= sizeof(struct btrfs_free_space_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) if (io_ctl->size >= sizeof(struct btrfs_free_space_entry))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) io_ctl_set_crc(io_ctl, io_ctl->index - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) /* No more pages to map */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) if (io_ctl->index >= io_ctl->num_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) /* map the next page */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) io_ctl_map_page(io_ctl, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) static int io_ctl_add_bitmap(struct btrfs_io_ctl *io_ctl, void *bitmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) if (!io_ctl->cur)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) * If we aren't at the start of the current page, unmap this one and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) * map the next one if there is any left.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) if (io_ctl->cur != io_ctl->orig) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) io_ctl_set_crc(io_ctl, io_ctl->index - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) if (io_ctl->index >= io_ctl->num_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) io_ctl_map_page(io_ctl, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) copy_page(io_ctl->cur, bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) io_ctl_set_crc(io_ctl, io_ctl->index - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) if (io_ctl->index < io_ctl->num_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) io_ctl_map_page(io_ctl, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) static void io_ctl_zero_remaining_pages(struct btrfs_io_ctl *io_ctl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) * If we're not on the boundary we know we've modified the page and we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) * need to crc the page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) if (io_ctl->cur != io_ctl->orig)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) io_ctl_set_crc(io_ctl, io_ctl->index - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) io_ctl_unmap_page(io_ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) while (io_ctl->index < io_ctl->num_pages) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) io_ctl_map_page(io_ctl, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) io_ctl_set_crc(io_ctl, io_ctl->index - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) static int io_ctl_read_entry(struct btrfs_io_ctl *io_ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) struct btrfs_free_space *entry, u8 *type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) struct btrfs_free_space_entry *e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) if (!io_ctl->cur) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) ret = io_ctl_check_crc(io_ctl, io_ctl->index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) e = io_ctl->cur;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) entry->offset = get_unaligned_le64(&e->offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) entry->bytes = get_unaligned_le64(&e->bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) *type = e->type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) io_ctl->cur += sizeof(struct btrfs_free_space_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) io_ctl->size -= sizeof(struct btrfs_free_space_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) if (io_ctl->size >= sizeof(struct btrfs_free_space_entry))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) io_ctl_unmap_page(io_ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) static int io_ctl_read_bitmap(struct btrfs_io_ctl *io_ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) struct btrfs_free_space *entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) ret = io_ctl_check_crc(io_ctl, io_ctl->index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) copy_page(entry->bitmap, io_ctl->cur);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) io_ctl_unmap_page(io_ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) * Since we attach pinned extents after the fact we can have contiguous sections
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) * of free space that are split up in entries. This poses a problem with the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) * tree logging stuff since it could have allocated across what appears to be 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) * entries since we would have merged the entries when adding the pinned extents
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) * back to the free space cache. So run through the space cache that we just
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) * loaded and merge contiguous entries. This will make the log replay stuff not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) * blow up and it will make for nicer allocator behavior.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) static void merge_space_tree(struct btrfs_free_space_ctl *ctl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) struct btrfs_free_space *e, *prev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) struct rb_node *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) spin_lock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) for (n = rb_first(&ctl->free_space_offset); n; n = rb_next(n)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) e = rb_entry(n, struct btrfs_free_space, offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) if (!prev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) goto next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) if (e->bitmap || prev->bitmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) goto next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) if (prev->offset + prev->bytes == e->offset) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) unlink_free_space(ctl, prev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) unlink_free_space(ctl, e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) prev->bytes += e->bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) kmem_cache_free(btrfs_free_space_cachep, e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) link_free_space(ctl, prev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) prev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) next:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) prev = e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) static int __load_free_space_cache(struct btrfs_root *root, struct inode *inode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) struct btrfs_free_space_ctl *ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) struct btrfs_path *path, u64 offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) struct btrfs_fs_info *fs_info = root->fs_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) struct btrfs_free_space_header *header;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) struct extent_buffer *leaf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) struct btrfs_io_ctl io_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) struct btrfs_key key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) struct btrfs_free_space *e, *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) LIST_HEAD(bitmaps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) u64 num_entries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) u64 num_bitmaps;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) u64 generation;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) u8 type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) /* Nothing in the space cache, goodbye */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) if (!i_size_read(inode))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) key.objectid = BTRFS_FREE_SPACE_OBJECTID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) key.offset = offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) key.type = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) else if (ret > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) btrfs_release_path(path);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) ret = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) leaf = path->nodes[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) header = btrfs_item_ptr(leaf, path->slots[0],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) struct btrfs_free_space_header);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) num_entries = btrfs_free_space_entries(leaf, header);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) num_bitmaps = btrfs_free_space_bitmaps(leaf, header);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) generation = btrfs_free_space_generation(leaf, header);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) btrfs_release_path(path);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) if (!BTRFS_I(inode)->generation) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) btrfs_info(fs_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) "the free space cache file (%llu) is invalid, skip it",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) if (BTRFS_I(inode)->generation != generation) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) btrfs_err(fs_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) "free space inode generation (%llu) did not match free space cache generation (%llu)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) BTRFS_I(inode)->generation, generation);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) if (!num_entries)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) ret = io_ctl_init(&io_ctl, inode, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) readahead_cache(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) ret = io_ctl_prepare_pages(&io_ctl, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) ret = io_ctl_check_crc(&io_ctl, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) goto free_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) ret = io_ctl_check_generation(&io_ctl, generation);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) goto free_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) while (num_entries) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) e = kmem_cache_zalloc(btrfs_free_space_cachep,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) GFP_NOFS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) if (!e) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) goto free_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) ret = io_ctl_read_entry(&io_ctl, e, &type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) kmem_cache_free(btrfs_free_space_cachep, e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) goto free_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) * Sync discard ensures that the free space cache is always
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) * trimmed. So when reading this in, the state should reflect
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) * that. We also do this for async as a stop gap for lack of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) * persistence.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) if (btrfs_test_opt(fs_info, DISCARD_SYNC) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) btrfs_test_opt(fs_info, DISCARD_ASYNC))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) e->trim_state = BTRFS_TRIM_STATE_TRIMMED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) if (!e->bytes) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) ret = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) kmem_cache_free(btrfs_free_space_cachep, e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) goto free_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) if (type == BTRFS_FREE_SPACE_EXTENT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) spin_lock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) ret = link_free_space(ctl, e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) btrfs_err(fs_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) "Duplicate entries in free space cache, dumping");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) kmem_cache_free(btrfs_free_space_cachep, e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) goto free_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) ASSERT(num_bitmaps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) num_bitmaps--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) e->bitmap = kmem_cache_zalloc(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) btrfs_free_space_bitmap_cachep, GFP_NOFS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) if (!e->bitmap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) kmem_cache_free(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) btrfs_free_space_cachep, e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) goto free_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) spin_lock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) ret = link_free_space(ctl, e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) ctl->total_bitmaps++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) ctl->op->recalc_thresholds(ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) btrfs_err(fs_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) "Duplicate entries in free space cache, dumping");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) kmem_cache_free(btrfs_free_space_cachep, e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) goto free_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) list_add_tail(&e->list, &bitmaps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) num_entries--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) io_ctl_unmap_page(&io_ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) * We add the bitmaps at the end of the entries in order that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) * the bitmap entries are added to the cache.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) list_for_each_entry_safe(e, n, &bitmaps, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) list_del_init(&e->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) ret = io_ctl_read_bitmap(&io_ctl, e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) goto free_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) e->bitmap_extents = count_bitmap_extents(ctl, e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) if (!btrfs_free_space_trimmed(e)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) ctl->discardable_extents[BTRFS_STAT_CURR] +=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) e->bitmap_extents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) ctl->discardable_bytes[BTRFS_STAT_CURR] += e->bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) io_ctl_drop_pages(&io_ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) merge_space_tree(ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) ret = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) btrfs_discard_update_discardable(ctl->private, ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) io_ctl_free(&io_ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) free_cache:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) io_ctl_drop_pages(&io_ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) __btrfs_remove_free_space_cache(ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) int load_free_space_cache(struct btrfs_block_group *block_group)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) struct btrfs_fs_info *fs_info = block_group->fs_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) struct inode *inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) struct btrfs_path *path;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) bool matched;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) u64 used = block_group->used;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) * If this block group has been marked to be cleared for one reason or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) * another then we can't trust the on disk cache, so just return.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) spin_lock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) if (block_group->disk_cache_state != BTRFS_DC_WRITTEN) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) spin_unlock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) spin_unlock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) path = btrfs_alloc_path();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) if (!path)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) path->search_commit_root = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) path->skip_locking = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) * We must pass a path with search_commit_root set to btrfs_iget in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) * order to avoid a deadlock when allocating extents for the tree root.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) * When we are COWing an extent buffer from the tree root, when looking
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) * for a free extent, at extent-tree.c:find_free_extent(), we can find
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) * block group without its free space cache loaded. When we find one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) * we must load its space cache which requires reading its free space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) * cache's inode item from the root tree. If this inode item is located
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) * in the same leaf that we started COWing before, then we end up in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) * deadlock on the extent buffer (trying to read lock it when we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) * previously write locked it).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) * It's safe to read the inode item using the commit root because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) * block groups, once loaded, stay in memory forever (until they are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) * removed) as well as their space caches once loaded. New block groups
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) * once created get their ->cached field set to BTRFS_CACHE_FINISHED so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) * we will never try to read their inode item while the fs is mounted.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) inode = lookup_free_space_inode(block_group, path);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) if (IS_ERR(inode)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) btrfs_free_path(path);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) /* We may have converted the inode and made the cache invalid. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) spin_lock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) if (block_group->disk_cache_state != BTRFS_DC_WRITTEN) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) spin_unlock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) btrfs_free_path(path);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) spin_unlock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) ret = __load_free_space_cache(fs_info->tree_root, inode, ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) path, block_group->start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) btrfs_free_path(path);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) if (ret <= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) spin_lock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) matched = (ctl->free_space == (block_group->length - used -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) block_group->bytes_super));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) if (!matched) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) __btrfs_remove_free_space_cache(ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) btrfs_warn(fs_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) "block group %llu has wrong amount of free space",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) block_group->start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) ret = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) /* This cache is bogus, make sure it gets cleared */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) spin_lock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) block_group->disk_cache_state = BTRFS_DC_CLEAR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) spin_unlock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) btrfs_warn(fs_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) "failed to load free space cache for block group %llu, rebuilding it now",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) block_group->start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) iput(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) static noinline_for_stack
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) int write_cache_extent_entries(struct btrfs_io_ctl *io_ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) struct btrfs_free_space_ctl *ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) struct btrfs_block_group *block_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) int *entries, int *bitmaps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) struct list_head *bitmap_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) struct btrfs_free_cluster *cluster = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) struct btrfs_free_cluster *cluster_locked = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) struct rb_node *node = rb_first(&ctl->free_space_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) struct btrfs_trim_range *trim_entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) /* Get the cluster for this block_group if it exists */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) if (block_group && !list_empty(&block_group->cluster_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) cluster = list_entry(block_group->cluster_list.next,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) struct btrfs_free_cluster,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) block_group_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) if (!node && cluster) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) cluster_locked = cluster;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) spin_lock(&cluster_locked->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) node = rb_first(&cluster->root);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) cluster = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) /* Write out the extent entries */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) while (node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) struct btrfs_free_space *e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) e = rb_entry(node, struct btrfs_free_space, offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) *entries += 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) ret = io_ctl_add_entry(io_ctl, e->offset, e->bytes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) e->bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) if (e->bitmap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) list_add_tail(&e->list, bitmap_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) *bitmaps += 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) node = rb_next(node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) if (!node && cluster) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) node = rb_first(&cluster->root);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) cluster_locked = cluster;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) spin_lock(&cluster_locked->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) cluster = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) if (cluster_locked) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) spin_unlock(&cluster_locked->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) cluster_locked = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) * Make sure we don't miss any range that was removed from our rbtree
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) * because trimming is running. Otherwise after a umount+mount (or crash
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) * after committing the transaction) we would leak free space and get
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) * an inconsistent free space cache report from fsck.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) list_for_each_entry(trim_entry, &ctl->trimming_ranges, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) ret = io_ctl_add_entry(io_ctl, trim_entry->start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) trim_entry->bytes, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) *entries += 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) fail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) if (cluster_locked)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) spin_unlock(&cluster_locked->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) static noinline_for_stack int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) update_cache_item(struct btrfs_trans_handle *trans,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) struct btrfs_root *root,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) struct inode *inode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) struct btrfs_path *path, u64 offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) int entries, int bitmaps)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) struct btrfs_key key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) struct btrfs_free_space_header *header;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) struct extent_buffer *leaf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) key.objectid = BTRFS_FREE_SPACE_OBJECTID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) key.offset = offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) key.type = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) clear_extent_bit(&BTRFS_I(inode)->io_tree, 0, inode->i_size - 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) EXTENT_DELALLOC, 0, 0, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) leaf = path->nodes[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) if (ret > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) struct btrfs_key found_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) ASSERT(path->slots[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) path->slots[0]--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) if (found_key.objectid != BTRFS_FREE_SPACE_OBJECTID ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) found_key.offset != offset) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) clear_extent_bit(&BTRFS_I(inode)->io_tree, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) inode->i_size - 1, EXTENT_DELALLOC, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) 0, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) btrfs_release_path(path);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) BTRFS_I(inode)->generation = trans->transid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) header = btrfs_item_ptr(leaf, path->slots[0],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) struct btrfs_free_space_header);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) btrfs_set_free_space_entries(leaf, header, entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) btrfs_set_free_space_bitmaps(leaf, header, bitmaps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) btrfs_set_free_space_generation(leaf, header, trans->transid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) btrfs_mark_buffer_dirty(leaf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) btrfs_release_path(path);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) fail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) static noinline_for_stack int write_pinned_extent_entries(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) struct btrfs_trans_handle *trans,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) struct btrfs_block_group *block_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) struct btrfs_io_ctl *io_ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) int *entries)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) u64 start, extent_start, extent_end, len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) struct extent_io_tree *unpin = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) if (!block_group)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) * We want to add any pinned extents to our free space cache
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) * so we don't leak the space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) * We shouldn't have switched the pinned extents yet so this is the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) * right one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) unpin = &trans->transaction->pinned_extents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) start = block_group->start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) while (start < block_group->start + block_group->length) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) ret = find_first_extent_bit(unpin, start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) &extent_start, &extent_end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) EXTENT_DIRTY, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) /* This pinned extent is out of our range */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) if (extent_start >= block_group->start + block_group->length)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) extent_start = max(extent_start, start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) extent_end = min(block_group->start + block_group->length,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) extent_end + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) len = extent_end - extent_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) *entries += 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) ret = io_ctl_add_entry(io_ctl, extent_start, len, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) start = extent_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) static noinline_for_stack int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) write_bitmap_entries(struct btrfs_io_ctl *io_ctl, struct list_head *bitmap_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) struct btrfs_free_space *entry, *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) /* Write out the bitmaps */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) list_for_each_entry_safe(entry, next, bitmap_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) ret = io_ctl_add_bitmap(io_ctl, entry->bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) list_del_init(&entry->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) static int flush_dirty_cache(struct inode *inode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) ret = btrfs_wait_ordered_range(inode, 0, (u64)-1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) clear_extent_bit(&BTRFS_I(inode)->io_tree, 0, inode->i_size - 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) EXTENT_DELALLOC, 0, 0, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) static void noinline_for_stack
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) cleanup_bitmap_list(struct list_head *bitmap_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) struct btrfs_free_space *entry, *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) list_for_each_entry_safe(entry, next, bitmap_list, list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) list_del_init(&entry->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) static void noinline_for_stack
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) cleanup_write_cache_enospc(struct inode *inode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) struct btrfs_io_ctl *io_ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) struct extent_state **cached_state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) io_ctl_drop_pages(io_ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) unlock_extent_cached(&BTRFS_I(inode)->io_tree, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) i_size_read(inode) - 1, cached_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) static int __btrfs_wait_cache_io(struct btrfs_root *root,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) struct btrfs_trans_handle *trans,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) struct btrfs_block_group *block_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) struct btrfs_io_ctl *io_ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) struct btrfs_path *path, u64 offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) struct inode *inode = io_ctl->inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) if (!inode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) /* Flush the dirty pages in the cache file. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) ret = flush_dirty_cache(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) /* Update the cache item to tell everyone this cache file is valid. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) ret = update_cache_item(trans, root, inode, path, offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) io_ctl->entries, io_ctl->bitmaps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) invalidate_inode_pages2(inode->i_mapping);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) BTRFS_I(inode)->generation = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) if (block_group)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) btrfs_debug(root->fs_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) "failed to write free space cache for block group %llu error %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) block_group->start, ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) btrfs_update_inode(trans, root, inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) if (block_group) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) /* the dirty list is protected by the dirty_bgs_lock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) spin_lock(&trans->transaction->dirty_bgs_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) /* the disk_cache_state is protected by the block group lock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) spin_lock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) * only mark this as written if we didn't get put back on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) * the dirty list while waiting for IO. Otherwise our
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) * cache state won't be right, and we won't get written again
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) if (!ret && list_empty(&block_group->dirty_list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) block_group->disk_cache_state = BTRFS_DC_WRITTEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) else if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) block_group->disk_cache_state = BTRFS_DC_ERROR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) spin_unlock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) spin_unlock(&trans->transaction->dirty_bgs_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) io_ctl->inode = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) iput(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) static int btrfs_wait_cache_io_root(struct btrfs_root *root,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) struct btrfs_trans_handle *trans,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) struct btrfs_io_ctl *io_ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) struct btrfs_path *path)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) return __btrfs_wait_cache_io(root, trans, NULL, io_ctl, path, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) int btrfs_wait_cache_io(struct btrfs_trans_handle *trans,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) struct btrfs_block_group *block_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) struct btrfs_path *path)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) return __btrfs_wait_cache_io(block_group->fs_info->tree_root, trans,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) block_group, &block_group->io_ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) path, block_group->start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) * __btrfs_write_out_cache - write out cached info to an inode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) * @root - the root the inode belongs to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) * @ctl - the free space cache we are going to write out
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) * @block_group - the block_group for this cache if it belongs to a block_group
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) * @trans - the trans handle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) * This function writes out a free space cache struct to disk for quick recovery
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) * on mount. This will return 0 if it was successful in writing the cache out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) * or an errno if it was not.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) static int __btrfs_write_out_cache(struct btrfs_root *root, struct inode *inode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) struct btrfs_free_space_ctl *ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) struct btrfs_block_group *block_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) struct btrfs_io_ctl *io_ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) struct btrfs_trans_handle *trans)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) struct extent_state *cached_state = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) LIST_HEAD(bitmap_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) int entries = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) int bitmaps = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) int must_iput = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) if (!i_size_read(inode))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) WARN_ON(io_ctl->pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) ret = io_ctl_init(io_ctl, inode, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) if (block_group && (block_group->flags & BTRFS_BLOCK_GROUP_DATA)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) down_write(&block_group->data_rwsem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) spin_lock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) if (block_group->delalloc_bytes) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) block_group->disk_cache_state = BTRFS_DC_WRITTEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) spin_unlock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) up_write(&block_group->data_rwsem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) BTRFS_I(inode)->generation = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) must_iput = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) spin_unlock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) /* Lock all pages first so we can lock the extent safely. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) ret = io_ctl_prepare_pages(io_ctl, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) lock_extent_bits(&BTRFS_I(inode)->io_tree, 0, i_size_read(inode) - 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) &cached_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) io_ctl_set_generation(io_ctl, trans->transid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) mutex_lock(&ctl->cache_writeout_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) /* Write out the extent entries in the free space cache */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) spin_lock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) ret = write_cache_extent_entries(io_ctl, ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) block_group, &entries, &bitmaps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) &bitmap_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) goto out_nospc_locked;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) * Some spaces that are freed in the current transaction are pinned,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) * they will be added into free space cache after the transaction is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) * committed, we shouldn't lose them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) * If this changes while we are working we'll get added back to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) * the dirty list and redo it. No locking needed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) ret = write_pinned_extent_entries(trans, block_group, io_ctl, &entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) goto out_nospc_locked;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) * At last, we write out all the bitmaps and keep cache_writeout_mutex
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) * locked while doing it because a concurrent trim can be manipulating
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) * or freeing the bitmap.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) ret = write_bitmap_entries(io_ctl, &bitmap_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) mutex_unlock(&ctl->cache_writeout_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) goto out_nospc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) /* Zero out the rest of the pages just to make sure */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) io_ctl_zero_remaining_pages(io_ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) /* Everything is written out, now we dirty the pages in the file. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) ret = btrfs_dirty_pages(BTRFS_I(inode), io_ctl->pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) io_ctl->num_pages, 0, i_size_read(inode),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) &cached_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) goto out_nospc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) if (block_group && (block_group->flags & BTRFS_BLOCK_GROUP_DATA))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) up_write(&block_group->data_rwsem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) * Release the pages and unlock the extent, we will flush
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) * them out later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) io_ctl_drop_pages(io_ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) io_ctl_free(io_ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) unlock_extent_cached(&BTRFS_I(inode)->io_tree, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) i_size_read(inode) - 1, &cached_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) * at this point the pages are under IO and we're happy,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) * The caller is responsible for waiting on them and updating
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) * the cache and the inode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) io_ctl->entries = entries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) io_ctl->bitmaps = bitmaps;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) ret = btrfs_fdatawrite_range(inode, 0, (u64)-1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) out_nospc_locked:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) cleanup_bitmap_list(&bitmap_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) mutex_unlock(&ctl->cache_writeout_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) out_nospc:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) cleanup_write_cache_enospc(inode, io_ctl, &cached_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) out_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) if (block_group && (block_group->flags & BTRFS_BLOCK_GROUP_DATA))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) up_write(&block_group->data_rwsem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) io_ctl->inode = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) io_ctl_free(io_ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) invalidate_inode_pages2(inode->i_mapping);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) BTRFS_I(inode)->generation = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) btrfs_update_inode(trans, root, inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) if (must_iput)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) iput(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) int btrfs_write_out_cache(struct btrfs_trans_handle *trans,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) struct btrfs_block_group *block_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) struct btrfs_path *path)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) struct btrfs_fs_info *fs_info = trans->fs_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) struct inode *inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) spin_lock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) if (block_group->disk_cache_state < BTRFS_DC_SETUP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) spin_unlock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) spin_unlock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) inode = lookup_free_space_inode(block_group, path);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) if (IS_ERR(inode))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) ret = __btrfs_write_out_cache(fs_info->tree_root, inode, ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) block_group, &block_group->io_ctl, trans);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) btrfs_debug(fs_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) "failed to write free space cache for block group %llu error %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) block_group->start, ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) spin_lock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) block_group->disk_cache_state = BTRFS_DC_ERROR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) spin_unlock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) block_group->io_ctl.inode = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) iput(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) * if ret == 0 the caller is expected to call btrfs_wait_cache_io
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) * to wait for IO and put the inode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) static inline unsigned long offset_to_bit(u64 bitmap_start, u32 unit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) u64 offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) ASSERT(offset >= bitmap_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) offset -= bitmap_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) return (unsigned long)(div_u64(offset, unit));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) static inline unsigned long bytes_to_bits(u64 bytes, u32 unit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) return (unsigned long)(div_u64(bytes, unit));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) static inline u64 offset_to_bitmap(struct btrfs_free_space_ctl *ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) u64 offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) u64 bitmap_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) u64 bytes_per_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) bytes_per_bitmap = BITS_PER_BITMAP * ctl->unit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) bitmap_start = offset - ctl->start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) bitmap_start = div64_u64(bitmap_start, bytes_per_bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) bitmap_start *= bytes_per_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) bitmap_start += ctl->start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) return bitmap_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) static int tree_insert_offset(struct rb_root *root, u64 offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) struct rb_node *node, int bitmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) struct rb_node **p = &root->rb_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) struct rb_node *parent = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) struct btrfs_free_space *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) while (*p) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) parent = *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) info = rb_entry(parent, struct btrfs_free_space, offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) if (offset < info->offset) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) p = &(*p)->rb_left;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) } else if (offset > info->offset) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) p = &(*p)->rb_right;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) * we could have a bitmap entry and an extent entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) * share the same offset. If this is the case, we want
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) * the extent entry to always be found first if we do a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) * linear search through the tree, since we want to have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) * the quickest allocation time, and allocating from an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) * extent is faster than allocating from a bitmap. So
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) * if we're inserting a bitmap and we find an entry at
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) * this offset, we want to go right, or after this entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) * logically. If we are inserting an extent and we've
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) * found a bitmap, we want to go left, or before
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) * logically.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) if (bitmap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) if (info->bitmap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) WARN_ON_ONCE(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) return -EEXIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) p = &(*p)->rb_right;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) if (!info->bitmap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) WARN_ON_ONCE(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) return -EEXIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) p = &(*p)->rb_left;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) rb_link_node(node, parent, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) rb_insert_color(node, root);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) * searches the tree for the given offset.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) * fuzzy - If this is set, then we are trying to make an allocation, and we just
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) * want a section that has at least bytes size and comes at or after the given
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) * offset.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) static struct btrfs_free_space *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) tree_search_offset(struct btrfs_free_space_ctl *ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) u64 offset, int bitmap_only, int fuzzy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) struct rb_node *n = ctl->free_space_offset.rb_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) struct btrfs_free_space *entry, *prev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) /* find entry that is closest to the 'offset' */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) if (!n) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) entry = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) entry = rb_entry(n, struct btrfs_free_space, offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) prev = entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) if (offset < entry->offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) n = n->rb_left;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) else if (offset > entry->offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) n = n->rb_right;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) if (bitmap_only) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) if (!entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) if (entry->bitmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) return entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) * bitmap entry and extent entry may share same offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) * in that case, bitmap entry comes after extent entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) n = rb_next(n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) if (!n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) entry = rb_entry(n, struct btrfs_free_space, offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) if (entry->offset != offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) WARN_ON(!entry->bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) return entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) } else if (entry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) if (entry->bitmap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) * if previous extent entry covers the offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) * we should return it instead of the bitmap entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) n = rb_prev(&entry->offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) if (n) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) prev = rb_entry(n, struct btrfs_free_space,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) if (!prev->bitmap &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) prev->offset + prev->bytes > offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) entry = prev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) return entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) if (!prev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) /* find last entry before the 'offset' */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) entry = prev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) if (entry->offset > offset) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) n = rb_prev(&entry->offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) if (n) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) entry = rb_entry(n, struct btrfs_free_space,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) ASSERT(entry->offset <= offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) if (fuzzy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) return entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) if (entry->bitmap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) n = rb_prev(&entry->offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) if (n) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) prev = rb_entry(n, struct btrfs_free_space,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) if (!prev->bitmap &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) prev->offset + prev->bytes > offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) return prev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) if (entry->offset + BITS_PER_BITMAP * ctl->unit > offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) return entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) } else if (entry->offset + entry->bytes > offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) return entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) if (!fuzzy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) if (entry->bitmap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) if (entry->offset + BITS_PER_BITMAP *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) ctl->unit > offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) if (entry->offset + entry->bytes > offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) n = rb_next(&entry->offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) if (!n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) entry = rb_entry(n, struct btrfs_free_space, offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) return entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) __unlink_free_space(struct btrfs_free_space_ctl *ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) struct btrfs_free_space *info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) rb_erase(&info->offset_index, &ctl->free_space_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) ctl->free_extents--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) if (!info->bitmap && !btrfs_free_space_trimmed(info)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) ctl->discardable_extents[BTRFS_STAT_CURR]--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) ctl->discardable_bytes[BTRFS_STAT_CURR] -= info->bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) static void unlink_free_space(struct btrfs_free_space_ctl *ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) struct btrfs_free_space *info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) __unlink_free_space(ctl, info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) ctl->free_space -= info->bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) static int link_free_space(struct btrfs_free_space_ctl *ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) struct btrfs_free_space *info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) ASSERT(info->bytes || info->bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) ret = tree_insert_offset(&ctl->free_space_offset, info->offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) &info->offset_index, (info->bitmap != NULL));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) if (!info->bitmap && !btrfs_free_space_trimmed(info)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) ctl->discardable_extents[BTRFS_STAT_CURR]++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) ctl->discardable_bytes[BTRFS_STAT_CURR] += info->bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) ctl->free_space += info->bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) ctl->free_extents++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) static void recalculate_thresholds(struct btrfs_free_space_ctl *ctl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) struct btrfs_block_group *block_group = ctl->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) u64 max_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) u64 bitmap_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) u64 extent_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) u64 size = block_group->length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) u64 bytes_per_bg = BITS_PER_BITMAP * ctl->unit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) u64 max_bitmaps = div64_u64(size + bytes_per_bg - 1, bytes_per_bg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689) max_bitmaps = max_t(u64, max_bitmaps, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) ASSERT(ctl->total_bitmaps <= max_bitmaps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) * We are trying to keep the total amount of memory used per 1GiB of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) * space to be MAX_CACHE_BYTES_PER_GIG. However, with a reclamation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) * mechanism of pulling extents >= FORCE_EXTENT_THRESHOLD out of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) * bitmaps, we may end up using more memory than this.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) if (size < SZ_1G)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700) max_bytes = MAX_CACHE_BYTES_PER_GIG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702) max_bytes = MAX_CACHE_BYTES_PER_GIG * div_u64(size, SZ_1G);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) bitmap_bytes = ctl->total_bitmaps * ctl->unit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) * we want the extent entry threshold to always be at most 1/2 the max
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) * bytes we can have, or whatever is less than that.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) extent_bytes = max_bytes - bitmap_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) extent_bytes = min_t(u64, extent_bytes, max_bytes >> 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) ctl->extents_thresh =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) div_u64(extent_bytes, sizeof(struct btrfs_free_space));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) static inline void __bitmap_clear_bits(struct btrfs_free_space_ctl *ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) struct btrfs_free_space *info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719) u64 offset, u64 bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721) unsigned long start, count, end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) int extent_delta = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) start = offset_to_bit(info->offset, ctl->unit, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725) count = bytes_to_bits(bytes, ctl->unit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) end = start + count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) ASSERT(end <= BITS_PER_BITMAP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) bitmap_clear(info->bitmap, start, count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) info->bytes -= bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) if (info->max_extent_size > ctl->unit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733) info->max_extent_size = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) if (start && test_bit(start - 1, info->bitmap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) extent_delta++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) if (end < BITS_PER_BITMAP && test_bit(end, info->bitmap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) extent_delta++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) info->bitmap_extents += extent_delta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742) if (!btrfs_free_space_trimmed(info)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) ctl->discardable_extents[BTRFS_STAT_CURR] += extent_delta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) ctl->discardable_bytes[BTRFS_STAT_CURR] -= bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) static void bitmap_clear_bits(struct btrfs_free_space_ctl *ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749) struct btrfs_free_space *info, u64 offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) u64 bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752) __bitmap_clear_bits(ctl, info, offset, bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) ctl->free_space -= bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756) static void bitmap_set_bits(struct btrfs_free_space_ctl *ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757) struct btrfs_free_space *info, u64 offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758) u64 bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760) unsigned long start, count, end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) int extent_delta = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) start = offset_to_bit(info->offset, ctl->unit, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764) count = bytes_to_bits(bytes, ctl->unit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) end = start + count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766) ASSERT(end <= BITS_PER_BITMAP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768) bitmap_set(info->bitmap, start, count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770) info->bytes += bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) ctl->free_space += bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) if (start && test_bit(start - 1, info->bitmap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774) extent_delta--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) if (end < BITS_PER_BITMAP && test_bit(end, info->bitmap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777) extent_delta--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779) info->bitmap_extents += extent_delta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780) if (!btrfs_free_space_trimmed(info)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781) ctl->discardable_extents[BTRFS_STAT_CURR] += extent_delta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782) ctl->discardable_bytes[BTRFS_STAT_CURR] += bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787) * If we can not find suitable extent, we will use bytes to record
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) * the size of the max extent.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790) static int search_bitmap(struct btrfs_free_space_ctl *ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791) struct btrfs_free_space *bitmap_info, u64 *offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792) u64 *bytes, bool for_alloc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794) unsigned long found_bits = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795) unsigned long max_bits = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796) unsigned long bits, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797) unsigned long next_zero;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798) unsigned long extent_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801) * Skip searching the bitmap if we don't have a contiguous section that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802) * is large enough for this allocation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804) if (for_alloc &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805) bitmap_info->max_extent_size &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806) bitmap_info->max_extent_size < *bytes) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807) *bytes = bitmap_info->max_extent_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811) i = offset_to_bit(bitmap_info->offset, ctl->unit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812) max_t(u64, *offset, bitmap_info->offset));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813) bits = bytes_to_bits(*bytes, ctl->unit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815) for_each_set_bit_from(i, bitmap_info->bitmap, BITS_PER_BITMAP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816) if (for_alloc && bits == 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817) found_bits = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820) next_zero = find_next_zero_bit(bitmap_info->bitmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821) BITS_PER_BITMAP, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822) extent_bits = next_zero - i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823) if (extent_bits >= bits) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824) found_bits = extent_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826) } else if (extent_bits > max_bits) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827) max_bits = extent_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829) i = next_zero;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832) if (found_bits) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833) *offset = (u64)(i * ctl->unit) + bitmap_info->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834) *bytes = (u64)(found_bits) * ctl->unit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838) *bytes = (u64)(max_bits) * ctl->unit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839) bitmap_info->max_extent_size = *bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843) static inline u64 get_max_extent_size(struct btrfs_free_space *entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845) if (entry->bitmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846) return entry->max_extent_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847) return entry->bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850) /* Cache the size of the max extent in bytes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851) static struct btrfs_free_space *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852) find_free_space(struct btrfs_free_space_ctl *ctl, u64 *offset, u64 *bytes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853) unsigned long align, u64 *max_extent_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855) struct btrfs_free_space *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856) struct rb_node *node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857) u64 tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858) u64 align_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1859) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1860)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1861) if (!ctl->free_space_offset.rb_node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1862) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1863)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1864) entry = tree_search_offset(ctl, offset_to_bitmap(ctl, *offset), 0, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1865) if (!entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1866) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1867)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1868) for (node = &entry->offset_index; node; node = rb_next(node)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1869) entry = rb_entry(node, struct btrfs_free_space, offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1870) if (entry->bytes < *bytes) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1871) *max_extent_size = max(get_max_extent_size(entry),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1872) *max_extent_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1873) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1874) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1875)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1876) /* make sure the space returned is big enough
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1877) * to match our requested alignment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1878) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1879) if (*bytes >= align) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1880) tmp = entry->offset - ctl->start + align - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1881) tmp = div64_u64(tmp, align);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1882) tmp = tmp * align + ctl->start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1883) align_off = tmp - entry->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1884) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1885) align_off = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1886) tmp = entry->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1887) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1889) if (entry->bytes < *bytes + align_off) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1890) *max_extent_size = max(get_max_extent_size(entry),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1891) *max_extent_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1892) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1893) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1894)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1895) if (entry->bitmap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1896) u64 size = *bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1897)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1898) ret = search_bitmap(ctl, entry, &tmp, &size, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1899) if (!ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1900) *offset = tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1901) *bytes = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1902) return entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1903) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1904) *max_extent_size =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1905) max(get_max_extent_size(entry),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1906) *max_extent_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1907) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1908) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1909) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1911) *offset = tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1912) *bytes = entry->bytes - align_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1913) return entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1914) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1915) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1916) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1917) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1918)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1919) static int count_bitmap_extents(struct btrfs_free_space_ctl *ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1920) struct btrfs_free_space *bitmap_info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1921) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1922) struct btrfs_block_group *block_group = ctl->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1923) u64 bytes = bitmap_info->bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1924) unsigned int rs, re;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1925) int count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1926)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1927) if (!block_group || !bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1928) return count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1929)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1930) bitmap_for_each_set_region(bitmap_info->bitmap, rs, re, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1931) BITS_PER_BITMAP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1932) bytes -= (rs - re) * ctl->unit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1933) count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1934)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1935) if (!bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1936) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1937) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1938)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1939) return count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1940) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1941)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1942) static void add_new_bitmap(struct btrfs_free_space_ctl *ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1943) struct btrfs_free_space *info, u64 offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1944) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1945) info->offset = offset_to_bitmap(ctl, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1946) info->bytes = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1947) info->bitmap_extents = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1948) INIT_LIST_HEAD(&info->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1949) link_free_space(ctl, info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1950) ctl->total_bitmaps++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1951)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1952) ctl->op->recalc_thresholds(ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1953) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1954)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1955) static void free_bitmap(struct btrfs_free_space_ctl *ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1956) struct btrfs_free_space *bitmap_info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1957) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1958) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1959) * Normally when this is called, the bitmap is completely empty. However,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1960) * if we are blowing up the free space cache for one reason or another
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1961) * via __btrfs_remove_free_space_cache(), then it may not be freed and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1962) * we may leave stats on the table.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1963) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1964) if (bitmap_info->bytes && !btrfs_free_space_trimmed(bitmap_info)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1965) ctl->discardable_extents[BTRFS_STAT_CURR] -=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1966) bitmap_info->bitmap_extents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1967) ctl->discardable_bytes[BTRFS_STAT_CURR] -= bitmap_info->bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1968)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1969) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1970) unlink_free_space(ctl, bitmap_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1971) kmem_cache_free(btrfs_free_space_bitmap_cachep, bitmap_info->bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1972) kmem_cache_free(btrfs_free_space_cachep, bitmap_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1973) ctl->total_bitmaps--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1974) ctl->op->recalc_thresholds(ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1975) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1976)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1977) static noinline int remove_from_bitmap(struct btrfs_free_space_ctl *ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1978) struct btrfs_free_space *bitmap_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1979) u64 *offset, u64 *bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1980) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1981) u64 end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1982) u64 search_start, search_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1983) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1984)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1985) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1986) end = bitmap_info->offset + (u64)(BITS_PER_BITMAP * ctl->unit) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1987)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1988) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1989) * We need to search for bits in this bitmap. We could only cover some
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1990) * of the extent in this bitmap thanks to how we add space, so we need
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1991) * to search for as much as it as we can and clear that amount, and then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1992) * go searching for the next bit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1993) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1994) search_start = *offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1995) search_bytes = ctl->unit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1996) search_bytes = min(search_bytes, end - search_start + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1997) ret = search_bitmap(ctl, bitmap_info, &search_start, &search_bytes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1998) false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1999) if (ret < 0 || search_start != *offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2000) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2001)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2002) /* We may have found more bits than what we need */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2003) search_bytes = min(search_bytes, *bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2004)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2005) /* Cannot clear past the end of the bitmap */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2006) search_bytes = min(search_bytes, end - search_start + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2007)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2008) bitmap_clear_bits(ctl, bitmap_info, search_start, search_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2009) *offset += search_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2010) *bytes -= search_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2011)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2012) if (*bytes) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2013) struct rb_node *next = rb_next(&bitmap_info->offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2014) if (!bitmap_info->bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2015) free_bitmap(ctl, bitmap_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2016)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2017) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2018) * no entry after this bitmap, but we still have bytes to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2019) * remove, so something has gone wrong.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2020) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2021) if (!next)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2022) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2023)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2024) bitmap_info = rb_entry(next, struct btrfs_free_space,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2025) offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2026)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2027) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2028) * if the next entry isn't a bitmap we need to return to let the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2029) * extent stuff do its work.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2030) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2031) if (!bitmap_info->bitmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2032) return -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2033)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2034) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2035) * Ok the next item is a bitmap, but it may not actually hold
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2036) * the information for the rest of this free space stuff, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2037) * look for it, and if we don't find it return so we can try
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2038) * everything over again.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2039) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2040) search_start = *offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2041) search_bytes = ctl->unit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2042) ret = search_bitmap(ctl, bitmap_info, &search_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2043) &search_bytes, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2044) if (ret < 0 || search_start != *offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2045) return -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2046)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2047) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2048) } else if (!bitmap_info->bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2049) free_bitmap(ctl, bitmap_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2050)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2051) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2052) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2053)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2054) static u64 add_bytes_to_bitmap(struct btrfs_free_space_ctl *ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2055) struct btrfs_free_space *info, u64 offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2056) u64 bytes, enum btrfs_trim_state trim_state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2057) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2058) u64 bytes_to_set = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2059) u64 end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2060)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2061) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2062) * This is a tradeoff to make bitmap trim state minimal. We mark the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2063) * whole bitmap untrimmed if at any point we add untrimmed regions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2064) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2065) if (trim_state == BTRFS_TRIM_STATE_UNTRIMMED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2066) if (btrfs_free_space_trimmed(info)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2067) ctl->discardable_extents[BTRFS_STAT_CURR] +=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2068) info->bitmap_extents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2069) ctl->discardable_bytes[BTRFS_STAT_CURR] += info->bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2070) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2071) info->trim_state = BTRFS_TRIM_STATE_UNTRIMMED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2072) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2073)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2074) end = info->offset + (u64)(BITS_PER_BITMAP * ctl->unit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2075)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2076) bytes_to_set = min(end - offset, bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2077)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2078) bitmap_set_bits(ctl, info, offset, bytes_to_set);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2079)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2080) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2081) * We set some bytes, we have no idea what the max extent size is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2082) * anymore.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2083) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2084) info->max_extent_size = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2085)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2086) return bytes_to_set;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2087)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2088) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2089)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2090) static bool use_bitmap(struct btrfs_free_space_ctl *ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2091) struct btrfs_free_space *info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2092) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2093) struct btrfs_block_group *block_group = ctl->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2094) struct btrfs_fs_info *fs_info = block_group->fs_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2095) bool forced = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2096)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2097) #ifdef CONFIG_BTRFS_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2098) if (btrfs_should_fragment_free_space(block_group))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2099) forced = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2100) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2102) /* This is a way to reclaim large regions from the bitmaps. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2103) if (!forced && info->bytes >= FORCE_EXTENT_THRESHOLD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2104) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2106) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2107) * If we are below the extents threshold then we can add this as an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2108) * extent, and don't have to deal with the bitmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2109) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2110) if (!forced && ctl->free_extents < ctl->extents_thresh) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2111) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2112) * If this block group has some small extents we don't want to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2113) * use up all of our free slots in the cache with them, we want
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2114) * to reserve them to larger extents, however if we have plenty
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2115) * of cache left then go ahead an dadd them, no sense in adding
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2116) * the overhead of a bitmap if we don't have to.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2117) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2118) if (info->bytes <= fs_info->sectorsize * 8) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2119) if (ctl->free_extents * 3 <= ctl->extents_thresh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2120) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2121) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2122) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2123) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2124) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2126) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2127) * The original block groups from mkfs can be really small, like 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2128) * megabytes, so don't bother with a bitmap for those entries. However
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2129) * some block groups can be smaller than what a bitmap would cover but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2130) * are still large enough that they could overflow the 32k memory limit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2131) * so allow those block groups to still be allowed to have a bitmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2132) * entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2133) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2134) if (((BITS_PER_BITMAP * ctl->unit) >> 1) > block_group->length)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2135) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2137) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2138) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2140) static const struct btrfs_free_space_op free_space_op = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2141) .recalc_thresholds = recalculate_thresholds,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2142) .use_bitmap = use_bitmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2143) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2145) static int insert_into_bitmap(struct btrfs_free_space_ctl *ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2146) struct btrfs_free_space *info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2147) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2148) struct btrfs_free_space *bitmap_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2149) struct btrfs_block_group *block_group = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2150) int added = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2151) u64 bytes, offset, bytes_added;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2152) enum btrfs_trim_state trim_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2153) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2155) bytes = info->bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2156) offset = info->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2157) trim_state = info->trim_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2159) if (!ctl->op->use_bitmap(ctl, info))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2160) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2162) if (ctl->op == &free_space_op)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2163) block_group = ctl->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2164) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2165) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2166) * Since we link bitmaps right into the cluster we need to see if we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2167) * have a cluster here, and if so and it has our bitmap we need to add
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2168) * the free space to that bitmap.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2169) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2170) if (block_group && !list_empty(&block_group->cluster_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2171) struct btrfs_free_cluster *cluster;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2172) struct rb_node *node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2173) struct btrfs_free_space *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2175) cluster = list_entry(block_group->cluster_list.next,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2176) struct btrfs_free_cluster,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2177) block_group_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2178) spin_lock(&cluster->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2179) node = rb_first(&cluster->root);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2180) if (!node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2181) spin_unlock(&cluster->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2182) goto no_cluster_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2183) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2185) entry = rb_entry(node, struct btrfs_free_space, offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2186) if (!entry->bitmap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2187) spin_unlock(&cluster->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2188) goto no_cluster_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2189) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2191) if (entry->offset == offset_to_bitmap(ctl, offset)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2192) bytes_added = add_bytes_to_bitmap(ctl, entry, offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2193) bytes, trim_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2194) bytes -= bytes_added;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2195) offset += bytes_added;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2196) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2197) spin_unlock(&cluster->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2198) if (!bytes) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2199) ret = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2200) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2201) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2202) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2204) no_cluster_bitmap:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2205) bitmap_info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2206) 1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2207) if (!bitmap_info) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2208) ASSERT(added == 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2209) goto new_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2210) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2212) bytes_added = add_bytes_to_bitmap(ctl, bitmap_info, offset, bytes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2213) trim_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2214) bytes -= bytes_added;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2215) offset += bytes_added;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2216) added = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2218) if (!bytes) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2219) ret = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2220) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2221) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2222) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2224) new_bitmap:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2225) if (info && info->bitmap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2226) add_new_bitmap(ctl, info, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2227) added = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2228) info = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2229) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2230) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2231) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2233) /* no pre-allocated info, allocate a new one */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2234) if (!info) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2235) info = kmem_cache_zalloc(btrfs_free_space_cachep,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2236) GFP_NOFS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2237) if (!info) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2238) spin_lock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2239) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2240) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2242) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2244) /* allocate the bitmap */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2245) info->bitmap = kmem_cache_zalloc(btrfs_free_space_bitmap_cachep,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2246) GFP_NOFS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2247) info->trim_state = BTRFS_TRIM_STATE_TRIMMED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2248) spin_lock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2249) if (!info->bitmap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2250) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2251) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2252) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2253) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2254) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2256) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2257) if (info) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2258) if (info->bitmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2259) kmem_cache_free(btrfs_free_space_bitmap_cachep,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2260) info->bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2261) kmem_cache_free(btrfs_free_space_cachep, info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2262) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2264) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2265) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2267) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2268) * Free space merging rules:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2269) * 1) Merge trimmed areas together
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2270) * 2) Let untrimmed areas coalesce with trimmed areas
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2271) * 3) Always pull neighboring regions from bitmaps
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2272) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2273) * The above rules are for when we merge free space based on btrfs_trim_state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2274) * Rules 2 and 3 are subtle because they are suboptimal, but are done for the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2275) * same reason: to promote larger extent regions which makes life easier for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2276) * find_free_extent(). Rule 2 enables coalescing based on the common path
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2277) * being returning free space from btrfs_finish_extent_commit(). So when free
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2278) * space is trimmed, it will prevent aggregating trimmed new region and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2279) * untrimmed regions in the rb_tree. Rule 3 is purely to obtain larger extents
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2280) * and provide find_free_extent() with the largest extents possible hoping for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2281) * the reuse path.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2282) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2283) static bool try_merge_free_space(struct btrfs_free_space_ctl *ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2284) struct btrfs_free_space *info, bool update_stat)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2285) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2286) struct btrfs_free_space *left_info = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2287) struct btrfs_free_space *right_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2288) bool merged = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2289) u64 offset = info->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2290) u64 bytes = info->bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2291) const bool is_trimmed = btrfs_free_space_trimmed(info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2293) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2294) * first we want to see if there is free space adjacent to the range we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2295) * are adding, if there is remove that struct and add a new one to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2296) * cover the entire range
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2297) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2298) right_info = tree_search_offset(ctl, offset + bytes, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2299) if (right_info && rb_prev(&right_info->offset_index))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2300) left_info = rb_entry(rb_prev(&right_info->offset_index),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2301) struct btrfs_free_space, offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2302) else if (!right_info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2303) left_info = tree_search_offset(ctl, offset - 1, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2305) /* See try_merge_free_space() comment. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2306) if (right_info && !right_info->bitmap &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2307) (!is_trimmed || btrfs_free_space_trimmed(right_info))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2308) if (update_stat)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2309) unlink_free_space(ctl, right_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2310) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2311) __unlink_free_space(ctl, right_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2312) info->bytes += right_info->bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2313) kmem_cache_free(btrfs_free_space_cachep, right_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2314) merged = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2315) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2317) /* See try_merge_free_space() comment. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2318) if (left_info && !left_info->bitmap &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2319) left_info->offset + left_info->bytes == offset &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2320) (!is_trimmed || btrfs_free_space_trimmed(left_info))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2321) if (update_stat)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2322) unlink_free_space(ctl, left_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2323) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2324) __unlink_free_space(ctl, left_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2325) info->offset = left_info->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2326) info->bytes += left_info->bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2327) kmem_cache_free(btrfs_free_space_cachep, left_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2328) merged = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2329) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2331) return merged;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2332) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2334) static bool steal_from_bitmap_to_end(struct btrfs_free_space_ctl *ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2335) struct btrfs_free_space *info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2336) bool update_stat)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2337) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2338) struct btrfs_free_space *bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2339) unsigned long i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2340) unsigned long j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2341) const u64 end = info->offset + info->bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2342) const u64 bitmap_offset = offset_to_bitmap(ctl, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2343) u64 bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2345) bitmap = tree_search_offset(ctl, bitmap_offset, 1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2346) if (!bitmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2347) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2349) i = offset_to_bit(bitmap->offset, ctl->unit, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2350) j = find_next_zero_bit(bitmap->bitmap, BITS_PER_BITMAP, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2351) if (j == i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2352) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2353) bytes = (j - i) * ctl->unit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2354) info->bytes += bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2356) /* See try_merge_free_space() comment. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2357) if (!btrfs_free_space_trimmed(bitmap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2358) info->trim_state = BTRFS_TRIM_STATE_UNTRIMMED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2360) if (update_stat)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2361) bitmap_clear_bits(ctl, bitmap, end, bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2362) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2363) __bitmap_clear_bits(ctl, bitmap, end, bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2364)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2365) if (!bitmap->bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2366) free_bitmap(ctl, bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2368) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2369) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2371) static bool steal_from_bitmap_to_front(struct btrfs_free_space_ctl *ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2372) struct btrfs_free_space *info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2373) bool update_stat)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2374) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2375) struct btrfs_free_space *bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2376) u64 bitmap_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2377) unsigned long i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2378) unsigned long j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2379) unsigned long prev_j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2380) u64 bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2382) bitmap_offset = offset_to_bitmap(ctl, info->offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2383) /* If we're on a boundary, try the previous logical bitmap. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2384) if (bitmap_offset == info->offset) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2385) if (info->offset == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2386) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2387) bitmap_offset = offset_to_bitmap(ctl, info->offset - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2388) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2390) bitmap = tree_search_offset(ctl, bitmap_offset, 1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2391) if (!bitmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2392) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2394) i = offset_to_bit(bitmap->offset, ctl->unit, info->offset) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2395) j = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2396) prev_j = (unsigned long)-1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2397) for_each_clear_bit_from(j, bitmap->bitmap, BITS_PER_BITMAP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2398) if (j > i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2399) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2400) prev_j = j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2401) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2402) if (prev_j == i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2403) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2405) if (prev_j == (unsigned long)-1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2406) bytes = (i + 1) * ctl->unit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2407) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2408) bytes = (i - prev_j) * ctl->unit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2410) info->offset -= bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2411) info->bytes += bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2413) /* See try_merge_free_space() comment. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2414) if (!btrfs_free_space_trimmed(bitmap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2415) info->trim_state = BTRFS_TRIM_STATE_UNTRIMMED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2417) if (update_stat)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2418) bitmap_clear_bits(ctl, bitmap, info->offset, bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2419) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2420) __bitmap_clear_bits(ctl, bitmap, info->offset, bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2422) if (!bitmap->bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2423) free_bitmap(ctl, bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2425) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2426) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2428) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2429) * We prefer always to allocate from extent entries, both for clustered and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2430) * non-clustered allocation requests. So when attempting to add a new extent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2431) * entry, try to see if there's adjacent free space in bitmap entries, and if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2432) * there is, migrate that space from the bitmaps to the extent.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2433) * Like this we get better chances of satisfying space allocation requests
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2434) * because we attempt to satisfy them based on a single cache entry, and never
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2435) * on 2 or more entries - even if the entries represent a contiguous free space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2436) * region (e.g. 1 extent entry + 1 bitmap entry starting where the extent entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2437) * ends).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2438) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2439) static void steal_from_bitmap(struct btrfs_free_space_ctl *ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2440) struct btrfs_free_space *info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2441) bool update_stat)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2442) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2443) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2444) * Only work with disconnected entries, as we can change their offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2445) * and must be extent entries.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2446) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2447) ASSERT(!info->bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2448) ASSERT(RB_EMPTY_NODE(&info->offset_index));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2450) if (ctl->total_bitmaps > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2451) bool stole_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2452) bool stole_front = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2454) stole_end = steal_from_bitmap_to_end(ctl, info, update_stat);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2455) if (ctl->total_bitmaps > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2456) stole_front = steal_from_bitmap_to_front(ctl, info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2457) update_stat);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2459) if (stole_end || stole_front)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2460) try_merge_free_space(ctl, info, update_stat);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2461) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2462) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2464) int __btrfs_add_free_space(struct btrfs_fs_info *fs_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2465) struct btrfs_free_space_ctl *ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2466) u64 offset, u64 bytes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2467) enum btrfs_trim_state trim_state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2468) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2469) struct btrfs_block_group *block_group = ctl->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2470) struct btrfs_free_space *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2471) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2472) u64 filter_bytes = bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2474) info = kmem_cache_zalloc(btrfs_free_space_cachep, GFP_NOFS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2475) if (!info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2476) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2478) info->offset = offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2479) info->bytes = bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2480) info->trim_state = trim_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2481) RB_CLEAR_NODE(&info->offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2483) spin_lock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2485) if (try_merge_free_space(ctl, info, true))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2486) goto link;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2488) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2489) * There was no extent directly to the left or right of this new
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2490) * extent then we know we're going to have to allocate a new extent, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2491) * before we do that see if we need to drop this into a bitmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2492) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2493) ret = insert_into_bitmap(ctl, info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2494) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2495) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2496) } else if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2497) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2498) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2499) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2500) link:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2501) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2502) * Only steal free space from adjacent bitmaps if we're sure we're not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2503) * going to add the new free space to existing bitmap entries - because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2504) * that would mean unnecessary work that would be reverted. Therefore
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2505) * attempt to steal space from bitmaps if we're adding an extent entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2506) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2507) steal_from_bitmap(ctl, info, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2509) filter_bytes = max(filter_bytes, info->bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2511) ret = link_free_space(ctl, info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2512) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2513) kmem_cache_free(btrfs_free_space_cachep, info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2514) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2515) btrfs_discard_update_discardable(block_group, ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2516) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2518) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2519) btrfs_crit(fs_info, "unable to add free space :%d", ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2520) ASSERT(ret != -EEXIST);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2521) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2523) if (trim_state != BTRFS_TRIM_STATE_TRIMMED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2524) btrfs_discard_check_filter(block_group, filter_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2525) btrfs_discard_queue_work(&fs_info->discard_ctl, block_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2526) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2528) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2529) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2530)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2531) int btrfs_add_free_space(struct btrfs_block_group *block_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2532) u64 bytenr, u64 size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2533) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2534) enum btrfs_trim_state trim_state = BTRFS_TRIM_STATE_UNTRIMMED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2536) if (btrfs_test_opt(block_group->fs_info, DISCARD_SYNC))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2537) trim_state = BTRFS_TRIM_STATE_TRIMMED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2539) return __btrfs_add_free_space(block_group->fs_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2540) block_group->free_space_ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2541) bytenr, size, trim_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2542) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2544) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2545) * This is a subtle distinction because when adding free space back in general,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2546) * we want it to be added as untrimmed for async. But in the case where we add
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2547) * it on loading of a block group, we want to consider it trimmed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2548) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2549) int btrfs_add_free_space_async_trimmed(struct btrfs_block_group *block_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2550) u64 bytenr, u64 size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2551) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2552) enum btrfs_trim_state trim_state = BTRFS_TRIM_STATE_UNTRIMMED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2554) if (btrfs_test_opt(block_group->fs_info, DISCARD_SYNC) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2555) btrfs_test_opt(block_group->fs_info, DISCARD_ASYNC))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2556) trim_state = BTRFS_TRIM_STATE_TRIMMED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2558) return __btrfs_add_free_space(block_group->fs_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2559) block_group->free_space_ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2560) bytenr, size, trim_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2561) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2563) int btrfs_remove_free_space(struct btrfs_block_group *block_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2564) u64 offset, u64 bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2565) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2566) struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2567) struct btrfs_free_space *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2568) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2569) bool re_search = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2571) spin_lock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2573) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2574) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2575) if (!bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2576) goto out_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2578) info = tree_search_offset(ctl, offset, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2579) if (!info) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2580) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2581) * oops didn't find an extent that matched the space we wanted
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2582) * to remove, look for a bitmap instead
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2583) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2584) info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2585) 1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2586) if (!info) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2587) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2588) * If we found a partial bit of our free space in a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2589) * bitmap but then couldn't find the other part this may
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2590) * be a problem, so WARN about it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2591) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2592) WARN_ON(re_search);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2593) goto out_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2594) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2595) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2597) re_search = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2598) if (!info->bitmap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2599) unlink_free_space(ctl, info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2600) if (offset == info->offset) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2601) u64 to_free = min(bytes, info->bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2603) info->bytes -= to_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2604) info->offset += to_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2605) if (info->bytes) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2606) ret = link_free_space(ctl, info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2607) WARN_ON(ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2608) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2609) kmem_cache_free(btrfs_free_space_cachep, info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2610) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2612) offset += to_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2613) bytes -= to_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2614) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2615) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2616) u64 old_end = info->bytes + info->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2618) info->bytes = offset - info->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2619) ret = link_free_space(ctl, info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2620) WARN_ON(ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2621) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2622) goto out_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2624) /* Not enough bytes in this entry to satisfy us */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2625) if (old_end < offset + bytes) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2626) bytes -= old_end - offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2627) offset = old_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2628) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2629) } else if (old_end == offset + bytes) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2630) /* all done */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2631) goto out_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2632) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2633) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2635) ret = __btrfs_add_free_space(block_group->fs_info, ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2636) offset + bytes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2637) old_end - (offset + bytes),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2638) info->trim_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2639) WARN_ON(ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2640) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2641) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2642) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2644) ret = remove_from_bitmap(ctl, info, &offset, &bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2645) if (ret == -EAGAIN) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2646) re_search = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2647) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2648) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2649) out_lock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2650) btrfs_discard_update_discardable(block_group, ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2651) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2652) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2653) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2654) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2656) void btrfs_dump_free_space(struct btrfs_block_group *block_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2657) u64 bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2658) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2659) struct btrfs_fs_info *fs_info = block_group->fs_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2660) struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2661) struct btrfs_free_space *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2662) struct rb_node *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2663) int count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2664)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2665) spin_lock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2666) for (n = rb_first(&ctl->free_space_offset); n; n = rb_next(n)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2667) info = rb_entry(n, struct btrfs_free_space, offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2668) if (info->bytes >= bytes && !block_group->ro)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2669) count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2670) btrfs_crit(fs_info, "entry offset %llu, bytes %llu, bitmap %s",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2671) info->offset, info->bytes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2672) (info->bitmap) ? "yes" : "no");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2673) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2674) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2675) btrfs_info(fs_info, "block group has cluster?: %s",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2676) list_empty(&block_group->cluster_list) ? "no" : "yes");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2677) btrfs_info(fs_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2678) "%d blocks of free space at or bigger than bytes is", count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2679) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2680)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2681) void btrfs_init_free_space_ctl(struct btrfs_block_group *block_group)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2682) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2683) struct btrfs_fs_info *fs_info = block_group->fs_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2684) struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2685)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2686) spin_lock_init(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2687) ctl->unit = fs_info->sectorsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2688) ctl->start = block_group->start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2689) ctl->private = block_group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2690) ctl->op = &free_space_op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2691) INIT_LIST_HEAD(&ctl->trimming_ranges);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2692) mutex_init(&ctl->cache_writeout_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2693)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2694) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2695) * we only want to have 32k of ram per block group for keeping
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2696) * track of free space, and if we pass 1/2 of that we want to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2697) * start converting things over to using bitmaps
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2698) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2699) ctl->extents_thresh = (SZ_32K / 2) / sizeof(struct btrfs_free_space);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2700) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2701)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2702) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2703) * for a given cluster, put all of its extents back into the free
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2704) * space cache. If the block group passed doesn't match the block group
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2705) * pointed to by the cluster, someone else raced in and freed the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2706) * cluster already. In that case, we just return without changing anything
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2707) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2708) static void __btrfs_return_cluster_to_free_space(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2709) struct btrfs_block_group *block_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2710) struct btrfs_free_cluster *cluster)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2711) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2712) struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2713) struct btrfs_free_space *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2714) struct rb_node *node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2715)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2716) spin_lock(&cluster->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2717) if (cluster->block_group != block_group) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2718) spin_unlock(&cluster->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2719) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2720) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2721)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2722) cluster->block_group = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2723) cluster->window_start = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2724) list_del_init(&cluster->block_group_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2726) node = rb_first(&cluster->root);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2727) while (node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2728) bool bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2729)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2730) entry = rb_entry(node, struct btrfs_free_space, offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2731) node = rb_next(&entry->offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2732) rb_erase(&entry->offset_index, &cluster->root);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2733) RB_CLEAR_NODE(&entry->offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2734)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2735) bitmap = (entry->bitmap != NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2736) if (!bitmap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2737) /* Merging treats extents as if they were new */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2738) if (!btrfs_free_space_trimmed(entry)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2739) ctl->discardable_extents[BTRFS_STAT_CURR]--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2740) ctl->discardable_bytes[BTRFS_STAT_CURR] -=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2741) entry->bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2742) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2743)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2744) try_merge_free_space(ctl, entry, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2745) steal_from_bitmap(ctl, entry, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2747) /* As we insert directly, update these statistics */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2748) if (!btrfs_free_space_trimmed(entry)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2749) ctl->discardable_extents[BTRFS_STAT_CURR]++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2750) ctl->discardable_bytes[BTRFS_STAT_CURR] +=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2751) entry->bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2752) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2753) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2754) tree_insert_offset(&ctl->free_space_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2755) entry->offset, &entry->offset_index, bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2756) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2757) cluster->root = RB_ROOT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2758) spin_unlock(&cluster->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2759) btrfs_put_block_group(block_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2760) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2761)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2762) static void __btrfs_remove_free_space_cache_locked(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2763) struct btrfs_free_space_ctl *ctl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2764) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2765) struct btrfs_free_space *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2766) struct rb_node *node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2767)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2768) while ((node = rb_last(&ctl->free_space_offset)) != NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2769) info = rb_entry(node, struct btrfs_free_space, offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2770) if (!info->bitmap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2771) unlink_free_space(ctl, info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2772) kmem_cache_free(btrfs_free_space_cachep, info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2773) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2774) free_bitmap(ctl, info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2775) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2776)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2777) cond_resched_lock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2778) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2779) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2781) void __btrfs_remove_free_space_cache(struct btrfs_free_space_ctl *ctl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2782) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2783) spin_lock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2784) __btrfs_remove_free_space_cache_locked(ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2785) if (ctl->private)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2786) btrfs_discard_update_discardable(ctl->private, ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2787) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2788) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2789)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2790) void btrfs_remove_free_space_cache(struct btrfs_block_group *block_group)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2791) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2792) struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2793) struct btrfs_free_cluster *cluster;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2794) struct list_head *head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2795)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2796) spin_lock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2797) while ((head = block_group->cluster_list.next) !=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2798) &block_group->cluster_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2799) cluster = list_entry(head, struct btrfs_free_cluster,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2800) block_group_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2801)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2802) WARN_ON(cluster->block_group != block_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2803) __btrfs_return_cluster_to_free_space(block_group, cluster);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2804)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2805) cond_resched_lock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2806) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2807) __btrfs_remove_free_space_cache_locked(ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2808) btrfs_discard_update_discardable(block_group, ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2809) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2810)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2811) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2812)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2813) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2814) * btrfs_is_free_space_trimmed - see if everything is trimmed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2815) * @block_group: block_group of interest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2816) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2817) * Walk @block_group's free space rb_tree to determine if everything is trimmed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2818) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2819) bool btrfs_is_free_space_trimmed(struct btrfs_block_group *block_group)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2820) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2821) struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2822) struct btrfs_free_space *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2823) struct rb_node *node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2824) bool ret = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2825)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2826) spin_lock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2827) node = rb_first(&ctl->free_space_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2828)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2829) while (node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2830) info = rb_entry(node, struct btrfs_free_space, offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2831)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2832) if (!btrfs_free_space_trimmed(info)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2833) ret = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2834) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2835) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2836)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2837) node = rb_next(node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2838) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2839)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2840) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2841) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2842) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2843)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2844) u64 btrfs_find_space_for_alloc(struct btrfs_block_group *block_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2845) u64 offset, u64 bytes, u64 empty_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2846) u64 *max_extent_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2847) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2848) struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2849) struct btrfs_discard_ctl *discard_ctl =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2850) &block_group->fs_info->discard_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2851) struct btrfs_free_space *entry = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2852) u64 bytes_search = bytes + empty_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2853) u64 ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2854) u64 align_gap = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2855) u64 align_gap_len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2856) enum btrfs_trim_state align_gap_trim_state = BTRFS_TRIM_STATE_UNTRIMMED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2857)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2858) spin_lock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2859) entry = find_free_space(ctl, &offset, &bytes_search,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2860) block_group->full_stripe_len, max_extent_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2861) if (!entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2862) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2863)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2864) ret = offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2865) if (entry->bitmap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2866) bitmap_clear_bits(ctl, entry, offset, bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2867)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2868) if (!btrfs_free_space_trimmed(entry))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2869) atomic64_add(bytes, &discard_ctl->discard_bytes_saved);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2870)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2871) if (!entry->bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2872) free_bitmap(ctl, entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2873) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2874) unlink_free_space(ctl, entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2875) align_gap_len = offset - entry->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2876) align_gap = entry->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2877) align_gap_trim_state = entry->trim_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2878)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2879) if (!btrfs_free_space_trimmed(entry))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2880) atomic64_add(bytes, &discard_ctl->discard_bytes_saved);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2881)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2882) entry->offset = offset + bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2883) WARN_ON(entry->bytes < bytes + align_gap_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2884)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2885) entry->bytes -= bytes + align_gap_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2886) if (!entry->bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2887) kmem_cache_free(btrfs_free_space_cachep, entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2888) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2889) link_free_space(ctl, entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2890) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2891) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2892) btrfs_discard_update_discardable(block_group, ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2893) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2894)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2895) if (align_gap_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2896) __btrfs_add_free_space(block_group->fs_info, ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2897) align_gap, align_gap_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2898) align_gap_trim_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2899) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2900) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2901)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2902) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2903) * given a cluster, put all of its extents back into the free space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2904) * cache. If a block group is passed, this function will only free
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2905) * a cluster that belongs to the passed block group.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2906) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2907) * Otherwise, it'll get a reference on the block group pointed to by the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2908) * cluster and remove the cluster from it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2909) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2910) void btrfs_return_cluster_to_free_space(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2911) struct btrfs_block_group *block_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2912) struct btrfs_free_cluster *cluster)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2913) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2914) struct btrfs_free_space_ctl *ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2915)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2916) /* first, get a safe pointer to the block group */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2917) spin_lock(&cluster->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2918) if (!block_group) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2919) block_group = cluster->block_group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2920) if (!block_group) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2921) spin_unlock(&cluster->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2922) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2923) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2924) } else if (cluster->block_group != block_group) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2925) /* someone else has already freed it don't redo their work */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2926) spin_unlock(&cluster->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2927) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2928) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2929) btrfs_get_block_group(block_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2930) spin_unlock(&cluster->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2931)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2932) ctl = block_group->free_space_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2933)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2934) /* now return any extents the cluster had on it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2935) spin_lock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2936) __btrfs_return_cluster_to_free_space(block_group, cluster);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2937) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2938)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2939) btrfs_discard_queue_work(&block_group->fs_info->discard_ctl, block_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2940)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2941) /* finally drop our ref */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2942) btrfs_put_block_group(block_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2943) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2944)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2945) static u64 btrfs_alloc_from_bitmap(struct btrfs_block_group *block_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2946) struct btrfs_free_cluster *cluster,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2947) struct btrfs_free_space *entry,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2948) u64 bytes, u64 min_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2949) u64 *max_extent_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2950) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2951) struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2952) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2953) u64 search_start = cluster->window_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2954) u64 search_bytes = bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2955) u64 ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2956)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2957) search_start = min_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2958) search_bytes = bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2959)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2960) err = search_bitmap(ctl, entry, &search_start, &search_bytes, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2961) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2962) *max_extent_size = max(get_max_extent_size(entry),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2963) *max_extent_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2964) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2965) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2966)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2967) ret = search_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2968) __bitmap_clear_bits(ctl, entry, ret, bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2969)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2970) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2971) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2972)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2973) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2974) * given a cluster, try to allocate 'bytes' from it, returns 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2975) * if it couldn't find anything suitably large, or a logical disk offset
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2976) * if things worked out
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2977) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2978) u64 btrfs_alloc_from_cluster(struct btrfs_block_group *block_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2979) struct btrfs_free_cluster *cluster, u64 bytes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2980) u64 min_start, u64 *max_extent_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2981) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2982) struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2983) struct btrfs_discard_ctl *discard_ctl =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2984) &block_group->fs_info->discard_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2985) struct btrfs_free_space *entry = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2986) struct rb_node *node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2987) u64 ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2988)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2989) spin_lock(&cluster->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2990) if (bytes > cluster->max_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2991) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2992)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2993) if (cluster->block_group != block_group)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2994) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2995)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2996) node = rb_first(&cluster->root);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2997) if (!node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2998) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2999)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3000) entry = rb_entry(node, struct btrfs_free_space, offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3001) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3002) if (entry->bytes < bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3003) *max_extent_size = max(get_max_extent_size(entry),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3004) *max_extent_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3005)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3006) if (entry->bytes < bytes ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3007) (!entry->bitmap && entry->offset < min_start)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3008) node = rb_next(&entry->offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3009) if (!node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3010) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3011) entry = rb_entry(node, struct btrfs_free_space,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3012) offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3013) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3014) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3016) if (entry->bitmap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3017) ret = btrfs_alloc_from_bitmap(block_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3018) cluster, entry, bytes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3019) cluster->window_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3020) max_extent_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3021) if (ret == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3022) node = rb_next(&entry->offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3023) if (!node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3024) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3025) entry = rb_entry(node, struct btrfs_free_space,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3026) offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3027) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3028) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3029) cluster->window_start += bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3030) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3031) ret = entry->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3032)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3033) entry->offset += bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3034) entry->bytes -= bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3035) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3036)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3037) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3038) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3039) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3040) spin_unlock(&cluster->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3041)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3042) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3043) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3044)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3045) spin_lock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3046)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3047) if (!btrfs_free_space_trimmed(entry))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3048) atomic64_add(bytes, &discard_ctl->discard_bytes_saved);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3049)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3050) ctl->free_space -= bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3051) if (!entry->bitmap && !btrfs_free_space_trimmed(entry))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3052) ctl->discardable_bytes[BTRFS_STAT_CURR] -= bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3053)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3054) spin_lock(&cluster->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3055) if (entry->bytes == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3056) rb_erase(&entry->offset_index, &cluster->root);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3057) ctl->free_extents--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3058) if (entry->bitmap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3059) kmem_cache_free(btrfs_free_space_bitmap_cachep,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3060) entry->bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3061) ctl->total_bitmaps--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3062) ctl->op->recalc_thresholds(ctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3063) } else if (!btrfs_free_space_trimmed(entry)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3064) ctl->discardable_extents[BTRFS_STAT_CURR]--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3065) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3066) kmem_cache_free(btrfs_free_space_cachep, entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3067) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3068)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3069) spin_unlock(&cluster->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3070) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3072) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3073) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3074)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3075) static int btrfs_bitmap_cluster(struct btrfs_block_group *block_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3076) struct btrfs_free_space *entry,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3077) struct btrfs_free_cluster *cluster,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3078) u64 offset, u64 bytes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3079) u64 cont1_bytes, u64 min_bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3080) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3081) struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3082) unsigned long next_zero;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3083) unsigned long i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3084) unsigned long want_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3085) unsigned long min_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3086) unsigned long found_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3087) unsigned long max_bits = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3088) unsigned long start = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3089) unsigned long total_found = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3090) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3091)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3092) i = offset_to_bit(entry->offset, ctl->unit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3093) max_t(u64, offset, entry->offset));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3094) want_bits = bytes_to_bits(bytes, ctl->unit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3095) min_bits = bytes_to_bits(min_bytes, ctl->unit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3096)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3097) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3098) * Don't bother looking for a cluster in this bitmap if it's heavily
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3099) * fragmented.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3100) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3101) if (entry->max_extent_size &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3102) entry->max_extent_size < cont1_bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3103) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3104) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3105) found_bits = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3106) for_each_set_bit_from(i, entry->bitmap, BITS_PER_BITMAP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3107) next_zero = find_next_zero_bit(entry->bitmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3108) BITS_PER_BITMAP, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3109) if (next_zero - i >= min_bits) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3110) found_bits = next_zero - i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3111) if (found_bits > max_bits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3112) max_bits = found_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3113) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3114) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3115) if (next_zero - i > max_bits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3116) max_bits = next_zero - i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3117) i = next_zero;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3118) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3120) if (!found_bits) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3121) entry->max_extent_size = (u64)max_bits * ctl->unit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3122) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3123) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3125) if (!total_found) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3126) start = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3127) cluster->max_size = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3128) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3130) total_found += found_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3132) if (cluster->max_size < found_bits * ctl->unit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3133) cluster->max_size = found_bits * ctl->unit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3135) if (total_found < want_bits || cluster->max_size < cont1_bytes) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3136) i = next_zero + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3137) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3138) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3140) cluster->window_start = start * ctl->unit + entry->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3141) rb_erase(&entry->offset_index, &ctl->free_space_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3142) ret = tree_insert_offset(&cluster->root, entry->offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3143) &entry->offset_index, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3144) ASSERT(!ret); /* -EEXIST; Logic error */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3146) trace_btrfs_setup_cluster(block_group, cluster,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3147) total_found * ctl->unit, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3148) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3149) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3151) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3152) * This searches the block group for just extents to fill the cluster with.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3153) * Try to find a cluster with at least bytes total bytes, at least one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3154) * extent of cont1_bytes, and other clusters of at least min_bytes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3155) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3156) static noinline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3157) setup_cluster_no_bitmap(struct btrfs_block_group *block_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3158) struct btrfs_free_cluster *cluster,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3159) struct list_head *bitmaps, u64 offset, u64 bytes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3160) u64 cont1_bytes, u64 min_bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3161) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3162) struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3163) struct btrfs_free_space *first = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3164) struct btrfs_free_space *entry = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3165) struct btrfs_free_space *last;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3166) struct rb_node *node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3167) u64 window_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3168) u64 max_extent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3169) u64 total_size = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3171) entry = tree_search_offset(ctl, offset, 0, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3172) if (!entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3173) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3175) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3176) * We don't want bitmaps, so just move along until we find a normal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3177) * extent entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3178) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3179) while (entry->bitmap || entry->bytes < min_bytes) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3180) if (entry->bitmap && list_empty(&entry->list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3181) list_add_tail(&entry->list, bitmaps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3182) node = rb_next(&entry->offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3183) if (!node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3184) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3185) entry = rb_entry(node, struct btrfs_free_space, offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3186) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3188) window_free = entry->bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3189) max_extent = entry->bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3190) first = entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3191) last = entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3193) for (node = rb_next(&entry->offset_index); node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3194) node = rb_next(&entry->offset_index)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3195) entry = rb_entry(node, struct btrfs_free_space, offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3197) if (entry->bitmap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3198) if (list_empty(&entry->list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3199) list_add_tail(&entry->list, bitmaps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3200) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3201) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3203) if (entry->bytes < min_bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3204) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3206) last = entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3207) window_free += entry->bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3208) if (entry->bytes > max_extent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3209) max_extent = entry->bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3210) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3212) if (window_free < bytes || max_extent < cont1_bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3213) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3215) cluster->window_start = first->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3217) node = &first->offset_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3219) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3220) * now we've found our entries, pull them out of the free space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3221) * cache and put them into the cluster rbtree
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3222) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3223) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3224) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3226) entry = rb_entry(node, struct btrfs_free_space, offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3227) node = rb_next(&entry->offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3228) if (entry->bitmap || entry->bytes < min_bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3229) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3231) rb_erase(&entry->offset_index, &ctl->free_space_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3232) ret = tree_insert_offset(&cluster->root, entry->offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3233) &entry->offset_index, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3234) total_size += entry->bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3235) ASSERT(!ret); /* -EEXIST; Logic error */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3236) } while (node && entry != last);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3238) cluster->max_size = max_extent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3239) trace_btrfs_setup_cluster(block_group, cluster, total_size, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3240) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3243) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3244) * This specifically looks for bitmaps that may work in the cluster, we assume
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3245) * that we have already failed to find extents that will work.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3246) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3247) static noinline int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3248) setup_cluster_bitmap(struct btrfs_block_group *block_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3249) struct btrfs_free_cluster *cluster,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3250) struct list_head *bitmaps, u64 offset, u64 bytes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3251) u64 cont1_bytes, u64 min_bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3252) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3253) struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3254) struct btrfs_free_space *entry = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3255) int ret = -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3256) u64 bitmap_offset = offset_to_bitmap(ctl, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3258) if (ctl->total_bitmaps == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3259) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3261) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3262) * The bitmap that covers offset won't be in the list unless offset
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3263) * is just its start offset.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3264) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3265) if (!list_empty(bitmaps))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3266) entry = list_first_entry(bitmaps, struct btrfs_free_space, list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3268) if (!entry || entry->offset != bitmap_offset) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3269) entry = tree_search_offset(ctl, bitmap_offset, 1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3270) if (entry && list_empty(&entry->list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3271) list_add(&entry->list, bitmaps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3272) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3274) list_for_each_entry(entry, bitmaps, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3275) if (entry->bytes < bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3276) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3277) ret = btrfs_bitmap_cluster(block_group, entry, cluster, offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3278) bytes, cont1_bytes, min_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3279) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3280) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3281) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3283) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3284) * The bitmaps list has all the bitmaps that record free space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3285) * starting after offset, so no more search is required.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3286) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3287) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3288) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3290) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3291) * here we try to find a cluster of blocks in a block group. The goal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3292) * is to find at least bytes+empty_size.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3293) * We might not find them all in one contiguous area.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3294) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3295) * returns zero and sets up cluster if things worked out, otherwise
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3296) * it returns -enospc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3297) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3298) int btrfs_find_space_cluster(struct btrfs_block_group *block_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3299) struct btrfs_free_cluster *cluster,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3300) u64 offset, u64 bytes, u64 empty_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3301) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3302) struct btrfs_fs_info *fs_info = block_group->fs_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3303) struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3304) struct btrfs_free_space *entry, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3305) LIST_HEAD(bitmaps);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3306) u64 min_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3307) u64 cont1_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3308) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3310) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3311) * Choose the minimum extent size we'll require for this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3312) * cluster. For SSD_SPREAD, don't allow any fragmentation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3313) * For metadata, allow allocates with smaller extents. For
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3314) * data, keep it dense.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3315) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3316) if (btrfs_test_opt(fs_info, SSD_SPREAD)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3317) cont1_bytes = min_bytes = bytes + empty_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3318) } else if (block_group->flags & BTRFS_BLOCK_GROUP_METADATA) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3319) cont1_bytes = bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3320) min_bytes = fs_info->sectorsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3321) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3322) cont1_bytes = max(bytes, (bytes + empty_size) >> 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3323) min_bytes = fs_info->sectorsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3324) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3326) spin_lock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3328) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3329) * If we know we don't have enough space to make a cluster don't even
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3330) * bother doing all the work to try and find one.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3331) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3332) if (ctl->free_space < bytes) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3333) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3334) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3335) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3337) spin_lock(&cluster->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3339) /* someone already found a cluster, hooray */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3340) if (cluster->block_group) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3341) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3342) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3343) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3345) trace_btrfs_find_cluster(block_group, offset, bytes, empty_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3346) min_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3348) ret = setup_cluster_no_bitmap(block_group, cluster, &bitmaps, offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3349) bytes + empty_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3350) cont1_bytes, min_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3351) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3352) ret = setup_cluster_bitmap(block_group, cluster, &bitmaps,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3353) offset, bytes + empty_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3354) cont1_bytes, min_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3356) /* Clear our temporary list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3357) list_for_each_entry_safe(entry, tmp, &bitmaps, list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3358) list_del_init(&entry->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3360) if (!ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3361) btrfs_get_block_group(block_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3362) list_add_tail(&cluster->block_group_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3363) &block_group->cluster_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3364) cluster->block_group = block_group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3365) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3366) trace_btrfs_failed_cluster_setup(block_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3367) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3368) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3369) spin_unlock(&cluster->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3370) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3372) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3373) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3375) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3376) * simple code to zero out a cluster
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3377) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3378) void btrfs_init_free_cluster(struct btrfs_free_cluster *cluster)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3379) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3380) spin_lock_init(&cluster->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3381) spin_lock_init(&cluster->refill_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3382) cluster->root = RB_ROOT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3383) cluster->max_size = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3384) cluster->fragmented = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3385) INIT_LIST_HEAD(&cluster->block_group_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3386) cluster->block_group = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3387) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3389) static int do_trimming(struct btrfs_block_group *block_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3390) u64 *total_trimmed, u64 start, u64 bytes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3391) u64 reserved_start, u64 reserved_bytes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3392) enum btrfs_trim_state reserved_trim_state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3393) struct btrfs_trim_range *trim_entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3394) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3395) struct btrfs_space_info *space_info = block_group->space_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3396) struct btrfs_fs_info *fs_info = block_group->fs_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3397) struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3398) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3399) int update = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3400) const u64 end = start + bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3401) const u64 reserved_end = reserved_start + reserved_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3402) enum btrfs_trim_state trim_state = BTRFS_TRIM_STATE_UNTRIMMED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3403) u64 trimmed = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3405) spin_lock(&space_info->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3406) spin_lock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3407) if (!block_group->ro) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3408) block_group->reserved += reserved_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3409) space_info->bytes_reserved += reserved_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3410) update = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3411) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3412) spin_unlock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3413) spin_unlock(&space_info->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3415) ret = btrfs_discard_extent(fs_info, start, bytes, &trimmed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3416) if (!ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3417) *total_trimmed += trimmed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3418) trim_state = BTRFS_TRIM_STATE_TRIMMED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3419) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3421) mutex_lock(&ctl->cache_writeout_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3422) if (reserved_start < start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3423) __btrfs_add_free_space(fs_info, ctl, reserved_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3424) start - reserved_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3425) reserved_trim_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3426) if (start + bytes < reserved_start + reserved_bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3427) __btrfs_add_free_space(fs_info, ctl, end, reserved_end - end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3428) reserved_trim_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3429) __btrfs_add_free_space(fs_info, ctl, start, bytes, trim_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3430) list_del(&trim_entry->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3431) mutex_unlock(&ctl->cache_writeout_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3433) if (update) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3434) spin_lock(&space_info->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3435) spin_lock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3436) if (block_group->ro)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3437) space_info->bytes_readonly += reserved_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3438) block_group->reserved -= reserved_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3439) space_info->bytes_reserved -= reserved_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3440) spin_unlock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3441) spin_unlock(&space_info->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3442) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3444) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3445) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3447) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3448) * If @async is set, then we will trim 1 region and return.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3449) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3450) static int trim_no_bitmap(struct btrfs_block_group *block_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3451) u64 *total_trimmed, u64 start, u64 end, u64 minlen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3452) bool async)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3453) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3454) struct btrfs_discard_ctl *discard_ctl =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3455) &block_group->fs_info->discard_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3456) struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3457) struct btrfs_free_space *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3458) struct rb_node *node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3459) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3460) u64 extent_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3461) u64 extent_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3462) enum btrfs_trim_state extent_trim_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3463) u64 bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3464) const u64 max_discard_size = READ_ONCE(discard_ctl->max_discard_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3466) while (start < end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3467) struct btrfs_trim_range trim_entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3469) mutex_lock(&ctl->cache_writeout_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3470) spin_lock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3472) if (ctl->free_space < minlen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3473) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3475) entry = tree_search_offset(ctl, start, 0, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3476) if (!entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3477) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3479) /* Skip bitmaps and if async, already trimmed entries */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3480) while (entry->bitmap ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3481) (async && btrfs_free_space_trimmed(entry))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3482) node = rb_next(&entry->offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3483) if (!node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3484) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3485) entry = rb_entry(node, struct btrfs_free_space,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3486) offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3487) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3489) if (entry->offset >= end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3490) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3492) extent_start = entry->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3493) extent_bytes = entry->bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3494) extent_trim_state = entry->trim_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3495) if (async) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3496) start = entry->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3497) bytes = entry->bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3498) if (bytes < minlen) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3499) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3500) mutex_unlock(&ctl->cache_writeout_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3501) goto next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3502) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3503) unlink_free_space(ctl, entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3504) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3505) * Let bytes = BTRFS_MAX_DISCARD_SIZE + X.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3506) * If X < BTRFS_ASYNC_DISCARD_MIN_FILTER, we won't trim
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3507) * X when we come back around. So trim it now.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3508) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3509) if (max_discard_size &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3510) bytes >= (max_discard_size +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3511) BTRFS_ASYNC_DISCARD_MIN_FILTER)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3512) bytes = max_discard_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3513) extent_bytes = max_discard_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3514) entry->offset += max_discard_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3515) entry->bytes -= max_discard_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3516) link_free_space(ctl, entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3517) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3518) kmem_cache_free(btrfs_free_space_cachep, entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3519) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3520) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3521) start = max(start, extent_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3522) bytes = min(extent_start + extent_bytes, end) - start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3523) if (bytes < minlen) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3524) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3525) mutex_unlock(&ctl->cache_writeout_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3526) goto next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3527) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3529) unlink_free_space(ctl, entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3530) kmem_cache_free(btrfs_free_space_cachep, entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3531) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3533) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3534) trim_entry.start = extent_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3535) trim_entry.bytes = extent_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3536) list_add_tail(&trim_entry.list, &ctl->trimming_ranges);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3537) mutex_unlock(&ctl->cache_writeout_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3539) ret = do_trimming(block_group, total_trimmed, start, bytes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3540) extent_start, extent_bytes, extent_trim_state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3541) &trim_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3542) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3543) block_group->discard_cursor = start + bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3544) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3545) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3546) next:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3547) start += bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3548) block_group->discard_cursor = start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3549) if (async && *total_trimmed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3550) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3552) if (fatal_signal_pending(current)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3553) ret = -ERESTARTSYS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3554) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3555) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3557) cond_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3558) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3560) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3561)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3562) out_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3563) block_group->discard_cursor = btrfs_block_group_end(block_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3564) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3565) mutex_unlock(&ctl->cache_writeout_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3567) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3568) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3570) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3571) * If we break out of trimming a bitmap prematurely, we should reset the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3572) * trimming bit. In a rather contrieved case, it's possible to race here so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3573) * reset the state to BTRFS_TRIM_STATE_UNTRIMMED.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3574) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3575) * start = start of bitmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3576) * end = near end of bitmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3577) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3578) * Thread 1: Thread 2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3579) * trim_bitmaps(start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3580) * trim_bitmaps(end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3581) * end_trimming_bitmap()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3582) * reset_trimming_bitmap()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3583) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3584) static void reset_trimming_bitmap(struct btrfs_free_space_ctl *ctl, u64 offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3585) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3586) struct btrfs_free_space *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3588) spin_lock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3589) entry = tree_search_offset(ctl, offset, 1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3590) if (entry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3591) if (btrfs_free_space_trimmed(entry)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3592) ctl->discardable_extents[BTRFS_STAT_CURR] +=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3593) entry->bitmap_extents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3594) ctl->discardable_bytes[BTRFS_STAT_CURR] += entry->bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3595) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3596) entry->trim_state = BTRFS_TRIM_STATE_UNTRIMMED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3597) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3599) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3600) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3602) static void end_trimming_bitmap(struct btrfs_free_space_ctl *ctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3603) struct btrfs_free_space *entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3604) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3605) if (btrfs_free_space_trimming_bitmap(entry)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3606) entry->trim_state = BTRFS_TRIM_STATE_TRIMMED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3607) ctl->discardable_extents[BTRFS_STAT_CURR] -=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3608) entry->bitmap_extents;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3609) ctl->discardable_bytes[BTRFS_STAT_CURR] -= entry->bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3610) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3611) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3612)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3613) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3614) * If @async is set, then we will trim 1 region and return.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3615) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3616) static int trim_bitmaps(struct btrfs_block_group *block_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3617) u64 *total_trimmed, u64 start, u64 end, u64 minlen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3618) u64 maxlen, bool async)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3619) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3620) struct btrfs_discard_ctl *discard_ctl =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3621) &block_group->fs_info->discard_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3622) struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3623) struct btrfs_free_space *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3624) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3625) int ret2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3626) u64 bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3627) u64 offset = offset_to_bitmap(ctl, start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3628) const u64 max_discard_size = READ_ONCE(discard_ctl->max_discard_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3630) while (offset < end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3631) bool next_bitmap = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3632) struct btrfs_trim_range trim_entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3634) mutex_lock(&ctl->cache_writeout_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3635) spin_lock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3636)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3637) if (ctl->free_space < minlen) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3638) block_group->discard_cursor =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3639) btrfs_block_group_end(block_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3640) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3641) mutex_unlock(&ctl->cache_writeout_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3642) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3643) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3645) entry = tree_search_offset(ctl, offset, 1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3646) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3647) * Bitmaps are marked trimmed lossily now to prevent constant
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3648) * discarding of the same bitmap (the reason why we are bound
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3649) * by the filters). So, retrim the block group bitmaps when we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3650) * are preparing to punt to the unused_bgs list. This uses
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3651) * @minlen to determine if we are in BTRFS_DISCARD_INDEX_UNUSED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3652) * which is the only discard index which sets minlen to 0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3653) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3654) if (!entry || (async && minlen && start == offset &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3655) btrfs_free_space_trimmed(entry))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3656) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3657) mutex_unlock(&ctl->cache_writeout_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3658) next_bitmap = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3659) goto next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3660) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3661)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3662) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3663) * Async discard bitmap trimming begins at by setting the start
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3664) * to be key.objectid and the offset_to_bitmap() aligns to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3665) * start of the bitmap. This lets us know we are fully
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3666) * scanning the bitmap rather than only some portion of it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3667) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3668) if (start == offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3669) entry->trim_state = BTRFS_TRIM_STATE_TRIMMING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3670)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3671) bytes = minlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3672) ret2 = search_bitmap(ctl, entry, &start, &bytes, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3673) if (ret2 || start >= end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3674) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3675) * We lossily consider a bitmap trimmed if we only skip
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3676) * over regions <= BTRFS_ASYNC_DISCARD_MIN_FILTER.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3677) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3678) if (ret2 && minlen <= BTRFS_ASYNC_DISCARD_MIN_FILTER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3679) end_trimming_bitmap(ctl, entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3680) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3681) entry->trim_state = BTRFS_TRIM_STATE_UNTRIMMED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3682) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3683) mutex_unlock(&ctl->cache_writeout_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3684) next_bitmap = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3685) goto next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3686) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3688) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3689) * We already trimmed a region, but are using the locking above
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3690) * to reset the trim_state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3691) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3692) if (async && *total_trimmed) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3693) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3694) mutex_unlock(&ctl->cache_writeout_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3695) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3696) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3697)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3698) bytes = min(bytes, end - start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3699) if (bytes < minlen || (async && maxlen && bytes > maxlen)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3700) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3701) mutex_unlock(&ctl->cache_writeout_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3702) goto next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3703) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3704)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3705) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3706) * Let bytes = BTRFS_MAX_DISCARD_SIZE + X.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3707) * If X < @minlen, we won't trim X when we come back around.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3708) * So trim it now. We differ here from trimming extents as we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3709) * don't keep individual state per bit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3710) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3711) if (async &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3712) max_discard_size &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3713) bytes > (max_discard_size + minlen))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3714) bytes = max_discard_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3715)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3716) bitmap_clear_bits(ctl, entry, start, bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3717) if (entry->bytes == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3718) free_bitmap(ctl, entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3719)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3720) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3721) trim_entry.start = start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3722) trim_entry.bytes = bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3723) list_add_tail(&trim_entry.list, &ctl->trimming_ranges);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3724) mutex_unlock(&ctl->cache_writeout_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3726) ret = do_trimming(block_group, total_trimmed, start, bytes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3727) start, bytes, 0, &trim_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3728) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3729) reset_trimming_bitmap(ctl, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3730) block_group->discard_cursor =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3731) btrfs_block_group_end(block_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3732) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3733) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3734) next:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3735) if (next_bitmap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3736) offset += BITS_PER_BITMAP * ctl->unit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3737) start = offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3738) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3739) start += bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3740) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3741) block_group->discard_cursor = start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3743) if (fatal_signal_pending(current)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3744) if (start != offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3745) reset_trimming_bitmap(ctl, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3746) ret = -ERESTARTSYS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3747) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3748) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3749)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3750) cond_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3751) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3753) if (offset >= end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3754) block_group->discard_cursor = end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3755)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3756) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3757) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3758) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3759)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3760) int btrfs_trim_block_group(struct btrfs_block_group *block_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3761) u64 *trimmed, u64 start, u64 end, u64 minlen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3762) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3763) struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3764) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3765) u64 rem = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3767) *trimmed = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3769) spin_lock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3770) if (block_group->removed) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3771) spin_unlock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3772) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3773) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3774) btrfs_freeze_block_group(block_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3775) spin_unlock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3776)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3777) ret = trim_no_bitmap(block_group, trimmed, start, end, minlen, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3778) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3779) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3781) ret = trim_bitmaps(block_group, trimmed, start, end, minlen, 0, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3782) div64_u64_rem(end, BITS_PER_BITMAP * ctl->unit, &rem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3783) /* If we ended in the middle of a bitmap, reset the trimming flag */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3784) if (rem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3785) reset_trimming_bitmap(ctl, offset_to_bitmap(ctl, end));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3786) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3787) btrfs_unfreeze_block_group(block_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3788) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3789) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3790)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3791) int btrfs_trim_block_group_extents(struct btrfs_block_group *block_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3792) u64 *trimmed, u64 start, u64 end, u64 minlen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3793) bool async)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3794) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3795) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3796)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3797) *trimmed = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3798)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3799) spin_lock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3800) if (block_group->removed) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3801) spin_unlock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3802) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3803) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3804) btrfs_freeze_block_group(block_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3805) spin_unlock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3806)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3807) ret = trim_no_bitmap(block_group, trimmed, start, end, minlen, async);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3808) btrfs_unfreeze_block_group(block_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3809)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3810) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3811) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3812)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3813) int btrfs_trim_block_group_bitmaps(struct btrfs_block_group *block_group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3814) u64 *trimmed, u64 start, u64 end, u64 minlen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3815) u64 maxlen, bool async)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3816) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3817) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3818)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3819) *trimmed = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3820)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3821) spin_lock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3822) if (block_group->removed) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3823) spin_unlock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3824) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3825) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3826) btrfs_freeze_block_group(block_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3827) spin_unlock(&block_group->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3828)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3829) ret = trim_bitmaps(block_group, trimmed, start, end, minlen, maxlen,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3830) async);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3831)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3832) btrfs_unfreeze_block_group(block_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3833)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3834) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3835) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3836)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3837) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3838) * Find the left-most item in the cache tree, and then return the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3839) * smallest inode number in the item.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3840) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3841) * Note: the returned inode number may not be the smallest one in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3842) * the tree, if the left-most item is a bitmap.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3843) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3844) u64 btrfs_find_ino_for_alloc(struct btrfs_root *fs_root)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3845) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3846) struct btrfs_free_space_ctl *ctl = fs_root->free_ino_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3847) struct btrfs_free_space *entry = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3848) u64 ino = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3849)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3850) spin_lock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3852) if (RB_EMPTY_ROOT(&ctl->free_space_offset))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3853) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3854)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3855) entry = rb_entry(rb_first(&ctl->free_space_offset),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3856) struct btrfs_free_space, offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3857)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3858) if (!entry->bitmap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3859) ino = entry->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3860)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3861) unlink_free_space(ctl, entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3862) entry->offset++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3863) entry->bytes--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3864) if (!entry->bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3865) kmem_cache_free(btrfs_free_space_cachep, entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3866) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3867) link_free_space(ctl, entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3868) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3869) u64 offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3870) u64 count = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3871) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3872)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3873) ret = search_bitmap(ctl, entry, &offset, &count, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3874) /* Logic error; Should be empty if it can't find anything */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3875) ASSERT(!ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3876)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3877) ino = offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3878) bitmap_clear_bits(ctl, entry, offset, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3879) if (entry->bytes == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3880) free_bitmap(ctl, entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3881) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3882) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3883) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3884)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3885) return ino;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3886) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3887)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3888) struct inode *lookup_free_ino_inode(struct btrfs_root *root,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3889) struct btrfs_path *path)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3890) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3891) struct inode *inode = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3892)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3893) spin_lock(&root->ino_cache_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3894) if (root->ino_cache_inode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3895) inode = igrab(root->ino_cache_inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3896) spin_unlock(&root->ino_cache_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3897) if (inode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3898) return inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3899)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3900) inode = __lookup_free_space_inode(root, path, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3901) if (IS_ERR(inode))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3902) return inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3904) spin_lock(&root->ino_cache_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3905) if (!btrfs_fs_closing(root->fs_info))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3906) root->ino_cache_inode = igrab(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3907) spin_unlock(&root->ino_cache_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3908)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3909) return inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3910) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3911)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3912) int create_free_ino_inode(struct btrfs_root *root,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3913) struct btrfs_trans_handle *trans,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3914) struct btrfs_path *path)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3915) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3916) return __create_free_space_inode(root, trans, path,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3917) BTRFS_FREE_INO_OBJECTID, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3918) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3919)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3920) int load_free_ino_cache(struct btrfs_fs_info *fs_info, struct btrfs_root *root)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3921) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3922) struct btrfs_free_space_ctl *ctl = root->free_ino_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3923) struct btrfs_path *path;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3924) struct inode *inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3925) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3926) u64 root_gen = btrfs_root_generation(&root->root_item);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3927)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3928) if (!btrfs_test_opt(fs_info, INODE_MAP_CACHE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3929) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3930)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3931) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3932) * If we're unmounting then just return, since this does a search on the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3933) * normal root and not the commit root and we could deadlock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3934) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3935) if (btrfs_fs_closing(fs_info))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3936) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3937)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3938) path = btrfs_alloc_path();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3939) if (!path)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3940) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3941)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3942) inode = lookup_free_ino_inode(root, path);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3943) if (IS_ERR(inode))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3944) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3945)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3946) if (root_gen != BTRFS_I(inode)->generation)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3947) goto out_put;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3948)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3949) ret = __load_free_space_cache(root, inode, ctl, path, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3950)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3951) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3952) btrfs_err(fs_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3953) "failed to load free ino cache for root %llu",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3954) root->root_key.objectid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3955) out_put:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3956) iput(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3957) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3958) btrfs_free_path(path);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3959) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3960) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3961)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3962) int btrfs_write_out_ino_cache(struct btrfs_root *root,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3963) struct btrfs_trans_handle *trans,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3964) struct btrfs_path *path,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3965) struct inode *inode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3966) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3967) struct btrfs_fs_info *fs_info = root->fs_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3968) struct btrfs_free_space_ctl *ctl = root->free_ino_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3969) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3970) struct btrfs_io_ctl io_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3971) bool release_metadata = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3972)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3973) if (!btrfs_test_opt(fs_info, INODE_MAP_CACHE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3974) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3975)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3976) memset(&io_ctl, 0, sizeof(io_ctl));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3977) ret = __btrfs_write_out_cache(root, inode, ctl, NULL, &io_ctl, trans);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3978) if (!ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3979) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3980) * At this point writepages() didn't error out, so our metadata
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3981) * reservation is released when the writeback finishes, at
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3982) * inode.c:btrfs_finish_ordered_io(), regardless of it finishing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3983) * with or without an error.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3984) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3985) release_metadata = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3986) ret = btrfs_wait_cache_io_root(root, trans, &io_ctl, path);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3987) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3988)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3989) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3990) if (release_metadata)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3991) btrfs_delalloc_release_metadata(BTRFS_I(inode),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3992) inode->i_size, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3993) btrfs_debug(fs_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3994) "failed to write free ino cache for root %llu error %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3995) root->root_key.objectid, ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3996) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3997)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3998) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3999) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4001) #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4002) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4003) * Use this if you need to make a bitmap or extent entry specifically, it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4004) * doesn't do any of the merging that add_free_space does, this acts a lot like
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4005) * how the free space cache loading stuff works, so you can get really weird
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4006) * configurations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4007) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4008) int test_add_free_space_entry(struct btrfs_block_group *cache,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4009) u64 offset, u64 bytes, bool bitmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4010) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4011) struct btrfs_free_space_ctl *ctl = cache->free_space_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4012) struct btrfs_free_space *info = NULL, *bitmap_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4013) void *map = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4014) enum btrfs_trim_state trim_state = BTRFS_TRIM_STATE_TRIMMED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4015) u64 bytes_added;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4016) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4017)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4018) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4019) if (!info) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4020) info = kmem_cache_zalloc(btrfs_free_space_cachep, GFP_NOFS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4021) if (!info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4022) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4023) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4024)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4025) if (!bitmap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4026) spin_lock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4027) info->offset = offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4028) info->bytes = bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4029) info->max_extent_size = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4030) ret = link_free_space(ctl, info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4031) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4032) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4033) kmem_cache_free(btrfs_free_space_cachep, info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4034) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4035) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4036)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4037) if (!map) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4038) map = kmem_cache_zalloc(btrfs_free_space_bitmap_cachep, GFP_NOFS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4039) if (!map) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4040) kmem_cache_free(btrfs_free_space_cachep, info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4041) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4042) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4043) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4044)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4045) spin_lock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4046) bitmap_info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4047) 1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4048) if (!bitmap_info) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4049) info->bitmap = map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4050) map = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4051) add_new_bitmap(ctl, info, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4052) bitmap_info = info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4053) info = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4054) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4055)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4056) bytes_added = add_bytes_to_bitmap(ctl, bitmap_info, offset, bytes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4057) trim_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4058)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4059) bytes -= bytes_added;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4060) offset += bytes_added;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4061) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4062)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4063) if (bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4064) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4065)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4066) if (info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4067) kmem_cache_free(btrfs_free_space_cachep, info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4068) if (map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4069) kmem_cache_free(btrfs_free_space_bitmap_cachep, map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4070) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4071) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4072)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4073) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4074) * Checks to see if the given range is in the free space cache. This is really
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4075) * just used to check the absence of space, so if there is free space in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4076) * range at all we will return 1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4077) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4078) int test_check_exists(struct btrfs_block_group *cache,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4079) u64 offset, u64 bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4080) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4081) struct btrfs_free_space_ctl *ctl = cache->free_space_ctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4082) struct btrfs_free_space *info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4083) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4084)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4085) spin_lock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4086) info = tree_search_offset(ctl, offset, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4087) if (!info) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4088) info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4089) 1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4090) if (!info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4091) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4092) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4093)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4094) have_info:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4095) if (info->bitmap) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4096) u64 bit_off, bit_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4097) struct rb_node *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4098) struct btrfs_free_space *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4099)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4100) bit_off = offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4101) bit_bytes = ctl->unit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4102) ret = search_bitmap(ctl, info, &bit_off, &bit_bytes, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4103) if (!ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4104) if (bit_off == offset) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4105) ret = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4106) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4107) } else if (bit_off > offset &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4108) offset + bytes > bit_off) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4109) ret = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4110) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4111) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4112) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4114) n = rb_prev(&info->offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4115) while (n) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4116) tmp = rb_entry(n, struct btrfs_free_space,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4117) offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4118) if (tmp->offset + tmp->bytes < offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4119) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4120) if (offset + bytes < tmp->offset) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4121) n = rb_prev(&tmp->offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4122) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4123) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4124) info = tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4125) goto have_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4126) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4128) n = rb_next(&info->offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4129) while (n) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4130) tmp = rb_entry(n, struct btrfs_free_space,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4131) offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4132) if (offset + bytes < tmp->offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4133) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4134) if (tmp->offset + tmp->bytes < offset) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4135) n = rb_next(&tmp->offset_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4136) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4137) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4138) info = tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4139) goto have_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4140) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4142) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4143) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4144) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4146) if (info->offset == offset) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4147) ret = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4148) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4149) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4151) if (offset > info->offset && offset < info->offset + info->bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4152) ret = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4153) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4154) spin_unlock(&ctl->tree_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4155) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4156) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4157) #endif /* CONFIG_BTRFS_FS_RUN_SANITY_TESTS */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags