^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * fs/f2fs/segment.c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (c) 2012 Samsung Electronics Co., Ltd.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * http://www.samsung.com/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/f2fs_fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/bio.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/blkdev.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/prefetch.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/kthread.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/swap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/timer.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/freezer.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/sched/signal.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include "f2fs.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include "segment.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include "node.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include "gc.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <trace/events/f2fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #define __reverse_ffz(x) __reverse_ffs(~(x))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) static struct kmem_cache *discard_entry_slab;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) static struct kmem_cache *discard_cmd_slab;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) static struct kmem_cache *sit_entry_set_slab;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) static struct kmem_cache *inmem_entry_slab;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) static unsigned long __reverse_ulong(unsigned char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) unsigned long tmp = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) int shift = 24, idx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #if BITS_PER_LONG == 64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) shift = 56;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) while (shift >= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) tmp |= (unsigned long)str[idx++] << shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) shift -= BITS_PER_BYTE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) return tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) * MSB and LSB are reversed in a byte by f2fs_set_bit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) static inline unsigned long __reverse_ffs(unsigned long word)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) int num = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #if BITS_PER_LONG == 64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) if ((word & 0xffffffff00000000UL) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) num += 32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) word >>= 32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) if ((word & 0xffff0000) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) num += 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) word >>= 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) if ((word & 0xff00) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) num += 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) word >>= 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) if ((word & 0xf0) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) num += 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) word >>= 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) if ((word & 0xc) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) num += 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) word >>= 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) if ((word & 0x2) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) num += 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) return num;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) * f2fs_set_bit makes MSB and LSB reversed in a byte.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) * @size must be integral times of unsigned long.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) * Example:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) * MSB <--> LSB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) * f2fs_set_bit(0, bitmap) => 1000 0000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) * f2fs_set_bit(7, bitmap) => 0000 0001
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) static unsigned long __find_rev_next_bit(const unsigned long *addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) unsigned long size, unsigned long offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) const unsigned long *p = addr + BIT_WORD(offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) unsigned long result = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) unsigned long tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) if (offset >= size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) return size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) size -= (offset & ~(BITS_PER_LONG - 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) offset %= BITS_PER_LONG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) if (*p == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) goto pass;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) tmp = __reverse_ulong((unsigned char *)p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) tmp &= ~0UL >> offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) if (size < BITS_PER_LONG)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) tmp &= (~0UL << (BITS_PER_LONG - size));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) if (tmp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) goto found;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) pass:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) if (size <= BITS_PER_LONG)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) size -= BITS_PER_LONG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) p++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) return result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) found:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) return result - size + __reverse_ffs(tmp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) static unsigned long __find_rev_next_zero_bit(const unsigned long *addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) unsigned long size, unsigned long offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) const unsigned long *p = addr + BIT_WORD(offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) unsigned long result = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) unsigned long tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) if (offset >= size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) return size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) size -= (offset & ~(BITS_PER_LONG - 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) offset %= BITS_PER_LONG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) if (*p == ~0UL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) goto pass;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) tmp = __reverse_ulong((unsigned char *)p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) if (offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) tmp |= ~0UL << (BITS_PER_LONG - offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) if (size < BITS_PER_LONG)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) tmp |= ~0UL >> size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) if (tmp != ~0UL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) goto found;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) pass:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) if (size <= BITS_PER_LONG)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) size -= BITS_PER_LONG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) p++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) return result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) found:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) return result - size + __reverse_ffz(tmp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) bool f2fs_need_SSR(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) if (f2fs_lfs_mode(sbi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) if (sbi->gc_mode == GC_URGENT_HIGH)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) return free_sections(sbi) <= (node_secs + 2 * dent_secs + imeta_secs +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) SM_I(sbi)->min_ssr_sections + reserved_sections(sbi));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) void f2fs_register_inmem_page(struct inode *inode, struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) struct inmem_pages *new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) set_page_private_atomic(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) new = f2fs_kmem_cache_alloc(inmem_entry_slab, GFP_NOFS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) /* add atomic page indices to the list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) new->page = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) INIT_LIST_HEAD(&new->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) /* increase reference count with clean state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) get_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) mutex_lock(&F2FS_I(inode)->inmem_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) list_add_tail(&new->list, &F2FS_I(inode)->inmem_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) inc_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) mutex_unlock(&F2FS_I(inode)->inmem_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) trace_f2fs_register_inmem_page(page, INMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) static int __revoke_inmem_pages(struct inode *inode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) struct list_head *head, bool drop, bool recover,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) bool trylock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) struct inmem_pages *cur, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) list_for_each_entry_safe(cur, tmp, head, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) struct page *page = cur->page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) if (drop)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) trace_f2fs_commit_inmem_page(page, INMEM_DROP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) if (trylock) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) * to avoid deadlock in between page lock and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) * inmem_lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) if (!trylock_page(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) lock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) f2fs_wait_on_page_writeback(page, DATA, true, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) if (recover) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) struct dnode_of_data dn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) struct node_info ni;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) trace_f2fs_commit_inmem_page(page, INMEM_REVOKE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) retry:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) set_new_dnode(&dn, inode, NULL, NULL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) err = f2fs_get_dnode_of_data(&dn, page->index,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) LOOKUP_NODE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) if (err == -ENOMEM) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) congestion_wait(BLK_RW_ASYNC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) DEFAULT_IO_TIMEOUT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) cond_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) err = -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) goto next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) err = f2fs_get_node_info(sbi, dn.nid, &ni, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) f2fs_put_dnode(&dn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) if (cur->old_addr == NEW_ADDR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) f2fs_invalidate_blocks(sbi, dn.data_blkaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) f2fs_update_data_blkaddr(&dn, NEW_ADDR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) f2fs_replace_block(sbi, &dn, dn.data_blkaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) cur->old_addr, ni.version, true, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) f2fs_put_dnode(&dn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) next:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) /* we don't need to invalidate this in the sccessful status */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) if (drop || recover) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) ClearPageUptodate(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) clear_page_private_gcing(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) detach_page_private(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) set_page_private(page, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) f2fs_put_page(page, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) list_del(&cur->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) kmem_cache_free(inmem_entry_slab, cur);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) dec_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) void f2fs_drop_inmem_pages_all(struct f2fs_sb_info *sbi, bool gc_failure)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) struct list_head *head = &sbi->inode_list[ATOMIC_FILE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) struct inode *inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) struct f2fs_inode_info *fi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) unsigned int count = sbi->atomic_files;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) unsigned int looped = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) next:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) spin_lock(&sbi->inode_lock[ATOMIC_FILE]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) if (list_empty(head)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) fi = list_first_entry(head, struct f2fs_inode_info, inmem_ilist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) inode = igrab(&fi->vfs_inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) if (inode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) list_move_tail(&fi->inmem_ilist, head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) if (inode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) if (gc_failure) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) if (!fi->i_gc_failures[GC_FAILURE_ATOMIC])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) goto skip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) set_inode_flag(inode, FI_ATOMIC_REVOKE_REQUEST);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) f2fs_drop_inmem_pages(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) skip:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) iput(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) congestion_wait(BLK_RW_ASYNC, DEFAULT_IO_TIMEOUT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) cond_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) if (gc_failure) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) if (++looped >= count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) goto next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) void f2fs_drop_inmem_pages(struct inode *inode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) struct f2fs_inode_info *fi = F2FS_I(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) mutex_lock(&fi->inmem_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) if (list_empty(&fi->inmem_pages)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) fi->i_gc_failures[GC_FAILURE_ATOMIC] = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) spin_lock(&sbi->inode_lock[ATOMIC_FILE]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) if (!list_empty(&fi->inmem_ilist))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) list_del_init(&fi->inmem_ilist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) if (f2fs_is_atomic_file(inode)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) clear_inode_flag(inode, FI_ATOMIC_FILE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) sbi->atomic_files--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) mutex_unlock(&fi->inmem_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) __revoke_inmem_pages(inode, &fi->inmem_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) true, false, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) mutex_unlock(&fi->inmem_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) } while (1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) void f2fs_drop_inmem_page(struct inode *inode, struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) struct f2fs_inode_info *fi = F2FS_I(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) struct list_head *head = &fi->inmem_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) struct inmem_pages *cur = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) f2fs_bug_on(sbi, !page_private_atomic(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) mutex_lock(&fi->inmem_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) list_for_each_entry(cur, head, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) if (cur->page == page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) f2fs_bug_on(sbi, list_empty(head) || cur->page != page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) list_del(&cur->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) mutex_unlock(&fi->inmem_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) dec_page_count(sbi, F2FS_INMEM_PAGES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) kmem_cache_free(inmem_entry_slab, cur);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) ClearPageUptodate(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) clear_page_private_atomic(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) f2fs_put_page(page, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) detach_page_private(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) set_page_private(page, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) trace_f2fs_commit_inmem_page(page, INMEM_INVALIDATE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) static int __f2fs_commit_inmem_pages(struct inode *inode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) struct f2fs_inode_info *fi = F2FS_I(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) struct inmem_pages *cur, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) struct f2fs_io_info fio = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) .sbi = sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) .ino = inode->i_ino,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) .type = DATA,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) .op = REQ_OP_WRITE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) .op_flags = REQ_SYNC | REQ_PRIO,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) .io_type = FS_DATA_IO,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) struct list_head revoke_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) bool submit_bio = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) INIT_LIST_HEAD(&revoke_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) list_for_each_entry_safe(cur, tmp, &fi->inmem_pages, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) struct page *page = cur->page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) lock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) if (page->mapping == inode->i_mapping) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) trace_f2fs_commit_inmem_page(page, INMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) f2fs_wait_on_page_writeback(page, DATA, true, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) set_page_dirty(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) if (clear_page_dirty_for_io(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) inode_dec_dirty_pages(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) f2fs_remove_dirty_inode(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) retry:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) fio.page = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) fio.old_blkaddr = NULL_ADDR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) fio.encrypted_page = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) fio.need_lock = LOCK_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) err = f2fs_do_write_data_page(&fio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) if (err == -ENOMEM) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) congestion_wait(BLK_RW_ASYNC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) DEFAULT_IO_TIMEOUT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) cond_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) /* record old blkaddr for revoking */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) cur->old_addr = fio.old_blkaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) submit_bio = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) list_move_tail(&cur->list, &revoke_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) if (submit_bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) f2fs_submit_merged_write_cond(sbi, inode, NULL, 0, DATA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) * try to revoke all committed pages, but still we could fail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) * due to no memory or other reason, if that happened, EAGAIN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) * will be returned, which means in such case, transaction is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) * already not integrity, caller should use journal to do the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) * recovery or rewrite & commit last transaction. For other
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) * error number, revoking was done by filesystem itself.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) err = __revoke_inmem_pages(inode, &revoke_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) false, true, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) /* drop all uncommitted pages */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) __revoke_inmem_pages(inode, &fi->inmem_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) true, false, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) __revoke_inmem_pages(inode, &revoke_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) false, false, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) int f2fs_commit_inmem_pages(struct inode *inode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) struct f2fs_inode_info *fi = F2FS_I(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) f2fs_balance_fs(sbi, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) f2fs_down_write(&fi->i_gc_rwsem[WRITE]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) f2fs_lock_op(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) set_inode_flag(inode, FI_ATOMIC_COMMIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) mutex_lock(&fi->inmem_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) err = __f2fs_commit_inmem_pages(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) mutex_unlock(&fi->inmem_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) clear_inode_flag(inode, FI_ATOMIC_COMMIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) f2fs_unlock_op(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) f2fs_up_write(&fi->i_gc_rwsem[WRITE]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) * This function balances dirty node and dentry pages.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) * In addition, it controls garbage collection.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) if (time_to_inject(sbi, FAULT_CHECKPOINT)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) f2fs_show_injection_info(sbi, FAULT_CHECKPOINT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) f2fs_stop_checkpoint(sbi, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) /* balance_fs_bg is able to be pending */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) if (need && excess_cached_nats(sbi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) f2fs_balance_fs_bg(sbi, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) if (!f2fs_is_checkpoint_ready(sbi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) * We should do GC or end up with checkpoint, if there are so many dirty
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) * dir/node pages without enough free segments.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) if (has_not_enough_free_secs(sbi, 0, 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) if (test_opt(sbi, GC_MERGE) && sbi->gc_thread &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) sbi->gc_thread->f2fs_gc_task) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) DEFINE_WAIT(wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) prepare_to_wait(&sbi->gc_thread->fggc_wq, &wait,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) TASK_UNINTERRUPTIBLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) wake_up(&sbi->gc_thread->gc_wait_queue_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) io_schedule();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) finish_wait(&sbi->gc_thread->fggc_wq, &wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) f2fs_down_write(&sbi->gc_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) f2fs_gc(sbi, false, false, false, NULL_SEGNO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi, bool from_bg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) /* try to shrink extent cache when there is no enough memory */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) if (!f2fs_available_free_memory(sbi, EXTENT_CACHE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) f2fs_shrink_extent_tree(sbi, EXTENT_CACHE_SHRINK_NUMBER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) /* check the # of cached NAT entries */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) if (!f2fs_available_free_memory(sbi, NAT_ENTRIES))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) f2fs_try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) if (!f2fs_available_free_memory(sbi, FREE_NIDS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) f2fs_try_to_free_nids(sbi, MAX_FREE_NIDS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) f2fs_build_free_nids(sbi, false, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) if (excess_dirty_nats(sbi) || excess_dirty_nodes(sbi) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) excess_prefree_segs(sbi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) goto do_sync;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) /* there is background inflight IO or foreground operation recently */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) if (is_inflight_io(sbi, REQ_TIME) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) (!f2fs_time_over(sbi, REQ_TIME) && f2fs_rwsem_is_locked(&sbi->cp_rwsem)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) /* exceed periodical checkpoint timeout threshold */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) if (f2fs_time_over(sbi, CP_TIME))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) goto do_sync;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) /* checkpoint is the only way to shrink partial cached entries */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) if (f2fs_available_free_memory(sbi, NAT_ENTRIES) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) f2fs_available_free_memory(sbi, INO_ENTRIES))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) do_sync:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) if (test_opt(sbi, DATA_FLUSH) && from_bg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) struct blk_plug plug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) mutex_lock(&sbi->flush_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) blk_start_plug(&plug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) f2fs_sync_dirty_inodes(sbi, FILE_INODE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) blk_finish_plug(&plug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) mutex_unlock(&sbi->flush_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) f2fs_sync_fs(sbi->sb, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) stat_inc_bg_cp_count(sbi->stat_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) static int __submit_flush_wait(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) struct block_device *bdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) int ret = blkdev_issue_flush(bdev, GFP_NOFS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) trace_f2fs_issue_flush(bdev, test_opt(sbi, NOBARRIER),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) test_opt(sbi, FLUSH_MERGE), ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) static int submit_flush_wait(struct f2fs_sb_info *sbi, nid_t ino)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) if (!f2fs_is_multi_device(sbi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) return __submit_flush_wait(sbi, sbi->sb->s_bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) for (i = 0; i < sbi->s_ndevs; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) if (!f2fs_is_dirty_device(sbi, ino, i, FLUSH_INO))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) ret = __submit_flush_wait(sbi, FDEV(i).bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) static int issue_flush_thread(void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) struct f2fs_sb_info *sbi = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) wait_queue_head_t *q = &fcc->flush_wait_queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) repeat:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) if (kthread_should_stop())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) if (!llist_empty(&fcc->issue_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) struct flush_cmd *cmd, *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) fcc->dispatch_list = llist_del_all(&fcc->issue_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) fcc->dispatch_list = llist_reverse_order(fcc->dispatch_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) cmd = llist_entry(fcc->dispatch_list, struct flush_cmd, llnode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) ret = submit_flush_wait(sbi, cmd->ino);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) atomic_inc(&fcc->issued_flush);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) llist_for_each_entry_safe(cmd, next,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) fcc->dispatch_list, llnode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) cmd->ret = ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) complete(&cmd->wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) fcc->dispatch_list = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) wait_event_interruptible(*q,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) kthread_should_stop() || !llist_empty(&fcc->issue_list));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) goto repeat;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) struct flush_cmd cmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) if (test_opt(sbi, NOBARRIER))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) if (!test_opt(sbi, FLUSH_MERGE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) atomic_inc(&fcc->queued_flush);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) ret = submit_flush_wait(sbi, ino);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) atomic_dec(&fcc->queued_flush);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) atomic_inc(&fcc->issued_flush);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) if (atomic_inc_return(&fcc->queued_flush) == 1 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) f2fs_is_multi_device(sbi)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) ret = submit_flush_wait(sbi, ino);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) atomic_dec(&fcc->queued_flush);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) atomic_inc(&fcc->issued_flush);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) cmd.ino = ino;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) init_completion(&cmd.wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) llist_add(&cmd.llnode, &fcc->issue_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) * update issue_list before we wake up issue_flush thread, this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) * smp_mb() pairs with another barrier in ___wait_event(), see
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) * more details in comments of waitqueue_active().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) if (waitqueue_active(&fcc->flush_wait_queue))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) wake_up(&fcc->flush_wait_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) if (fcc->f2fs_issue_flush) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) wait_for_completion(&cmd.wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) atomic_dec(&fcc->queued_flush);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) struct llist_node *list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) list = llist_del_all(&fcc->issue_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) if (!list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) wait_for_completion(&cmd.wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) atomic_dec(&fcc->queued_flush);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) struct flush_cmd *tmp, *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) ret = submit_flush_wait(sbi, ino);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) llist_for_each_entry_safe(tmp, next, list, llnode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) if (tmp == &cmd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) cmd.ret = ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) atomic_dec(&fcc->queued_flush);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) tmp->ret = ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) complete(&tmp->wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) return cmd.ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) int f2fs_create_flush_cmd_control(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) dev_t dev = sbi->sb->s_bdev->bd_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) struct flush_cmd_control *fcc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) if (SM_I(sbi)->fcc_info) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) fcc = SM_I(sbi)->fcc_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) if (fcc->f2fs_issue_flush)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) goto init_thread;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) fcc = f2fs_kzalloc(sbi, sizeof(struct flush_cmd_control), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) if (!fcc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) atomic_set(&fcc->issued_flush, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) atomic_set(&fcc->queued_flush, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) init_waitqueue_head(&fcc->flush_wait_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) init_llist_head(&fcc->issue_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) SM_I(sbi)->fcc_info = fcc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) if (!test_opt(sbi, FLUSH_MERGE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) init_thread:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) "f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) if (IS_ERR(fcc->f2fs_issue_flush)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) err = PTR_ERR(fcc->f2fs_issue_flush);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) kfree(fcc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) SM_I(sbi)->fcc_info = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) if (fcc && fcc->f2fs_issue_flush) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) struct task_struct *flush_thread = fcc->f2fs_issue_flush;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) fcc->f2fs_issue_flush = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) kthread_stop(flush_thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) if (free) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) kfree(fcc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) SM_I(sbi)->fcc_info = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) int f2fs_flush_device_cache(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) int ret = 0, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) if (!f2fs_is_multi_device(sbi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) if (test_opt(sbi, NOBARRIER))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) for (i = 1; i < sbi->s_ndevs; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) if (!f2fs_test_bit(i, (char *)&sbi->dirty_device))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) ret = __submit_flush_wait(sbi, FDEV(i).bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) spin_lock(&sbi->dev_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) f2fs_clear_bit(i, (char *)&sbi->dirty_device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) spin_unlock(&sbi->dev_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) enum dirty_type dirty_type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) /* need not be added */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) if (IS_CURSEG(sbi, segno))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) dirty_i->nr_dirty[dirty_type]++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) if (dirty_type == DIRTY) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) struct seg_entry *sentry = get_seg_entry(sbi, segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) enum dirty_type t = sentry->type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) if (unlikely(t >= DIRTY)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) f2fs_bug_on(sbi, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) dirty_i->nr_dirty[t]++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) if (__is_large_section(sbi)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) block_t valid_blocks =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) get_valid_blocks(sbi, segno, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) f2fs_bug_on(sbi, unlikely(!valid_blocks ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) valid_blocks == BLKS_PER_SEC(sbi)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) if (!IS_CURSEC(sbi, secno))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) set_bit(secno, dirty_i->dirty_secmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) enum dirty_type dirty_type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) block_t valid_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) dirty_i->nr_dirty[dirty_type]--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) if (dirty_type == DIRTY) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) struct seg_entry *sentry = get_seg_entry(sbi, segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) enum dirty_type t = sentry->type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) dirty_i->nr_dirty[t]--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) valid_blocks = get_valid_blocks(sbi, segno, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) if (valid_blocks == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) clear_bit(GET_SEC_FROM_SEG(sbi, segno),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) dirty_i->victim_secmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) #ifdef CONFIG_F2FS_CHECK_FS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) clear_bit(segno, SIT_I(sbi)->invalid_segmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) if (__is_large_section(sbi)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) if (!valid_blocks ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) valid_blocks == BLKS_PER_SEC(sbi)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) clear_bit(secno, dirty_i->dirty_secmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) if (!IS_CURSEC(sbi, secno))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) set_bit(secno, dirty_i->dirty_secmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) * Should not occur error such as -ENOMEM.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) * Adding dirty entry into seglist is not critical operation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) * If a given segment is one of current working segments, it won't be added.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) unsigned short valid_blocks, ckpt_valid_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) unsigned int usable_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) usable_blocks = f2fs_usable_blks_in_seg(sbi, segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) mutex_lock(&dirty_i->seglist_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) valid_blocks = get_valid_blocks(sbi, segno, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) ckpt_valid_blocks = get_ckpt_valid_blocks(sbi, segno, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) if (valid_blocks == 0 && (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) ckpt_valid_blocks == usable_blocks)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) __locate_dirty_segment(sbi, segno, PRE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) __remove_dirty_segment(sbi, segno, DIRTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) } else if (valid_blocks < usable_blocks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) __locate_dirty_segment(sbi, segno, DIRTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) /* Recovery routine with SSR needs this */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) __remove_dirty_segment(sbi, segno, DIRTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) mutex_unlock(&dirty_i->seglist_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) /* This moves currently empty dirty blocks to prefree. Must hold seglist_lock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) unsigned int segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) mutex_lock(&dirty_i->seglist_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) if (get_valid_blocks(sbi, segno, false))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) if (IS_CURSEG(sbi, segno))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) __locate_dirty_segment(sbi, segno, PRE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) __remove_dirty_segment(sbi, segno, DIRTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) mutex_unlock(&dirty_i->seglist_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) int ovp_hole_segs =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) (overprovision_segments(sbi) - reserved_segments(sbi));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) block_t ovp_holes = ovp_hole_segs << sbi->log_blocks_per_seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) block_t holes[2] = {0, 0}; /* DATA and NODE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) block_t unusable;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) struct seg_entry *se;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) unsigned int segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) mutex_lock(&dirty_i->seglist_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) se = get_seg_entry(sbi, segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) if (IS_NODESEG(se->type))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) holes[NODE] += f2fs_usable_blks_in_seg(sbi, segno) -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) se->valid_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) holes[DATA] += f2fs_usable_blks_in_seg(sbi, segno) -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) se->valid_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) mutex_unlock(&dirty_i->seglist_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) unusable = holes[DATA] > holes[NODE] ? holes[DATA] : holes[NODE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) if (unusable > ovp_holes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) return unusable - ovp_holes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) int ovp_hole_segs =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) (overprovision_segments(sbi) - reserved_segments(sbi));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) if (unusable > F2FS_OPTION(sbi).unusable_cap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) return -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) if (is_sbi_flag_set(sbi, SBI_CP_DISABLED_QUICK) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) dirty_segments(sbi) > ovp_hole_segs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) return -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) /* This is only used by SBI_CP_DISABLED */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) static unsigned int get_free_segment(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) unsigned int segno = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) mutex_lock(&dirty_i->seglist_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) if (get_valid_blocks(sbi, segno, false))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) if (get_ckpt_valid_blocks(sbi, segno, false))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) mutex_unlock(&dirty_i->seglist_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) return segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) mutex_unlock(&dirty_i->seglist_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) return NULL_SEGNO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) struct block_device *bdev, block_t lstart,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) block_t start, block_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) struct list_head *pend_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) struct discard_cmd *dc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) f2fs_bug_on(sbi, !len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) pend_list = &dcc->pend_list[plist_idx(len)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) dc = f2fs_kmem_cache_alloc(discard_cmd_slab, GFP_NOFS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) INIT_LIST_HEAD(&dc->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) dc->bdev = bdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) dc->lstart = lstart;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) dc->start = start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) dc->len = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) dc->ref = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) dc->state = D_PREP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) dc->queued = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) dc->error = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) init_completion(&dc->wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) list_add_tail(&dc->list, pend_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) spin_lock_init(&dc->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) dc->bio_ref = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) atomic_inc(&dcc->discard_cmd_cnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) dcc->undiscard_blks += len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) return dc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) static struct discard_cmd *__attach_discard_cmd(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) struct block_device *bdev, block_t lstart,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) block_t start, block_t len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) struct rb_node *parent, struct rb_node **p,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) bool leftmost)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) struct discard_cmd *dc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) dc = __create_discard_cmd(sbi, bdev, lstart, start, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) rb_link_node(&dc->rb_node, parent, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) rb_insert_color_cached(&dc->rb_node, &dcc->root, leftmost);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) return dc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) static void __detach_discard_cmd(struct discard_cmd_control *dcc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) struct discard_cmd *dc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) if (dc->state == D_DONE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) atomic_sub(dc->queued, &dcc->queued_discard);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) list_del(&dc->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) rb_erase_cached(&dc->rb_node, &dcc->root);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) dcc->undiscard_blks -= dc->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) kmem_cache_free(discard_cmd_slab, dc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) atomic_dec(&dcc->discard_cmd_cnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) static void __remove_discard_cmd(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) struct discard_cmd *dc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) trace_f2fs_remove_discard(dc->bdev, dc->start, dc->len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) spin_lock_irqsave(&dc->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) if (dc->bio_ref) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) spin_unlock_irqrestore(&dc->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) spin_unlock_irqrestore(&dc->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) f2fs_bug_on(sbi, dc->ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) if (dc->error == -EOPNOTSUPP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) dc->error = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) if (dc->error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) printk_ratelimited(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) "%sF2FS-fs (%s): Issue discard(%u, %u, %u) failed, ret: %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) KERN_INFO, sbi->sb->s_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) dc->lstart, dc->start, dc->len, dc->error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) __detach_discard_cmd(dcc, dc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) static void f2fs_submit_discard_endio(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) struct discard_cmd *dc = (struct discard_cmd *)bio->bi_private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) spin_lock_irqsave(&dc->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) if (!dc->error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) dc->error = blk_status_to_errno(bio->bi_status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) dc->bio_ref--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) if (!dc->bio_ref && dc->state == D_SUBMIT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) dc->state = D_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) complete_all(&dc->wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) spin_unlock_irqrestore(&dc->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) bio_put(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) static void __check_sit_bitmap(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) block_t start, block_t end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) #ifdef CONFIG_F2FS_CHECK_FS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) struct seg_entry *sentry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) unsigned int segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) block_t blk = start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) unsigned long offset, size, max_blocks = sbi->blocks_per_seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) unsigned long *map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) while (blk < end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) segno = GET_SEGNO(sbi, blk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) sentry = get_seg_entry(sbi, segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) offset = GET_BLKOFF_FROM_SEG0(sbi, blk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) if (end < START_BLOCK(sbi, segno + 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) size = GET_BLKOFF_FROM_SEG0(sbi, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) size = max_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) map = (unsigned long *)(sentry->cur_valid_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) offset = __find_rev_next_bit(map, size, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) f2fs_bug_on(sbi, offset != size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) blk = START_BLOCK(sbi, segno + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) static void __init_discard_policy(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) struct discard_policy *dpolicy,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) int discard_type, unsigned int granularity)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) /* common policy */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) dpolicy->type = discard_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) dpolicy->sync = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) dpolicy->ordered = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) dpolicy->granularity = granularity;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) dpolicy->max_requests = DEF_MAX_DISCARD_REQUEST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) dpolicy->io_aware_gran = MAX_PLIST_NUM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) dpolicy->timeout = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) if (discard_type == DPOLICY_BG) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) dpolicy->min_interval = DEF_MIN_DISCARD_ISSUE_TIME;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) dpolicy->mid_interval = DEF_MID_DISCARD_ISSUE_TIME;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) dpolicy->max_interval = DEF_MAX_DISCARD_ISSUE_TIME;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) dpolicy->io_aware = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) dpolicy->sync = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) dpolicy->ordered = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) if (utilization(sbi) > DEF_DISCARD_URGENT_UTIL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) dpolicy->granularity = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) if (atomic_read(&dcc->discard_cmd_cnt))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) dpolicy->max_interval =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) DEF_MIN_DISCARD_ISSUE_TIME;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) } else if (discard_type == DPOLICY_FORCE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) dpolicy->min_interval = DEF_MIN_DISCARD_ISSUE_TIME;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) dpolicy->mid_interval = DEF_MID_DISCARD_ISSUE_TIME;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) dpolicy->max_interval = DEF_MAX_DISCARD_ISSUE_TIME;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) dpolicy->io_aware = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) } else if (discard_type == DPOLICY_FSTRIM) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) dpolicy->io_aware = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) } else if (discard_type == DPOLICY_UMOUNT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) dpolicy->io_aware = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) /* we need to issue all to keep CP_TRIMMED_FLAG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) dpolicy->granularity = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) dpolicy->timeout = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) struct block_device *bdev, block_t lstart,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) block_t start, block_t len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) /* this function is copied from blkdev_issue_discard from block/blk-lib.c */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) static int __submit_discard_cmd(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) struct discard_policy *dpolicy,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) struct discard_cmd *dc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) unsigned int *issued)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) struct block_device *bdev = dc->bdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) struct request_queue *q = bdev_get_queue(bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) unsigned int max_discard_blocks =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) SECTOR_TO_BLOCK(q->limits.max_discard_sectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) &(dcc->fstrim_list) : &(dcc->wait_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) int flag = dpolicy->sync ? REQ_SYNC : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) block_t lstart, start, len, total_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) if (dc->state != D_PREP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) if (is_sbi_flag_set(sbi, SBI_NEED_FSCK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) trace_f2fs_issue_discard(bdev, dc->start, dc->len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) lstart = dc->lstart;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) start = dc->start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) len = dc->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) total_len = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) dc->len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) while (total_len && *issued < dpolicy->max_requests && !err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) struct bio *bio = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) bool last = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) if (len > max_discard_blocks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) len = max_discard_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) last = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) (*issued)++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) if (*issued == dpolicy->max_requests)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) last = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) dc->len += len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) if (time_to_inject(sbi, FAULT_DISCARD)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) f2fs_show_injection_info(sbi, FAULT_DISCARD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) err = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) goto submit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) err = __blkdev_issue_discard(bdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) SECTOR_FROM_BLOCK(start),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) SECTOR_FROM_BLOCK(len),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) GFP_NOFS, 0, &bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) submit:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) spin_lock_irqsave(&dc->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) if (dc->state == D_PARTIAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) dc->state = D_SUBMIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) spin_unlock_irqrestore(&dc->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) f2fs_bug_on(sbi, !bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) * should keep before submission to avoid D_DONE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) * right away
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) spin_lock_irqsave(&dc->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) if (last)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) dc->state = D_SUBMIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) dc->state = D_PARTIAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) dc->bio_ref++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) spin_unlock_irqrestore(&dc->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) atomic_inc(&dcc->queued_discard);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) dc->queued++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) list_move_tail(&dc->list, wait_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) /* sanity check on discard range */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) __check_sit_bitmap(sbi, lstart, lstart + len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) bio->bi_private = dc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) bio->bi_end_io = f2fs_submit_discard_endio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) bio->bi_opf |= flag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) submit_bio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) atomic_inc(&dcc->issued_discard);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) f2fs_update_iostat(sbi, FS_DISCARD, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) lstart += len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) start += len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) total_len -= len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) len = total_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) if (!err && len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) dcc->undiscard_blks -= len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) __update_discard_tree_range(sbi, bdev, lstart, start, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) static void __insert_discard_tree(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) struct block_device *bdev, block_t lstart,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) block_t start, block_t len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) struct rb_node **insert_p,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) struct rb_node *insert_parent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) struct rb_node **p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) struct rb_node *parent = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) bool leftmost = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) if (insert_p && insert_parent) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) parent = insert_parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) p = insert_p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) goto do_insert;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) p = f2fs_lookup_rb_tree_for_insert(sbi, &dcc->root, &parent,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) lstart, &leftmost);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) do_insert:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) __attach_discard_cmd(sbi, bdev, lstart, start, len, parent,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) p, leftmost);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) static void __relocate_discard_cmd(struct discard_cmd_control *dcc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) struct discard_cmd *dc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) list_move_tail(&dc->list, &dcc->pend_list[plist_idx(dc->len)]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) static void __punch_discard_cmd(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) struct discard_cmd *dc, block_t blkaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) struct discard_info di = dc->di;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) bool modified = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) if (dc->state == D_DONE || dc->len == 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) __remove_discard_cmd(sbi, dc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) dcc->undiscard_blks -= di.len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) if (blkaddr > di.lstart) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) dc->len = blkaddr - dc->lstart;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) dcc->undiscard_blks += dc->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) __relocate_discard_cmd(dcc, dc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) modified = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) if (blkaddr < di.lstart + di.len - 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) if (modified) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) __insert_discard_tree(sbi, dc->bdev, blkaddr + 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) di.start + blkaddr + 1 - di.lstart,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) di.lstart + di.len - 1 - blkaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) NULL, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) dc->lstart++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) dc->len--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) dc->start++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) dcc->undiscard_blks += dc->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) __relocate_discard_cmd(dcc, dc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) struct block_device *bdev, block_t lstart,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) block_t start, block_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) struct discard_cmd *dc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) struct discard_info di = {0};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) struct rb_node **insert_p = NULL, *insert_parent = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) struct request_queue *q = bdev_get_queue(bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) unsigned int max_discard_blocks =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) SECTOR_TO_BLOCK(q->limits.max_discard_sectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) block_t end = lstart + len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) NULL, lstart,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) (struct rb_entry **)&prev_dc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) (struct rb_entry **)&next_dc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) &insert_p, &insert_parent, true, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) if (dc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) prev_dc = dc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) if (!prev_dc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) di.lstart = lstart;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) di.len = next_dc ? next_dc->lstart - lstart : len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) di.len = min(di.len, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) di.start = start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) struct rb_node *node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) bool merged = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) struct discard_cmd *tdc = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) if (prev_dc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) di.lstart = prev_dc->lstart + prev_dc->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) if (di.lstart < lstart)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) di.lstart = lstart;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) if (di.lstart >= end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) if (!next_dc || next_dc->lstart > end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) di.len = end - di.lstart;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) di.len = next_dc->lstart - di.lstart;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) di.start = start + di.lstart - lstart;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) if (!di.len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) goto next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) if (prev_dc && prev_dc->state == D_PREP &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) prev_dc->bdev == bdev &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) __is_discard_back_mergeable(&di, &prev_dc->di,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) max_discard_blocks)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) prev_dc->di.len += di.len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) dcc->undiscard_blks += di.len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) __relocate_discard_cmd(dcc, prev_dc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) di = prev_dc->di;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) tdc = prev_dc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) merged = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) if (next_dc && next_dc->state == D_PREP &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) next_dc->bdev == bdev &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) __is_discard_front_mergeable(&di, &next_dc->di,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) max_discard_blocks)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) next_dc->di.lstart = di.lstart;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) next_dc->di.len += di.len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) next_dc->di.start = di.start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) dcc->undiscard_blks += di.len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) __relocate_discard_cmd(dcc, next_dc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) if (tdc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) __remove_discard_cmd(sbi, tdc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) merged = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) if (!merged) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) __insert_discard_tree(sbi, bdev, di.lstart, di.start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) di.len, NULL, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) next:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) prev_dc = next_dc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) if (!prev_dc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) node = rb_next(&prev_dc->rb_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) next_dc = rb_entry_safe(node, struct discard_cmd, rb_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) static int __queue_discard_cmd(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) struct block_device *bdev, block_t blkstart, block_t blklen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) block_t lblkstart = blkstart;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) if (!f2fs_bdev_support_discard(bdev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) trace_f2fs_queue_discard(bdev, blkstart, blklen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) if (f2fs_is_multi_device(sbi)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) int devi = f2fs_target_device_index(sbi, blkstart);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) blkstart -= FDEV(devi).start_blk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) __update_discard_tree_range(sbi, bdev, lblkstart, blkstart, blklen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) static unsigned int __issue_discard_cmd_orderly(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) struct discard_policy *dpolicy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) struct rb_node **insert_p = NULL, *insert_parent = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) struct discard_cmd *dc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) struct blk_plug plug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) unsigned int pos = dcc->next_pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) unsigned int issued = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) bool io_interrupted = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) mutex_lock(&dcc->cmd_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) NULL, pos,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) (struct rb_entry **)&prev_dc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) (struct rb_entry **)&next_dc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) &insert_p, &insert_parent, true, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) if (!dc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) dc = next_dc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) blk_start_plug(&plug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) while (dc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) struct rb_node *node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) if (dc->state != D_PREP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) goto next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) if (dpolicy->io_aware && !is_idle(sbi, DISCARD_TIME)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) io_interrupted = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) dcc->next_pos = dc->lstart + dc->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) err = __submit_discard_cmd(sbi, dpolicy, dc, &issued);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) if (issued >= dpolicy->max_requests)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) next:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) node = rb_next(&dc->rb_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) __remove_discard_cmd(sbi, dc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) dc = rb_entry_safe(node, struct discard_cmd, rb_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) blk_finish_plug(&plug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) if (!dc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) dcc->next_pos = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) mutex_unlock(&dcc->cmd_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) if (!issued && io_interrupted)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) issued = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) return issued;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) struct discard_policy *dpolicy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) static int __issue_discard_cmd(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) struct discard_policy *dpolicy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) struct list_head *pend_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) struct discard_cmd *dc, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) struct blk_plug plug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) int i, issued;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) bool io_interrupted = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) if (dpolicy->timeout)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) f2fs_update_time(sbi, UMOUNT_DISCARD_TIMEOUT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) retry:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) issued = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) for (i = MAX_PLIST_NUM - 1; i >= 0; i--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) if (dpolicy->timeout &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) if (i + 1 < dpolicy->granularity)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) if (i < DEFAULT_DISCARD_GRANULARITY && dpolicy->ordered)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) return __issue_discard_cmd_orderly(sbi, dpolicy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) pend_list = &dcc->pend_list[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) mutex_lock(&dcc->cmd_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) if (list_empty(pend_list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) goto next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) if (unlikely(dcc->rbtree_check))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) &dcc->root, false));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) blk_start_plug(&plug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) list_for_each_entry_safe(dc, tmp, pend_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) f2fs_bug_on(sbi, dc->state != D_PREP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) if (dpolicy->timeout &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) if (dpolicy->io_aware && i < dpolicy->io_aware_gran &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) !is_idle(sbi, DISCARD_TIME)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) io_interrupted = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) __submit_discard_cmd(sbi, dpolicy, dc, &issued);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) if (issued >= dpolicy->max_requests)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) blk_finish_plug(&plug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) next:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) mutex_unlock(&dcc->cmd_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) if (issued >= dpolicy->max_requests || io_interrupted)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) if (dpolicy->type == DPOLICY_UMOUNT && issued) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) __wait_all_discard_cmd(sbi, dpolicy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) if (!issued && io_interrupted)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) issued = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) return issued;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) static bool __drop_discard_cmd(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) struct list_head *pend_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) struct discard_cmd *dc, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) bool dropped = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) mutex_lock(&dcc->cmd_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) for (i = MAX_PLIST_NUM - 1; i >= 0; i--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) pend_list = &dcc->pend_list[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) list_for_each_entry_safe(dc, tmp, pend_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) f2fs_bug_on(sbi, dc->state != D_PREP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) __remove_discard_cmd(sbi, dc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) dropped = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) mutex_unlock(&dcc->cmd_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) return dropped;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) __drop_discard_cmd(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) static unsigned int __wait_one_discard_bio(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) struct discard_cmd *dc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) unsigned int len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) wait_for_completion_io(&dc->wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) mutex_lock(&dcc->cmd_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) f2fs_bug_on(sbi, dc->state != D_DONE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) dc->ref--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) if (!dc->ref) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) if (!dc->error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) len = dc->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) __remove_discard_cmd(sbi, dc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) mutex_unlock(&dcc->cmd_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) return len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) static unsigned int __wait_discard_cmd_range(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) struct discard_policy *dpolicy,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) block_t start, block_t end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) &(dcc->fstrim_list) : &(dcc->wait_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) struct discard_cmd *dc, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) bool need_wait;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) unsigned int trimmed = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) next:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) need_wait = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) mutex_lock(&dcc->cmd_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) list_for_each_entry_safe(dc, tmp, wait_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) if (dc->lstart + dc->len <= start || end <= dc->lstart)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) if (dc->len < dpolicy->granularity)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) if (dc->state == D_DONE && !dc->ref) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) wait_for_completion_io(&dc->wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) if (!dc->error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) trimmed += dc->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) __remove_discard_cmd(sbi, dc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) dc->ref++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) need_wait = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) mutex_unlock(&dcc->cmd_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) if (need_wait) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) trimmed += __wait_one_discard_bio(sbi, dc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) goto next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) return trimmed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) struct discard_policy *dpolicy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) struct discard_policy dp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) unsigned int discard_blks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) if (dpolicy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) return __wait_discard_cmd_range(sbi, dpolicy, 0, UINT_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) /* wait all */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) __init_discard_policy(sbi, &dp, DPOLICY_FSTRIM, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) discard_blks = __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) __init_discard_policy(sbi, &dp, DPOLICY_UMOUNT, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) discard_blks += __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690) return discard_blks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) /* This should be covered by global mutex, &sit_i->sentry_lock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) static void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) struct discard_cmd *dc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) bool need_wait = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700) mutex_lock(&dcc->cmd_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) dc = (struct discard_cmd *)f2fs_lookup_rb_tree(&dcc->root,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702) NULL, blkaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) if (dc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) if (dc->state == D_PREP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705) __punch_discard_cmd(sbi, dc, blkaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) dc->ref++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) need_wait = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) mutex_unlock(&dcc->cmd_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) if (need_wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) __wait_one_discard_bio(sbi, dc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719) struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721) if (dcc && dcc->f2fs_issue_discard) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) struct task_struct *discard_thread = dcc->f2fs_issue_discard;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) dcc->f2fs_issue_discard = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725) kthread_stop(discard_thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) /* This comes from f2fs_put_super */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730) bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733) struct discard_policy dpolicy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734) bool dropped;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) __init_discard_policy(sbi, &dpolicy, DPOLICY_UMOUNT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) dcc->discard_granularity);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) __issue_discard_cmd(sbi, &dpolicy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) dropped = __drop_discard_cmd(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) /* just to make sure there is no pending discard commands */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742) __wait_all_discard_cmd(sbi, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) f2fs_bug_on(sbi, atomic_read(&dcc->discard_cmd_cnt));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) return dropped;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) static int issue_discard_thread(void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) struct f2fs_sb_info *sbi = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751) struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752) wait_queue_head_t *q = &dcc->discard_wait_queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) struct discard_policy dpolicy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754) unsigned int wait_ms = DEF_MIN_DISCARD_ISSUE_TIME;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755) int issued;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757) set_freezable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760) if (sbi->gc_mode == GC_URGENT_HIGH ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) !f2fs_available_free_memory(sbi, DISCARD_CACHE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762) __init_discard_policy(sbi, &dpolicy, DPOLICY_FORCE, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764) __init_discard_policy(sbi, &dpolicy, DPOLICY_BG,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) dcc->discard_granularity);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) if (!atomic_read(&dcc->discard_cmd_cnt))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768) wait_ms = dpolicy.max_interval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770) wait_event_interruptible_timeout(*q,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) kthread_should_stop() || freezing(current) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772) dcc->discard_wake,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) msecs_to_jiffies(wait_ms));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775) if (dcc->discard_wake)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) dcc->discard_wake = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778) /* clean up pending candidates before going to sleep */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779) if (atomic_read(&dcc->queued_discard))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780) __wait_all_discard_cmd(sbi, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782) if (try_to_freeze())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784) if (f2fs_readonly(sbi->sb))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786) if (kthread_should_stop())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789) wait_ms = dpolicy.max_interval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792) if (!atomic_read(&dcc->discard_cmd_cnt))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795) sb_start_intwrite(sbi->sb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797) issued = __issue_discard_cmd(sbi, &dpolicy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798) if (issued > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799) __wait_all_discard_cmd(sbi, &dpolicy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800) wait_ms = dpolicy.min_interval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801) } else if (issued == -1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802) wait_ms = f2fs_time_to_wait(sbi, DISCARD_TIME);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803) if (!wait_ms)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804) wait_ms = dpolicy.mid_interval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806) wait_ms = dpolicy.max_interval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809) sb_end_intwrite(sbi->sb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811) } while (!kthread_should_stop());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815) #ifdef CONFIG_BLK_DEV_ZONED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816) static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817) struct block_device *bdev, block_t blkstart, block_t blklen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819) sector_t sector, nr_sects;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820) block_t lblkstart = blkstart;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821) int devi = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823) if (f2fs_is_multi_device(sbi)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824) devi = f2fs_target_device_index(sbi, blkstart);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825) if (blkstart < FDEV(devi).start_blk ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826) blkstart > FDEV(devi).end_blk) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827) f2fs_err(sbi, "Invalid block %x", blkstart);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830) blkstart -= FDEV(devi).start_blk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833) /* For sequential zones, reset the zone write pointer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834) if (f2fs_blkz_is_seq(sbi, devi, blkstart)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835) sector = SECTOR_FROM_BLOCK(blkstart);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836) nr_sects = SECTOR_FROM_BLOCK(blklen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838) if (sector & (bdev_zone_sectors(bdev) - 1) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839) nr_sects != bdev_zone_sectors(bdev)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840) f2fs_err(sbi, "(%d) %s: Unaligned zone reset attempted (block %x + %x)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841) devi, sbi->s_ndevs ? FDEV(devi).path : "",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842) blkstart, blklen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845) trace_f2fs_issue_reset_zone(bdev, blkstart);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846) return blkdev_zone_mgmt(bdev, REQ_OP_ZONE_RESET,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847) sector, nr_sects, GFP_NOFS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850) /* For conventional zones, use regular discard if supported */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851) return __queue_discard_cmd(sbi, bdev, lblkstart, blklen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855) static int __issue_discard_async(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856) struct block_device *bdev, block_t blkstart, block_t blklen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858) #ifdef CONFIG_BLK_DEV_ZONED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1859) if (f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(bdev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1860) return __f2fs_issue_discard_zone(sbi, bdev, blkstart, blklen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1861) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1862) return __queue_discard_cmd(sbi, bdev, blkstart, blklen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1863) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1864)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1865) static int f2fs_issue_discard(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1866) block_t blkstart, block_t blklen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1867) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1868) sector_t start = blkstart, len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1869) struct block_device *bdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1870) struct seg_entry *se;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1871) unsigned int offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1872) block_t i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1873) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1874)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1875) bdev = f2fs_target_device(sbi, blkstart, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1876)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1877) for (i = blkstart; i < blkstart + blklen; i++, len++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1878) if (i != start) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1879) struct block_device *bdev2 =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1880) f2fs_target_device(sbi, i, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1881)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1882) if (bdev2 != bdev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1883) err = __issue_discard_async(sbi, bdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1884) start, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1885) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1886) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1887) bdev = bdev2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1888) start = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1889) len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1890) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1891) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1892)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1893) se = get_seg_entry(sbi, GET_SEGNO(sbi, i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1894) offset = GET_BLKOFF_FROM_SEG0(sbi, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1895)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1896) if (!f2fs_test_and_set_bit(offset, se->discard_map))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1897) sbi->discard_blks--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1898) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1899)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1900) if (len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1901) err = __issue_discard_async(sbi, bdev, start, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1902) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1903) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1904)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1905) static bool add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1906) bool check_only)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1907) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1908) int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1909) int max_blocks = sbi->blocks_per_seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1910) struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1911) unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1912) unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1913) unsigned long *discard_map = (unsigned long *)se->discard_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1914) unsigned long *dmap = SIT_I(sbi)->tmp_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1915) unsigned int start = 0, end = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1916) bool force = (cpc->reason & CP_DISCARD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1917) struct discard_entry *de = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1918) struct list_head *head = &SM_I(sbi)->dcc_info->entry_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1919) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1920)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1921) if (se->valid_blocks == max_blocks || !f2fs_hw_support_discard(sbi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1922) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1923)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1924) if (!force) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1925) if (!f2fs_realtime_discard_enable(sbi) || !se->valid_blocks ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1926) SM_I(sbi)->dcc_info->nr_discards >=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1927) SM_I(sbi)->dcc_info->max_discards)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1928) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1929) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1930)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1931) /* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1932) for (i = 0; i < entries; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1933) dmap[i] = force ? ~ckpt_map[i] & ~discard_map[i] :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1934) (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1935)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1936) while (force || SM_I(sbi)->dcc_info->nr_discards <=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1937) SM_I(sbi)->dcc_info->max_discards) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1938) start = __find_rev_next_bit(dmap, max_blocks, end + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1939) if (start >= max_blocks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1940) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1941)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1942) end = __find_rev_next_zero_bit(dmap, max_blocks, start + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1943) if (force && start && end != max_blocks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1944) && (end - start) < cpc->trim_minlen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1945) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1946)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1947) if (check_only)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1948) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1949)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1950) if (!de) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1951) de = f2fs_kmem_cache_alloc(discard_entry_slab,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1952) GFP_F2FS_ZERO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1953) de->start_blkaddr = START_BLOCK(sbi, cpc->trim_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1954) list_add_tail(&de->list, head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1955) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1956)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1957) for (i = start; i < end; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1958) __set_bit_le(i, (void *)de->discard_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1959)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1960) SM_I(sbi)->dcc_info->nr_discards += end - start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1961) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1962) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1963) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1964)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1965) static void release_discard_addr(struct discard_entry *entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1966) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1967) list_del(&entry->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1968) kmem_cache_free(discard_entry_slab, entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1969) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1970)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1971) void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1972) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1973) struct list_head *head = &(SM_I(sbi)->dcc_info->entry_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1974) struct discard_entry *entry, *this;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1975)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1976) /* drop caches */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1977) list_for_each_entry_safe(entry, this, head, list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1978) release_discard_addr(entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1979) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1980)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1981) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1982) * Should call f2fs_clear_prefree_segments after checkpoint is done.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1983) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1984) static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1985) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1986) struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1987) unsigned int segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1988)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1989) mutex_lock(&dirty_i->seglist_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1990) for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1991) __set_test_and_free(sbi, segno, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1992) mutex_unlock(&dirty_i->seglist_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1993) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1994)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1995) void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1996) struct cp_control *cpc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1997) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1998) struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1999) struct list_head *head = &dcc->entry_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2000) struct discard_entry *entry, *this;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2001) struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2002) unsigned long *prefree_map = dirty_i->dirty_segmap[PRE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2003) unsigned int start = 0, end = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2004) unsigned int secno, start_segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2005) bool force = (cpc->reason & CP_DISCARD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2006) bool need_align = f2fs_lfs_mode(sbi) && __is_large_section(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2007)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2008) mutex_lock(&dirty_i->seglist_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2009)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2010) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2011) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2012)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2013) if (need_align && end != -1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2014) end--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2015) start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2016) if (start >= MAIN_SEGS(sbi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2017) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2018) end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2019) start + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2020)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2021) if (need_align) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2022) start = rounddown(start, sbi->segs_per_sec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2023) end = roundup(end, sbi->segs_per_sec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2024) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2025)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2026) for (i = start; i < end; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2027) if (test_and_clear_bit(i, prefree_map))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2028) dirty_i->nr_dirty[PRE]--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2029) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2030)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2031) if (!f2fs_realtime_discard_enable(sbi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2032) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2033)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2034) if (force && start >= cpc->trim_start &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2035) (end - 1) <= cpc->trim_end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2036) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2037)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2038) if (!f2fs_lfs_mode(sbi) || !__is_large_section(sbi)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2039) f2fs_issue_discard(sbi, START_BLOCK(sbi, start),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2040) (end - start) << sbi->log_blocks_per_seg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2041) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2042) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2043) next:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2044) secno = GET_SEC_FROM_SEG(sbi, start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2045) start_segno = GET_SEG_FROM_SEC(sbi, secno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2046) if (!IS_CURSEC(sbi, secno) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2047) !get_valid_blocks(sbi, start, true))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2048) f2fs_issue_discard(sbi, START_BLOCK(sbi, start_segno),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2049) sbi->segs_per_sec << sbi->log_blocks_per_seg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2050)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2051) start = start_segno + sbi->segs_per_sec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2052) if (start < end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2053) goto next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2054) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2055) end = start - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2056) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2057) mutex_unlock(&dirty_i->seglist_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2058)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2059) /* send small discards */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2060) list_for_each_entry_safe(entry, this, head, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2061) unsigned int cur_pos = 0, next_pos, len, total_len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2062) bool is_valid = test_bit_le(0, entry->discard_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2063)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2064) find_next:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2065) if (is_valid) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2066) next_pos = find_next_zero_bit_le(entry->discard_map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2067) sbi->blocks_per_seg, cur_pos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2068) len = next_pos - cur_pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2069)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2070) if (f2fs_sb_has_blkzoned(sbi) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2071) (force && len < cpc->trim_minlen))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2072) goto skip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2073)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2074) f2fs_issue_discard(sbi, entry->start_blkaddr + cur_pos,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2075) len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2076) total_len += len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2077) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2078) next_pos = find_next_bit_le(entry->discard_map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2079) sbi->blocks_per_seg, cur_pos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2080) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2081) skip:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2082) cur_pos = next_pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2083) is_valid = !is_valid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2084)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2085) if (cur_pos < sbi->blocks_per_seg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2086) goto find_next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2087)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2088) release_discard_addr(entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2089) dcc->nr_discards -= total_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2090) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2091)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2092) wake_up_discard_thread(sbi, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2093) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2094)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2095) static int create_discard_cmd_control(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2096) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2097) dev_t dev = sbi->sb->s_bdev->bd_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2098) struct discard_cmd_control *dcc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2099) int err = 0, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2101) if (SM_I(sbi)->dcc_info) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2102) dcc = SM_I(sbi)->dcc_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2103) goto init_thread;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2104) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2106) dcc = f2fs_kzalloc(sbi, sizeof(struct discard_cmd_control), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2107) if (!dcc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2108) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2110) dcc->discard_granularity = DEFAULT_DISCARD_GRANULARITY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2111) INIT_LIST_HEAD(&dcc->entry_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2112) for (i = 0; i < MAX_PLIST_NUM; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2113) INIT_LIST_HEAD(&dcc->pend_list[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2114) INIT_LIST_HEAD(&dcc->wait_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2115) INIT_LIST_HEAD(&dcc->fstrim_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2116) mutex_init(&dcc->cmd_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2117) atomic_set(&dcc->issued_discard, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2118) atomic_set(&dcc->queued_discard, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2119) atomic_set(&dcc->discard_cmd_cnt, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2120) dcc->nr_discards = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2121) dcc->max_discards = MAIN_SEGS(sbi) << sbi->log_blocks_per_seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2122) dcc->undiscard_blks = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2123) dcc->next_pos = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2124) dcc->root = RB_ROOT_CACHED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2125) dcc->rbtree_check = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2127) init_waitqueue_head(&dcc->discard_wait_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2128) SM_I(sbi)->dcc_info = dcc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2129) init_thread:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2130) dcc->f2fs_issue_discard = kthread_run(issue_discard_thread, sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2131) "f2fs_discard-%u:%u", MAJOR(dev), MINOR(dev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2132) if (IS_ERR(dcc->f2fs_issue_discard)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2133) err = PTR_ERR(dcc->f2fs_issue_discard);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2134) kfree(dcc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2135) SM_I(sbi)->dcc_info = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2136) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2137) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2139) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2140) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2142) static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2143) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2144) struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2146) if (!dcc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2147) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2149) f2fs_stop_discard_thread(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2151) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2152) * Recovery can cache discard commands, so in error path of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2153) * fill_super(), it needs to give a chance to handle them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2154) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2155) if (unlikely(atomic_read(&dcc->discard_cmd_cnt)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2156) f2fs_issue_discard_timeout(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2158) kfree(dcc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2159) SM_I(sbi)->dcc_info = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2160) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2162) static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2163) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2164) struct sit_info *sit_i = SIT_I(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2166) if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2167) sit_i->dirty_sentries++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2168) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2169) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2171) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2172) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2174) static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2175) unsigned int segno, int modified)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2176) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2177) struct seg_entry *se = get_seg_entry(sbi, segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2179) se->type = type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2180) if (modified)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2181) __mark_sit_entry_dirty(sbi, segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2182) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2184) static inline unsigned long long get_segment_mtime(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2185) block_t blkaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2186) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2187) unsigned int segno = GET_SEGNO(sbi, blkaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2189) if (segno == NULL_SEGNO)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2190) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2191) return get_seg_entry(sbi, segno)->mtime;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2192) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2194) static void update_segment_mtime(struct f2fs_sb_info *sbi, block_t blkaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2195) unsigned long long old_mtime)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2196) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2197) struct seg_entry *se;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2198) unsigned int segno = GET_SEGNO(sbi, blkaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2199) unsigned long long ctime = get_mtime(sbi, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2200) unsigned long long mtime = old_mtime ? old_mtime : ctime;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2202) if (segno == NULL_SEGNO)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2203) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2205) se = get_seg_entry(sbi, segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2207) if (!se->mtime)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2208) se->mtime = mtime;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2209) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2210) se->mtime = div_u64(se->mtime * se->valid_blocks + mtime,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2211) se->valid_blocks + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2213) if (ctime > SIT_I(sbi)->max_mtime)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2214) SIT_I(sbi)->max_mtime = ctime;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2215) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2217) static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2218) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2219) struct seg_entry *se;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2220) unsigned int segno, offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2221) long int new_vblocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2222) bool exist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2223) #ifdef CONFIG_F2FS_CHECK_FS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2224) bool mir_exist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2225) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2227) segno = GET_SEGNO(sbi, blkaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2229) se = get_seg_entry(sbi, segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2230) new_vblocks = se->valid_blocks + del;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2231) offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2233) f2fs_bug_on(sbi, (new_vblocks < 0 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2234) (new_vblocks > f2fs_usable_blks_in_seg(sbi, segno))));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2236) se->valid_blocks = new_vblocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2238) /* Update valid block bitmap */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2239) if (del > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2240) exist = f2fs_test_and_set_bit(offset, se->cur_valid_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2241) #ifdef CONFIG_F2FS_CHECK_FS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2242) mir_exist = f2fs_test_and_set_bit(offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2243) se->cur_valid_map_mir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2244) if (unlikely(exist != mir_exist)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2245) f2fs_err(sbi, "Inconsistent error when setting bitmap, blk:%u, old bit:%d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2246) blkaddr, exist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2247) f2fs_bug_on(sbi, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2248) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2249) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2250) if (unlikely(exist)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2251) f2fs_err(sbi, "Bitmap was wrongly set, blk:%u",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2252) blkaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2253) f2fs_bug_on(sbi, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2254) se->valid_blocks--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2255) del = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2256) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2258) if (!f2fs_test_and_set_bit(offset, se->discard_map))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2259) sbi->discard_blks--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2261) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2262) * SSR should never reuse block which is checkpointed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2263) * or newly invalidated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2264) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2265) if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2266) if (!f2fs_test_and_set_bit(offset, se->ckpt_valid_map))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2267) se->ckpt_valid_blocks++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2268) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2269) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2270) exist = f2fs_test_and_clear_bit(offset, se->cur_valid_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2271) #ifdef CONFIG_F2FS_CHECK_FS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2272) mir_exist = f2fs_test_and_clear_bit(offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2273) se->cur_valid_map_mir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2274) if (unlikely(exist != mir_exist)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2275) f2fs_err(sbi, "Inconsistent error when clearing bitmap, blk:%u, old bit:%d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2276) blkaddr, exist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2277) f2fs_bug_on(sbi, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2278) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2279) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2280) if (unlikely(!exist)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2281) f2fs_err(sbi, "Bitmap was wrongly cleared, blk:%u",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2282) blkaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2283) f2fs_bug_on(sbi, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2284) se->valid_blocks++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2285) del = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2286) } else if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2287) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2288) * If checkpoints are off, we must not reuse data that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2289) * was used in the previous checkpoint. If it was used
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2290) * before, we must track that to know how much space we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2291) * really have.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2292) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2293) if (f2fs_test_bit(offset, se->ckpt_valid_map)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2294) spin_lock(&sbi->stat_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2295) sbi->unusable_block_count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2296) spin_unlock(&sbi->stat_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2297) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2298) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2300) if (f2fs_test_and_clear_bit(offset, se->discard_map))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2301) sbi->discard_blks++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2302) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2303) if (!f2fs_test_bit(offset, se->ckpt_valid_map))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2304) se->ckpt_valid_blocks += del;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2306) __mark_sit_entry_dirty(sbi, segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2308) /* update total number of valid blocks to be written in ckpt area */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2309) SIT_I(sbi)->written_valid_blocks += del;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2311) if (__is_large_section(sbi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2312) get_sec_entry(sbi, segno)->valid_blocks += del;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2313) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2315) void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2316) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2317) unsigned int segno = GET_SEGNO(sbi, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2318) struct sit_info *sit_i = SIT_I(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2320) f2fs_bug_on(sbi, addr == NULL_ADDR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2321) if (addr == NEW_ADDR || addr == COMPRESS_ADDR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2322) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2324) invalidate_mapping_pages(META_MAPPING(sbi), addr, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2325) f2fs_invalidate_compress_page(sbi, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2327) /* add it into sit main buffer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2328) down_write(&sit_i->sentry_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2330) update_segment_mtime(sbi, addr, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2331) update_sit_entry(sbi, addr, -1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2333) /* add it into dirty seglist */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2334) locate_dirty_segment(sbi, segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2336) up_write(&sit_i->sentry_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2337) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2339) bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2340) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2341) struct sit_info *sit_i = SIT_I(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2342) unsigned int segno, offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2343) struct seg_entry *se;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2344) bool is_cp = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2346) if (!__is_valid_data_blkaddr(blkaddr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2347) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2349) down_read(&sit_i->sentry_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2351) segno = GET_SEGNO(sbi, blkaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2352) se = get_seg_entry(sbi, segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2353) offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2355) if (f2fs_test_bit(offset, se->ckpt_valid_map))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2356) is_cp = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2358) up_read(&sit_i->sentry_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2360) return is_cp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2361) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2363) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2364) * This function should be resided under the curseg_mutex lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2365) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2366) static void __add_sum_entry(struct f2fs_sb_info *sbi, int type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2367) struct f2fs_summary *sum)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2368) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2369) struct curseg_info *curseg = CURSEG_I(sbi, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2370) void *addr = curseg->sum_blk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2372) addr += curseg->next_blkoff * sizeof(struct f2fs_summary);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2373) memcpy(addr, sum, sizeof(struct f2fs_summary));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2374) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2376) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2377) * Calculate the number of current summary pages for writing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2378) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2379) int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2380) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2381) int valid_sum_count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2382) int i, sum_in_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2384) for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2385) if (sbi->ckpt->alloc_type[i] == SSR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2386) valid_sum_count += sbi->blocks_per_seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2387) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2388) if (for_ra)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2389) valid_sum_count += le16_to_cpu(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2390) F2FS_CKPT(sbi)->cur_data_blkoff[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2391) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2392) valid_sum_count += curseg_blkoff(sbi, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2393) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2394) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2396) sum_in_page = (PAGE_SIZE - 2 * SUM_JOURNAL_SIZE -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2397) SUM_FOOTER_SIZE) / SUMMARY_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2398) if (valid_sum_count <= sum_in_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2399) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2400) else if ((valid_sum_count - sum_in_page) <=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2401) (PAGE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2402) return 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2403) return 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2404) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2406) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2407) * Caller should put this summary page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2408) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2409) struct page *f2fs_get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2410) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2411) if (unlikely(f2fs_cp_error(sbi)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2412) return ERR_PTR(-EIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2413) return f2fs_get_meta_page_retry(sbi, GET_SUM_BLOCK(sbi, segno));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2414) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2416) void f2fs_update_meta_page(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2417) void *src, block_t blk_addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2418) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2419) struct page *page = f2fs_grab_meta_page(sbi, blk_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2421) memcpy(page_address(page), src, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2422) set_page_dirty(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2423) f2fs_put_page(page, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2424) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2426) static void write_sum_page(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2427) struct f2fs_summary_block *sum_blk, block_t blk_addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2428) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2429) f2fs_update_meta_page(sbi, (void *)sum_blk, blk_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2430) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2432) static void write_current_sum_page(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2433) int type, block_t blk_addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2434) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2435) struct curseg_info *curseg = CURSEG_I(sbi, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2436) struct page *page = f2fs_grab_meta_page(sbi, blk_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2437) struct f2fs_summary_block *src = curseg->sum_blk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2438) struct f2fs_summary_block *dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2440) dst = (struct f2fs_summary_block *)page_address(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2441) memset(dst, 0, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2442)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2443) mutex_lock(&curseg->curseg_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2445) down_read(&curseg->journal_rwsem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2446) memcpy(&dst->journal, curseg->journal, SUM_JOURNAL_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2447) up_read(&curseg->journal_rwsem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2449) memcpy(dst->entries, src->entries, SUM_ENTRY_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2450) memcpy(&dst->footer, &src->footer, SUM_FOOTER_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2452) mutex_unlock(&curseg->curseg_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2454) set_page_dirty(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2455) f2fs_put_page(page, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2456) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2458) static int is_next_segment_free(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2459) struct curseg_info *curseg, int type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2460) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2461) unsigned int segno = curseg->segno + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2462) struct free_segmap_info *free_i = FREE_I(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2464) if (segno < MAIN_SEGS(sbi) && segno % sbi->segs_per_sec)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2465) return !test_bit(segno, free_i->free_segmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2466) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2467) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2469) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2470) * Find a new segment from the free segments bitmap to right order
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2471) * This function should be returned with success, otherwise BUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2472) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2473) static void get_new_segment(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2474) unsigned int *newseg, bool new_sec, int dir)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2475) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2476) struct free_segmap_info *free_i = FREE_I(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2477) unsigned int segno, secno, zoneno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2478) unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2479) unsigned int hint = GET_SEC_FROM_SEG(sbi, *newseg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2480) unsigned int old_zoneno = GET_ZONE_FROM_SEG(sbi, *newseg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2481) unsigned int left_start = hint;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2482) bool init = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2483) int go_left = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2484) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2486) spin_lock(&free_i->segmap_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2488) if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2489) segno = find_next_zero_bit(free_i->free_segmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2490) GET_SEG_FROM_SEC(sbi, hint + 1), *newseg + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2491) if (segno < GET_SEG_FROM_SEC(sbi, hint + 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2492) goto got_it;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2493) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2494) find_other_zone:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2495) secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2496) if (secno >= MAIN_SECS(sbi)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2497) if (dir == ALLOC_RIGHT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2498) secno = find_next_zero_bit(free_i->free_secmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2499) MAIN_SECS(sbi), 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2500) f2fs_bug_on(sbi, secno >= MAIN_SECS(sbi));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2501) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2502) go_left = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2503) left_start = hint - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2504) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2505) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2506) if (go_left == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2507) goto skip_left;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2509) while (test_bit(left_start, free_i->free_secmap)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2510) if (left_start > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2511) left_start--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2512) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2513) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2514) left_start = find_next_zero_bit(free_i->free_secmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2515) MAIN_SECS(sbi), 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2516) f2fs_bug_on(sbi, left_start >= MAIN_SECS(sbi));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2517) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2518) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2519) secno = left_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2520) skip_left:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2521) segno = GET_SEG_FROM_SEC(sbi, secno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2522) zoneno = GET_ZONE_FROM_SEC(sbi, secno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2524) /* give up on finding another zone */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2525) if (!init)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2526) goto got_it;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2527) if (sbi->secs_per_zone == 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2528) goto got_it;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2529) if (zoneno == old_zoneno)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2530) goto got_it;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2531) if (dir == ALLOC_LEFT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2532) if (!go_left && zoneno + 1 >= total_zones)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2533) goto got_it;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2534) if (go_left && zoneno == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2535) goto got_it;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2536) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2537) for (i = 0; i < NR_CURSEG_TYPE; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2538) if (CURSEG_I(sbi, i)->zone == zoneno)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2539) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2541) if (i < NR_CURSEG_TYPE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2542) /* zone is in user, try another */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2543) if (go_left)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2544) hint = zoneno * sbi->secs_per_zone - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2545) else if (zoneno + 1 >= total_zones)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2546) hint = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2547) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2548) hint = (zoneno + 1) * sbi->secs_per_zone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2549) init = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2550) goto find_other_zone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2551) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2552) got_it:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2553) /* set it as dirty segment in free segmap */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2554) f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2555) __set_inuse(sbi, segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2556) *newseg = segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2557) spin_unlock(&free_i->segmap_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2558) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2560) static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2561) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2562) struct curseg_info *curseg = CURSEG_I(sbi, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2563) struct summary_footer *sum_footer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2564) unsigned short seg_type = curseg->seg_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2566) curseg->inited = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2567) curseg->segno = curseg->next_segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2568) curseg->zone = GET_ZONE_FROM_SEG(sbi, curseg->segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2569) curseg->next_blkoff = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2570) curseg->next_segno = NULL_SEGNO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2572) sum_footer = &(curseg->sum_blk->footer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2573) memset(sum_footer, 0, sizeof(struct summary_footer));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2575) sanity_check_seg_type(sbi, seg_type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2576)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2577) if (IS_DATASEG(seg_type))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2578) SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2579) if (IS_NODESEG(seg_type))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2580) SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2581) __set_sit_entry_type(sbi, seg_type, curseg->segno, modified);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2582) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2584) static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2585) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2586) struct curseg_info *curseg = CURSEG_I(sbi, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2587) unsigned short seg_type = curseg->seg_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2589) sanity_check_seg_type(sbi, seg_type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2591) /* if segs_per_sec is large than 1, we need to keep original policy. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2592) if (__is_large_section(sbi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2593) return curseg->segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2595) /* inmem log may not locate on any segment after mount */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2596) if (!curseg->inited)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2597) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2599) if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2600) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2602) if (test_opt(sbi, NOHEAP) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2603) (seg_type == CURSEG_HOT_DATA || IS_NODESEG(seg_type)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2604) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2606) if (SIT_I(sbi)->last_victim[ALLOC_NEXT])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2607) return SIT_I(sbi)->last_victim[ALLOC_NEXT];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2609) /* find segments from 0 to reuse freed segments */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2610) if (F2FS_OPTION(sbi).alloc_mode == ALLOC_MODE_REUSE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2611) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2612)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2613) return curseg->segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2614) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2616) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2617) * Allocate a current working segment.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2618) * This function always allocates a free segment in LFS manner.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2619) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2620) static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2621) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2622) struct curseg_info *curseg = CURSEG_I(sbi, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2623) unsigned short seg_type = curseg->seg_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2624) unsigned int segno = curseg->segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2625) int dir = ALLOC_LEFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2627) if (curseg->inited)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2628) write_sum_page(sbi, curseg->sum_blk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2629) GET_SUM_BLOCK(sbi, segno));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2630) if (seg_type == CURSEG_WARM_DATA || seg_type == CURSEG_COLD_DATA)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2631) dir = ALLOC_RIGHT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2632)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2633) if (test_opt(sbi, NOHEAP))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2634) dir = ALLOC_RIGHT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2635)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2636) segno = __get_next_segno(sbi, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2637) get_new_segment(sbi, &segno, new_sec, dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2638) curseg->next_segno = segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2639) reset_curseg(sbi, type, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2640) curseg->alloc_type = LFS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2641) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2643) static int __next_free_blkoff(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2644) int segno, block_t start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2645) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2646) struct seg_entry *se = get_seg_entry(sbi, segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2647) int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2648) unsigned long *target_map = SIT_I(sbi)->tmp_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2649) unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2650) unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2651) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2653) for (i = 0; i < entries; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2654) target_map[i] = ckpt_map[i] | cur_map[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2656) return __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2657) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2659) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2660) * If a segment is written by LFS manner, next block offset is just obtained
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2661) * by increasing the current block offset. However, if a segment is written by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2662) * SSR manner, next block offset obtained by calling __next_free_blkoff
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2663) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2664) static void __refresh_next_blkoff(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2665) struct curseg_info *seg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2666) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2667) if (seg->alloc_type == SSR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2668) seg->next_blkoff =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2669) __next_free_blkoff(sbi, seg->segno,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2670) seg->next_blkoff + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2671) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2672) seg->next_blkoff++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2673) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2674)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2675) bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2676) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2677) return __next_free_blkoff(sbi, segno, 0) < sbi->blocks_per_seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2678) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2680) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2681) * This function always allocates a used segment(from dirty seglist) by SSR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2682) * manner, so it should recover the existing segment information of valid blocks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2683) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2684) static void change_curseg(struct f2fs_sb_info *sbi, int type, bool flush)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2685) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2686) struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2687) struct curseg_info *curseg = CURSEG_I(sbi, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2688) unsigned int new_segno = curseg->next_segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2689) struct f2fs_summary_block *sum_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2690) struct page *sum_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2692) if (flush)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2693) write_sum_page(sbi, curseg->sum_blk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2694) GET_SUM_BLOCK(sbi, curseg->segno));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2696) __set_test_and_inuse(sbi, new_segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2697)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2698) mutex_lock(&dirty_i->seglist_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2699) __remove_dirty_segment(sbi, new_segno, PRE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2700) __remove_dirty_segment(sbi, new_segno, DIRTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2701) mutex_unlock(&dirty_i->seglist_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2703) reset_curseg(sbi, type, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2704) curseg->alloc_type = SSR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2705) curseg->next_blkoff = __next_free_blkoff(sbi, curseg->segno, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2706)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2707) sum_page = f2fs_get_sum_page(sbi, new_segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2708) if (IS_ERR(sum_page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2709) /* GC won't be able to use stale summary pages by cp_error */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2710) memset(curseg->sum_blk, 0, SUM_ENTRY_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2711) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2712) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2713) sum_node = (struct f2fs_summary_block *)page_address(sum_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2714) memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2715) f2fs_put_page(sum_page, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2716) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2718) static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2719) int alloc_mode, unsigned long long age);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2720)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2721) static void get_atssr_segment(struct f2fs_sb_info *sbi, int type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2722) int target_type, int alloc_mode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2723) unsigned long long age)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2724) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2725) struct curseg_info *curseg = CURSEG_I(sbi, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2726)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2727) curseg->seg_type = target_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2728)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2729) if (get_ssr_segment(sbi, type, alloc_mode, age)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2730) struct seg_entry *se = get_seg_entry(sbi, curseg->next_segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2731)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2732) curseg->seg_type = se->type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2733) change_curseg(sbi, type, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2734) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2735) /* allocate cold segment by default */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2736) curseg->seg_type = CURSEG_COLD_DATA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2737) new_curseg(sbi, type, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2738) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2739) stat_inc_seg_type(sbi, curseg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2740) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2741)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2742) static void __f2fs_init_atgc_curseg(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2743) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2744) struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_ALL_DATA_ATGC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2745)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2746) if (!sbi->am.atgc_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2747) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2749) f2fs_down_read(&SM_I(sbi)->curseg_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2750)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2751) mutex_lock(&curseg->curseg_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2752) down_write(&SIT_I(sbi)->sentry_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2753)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2754) get_atssr_segment(sbi, CURSEG_ALL_DATA_ATGC, CURSEG_COLD_DATA, SSR, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2755)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2756) up_write(&SIT_I(sbi)->sentry_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2757) mutex_unlock(&curseg->curseg_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2758)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2759) f2fs_up_read(&SM_I(sbi)->curseg_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2761) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2762) void f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2763) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2764) __f2fs_init_atgc_curseg(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2765) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2767) static void __f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi, int type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2768) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2769) struct curseg_info *curseg = CURSEG_I(sbi, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2770)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2771) mutex_lock(&curseg->curseg_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2772) if (!curseg->inited)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2773) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2774)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2775) if (get_valid_blocks(sbi, curseg->segno, false)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2776) write_sum_page(sbi, curseg->sum_blk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2777) GET_SUM_BLOCK(sbi, curseg->segno));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2778) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2779) mutex_lock(&DIRTY_I(sbi)->seglist_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2780) __set_test_and_free(sbi, curseg->segno, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2781) mutex_unlock(&DIRTY_I(sbi)->seglist_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2782) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2783) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2784) mutex_unlock(&curseg->curseg_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2785) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2786)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2787) void f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2788) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2789) __f2fs_save_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2790)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2791) if (sbi->am.atgc_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2792) __f2fs_save_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2793) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2795) static void __f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi, int type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2796) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2797) struct curseg_info *curseg = CURSEG_I(sbi, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2798)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2799) mutex_lock(&curseg->curseg_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2800) if (!curseg->inited)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2801) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2802) if (get_valid_blocks(sbi, curseg->segno, false))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2803) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2804)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2805) mutex_lock(&DIRTY_I(sbi)->seglist_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2806) __set_test_and_inuse(sbi, curseg->segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2807) mutex_unlock(&DIRTY_I(sbi)->seglist_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2808) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2809) mutex_unlock(&curseg->curseg_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2810) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2812) void f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2813) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2814) __f2fs_restore_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2815)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2816) if (sbi->am.atgc_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2817) __f2fs_restore_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2818) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2819)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2820) static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2821) int alloc_mode, unsigned long long age)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2822) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2823) struct curseg_info *curseg = CURSEG_I(sbi, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2824) const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2825) unsigned segno = NULL_SEGNO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2826) unsigned short seg_type = curseg->seg_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2827) int i, cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2828) bool reversed = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2829)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2830) sanity_check_seg_type(sbi, seg_type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2831)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2832) /* f2fs_need_SSR() already forces to do this */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2833) if (!v_ops->get_victim(sbi, &segno, BG_GC, seg_type, alloc_mode, age)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2834) curseg->next_segno = segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2835) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2836) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2837)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2838) /* For node segments, let's do SSR more intensively */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2839) if (IS_NODESEG(seg_type)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2840) if (seg_type >= CURSEG_WARM_NODE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2841) reversed = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2842) i = CURSEG_COLD_NODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2843) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2844) i = CURSEG_HOT_NODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2845) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2846) cnt = NR_CURSEG_NODE_TYPE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2847) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2848) if (seg_type >= CURSEG_WARM_DATA) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2849) reversed = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2850) i = CURSEG_COLD_DATA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2851) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2852) i = CURSEG_HOT_DATA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2853) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2854) cnt = NR_CURSEG_DATA_TYPE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2855) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2856)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2857) for (; cnt-- > 0; reversed ? i-- : i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2858) if (i == seg_type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2859) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2860) if (!v_ops->get_victim(sbi, &segno, BG_GC, i, alloc_mode, age)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2861) curseg->next_segno = segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2862) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2863) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2864) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2865)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2866) /* find valid_blocks=0 in dirty list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2867) if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2868) segno = get_free_segment(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2869) if (segno != NULL_SEGNO) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2870) curseg->next_segno = segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2871) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2872) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2873) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2874) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2875) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2876)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2877) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2878) * flush out current segment and replace it with new segment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2879) * This function should be returned with success, otherwise BUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2880) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2881) static void allocate_segment_by_default(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2882) int type, bool force)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2883) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2884) struct curseg_info *curseg = CURSEG_I(sbi, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2885)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2886) if (force)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2887) new_curseg(sbi, type, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2888) else if (!is_set_ckpt_flags(sbi, CP_CRC_RECOVERY_FLAG) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2889) curseg->seg_type == CURSEG_WARM_NODE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2890) new_curseg(sbi, type, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2891) else if (curseg->alloc_type == LFS &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2892) is_next_segment_free(sbi, curseg, type) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2893) likely(!is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2894) new_curseg(sbi, type, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2895) else if (f2fs_need_SSR(sbi) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2896) get_ssr_segment(sbi, type, SSR, 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2897) change_curseg(sbi, type, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2898) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2899) new_curseg(sbi, type, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2900)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2901) stat_inc_seg_type(sbi, curseg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2902) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2904) void f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2905) unsigned int start, unsigned int end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2906) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2907) struct curseg_info *curseg = CURSEG_I(sbi, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2908) unsigned int segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2909)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2910) f2fs_down_read(&SM_I(sbi)->curseg_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2911) mutex_lock(&curseg->curseg_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2912) down_write(&SIT_I(sbi)->sentry_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2913)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2914) segno = CURSEG_I(sbi, type)->segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2915) if (segno < start || segno > end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2916) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2917)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2918) if (f2fs_need_SSR(sbi) && get_ssr_segment(sbi, type, SSR, 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2919) change_curseg(sbi, type, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2920) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2921) new_curseg(sbi, type, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2922)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2923) stat_inc_seg_type(sbi, curseg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2925) locate_dirty_segment(sbi, segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2926) unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2927) up_write(&SIT_I(sbi)->sentry_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2928)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2929) if (segno != curseg->segno)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2930) f2fs_notice(sbi, "For resize: curseg of type %d: %u ==> %u",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2931) type, segno, curseg->segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2932)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2933) mutex_unlock(&curseg->curseg_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2934) f2fs_up_read(&SM_I(sbi)->curseg_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2935) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2936)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2937) static void __allocate_new_segment(struct f2fs_sb_info *sbi, int type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2938) bool new_sec, bool force)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2939) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2940) struct curseg_info *curseg = CURSEG_I(sbi, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2941) unsigned int old_segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2942)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2943) if (!curseg->inited)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2944) goto alloc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2945)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2946) if (force || curseg->next_blkoff ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2947) get_valid_blocks(sbi, curseg->segno, new_sec))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2948) goto alloc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2949)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2950) if (!get_ckpt_valid_blocks(sbi, curseg->segno, new_sec))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2951) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2952) alloc:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2953) old_segno = curseg->segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2954) SIT_I(sbi)->s_ops->allocate_segment(sbi, type, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2955) locate_dirty_segment(sbi, old_segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2956) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2957)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2958) static void __allocate_new_section(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2959) int type, bool force)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2960) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2961) __allocate_new_segment(sbi, type, true, force);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2962) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2963)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2964) void f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type, bool force)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2965) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2966) f2fs_down_read(&SM_I(sbi)->curseg_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2967) down_write(&SIT_I(sbi)->sentry_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2968) __allocate_new_section(sbi, type, force);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2969) up_write(&SIT_I(sbi)->sentry_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2970) f2fs_up_read(&SM_I(sbi)->curseg_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2971) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2972)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2973) void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2974) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2975) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2976)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2977) f2fs_down_read(&SM_I(sbi)->curseg_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2978) down_write(&SIT_I(sbi)->sentry_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2979) for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2980) __allocate_new_segment(sbi, i, false, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2981) up_write(&SIT_I(sbi)->sentry_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2982) f2fs_up_read(&SM_I(sbi)->curseg_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2983) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2984)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2985) static const struct segment_allocation default_salloc_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2986) .allocate_segment = allocate_segment_by_default,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2987) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2988)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2989) bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2990) struct cp_control *cpc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2991) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2992) __u64 trim_start = cpc->trim_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2993) bool has_candidate = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2994)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2995) down_write(&SIT_I(sbi)->sentry_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2996) for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2997) if (add_discard_addrs(sbi, cpc, true)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2998) has_candidate = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2999) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3000) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3001) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3002) up_write(&SIT_I(sbi)->sentry_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3003)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3004) cpc->trim_start = trim_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3005) return has_candidate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3006) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3007)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3008) static unsigned int __issue_discard_cmd_range(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3009) struct discard_policy *dpolicy,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3010) unsigned int start, unsigned int end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3011) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3012) struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3013) struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3014) struct rb_node **insert_p = NULL, *insert_parent = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3015) struct discard_cmd *dc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3016) struct blk_plug plug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3017) int issued;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3018) unsigned int trimmed = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3019)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3020) next:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3021) issued = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3022)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3023) mutex_lock(&dcc->cmd_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3024) if (unlikely(dcc->rbtree_check))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3025) f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3026) &dcc->root, false));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3027)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3028) dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3029) NULL, start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3030) (struct rb_entry **)&prev_dc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3031) (struct rb_entry **)&next_dc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3032) &insert_p, &insert_parent, true, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3033) if (!dc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3034) dc = next_dc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3035)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3036) blk_start_plug(&plug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3037)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3038) while (dc && dc->lstart <= end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3039) struct rb_node *node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3040) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3041)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3042) if (dc->len < dpolicy->granularity)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3043) goto skip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3044)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3045) if (dc->state != D_PREP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3046) list_move_tail(&dc->list, &dcc->fstrim_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3047) goto skip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3048) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3049)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3050) err = __submit_discard_cmd(sbi, dpolicy, dc, &issued);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3051)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3052) if (issued >= dpolicy->max_requests) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3053) start = dc->lstart + dc->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3054)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3055) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3056) __remove_discard_cmd(sbi, dc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3057)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3058) blk_finish_plug(&plug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3059) mutex_unlock(&dcc->cmd_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3060) trimmed += __wait_all_discard_cmd(sbi, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3061) congestion_wait(BLK_RW_ASYNC, DEFAULT_IO_TIMEOUT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3062) goto next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3063) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3064) skip:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3065) node = rb_next(&dc->rb_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3066) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3067) __remove_discard_cmd(sbi, dc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3068) dc = rb_entry_safe(node, struct discard_cmd, rb_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3069)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3070) if (fatal_signal_pending(current))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3071) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3072) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3073)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3074) blk_finish_plug(&plug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3075) mutex_unlock(&dcc->cmd_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3076)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3077) return trimmed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3078) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3079)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3080) int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3081) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3082) __u64 start = F2FS_BYTES_TO_BLK(range->start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3083) __u64 end = start + F2FS_BYTES_TO_BLK(range->len) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3084) unsigned int start_segno, end_segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3085) block_t start_block, end_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3086) struct cp_control cpc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3087) struct discard_policy dpolicy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3088) unsigned long long trimmed = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3089) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3090) bool need_align = f2fs_lfs_mode(sbi) && __is_large_section(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3091)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3092) if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3093) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3094)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3095) if (end < MAIN_BLKADDR(sbi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3096) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3097)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3098) if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3099) f2fs_warn(sbi, "Found FS corruption, run fsck to fix.");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3100) return -EFSCORRUPTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3101) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3103) /* start/end segment number in main_area */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3104) start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3105) end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3106) GET_SEGNO(sbi, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3107) if (need_align) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3108) start_segno = rounddown(start_segno, sbi->segs_per_sec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3109) end_segno = roundup(end_segno + 1, sbi->segs_per_sec) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3110) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3112) cpc.reason = CP_DISCARD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3113) cpc.trim_minlen = max_t(__u64, 1, F2FS_BYTES_TO_BLK(range->minlen));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3114) cpc.trim_start = start_segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3115) cpc.trim_end = end_segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3117) if (sbi->discard_blks == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3118) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3120) f2fs_down_write(&sbi->gc_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3121) err = f2fs_write_checkpoint(sbi, &cpc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3122) f2fs_up_write(&sbi->gc_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3123) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3124) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3126) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3127) * We filed discard candidates, but actually we don't need to wait for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3128) * all of them, since they'll be issued in idle time along with runtime
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3129) * discard option. User configuration looks like using runtime discard
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3130) * or periodic fstrim instead of it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3131) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3132) if (f2fs_realtime_discard_enable(sbi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3133) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3135) start_block = START_BLOCK(sbi, start_segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3136) end_block = START_BLOCK(sbi, end_segno + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3138) __init_discard_policy(sbi, &dpolicy, DPOLICY_FSTRIM, cpc.trim_minlen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3139) trimmed = __issue_discard_cmd_range(sbi, &dpolicy,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3140) start_block, end_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3142) trimmed += __wait_discard_cmd_range(sbi, &dpolicy,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3143) start_block, end_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3144) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3145) if (!err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3146) range->len = F2FS_BLK_TO_BYTES(trimmed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3147) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3148) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3150) static bool __has_curseg_space(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3151) struct curseg_info *curseg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3152) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3153) return curseg->next_blkoff < f2fs_usable_blks_in_seg(sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3154) curseg->segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3155) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3157) int f2fs_rw_hint_to_seg_type(enum rw_hint hint)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3158) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3159) switch (hint) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3160) case WRITE_LIFE_SHORT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3161) return CURSEG_HOT_DATA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3162) case WRITE_LIFE_EXTREME:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3163) return CURSEG_COLD_DATA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3164) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3165) return CURSEG_WARM_DATA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3166) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3167) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3169) /* This returns write hints for each segment type. This hints will be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3170) * passed down to block layer. There are mapping tables which depend on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3171) * the mount option 'whint_mode'.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3172) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3173) * 1) whint_mode=off. F2FS only passes down WRITE_LIFE_NOT_SET.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3174) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3175) * 2) whint_mode=user-based. F2FS tries to pass down hints given by users.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3176) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3177) * User F2FS Block
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3178) * ---- ---- -----
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3179) * META WRITE_LIFE_NOT_SET
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3180) * HOT_NODE "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3181) * WARM_NODE "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3182) * COLD_NODE "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3183) * ioctl(COLD) COLD_DATA WRITE_LIFE_EXTREME
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3184) * extension list " "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3185) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3186) * -- buffered io
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3187) * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3188) * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3189) * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_NOT_SET
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3190) * WRITE_LIFE_NONE " "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3191) * WRITE_LIFE_MEDIUM " "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3192) * WRITE_LIFE_LONG " "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3193) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3194) * -- direct io
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3195) * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3196) * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3197) * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_NOT_SET
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3198) * WRITE_LIFE_NONE " WRITE_LIFE_NONE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3199) * WRITE_LIFE_MEDIUM " WRITE_LIFE_MEDIUM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3200) * WRITE_LIFE_LONG " WRITE_LIFE_LONG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3201) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3202) * 3) whint_mode=fs-based. F2FS passes down hints with its policy.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3203) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3204) * User F2FS Block
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3205) * ---- ---- -----
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3206) * META WRITE_LIFE_MEDIUM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3207) * HOT_NODE WRITE_LIFE_NOT_SET
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3208) * WARM_NODE "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3209) * COLD_NODE WRITE_LIFE_NONE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3210) * ioctl(COLD) COLD_DATA WRITE_LIFE_EXTREME
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3211) * extension list " "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3212) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3213) * -- buffered io
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3214) * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3215) * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3216) * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_LONG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3217) * WRITE_LIFE_NONE " "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3218) * WRITE_LIFE_MEDIUM " "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3219) * WRITE_LIFE_LONG " "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3220) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3221) * -- direct io
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3222) * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3223) * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3224) * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_NOT_SET
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3225) * WRITE_LIFE_NONE " WRITE_LIFE_NONE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3226) * WRITE_LIFE_MEDIUM " WRITE_LIFE_MEDIUM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3227) * WRITE_LIFE_LONG " WRITE_LIFE_LONG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3228) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3230) enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3231) enum page_type type, enum temp_type temp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3232) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3233) if (F2FS_OPTION(sbi).whint_mode == WHINT_MODE_USER) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3234) if (type == DATA) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3235) if (temp == WARM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3236) return WRITE_LIFE_NOT_SET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3237) else if (temp == HOT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3238) return WRITE_LIFE_SHORT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3239) else if (temp == COLD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3240) return WRITE_LIFE_EXTREME;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3241) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3242) return WRITE_LIFE_NOT_SET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3243) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3244) } else if (F2FS_OPTION(sbi).whint_mode == WHINT_MODE_FS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3245) if (type == DATA) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3246) if (temp == WARM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3247) return WRITE_LIFE_LONG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3248) else if (temp == HOT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3249) return WRITE_LIFE_SHORT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3250) else if (temp == COLD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3251) return WRITE_LIFE_EXTREME;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3252) } else if (type == NODE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3253) if (temp == WARM || temp == HOT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3254) return WRITE_LIFE_NOT_SET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3255) else if (temp == COLD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3256) return WRITE_LIFE_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3257) } else if (type == META) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3258) return WRITE_LIFE_MEDIUM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3259) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3260) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3261) return WRITE_LIFE_NOT_SET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3262) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3264) static int __get_segment_type_2(struct f2fs_io_info *fio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3265) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3266) if (fio->type == DATA)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3267) return CURSEG_HOT_DATA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3268) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3269) return CURSEG_HOT_NODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3270) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3272) static int __get_segment_type_4(struct f2fs_io_info *fio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3273) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3274) if (fio->type == DATA) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3275) struct inode *inode = fio->page->mapping->host;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3277) if (S_ISDIR(inode->i_mode))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3278) return CURSEG_HOT_DATA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3279) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3280) return CURSEG_COLD_DATA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3281) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3282) if (IS_DNODE(fio->page) && is_cold_node(fio->page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3283) return CURSEG_WARM_NODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3284) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3285) return CURSEG_COLD_NODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3286) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3287) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3289) static int __get_segment_type_6(struct f2fs_io_info *fio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3290) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3291) if (fio->type == DATA) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3292) struct inode *inode = fio->page->mapping->host;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3294) if (is_inode_flag_set(inode, FI_ALIGNED_WRITE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3295) return CURSEG_COLD_DATA_PINNED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3297) if (page_private_gcing(fio->page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3298) if (fio->sbi->am.atgc_enabled &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3299) (fio->io_type == FS_DATA_IO) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3300) (fio->sbi->gc_mode != GC_URGENT_HIGH))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3301) return CURSEG_ALL_DATA_ATGC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3302) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3303) return CURSEG_COLD_DATA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3304) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3305) if (file_is_cold(inode) || f2fs_need_compress_data(inode))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3306) return CURSEG_COLD_DATA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3307) if (file_is_hot(inode) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3308) is_inode_flag_set(inode, FI_HOT_DATA) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3309) f2fs_is_atomic_file(inode) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3310) f2fs_is_volatile_file(inode))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3311) return CURSEG_HOT_DATA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3312) return f2fs_rw_hint_to_seg_type(inode->i_write_hint);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3313) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3314) if (IS_DNODE(fio->page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3315) return is_cold_node(fio->page) ? CURSEG_WARM_NODE :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3316) CURSEG_HOT_NODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3317) return CURSEG_COLD_NODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3318) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3319) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3321) static int __get_segment_type(struct f2fs_io_info *fio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3322) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3323) int type = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3325) switch (F2FS_OPTION(fio->sbi).active_logs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3326) case 2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3327) type = __get_segment_type_2(fio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3328) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3329) case 4:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3330) type = __get_segment_type_4(fio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3331) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3332) case 6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3333) type = __get_segment_type_6(fio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3334) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3335) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3336) f2fs_bug_on(fio->sbi, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3337) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3339) if (IS_HOT(type))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3340) fio->temp = HOT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3341) else if (IS_WARM(type))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3342) fio->temp = WARM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3343) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3344) fio->temp = COLD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3345) return type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3346) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3348) void f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3349) block_t old_blkaddr, block_t *new_blkaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3350) struct f2fs_summary *sum, int type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3351) struct f2fs_io_info *fio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3352) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3353) struct sit_info *sit_i = SIT_I(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3354) struct curseg_info *curseg = CURSEG_I(sbi, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3355) unsigned long long old_mtime;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3356) bool from_gc = (type == CURSEG_ALL_DATA_ATGC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3357) struct seg_entry *se = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3358)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3359) f2fs_down_read(&SM_I(sbi)->curseg_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3361) mutex_lock(&curseg->curseg_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3362) down_write(&sit_i->sentry_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3364) if (from_gc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3365) f2fs_bug_on(sbi, GET_SEGNO(sbi, old_blkaddr) == NULL_SEGNO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3366) se = get_seg_entry(sbi, GET_SEGNO(sbi, old_blkaddr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3367) sanity_check_seg_type(sbi, se->type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3368) f2fs_bug_on(sbi, IS_NODESEG(se->type));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3369) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3370) *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3372) f2fs_bug_on(sbi, curseg->next_blkoff >= sbi->blocks_per_seg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3374) f2fs_wait_discard_bio(sbi, *new_blkaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3376) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3377) * __add_sum_entry should be resided under the curseg_mutex
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3378) * because, this function updates a summary entry in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3379) * current summary block.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3380) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3381) __add_sum_entry(sbi, type, sum);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3383) __refresh_next_blkoff(sbi, curseg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3385) stat_inc_block_count(sbi, curseg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3387) if (from_gc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3388) old_mtime = get_segment_mtime(sbi, old_blkaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3389) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3390) update_segment_mtime(sbi, old_blkaddr, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3391) old_mtime = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3392) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3393) update_segment_mtime(sbi, *new_blkaddr, old_mtime);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3395) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3396) * SIT information should be updated before segment allocation,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3397) * since SSR needs latest valid block information.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3398) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3399) update_sit_entry(sbi, *new_blkaddr, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3400) if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3401) update_sit_entry(sbi, old_blkaddr, -1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3403) if (!__has_curseg_space(sbi, curseg)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3404) if (from_gc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3405) get_atssr_segment(sbi, type, se->type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3406) AT_SSR, se->mtime);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3407) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3408) sit_i->s_ops->allocate_segment(sbi, type, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3409) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3410) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3411) * segment dirty status should be updated after segment allocation,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3412) * so we just need to update status only one time after previous
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3413) * segment being closed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3414) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3415) locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3416) locate_dirty_segment(sbi, GET_SEGNO(sbi, *new_blkaddr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3418) up_write(&sit_i->sentry_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3420) if (page && IS_NODESEG(type)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3421) fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3423) f2fs_inode_chksum_set(sbi, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3424) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3426) if (fio) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3427) struct f2fs_bio_info *io;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3428)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3429) if (F2FS_IO_ALIGNED(sbi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3430) fio->retry = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3432) INIT_LIST_HEAD(&fio->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3433) fio->in_list = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3434) io = sbi->write_io[fio->type] + fio->temp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3435) spin_lock(&io->io_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3436) list_add_tail(&fio->list, &io->io_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3437) spin_unlock(&io->io_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3438) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3440) mutex_unlock(&curseg->curseg_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3442) f2fs_up_read(&SM_I(sbi)->curseg_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3443) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3445) static void update_device_state(struct f2fs_io_info *fio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3446) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3447) struct f2fs_sb_info *sbi = fio->sbi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3448) unsigned int devidx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3450) if (!f2fs_is_multi_device(sbi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3451) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3453) devidx = f2fs_target_device_index(sbi, fio->new_blkaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3455) /* update device state for fsync */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3456) f2fs_set_dirty_device(sbi, fio->ino, devidx, FLUSH_INO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3458) /* update device state for checkpoint */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3459) if (!f2fs_test_bit(devidx, (char *)&sbi->dirty_device)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3460) spin_lock(&sbi->dev_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3461) f2fs_set_bit(devidx, (char *)&sbi->dirty_device);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3462) spin_unlock(&sbi->dev_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3463) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3464) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3466) static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3467) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3468) int type = __get_segment_type(fio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3469) bool keep_order = (f2fs_lfs_mode(fio->sbi) && type == CURSEG_COLD_DATA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3471) if (keep_order)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3472) f2fs_down_read(&fio->sbi->io_order_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3473) reallocate:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3474) f2fs_allocate_data_block(fio->sbi, fio->page, fio->old_blkaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3475) &fio->new_blkaddr, sum, type, fio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3476) if (GET_SEGNO(fio->sbi, fio->old_blkaddr) != NULL_SEGNO) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3477) invalidate_mapping_pages(META_MAPPING(fio->sbi),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3478) fio->old_blkaddr, fio->old_blkaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3479) f2fs_invalidate_compress_page(fio->sbi, fio->old_blkaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3480) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3482) /* writeout dirty page into bdev */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3483) f2fs_submit_page_write(fio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3484) if (fio->retry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3485) fio->old_blkaddr = fio->new_blkaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3486) goto reallocate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3487) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3489) update_device_state(fio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3491) if (keep_order)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3492) f2fs_up_read(&fio->sbi->io_order_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3493) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3495) void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3496) enum iostat_type io_type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3497) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3498) struct f2fs_io_info fio = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3499) .sbi = sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3500) .type = META,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3501) .temp = HOT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3502) .op = REQ_OP_WRITE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3503) .op_flags = REQ_SYNC | REQ_META | REQ_PRIO,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3504) .old_blkaddr = page->index,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3505) .new_blkaddr = page->index,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3506) .page = page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3507) .encrypted_page = NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3508) .in_list = false,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3509) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3511) if (unlikely(page->index >= MAIN_BLKADDR(sbi)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3512) fio.op_flags &= ~REQ_META;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3514) set_page_writeback(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3515) ClearPageError(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3516) f2fs_submit_page_write(&fio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3518) stat_inc_meta_count(sbi, page->index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3519) f2fs_update_iostat(sbi, io_type, F2FS_BLKSIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3520) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3521)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3522) void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3523) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3524) struct f2fs_summary sum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3526) set_summary(&sum, nid, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3527) do_write_page(&sum, fio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3529) f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3530) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3532) void f2fs_outplace_write_data(struct dnode_of_data *dn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3533) struct f2fs_io_info *fio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3534) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3535) struct f2fs_sb_info *sbi = fio->sbi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3536) struct f2fs_summary sum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3538) f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3539) set_summary(&sum, dn->nid, dn->ofs_in_node, fio->version);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3540) do_write_page(&sum, fio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3541) f2fs_update_data_blkaddr(dn, fio->new_blkaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3543) f2fs_update_iostat(sbi, fio->io_type, F2FS_BLKSIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3544) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3546) int f2fs_inplace_write_data(struct f2fs_io_info *fio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3547) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3548) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3549) struct f2fs_sb_info *sbi = fio->sbi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3550) unsigned int segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3552) fio->new_blkaddr = fio->old_blkaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3553) /* i/o temperature is needed for passing down write hints */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3554) __get_segment_type(fio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3555)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3556) segno = GET_SEGNO(sbi, fio->new_blkaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3558) if (!IS_DATASEG(get_seg_entry(sbi, segno)->type)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3559) set_sbi_flag(sbi, SBI_NEED_FSCK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3560) f2fs_warn(sbi, "%s: incorrect segment(%u) type, run fsck to fix.",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3561) __func__, segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3562) err = -EFSCORRUPTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3563) goto drop_bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3564) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3566) if (f2fs_cp_error(sbi)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3567) err = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3568) goto drop_bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3569) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3571) stat_inc_inplace_blocks(fio->sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3573) if (fio->bio && !(SM_I(sbi)->ipu_policy & (1 << F2FS_IPU_NOCACHE)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3574) err = f2fs_merge_page_bio(fio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3575) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3576) err = f2fs_submit_page_bio(fio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3577) if (!err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3578) update_device_state(fio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3579) f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3580) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3582) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3583) drop_bio:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3584) if (fio->bio && *(fio->bio)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3585) struct bio *bio = *(fio->bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3587) bio->bi_status = BLK_STS_IOERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3588) bio_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3589) *(fio->bio) = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3590) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3591) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3592) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3593)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3594) static inline int __f2fs_get_curseg(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3595) unsigned int segno)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3596) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3597) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3599) for (i = CURSEG_HOT_DATA; i < NO_CHECK_TYPE; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3600) if (CURSEG_I(sbi, i)->segno == segno)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3601) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3602) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3603) return i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3604) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3606) void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3607) block_t old_blkaddr, block_t new_blkaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3608) bool recover_curseg, bool recover_newaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3609) bool from_gc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3610) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3611) struct sit_info *sit_i = SIT_I(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3612) struct curseg_info *curseg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3613) unsigned int segno, old_cursegno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3614) struct seg_entry *se;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3615) int type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3616) unsigned short old_blkoff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3617) unsigned char old_alloc_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3619) segno = GET_SEGNO(sbi, new_blkaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3620) se = get_seg_entry(sbi, segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3621) type = se->type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3623) f2fs_down_write(&SM_I(sbi)->curseg_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3625) if (!recover_curseg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3626) /* for recovery flow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3627) if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3628) if (old_blkaddr == NULL_ADDR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3629) type = CURSEG_COLD_DATA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3630) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3631) type = CURSEG_WARM_DATA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3632) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3633) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3634) if (IS_CURSEG(sbi, segno)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3635) /* se->type is volatile as SSR allocation */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3636) type = __f2fs_get_curseg(sbi, segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3637) f2fs_bug_on(sbi, type == NO_CHECK_TYPE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3638) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3639) type = CURSEG_WARM_DATA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3640) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3641) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3643) f2fs_bug_on(sbi, !IS_DATASEG(type));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3644) curseg = CURSEG_I(sbi, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3646) mutex_lock(&curseg->curseg_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3647) down_write(&sit_i->sentry_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3649) old_cursegno = curseg->segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3650) old_blkoff = curseg->next_blkoff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3651) old_alloc_type = curseg->alloc_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3653) /* change the current segment */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3654) if (segno != curseg->segno) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3655) curseg->next_segno = segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3656) change_curseg(sbi, type, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3657) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3659) curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3660) __add_sum_entry(sbi, type, sum);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3661)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3662) if (!recover_curseg || recover_newaddr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3663) if (!from_gc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3664) update_segment_mtime(sbi, new_blkaddr, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3665) update_sit_entry(sbi, new_blkaddr, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3666) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3667) if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3668) invalidate_mapping_pages(META_MAPPING(sbi),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3669) old_blkaddr, old_blkaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3670) f2fs_invalidate_compress_page(sbi, old_blkaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3671) if (!from_gc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3672) update_segment_mtime(sbi, old_blkaddr, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3673) update_sit_entry(sbi, old_blkaddr, -1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3674) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3676) locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3677) locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3679) locate_dirty_segment(sbi, old_cursegno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3680)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3681) if (recover_curseg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3682) if (old_cursegno != curseg->segno) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3683) curseg->next_segno = old_cursegno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3684) change_curseg(sbi, type, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3685) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3686) curseg->next_blkoff = old_blkoff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3687) curseg->alloc_type = old_alloc_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3688) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3690) up_write(&sit_i->sentry_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3691) mutex_unlock(&curseg->curseg_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3692) f2fs_up_write(&SM_I(sbi)->curseg_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3693) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3695) void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3696) block_t old_addr, block_t new_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3697) unsigned char version, bool recover_curseg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3698) bool recover_newaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3699) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3700) struct f2fs_summary sum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3701)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3702) set_summary(&sum, dn->nid, dn->ofs_in_node, version);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3703)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3704) f2fs_do_replace_block(sbi, &sum, old_addr, new_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3705) recover_curseg, recover_newaddr, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3706)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3707) f2fs_update_data_blkaddr(dn, new_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3708) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3709)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3710) void f2fs_wait_on_page_writeback(struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3711) enum page_type type, bool ordered, bool locked)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3712) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3713) if (PageWriteback(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3714) struct f2fs_sb_info *sbi = F2FS_P_SB(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3715)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3716) /* submit cached LFS IO */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3717) f2fs_submit_merged_write_cond(sbi, NULL, page, 0, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3718) /* sbumit cached IPU IO */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3719) f2fs_submit_merged_ipu_write(sbi, NULL, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3720) if (ordered) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3721) wait_on_page_writeback(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3722) f2fs_bug_on(sbi, locked && PageWriteback(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3723) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3724) wait_for_stable_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3725) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3726) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3727) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3728)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3729) void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3730) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3731) struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3732) struct page *cpage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3734) if (!f2fs_post_read_required(inode))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3735) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3737) if (!__is_valid_data_blkaddr(blkaddr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3738) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3739)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3740) cpage = find_lock_page(META_MAPPING(sbi), blkaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3741) if (cpage) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3742) f2fs_wait_on_page_writeback(cpage, DATA, true, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3743) f2fs_put_page(cpage, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3744) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3745) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3747) void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3748) block_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3749) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3750) block_t i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3752) for (i = 0; i < len; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3753) f2fs_wait_on_block_writeback(inode, blkaddr + i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3754) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3755)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3756) static int read_compacted_summaries(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3757) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3758) struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3759) struct curseg_info *seg_i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3760) unsigned char *kaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3761) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3762) block_t start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3763) int i, j, offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3764)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3765) start = start_sum_block(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3767) page = f2fs_get_meta_page(sbi, start++);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3768) if (IS_ERR(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3769) return PTR_ERR(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3770) kaddr = (unsigned char *)page_address(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3771)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3772) /* Step 1: restore nat cache */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3773) seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3774) memcpy(seg_i->journal, kaddr, SUM_JOURNAL_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3775)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3776) /* Step 2: restore sit cache */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3777) seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3778) memcpy(seg_i->journal, kaddr + SUM_JOURNAL_SIZE, SUM_JOURNAL_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3779) offset = 2 * SUM_JOURNAL_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3781) /* Step 3: restore summary entries */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3782) for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3783) unsigned short blk_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3784) unsigned int segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3785)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3786) seg_i = CURSEG_I(sbi, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3787) segno = le32_to_cpu(ckpt->cur_data_segno[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3788) blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3789) seg_i->next_segno = segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3790) reset_curseg(sbi, i, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3791) seg_i->alloc_type = ckpt->alloc_type[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3792) seg_i->next_blkoff = blk_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3793)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3794) if (seg_i->alloc_type == SSR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3795) blk_off = sbi->blocks_per_seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3796)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3797) for (j = 0; j < blk_off; j++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3798) struct f2fs_summary *s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3799)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3800) s = (struct f2fs_summary *)(kaddr + offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3801) seg_i->sum_blk->entries[j] = *s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3802) offset += SUMMARY_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3803) if (offset + SUMMARY_SIZE <= PAGE_SIZE -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3804) SUM_FOOTER_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3805) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3806)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3807) f2fs_put_page(page, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3808) page = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3809)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3810) page = f2fs_get_meta_page(sbi, start++);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3811) if (IS_ERR(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3812) return PTR_ERR(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3813) kaddr = (unsigned char *)page_address(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3814) offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3815) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3816) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3817) f2fs_put_page(page, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3818) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3819) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3820)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3821) static int read_normal_summaries(struct f2fs_sb_info *sbi, int type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3822) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3823) struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3824) struct f2fs_summary_block *sum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3825) struct curseg_info *curseg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3826) struct page *new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3827) unsigned short blk_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3828) unsigned int segno = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3829) block_t blk_addr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3830) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3831)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3832) /* get segment number and block addr */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3833) if (IS_DATASEG(type)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3834) segno = le32_to_cpu(ckpt->cur_data_segno[type]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3835) blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3836) CURSEG_HOT_DATA]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3837) if (__exist_node_summaries(sbi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3838) blk_addr = sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3839) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3840) blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3841) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3842) segno = le32_to_cpu(ckpt->cur_node_segno[type -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3843) CURSEG_HOT_NODE]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3844) blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3845) CURSEG_HOT_NODE]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3846) if (__exist_node_summaries(sbi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3847) blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3848) type - CURSEG_HOT_NODE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3849) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3850) blk_addr = GET_SUM_BLOCK(sbi, segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3851) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3852)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3853) new = f2fs_get_meta_page(sbi, blk_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3854) if (IS_ERR(new))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3855) return PTR_ERR(new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3856) sum = (struct f2fs_summary_block *)page_address(new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3857)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3858) if (IS_NODESEG(type)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3859) if (__exist_node_summaries(sbi)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3860) struct f2fs_summary *ns = &sum->entries[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3861) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3863) for (i = 0; i < sbi->blocks_per_seg; i++, ns++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3864) ns->version = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3865) ns->ofs_in_node = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3866) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3867) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3868) err = f2fs_restore_node_summary(sbi, segno, sum);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3869) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3870) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3871) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3872) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3873)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3874) /* set uncompleted segment to curseg */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3875) curseg = CURSEG_I(sbi, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3876) mutex_lock(&curseg->curseg_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3877)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3878) /* update journal info */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3879) down_write(&curseg->journal_rwsem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3880) memcpy(curseg->journal, &sum->journal, SUM_JOURNAL_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3881) up_write(&curseg->journal_rwsem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3882)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3883) memcpy(curseg->sum_blk->entries, sum->entries, SUM_ENTRY_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3884) memcpy(&curseg->sum_blk->footer, &sum->footer, SUM_FOOTER_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3885) curseg->next_segno = segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3886) reset_curseg(sbi, type, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3887) curseg->alloc_type = ckpt->alloc_type[type];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3888) curseg->next_blkoff = blk_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3889) mutex_unlock(&curseg->curseg_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3890) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3891) f2fs_put_page(new, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3892) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3893) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3894)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3895) static int restore_curseg_summaries(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3896) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3897) struct f2fs_journal *sit_j = CURSEG_I(sbi, CURSEG_COLD_DATA)->journal;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3898) struct f2fs_journal *nat_j = CURSEG_I(sbi, CURSEG_HOT_DATA)->journal;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3899) int type = CURSEG_HOT_DATA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3900) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3901)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3902) if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3903) int npages = f2fs_npages_for_summary_flush(sbi, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3904)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3905) if (npages >= 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3906) f2fs_ra_meta_pages(sbi, start_sum_block(sbi), npages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3907) META_CP, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3908)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3909) /* restore for compacted data summary */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3910) err = read_compacted_summaries(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3911) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3912) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3913) type = CURSEG_HOT_NODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3914) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3915)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3916) if (__exist_node_summaries(sbi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3917) f2fs_ra_meta_pages(sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3918) sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3919) NR_CURSEG_PERSIST_TYPE - type, META_CP, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3920)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3921) for (; type <= CURSEG_COLD_NODE; type++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3922) err = read_normal_summaries(sbi, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3923) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3924) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3925) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3926)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3927) /* sanity check for summary blocks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3928) if (nats_in_cursum(nat_j) > NAT_JOURNAL_ENTRIES ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3929) sits_in_cursum(sit_j) > SIT_JOURNAL_ENTRIES) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3930) f2fs_err(sbi, "invalid journal entries nats %u sits %u",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3931) nats_in_cursum(nat_j), sits_in_cursum(sit_j));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3932) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3933) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3934)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3935) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3936) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3937)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3938) static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3939) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3940) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3941) unsigned char *kaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3942) struct f2fs_summary *summary;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3943) struct curseg_info *seg_i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3944) int written_size = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3945) int i, j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3946)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3947) page = f2fs_grab_meta_page(sbi, blkaddr++);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3948) kaddr = (unsigned char *)page_address(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3949) memset(kaddr, 0, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3950)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3951) /* Step 1: write nat cache */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3952) seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3953) memcpy(kaddr, seg_i->journal, SUM_JOURNAL_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3954) written_size += SUM_JOURNAL_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3955)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3956) /* Step 2: write sit cache */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3957) seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3958) memcpy(kaddr + written_size, seg_i->journal, SUM_JOURNAL_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3959) written_size += SUM_JOURNAL_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3960)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3961) /* Step 3: write summary entries */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3962) for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3963) unsigned short blkoff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3964)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3965) seg_i = CURSEG_I(sbi, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3966) if (sbi->ckpt->alloc_type[i] == SSR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3967) blkoff = sbi->blocks_per_seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3968) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3969) blkoff = curseg_blkoff(sbi, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3970)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3971) for (j = 0; j < blkoff; j++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3972) if (!page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3973) page = f2fs_grab_meta_page(sbi, blkaddr++);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3974) kaddr = (unsigned char *)page_address(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3975) memset(kaddr, 0, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3976) written_size = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3977) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3978) summary = (struct f2fs_summary *)(kaddr + written_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3979) *summary = seg_i->sum_blk->entries[j];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3980) written_size += SUMMARY_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3981)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3982) if (written_size + SUMMARY_SIZE <= PAGE_SIZE -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3983) SUM_FOOTER_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3984) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3985)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3986) set_page_dirty(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3987) f2fs_put_page(page, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3988) page = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3989) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3990) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3991) if (page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3992) set_page_dirty(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3993) f2fs_put_page(page, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3994) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3995) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3996)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3997) static void write_normal_summaries(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3998) block_t blkaddr, int type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3999) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4000) int i, end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4001)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4002) if (IS_DATASEG(type))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4003) end = type + NR_CURSEG_DATA_TYPE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4004) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4005) end = type + NR_CURSEG_NODE_TYPE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4006)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4007) for (i = type; i < end; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4008) write_current_sum_page(sbi, i, blkaddr + (i - type));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4009) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4010)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4011) void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4012) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4013) if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4014) write_compacted_summaries(sbi, start_blk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4015) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4016) write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4017) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4018)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4019) void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4020) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4021) write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4022) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4023)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4024) int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4025) unsigned int val, int alloc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4026) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4027) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4028)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4029) if (type == NAT_JOURNAL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4030) for (i = 0; i < nats_in_cursum(journal); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4031) if (le32_to_cpu(nid_in_journal(journal, i)) == val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4032) return i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4033) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4034) if (alloc && __has_cursum_space(journal, 1, NAT_JOURNAL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4035) return update_nats_in_cursum(journal, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4036) } else if (type == SIT_JOURNAL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4037) for (i = 0; i < sits_in_cursum(journal); i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4038) if (le32_to_cpu(segno_in_journal(journal, i)) == val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4039) return i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4040) if (alloc && __has_cursum_space(journal, 1, SIT_JOURNAL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4041) return update_sits_in_cursum(journal, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4042) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4043) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4044) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4045)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4046) static struct page *get_current_sit_page(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4047) unsigned int segno)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4048) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4049) return f2fs_get_meta_page(sbi, current_sit_addr(sbi, segno));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4050) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4051)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4052) static struct page *get_next_sit_page(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4053) unsigned int start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4054) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4055) struct sit_info *sit_i = SIT_I(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4056) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4057) pgoff_t src_off, dst_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4058)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4059) src_off = current_sit_addr(sbi, start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4060) dst_off = next_sit_addr(sbi, src_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4061)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4062) page = f2fs_grab_meta_page(sbi, dst_off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4063) seg_info_to_sit_page(sbi, page, start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4064)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4065) set_page_dirty(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4066) set_to_next_sit(sit_i, start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4067)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4068) return page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4069) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4070)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4071) static struct sit_entry_set *grab_sit_entry_set(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4072) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4073) struct sit_entry_set *ses =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4074) f2fs_kmem_cache_alloc(sit_entry_set_slab, GFP_NOFS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4075)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4076) ses->entry_cnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4077) INIT_LIST_HEAD(&ses->set_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4078) return ses;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4079) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4080)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4081) static void release_sit_entry_set(struct sit_entry_set *ses)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4082) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4083) list_del(&ses->set_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4084) kmem_cache_free(sit_entry_set_slab, ses);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4085) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4086)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4087) static void adjust_sit_entry_set(struct sit_entry_set *ses,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4088) struct list_head *head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4089) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4090) struct sit_entry_set *next = ses;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4091)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4092) if (list_is_last(&ses->set_list, head))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4093) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4094)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4095) list_for_each_entry_continue(next, head, set_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4096) if (ses->entry_cnt <= next->entry_cnt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4097) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4098)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4099) list_move_tail(&ses->set_list, &next->set_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4100) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4102) static void add_sit_entry(unsigned int segno, struct list_head *head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4103) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4104) struct sit_entry_set *ses;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4105) unsigned int start_segno = START_SEGNO(segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4107) list_for_each_entry(ses, head, set_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4108) if (ses->start_segno == start_segno) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4109) ses->entry_cnt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4110) adjust_sit_entry_set(ses, head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4111) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4112) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4113) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4115) ses = grab_sit_entry_set();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4117) ses->start_segno = start_segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4118) ses->entry_cnt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4119) list_add(&ses->set_list, head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4120) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4122) static void add_sits_in_set(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4123) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4124) struct f2fs_sm_info *sm_info = SM_I(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4125) struct list_head *set_list = &sm_info->sit_entry_set;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4126) unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4127) unsigned int segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4129) for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4130) add_sit_entry(segno, set_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4131) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4133) static void remove_sits_in_journal(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4134) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4135) struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4136) struct f2fs_journal *journal = curseg->journal;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4137) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4139) down_write(&curseg->journal_rwsem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4140) for (i = 0; i < sits_in_cursum(journal); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4141) unsigned int segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4142) bool dirtied;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4144) segno = le32_to_cpu(segno_in_journal(journal, i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4145) dirtied = __mark_sit_entry_dirty(sbi, segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4147) if (!dirtied)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4148) add_sit_entry(segno, &SM_I(sbi)->sit_entry_set);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4149) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4150) update_sits_in_cursum(journal, -i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4151) up_write(&curseg->journal_rwsem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4154) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4155) * CP calls this function, which flushes SIT entries including sit_journal,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4156) * and moves prefree segs to free segs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4157) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4158) void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4159) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4160) struct sit_info *sit_i = SIT_I(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4161) unsigned long *bitmap = sit_i->dirty_sentries_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4162) struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4163) struct f2fs_journal *journal = curseg->journal;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4164) struct sit_entry_set *ses, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4165) struct list_head *head = &SM_I(sbi)->sit_entry_set;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4166) bool to_journal = !is_sbi_flag_set(sbi, SBI_IS_RESIZEFS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4167) struct seg_entry *se;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4169) down_write(&sit_i->sentry_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4171) if (!sit_i->dirty_sentries)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4172) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4174) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4175) * add and account sit entries of dirty bitmap in sit entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4176) * set temporarily
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4177) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4178) add_sits_in_set(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4180) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4181) * if there are no enough space in journal to store dirty sit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4182) * entries, remove all entries from journal and add and account
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4183) * them in sit entry set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4184) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4185) if (!__has_cursum_space(journal, sit_i->dirty_sentries, SIT_JOURNAL) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4186) !to_journal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4187) remove_sits_in_journal(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4189) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4190) * there are two steps to flush sit entries:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4191) * #1, flush sit entries to journal in current cold data summary block.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4192) * #2, flush sit entries to sit page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4193) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4194) list_for_each_entry_safe(ses, tmp, head, set_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4195) struct page *page = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4196) struct f2fs_sit_block *raw_sit = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4197) unsigned int start_segno = ses->start_segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4198) unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4199) (unsigned long)MAIN_SEGS(sbi));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4200) unsigned int segno = start_segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4202) if (to_journal &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4203) !__has_cursum_space(journal, ses->entry_cnt, SIT_JOURNAL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4204) to_journal = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4206) if (to_journal) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4207) down_write(&curseg->journal_rwsem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4208) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4209) page = get_next_sit_page(sbi, start_segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4210) raw_sit = page_address(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4211) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4213) /* flush dirty sit entries in region of current sit set */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4214) for_each_set_bit_from(segno, bitmap, end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4215) int offset, sit_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4217) se = get_seg_entry(sbi, segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4218) #ifdef CONFIG_F2FS_CHECK_FS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4219) if (memcmp(se->cur_valid_map, se->cur_valid_map_mir,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4220) SIT_VBLOCK_MAP_SIZE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4221) f2fs_bug_on(sbi, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4222) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4224) /* add discard candidates */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4225) if (!(cpc->reason & CP_DISCARD)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4226) cpc->trim_start = segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4227) add_discard_addrs(sbi, cpc, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4228) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4230) if (to_journal) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4231) offset = f2fs_lookup_journal_in_cursum(journal,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4232) SIT_JOURNAL, segno, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4233) f2fs_bug_on(sbi, offset < 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4234) segno_in_journal(journal, offset) =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4235) cpu_to_le32(segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4236) seg_info_to_raw_sit(se,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4237) &sit_in_journal(journal, offset));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4238) check_block_count(sbi, segno,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4239) &sit_in_journal(journal, offset));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4240) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4241) sit_offset = SIT_ENTRY_OFFSET(sit_i, segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4242) seg_info_to_raw_sit(se,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4243) &raw_sit->entries[sit_offset]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4244) check_block_count(sbi, segno,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4245) &raw_sit->entries[sit_offset]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4246) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4248) __clear_bit(segno, bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4249) sit_i->dirty_sentries--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4250) ses->entry_cnt--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4251) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4253) if (to_journal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4254) up_write(&curseg->journal_rwsem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4255) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4256) f2fs_put_page(page, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4258) f2fs_bug_on(sbi, ses->entry_cnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4259) release_sit_entry_set(ses);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4260) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4262) f2fs_bug_on(sbi, !list_empty(head));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4263) f2fs_bug_on(sbi, sit_i->dirty_sentries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4264) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4265) if (cpc->reason & CP_DISCARD) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4266) __u64 trim_start = cpc->trim_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4268) for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4269) add_discard_addrs(sbi, cpc, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4271) cpc->trim_start = trim_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4272) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4273) up_write(&sit_i->sentry_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4275) set_prefree_as_free_segments(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4276) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4278) static int build_sit_info(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4279) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4280) struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4281) struct sit_info *sit_i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4282) unsigned int sit_segs, start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4283) char *src_bitmap, *bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4284) unsigned int bitmap_size, main_bitmap_size, sit_bitmap_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4286) /* allocate memory for SIT information */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4287) sit_i = f2fs_kzalloc(sbi, sizeof(struct sit_info), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4288) if (!sit_i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4289) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4291) SM_I(sbi)->sit_info = sit_i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4293) sit_i->sentries =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4294) f2fs_kvzalloc(sbi, array_size(sizeof(struct seg_entry),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4295) MAIN_SEGS(sbi)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4296) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4297) if (!sit_i->sentries)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4298) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4300) main_bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4301) sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(sbi, main_bitmap_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4302) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4303) if (!sit_i->dirty_sentries_bitmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4304) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4306) #ifdef CONFIG_F2FS_CHECK_FS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4307) bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4308) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4309) bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4310) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4311) sit_i->bitmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4312) if (!sit_i->bitmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4313) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4315) bitmap = sit_i->bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4317) for (start = 0; start < MAIN_SEGS(sbi); start++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4318) sit_i->sentries[start].cur_valid_map = bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4319) bitmap += SIT_VBLOCK_MAP_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4321) sit_i->sentries[start].ckpt_valid_map = bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4322) bitmap += SIT_VBLOCK_MAP_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4324) #ifdef CONFIG_F2FS_CHECK_FS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4325) sit_i->sentries[start].cur_valid_map_mir = bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4326) bitmap += SIT_VBLOCK_MAP_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4327) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4329) sit_i->sentries[start].discard_map = bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4330) bitmap += SIT_VBLOCK_MAP_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4331) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4333) sit_i->tmp_map = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4334) if (!sit_i->tmp_map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4335) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4337) if (__is_large_section(sbi)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4338) sit_i->sec_entries =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4339) f2fs_kvzalloc(sbi, array_size(sizeof(struct sec_entry),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4340) MAIN_SECS(sbi)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4341) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4342) if (!sit_i->sec_entries)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4343) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4344) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4346) /* get information related with SIT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4347) sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4349) /* setup SIT bitmap from ckeckpoint pack */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4350) sit_bitmap_size = __bitmap_size(sbi, SIT_BITMAP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4351) src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4353) sit_i->sit_bitmap = kmemdup(src_bitmap, sit_bitmap_size, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4354) if (!sit_i->sit_bitmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4355) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4357) #ifdef CONFIG_F2FS_CHECK_FS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4358) sit_i->sit_bitmap_mir = kmemdup(src_bitmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4359) sit_bitmap_size, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4360) if (!sit_i->sit_bitmap_mir)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4361) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4363) sit_i->invalid_segmap = f2fs_kvzalloc(sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4364) main_bitmap_size, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4365) if (!sit_i->invalid_segmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4366) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4367) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4369) /* init SIT information */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4370) sit_i->s_ops = &default_salloc_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4372) sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4373) sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4374) sit_i->written_valid_blocks = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4375) sit_i->bitmap_size = sit_bitmap_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4376) sit_i->dirty_sentries = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4377) sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4378) sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4379) sit_i->mounted_time = ktime_get_boottime_seconds();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4380) init_rwsem(&sit_i->sentry_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4381) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4382) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4384) static int build_free_segmap(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4385) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4386) struct free_segmap_info *free_i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4387) unsigned int bitmap_size, sec_bitmap_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4389) /* allocate memory for free segmap information */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4390) free_i = f2fs_kzalloc(sbi, sizeof(struct free_segmap_info), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4391) if (!free_i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4392) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4394) SM_I(sbi)->free_info = free_i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4396) bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4397) free_i->free_segmap = f2fs_kvmalloc(sbi, bitmap_size, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4398) if (!free_i->free_segmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4399) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4401) sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4402) free_i->free_secmap = f2fs_kvmalloc(sbi, sec_bitmap_size, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4403) if (!free_i->free_secmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4404) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4406) /* set all segments as dirty temporarily */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4407) memset(free_i->free_segmap, 0xff, bitmap_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4408) memset(free_i->free_secmap, 0xff, sec_bitmap_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4410) /* init free segmap information */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4411) free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4412) free_i->free_segments = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4413) free_i->free_sections = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4414) spin_lock_init(&free_i->segmap_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4415) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4416) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4418) static int build_curseg(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4419) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4420) struct curseg_info *array;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4421) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4423) array = f2fs_kzalloc(sbi, array_size(NR_CURSEG_TYPE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4424) sizeof(*array)), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4425) if (!array)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4426) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4428) SM_I(sbi)->curseg_array = array;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4430) for (i = 0; i < NO_CHECK_TYPE; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4431) mutex_init(&array[i].curseg_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4432) array[i].sum_blk = f2fs_kzalloc(sbi, PAGE_SIZE, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4433) if (!array[i].sum_blk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4434) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4435) init_rwsem(&array[i].journal_rwsem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4436) array[i].journal = f2fs_kzalloc(sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4437) sizeof(struct f2fs_journal), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4438) if (!array[i].journal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4439) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4440) if (i < NR_PERSISTENT_LOG)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4441) array[i].seg_type = CURSEG_HOT_DATA + i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4442) else if (i == CURSEG_COLD_DATA_PINNED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4443) array[i].seg_type = CURSEG_COLD_DATA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4444) else if (i == CURSEG_ALL_DATA_ATGC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4445) array[i].seg_type = CURSEG_COLD_DATA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4446) array[i].segno = NULL_SEGNO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4447) array[i].next_blkoff = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4448) array[i].inited = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4449) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4450) return restore_curseg_summaries(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4451) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4453) static int build_sit_entries(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4454) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4455) struct sit_info *sit_i = SIT_I(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4456) struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4457) struct f2fs_journal *journal = curseg->journal;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4458) struct seg_entry *se;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4459) struct f2fs_sit_entry sit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4460) int sit_blk_cnt = SIT_BLK_CNT(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4461) unsigned int i, start, end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4462) unsigned int readed, start_blk = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4463) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4464) block_t total_node_blocks = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4466) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4467) readed = f2fs_ra_meta_pages(sbi, start_blk, BIO_MAX_PAGES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4468) META_SIT, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4470) start = start_blk * sit_i->sents_per_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4471) end = (start_blk + readed) * sit_i->sents_per_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4473) for (; start < end && start < MAIN_SEGS(sbi); start++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4474) struct f2fs_sit_block *sit_blk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4475) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4477) se = &sit_i->sentries[start];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4478) page = get_current_sit_page(sbi, start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4479) if (IS_ERR(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4480) return PTR_ERR(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4481) sit_blk = (struct f2fs_sit_block *)page_address(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4482) sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4483) f2fs_put_page(page, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4485) err = check_block_count(sbi, start, &sit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4486) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4487) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4488) seg_info_from_raw_sit(se, &sit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4489) if (IS_NODESEG(se->type))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4490) total_node_blocks += se->valid_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4492) /* build discard map only one time */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4493) if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4494) memset(se->discard_map, 0xff,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4495) SIT_VBLOCK_MAP_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4496) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4497) memcpy(se->discard_map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4498) se->cur_valid_map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4499) SIT_VBLOCK_MAP_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4500) sbi->discard_blks +=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4501) sbi->blocks_per_seg -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4502) se->valid_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4503) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4505) if (__is_large_section(sbi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4506) get_sec_entry(sbi, start)->valid_blocks +=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4507) se->valid_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4508) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4509) start_blk += readed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4510) } while (start_blk < sit_blk_cnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4512) down_read(&curseg->journal_rwsem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4513) for (i = 0; i < sits_in_cursum(journal); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4514) unsigned int old_valid_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4516) start = le32_to_cpu(segno_in_journal(journal, i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4517) if (start >= MAIN_SEGS(sbi)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4518) f2fs_err(sbi, "Wrong journal entry on segno %u",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4519) start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4520) err = -EFSCORRUPTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4521) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4522) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4524) se = &sit_i->sentries[start];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4525) sit = sit_in_journal(journal, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4527) old_valid_blocks = se->valid_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4528) if (IS_NODESEG(se->type))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4529) total_node_blocks -= old_valid_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4530)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4531) err = check_block_count(sbi, start, &sit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4532) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4533) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4534) seg_info_from_raw_sit(se, &sit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4535) if (IS_NODESEG(se->type))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4536) total_node_blocks += se->valid_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4538) if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4539) memset(se->discard_map, 0xff, SIT_VBLOCK_MAP_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4540) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4541) memcpy(se->discard_map, se->cur_valid_map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4542) SIT_VBLOCK_MAP_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4543) sbi->discard_blks += old_valid_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4544) sbi->discard_blks -= se->valid_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4545) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4547) if (__is_large_section(sbi)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4548) get_sec_entry(sbi, start)->valid_blocks +=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4549) se->valid_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4550) get_sec_entry(sbi, start)->valid_blocks -=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4551) old_valid_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4552) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4553) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4554) up_read(&curseg->journal_rwsem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4555)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4556) if (!err && total_node_blocks != valid_node_count(sbi)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4557) f2fs_err(sbi, "SIT is corrupted node# %u vs %u",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4558) total_node_blocks, valid_node_count(sbi));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4559) err = -EFSCORRUPTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4560) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4561)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4562) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4563) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4564)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4565) static void init_free_segmap(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4566) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4567) unsigned int start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4568) int type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4569) struct seg_entry *sentry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4571) for (start = 0; start < MAIN_SEGS(sbi); start++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4572) if (f2fs_usable_blks_in_seg(sbi, start) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4573) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4574) sentry = get_seg_entry(sbi, start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4575) if (!sentry->valid_blocks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4576) __set_free(sbi, start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4577) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4578) SIT_I(sbi)->written_valid_blocks +=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4579) sentry->valid_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4580) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4582) /* set use the current segments */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4583) for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4584) struct curseg_info *curseg_t = CURSEG_I(sbi, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4586) __set_test_and_inuse(sbi, curseg_t->segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4587) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4588) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4590) static void init_dirty_segmap(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4591) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4592) struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4593) struct free_segmap_info *free_i = FREE_I(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4594) unsigned int segno = 0, offset = 0, secno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4595) block_t valid_blocks, usable_blks_in_seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4596) block_t blks_per_sec = BLKS_PER_SEC(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4597)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4598) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4599) /* find dirty segment based on free segmap */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4600) segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4601) if (segno >= MAIN_SEGS(sbi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4602) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4603) offset = segno + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4604) valid_blocks = get_valid_blocks(sbi, segno, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4605) usable_blks_in_seg = f2fs_usable_blks_in_seg(sbi, segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4606) if (valid_blocks == usable_blks_in_seg || !valid_blocks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4607) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4608) if (valid_blocks > usable_blks_in_seg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4609) f2fs_bug_on(sbi, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4610) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4611) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4612) mutex_lock(&dirty_i->seglist_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4613) __locate_dirty_segment(sbi, segno, DIRTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4614) mutex_unlock(&dirty_i->seglist_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4615) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4617) if (!__is_large_section(sbi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4618) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4620) mutex_lock(&dirty_i->seglist_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4621) for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4622) valid_blocks = get_valid_blocks(sbi, segno, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4623) secno = GET_SEC_FROM_SEG(sbi, segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4625) if (!valid_blocks || valid_blocks == blks_per_sec)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4626) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4627) if (IS_CURSEC(sbi, secno))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4628) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4629) set_bit(secno, dirty_i->dirty_secmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4630) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4631) mutex_unlock(&dirty_i->seglist_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4632) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4634) static int init_victim_secmap(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4635) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4636) struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4637) unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4639) dirty_i->victim_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4640) if (!dirty_i->victim_secmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4641) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4642) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4643) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4645) static int build_dirty_segmap(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4646) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4647) struct dirty_seglist_info *dirty_i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4648) unsigned int bitmap_size, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4650) /* allocate memory for dirty segments list information */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4651) dirty_i = f2fs_kzalloc(sbi, sizeof(struct dirty_seglist_info),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4652) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4653) if (!dirty_i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4654) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4656) SM_I(sbi)->dirty_info = dirty_i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4657) mutex_init(&dirty_i->seglist_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4659) bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4660)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4661) for (i = 0; i < NR_DIRTY_TYPE; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4662) dirty_i->dirty_segmap[i] = f2fs_kvzalloc(sbi, bitmap_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4663) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4664) if (!dirty_i->dirty_segmap[i])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4665) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4666) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4667)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4668) if (__is_large_section(sbi)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4669) bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4670) dirty_i->dirty_secmap = f2fs_kvzalloc(sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4671) bitmap_size, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4672) if (!dirty_i->dirty_secmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4673) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4674) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4676) init_dirty_segmap(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4677) return init_victim_secmap(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4678) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4680) static int sanity_check_curseg(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4681) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4682) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4683)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4684) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4685) * In LFS/SSR curseg, .next_blkoff should point to an unused blkaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4686) * In LFS curseg, all blkaddr after .next_blkoff should be unused.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4687) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4688) for (i = 0; i < NR_PERSISTENT_LOG; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4689) struct curseg_info *curseg = CURSEG_I(sbi, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4690) struct seg_entry *se = get_seg_entry(sbi, curseg->segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4691) unsigned int blkofs = curseg->next_blkoff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4693) if (f2fs_sb_has_readonly(sbi) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4694) i != CURSEG_HOT_DATA && i != CURSEG_HOT_NODE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4695) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4696)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4697) sanity_check_seg_type(sbi, curseg->seg_type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4699) if (curseg->alloc_type != LFS && curseg->alloc_type != SSR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4700) f2fs_err(sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4701) "Current segment has invalid alloc_type:%d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4702) curseg->alloc_type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4703) return -EFSCORRUPTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4704) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4705)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4706) if (f2fs_test_bit(blkofs, se->cur_valid_map))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4707) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4708)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4709) if (curseg->alloc_type == SSR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4710) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4711)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4712) for (blkofs += 1; blkofs < sbi->blocks_per_seg; blkofs++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4713) if (!f2fs_test_bit(blkofs, se->cur_valid_map))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4714) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4715) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4716) f2fs_err(sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4717) "Current segment's next free block offset is inconsistent with bitmap, logtype:%u, segno:%u, type:%u, next_blkoff:%u, blkofs:%u",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4718) i, curseg->segno, curseg->alloc_type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4719) curseg->next_blkoff, blkofs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4720) return -EFSCORRUPTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4721) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4722) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4723) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4724) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4726) #ifdef CONFIG_BLK_DEV_ZONED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4727)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4728) static int check_zone_write_pointer(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4729) struct f2fs_dev_info *fdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4730) struct blk_zone *zone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4731) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4732) unsigned int wp_segno, wp_blkoff, zone_secno, zone_segno, segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4733) block_t zone_block, wp_block, last_valid_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4734) unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4735) int i, s, b, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4736) struct seg_entry *se;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4738) if (zone->type != BLK_ZONE_TYPE_SEQWRITE_REQ)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4739) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4741) wp_block = fdev->start_blk + (zone->wp >> log_sectors_per_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4742) wp_segno = GET_SEGNO(sbi, wp_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4743) wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4744) zone_block = fdev->start_blk + (zone->start >> log_sectors_per_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4745) zone_segno = GET_SEGNO(sbi, zone_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4746) zone_secno = GET_SEC_FROM_SEG(sbi, zone_segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4747)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4748) if (zone_segno >= MAIN_SEGS(sbi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4749) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4750)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4751) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4752) * Skip check of zones cursegs point to, since
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4753) * fix_curseg_write_pointer() checks them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4754) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4755) for (i = 0; i < NO_CHECK_TYPE; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4756) if (zone_secno == GET_SEC_FROM_SEG(sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4757) CURSEG_I(sbi, i)->segno))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4758) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4759)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4760) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4761) * Get last valid block of the zone.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4762) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4763) last_valid_block = zone_block - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4764) for (s = sbi->segs_per_sec - 1; s >= 0; s--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4765) segno = zone_segno + s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4766) se = get_seg_entry(sbi, segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4767) for (b = sbi->blocks_per_seg - 1; b >= 0; b--)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4768) if (f2fs_test_bit(b, se->cur_valid_map)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4769) last_valid_block = START_BLOCK(sbi, segno) + b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4770) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4771) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4772) if (last_valid_block >= zone_block)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4773) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4774) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4775)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4776) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4777) * If last valid block is beyond the write pointer, report the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4778) * inconsistency. This inconsistency does not cause write error
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4779) * because the zone will not be selected for write operation until
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4780) * it get discarded. Just report it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4781) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4782) if (last_valid_block >= wp_block) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4783) f2fs_notice(sbi, "Valid block beyond write pointer: "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4784) "valid block[0x%x,0x%x] wp[0x%x,0x%x]",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4785) GET_SEGNO(sbi, last_valid_block),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4786) GET_BLKOFF_FROM_SEG0(sbi, last_valid_block),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4787) wp_segno, wp_blkoff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4788) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4789) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4790)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4791) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4792) * If there is no valid block in the zone and if write pointer is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4793) * not at zone start, reset the write pointer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4794) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4795) if (last_valid_block + 1 == zone_block && zone->wp != zone->start) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4796) f2fs_notice(sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4797) "Zone without valid block has non-zero write "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4798) "pointer. Reset the write pointer: wp[0x%x,0x%x]",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4799) wp_segno, wp_blkoff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4800) ret = __f2fs_issue_discard_zone(sbi, fdev->bdev, zone_block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4801) zone->len >> log_sectors_per_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4802) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4803) f2fs_err(sbi, "Discard zone failed: %s (errno=%d)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4804) fdev->path, ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4805) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4806) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4807) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4808)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4809) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4810) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4812) static struct f2fs_dev_info *get_target_zoned_dev(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4813) block_t zone_blkaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4814) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4815) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4816)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4817) for (i = 0; i < sbi->s_ndevs; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4818) if (!bdev_is_zoned(FDEV(i).bdev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4819) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4820) if (sbi->s_ndevs == 1 || (FDEV(i).start_blk <= zone_blkaddr &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4821) zone_blkaddr <= FDEV(i).end_blk))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4822) return &FDEV(i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4823) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4824)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4825) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4826) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4827)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4828) static int report_one_zone_cb(struct blk_zone *zone, unsigned int idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4829) void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4830) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4831) memcpy(data, zone, sizeof(struct blk_zone));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4832) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4833) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4834)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4835) static int fix_curseg_write_pointer(struct f2fs_sb_info *sbi, int type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4836) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4837) struct curseg_info *cs = CURSEG_I(sbi, type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4838) struct f2fs_dev_info *zbd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4839) struct blk_zone zone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4840) unsigned int cs_section, wp_segno, wp_blkoff, wp_sector_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4841) block_t cs_zone_block, wp_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4842) unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4843) sector_t zone_sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4844) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4845)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4846) cs_section = GET_SEC_FROM_SEG(sbi, cs->segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4847) cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4848)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4849) zbd = get_target_zoned_dev(sbi, cs_zone_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4850) if (!zbd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4851) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4852)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4853) /* report zone for the sector the curseg points to */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4854) zone_sector = (sector_t)(cs_zone_block - zbd->start_blk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4855) << log_sectors_per_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4856) err = blkdev_report_zones(zbd->bdev, zone_sector, 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4857) report_one_zone_cb, &zone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4858) if (err != 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4859) f2fs_err(sbi, "Report zone failed: %s errno=(%d)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4860) zbd->path, err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4861) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4862) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4863)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4864) if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4865) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4866)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4867) wp_block = zbd->start_blk + (zone.wp >> log_sectors_per_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4868) wp_segno = GET_SEGNO(sbi, wp_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4869) wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4870) wp_sector_off = zone.wp & GENMASK(log_sectors_per_block - 1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4871)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4872) if (cs->segno == wp_segno && cs->next_blkoff == wp_blkoff &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4873) wp_sector_off == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4874) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4875)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4876) f2fs_notice(sbi, "Unaligned curseg[%d] with write pointer: "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4877) "curseg[0x%x,0x%x] wp[0x%x,0x%x]",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4878) type, cs->segno, cs->next_blkoff, wp_segno, wp_blkoff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4879)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4880) f2fs_notice(sbi, "Assign new section to curseg[%d]: "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4881) "curseg[0x%x,0x%x]", type, cs->segno, cs->next_blkoff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4882)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4883) f2fs_allocate_new_section(sbi, type, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4884)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4885) /* check consistency of the zone curseg pointed to */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4886) if (check_zone_write_pointer(sbi, zbd, &zone))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4887) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4889) /* check newly assigned zone */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4890) cs_section = GET_SEC_FROM_SEG(sbi, cs->segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4891) cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4892)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4893) zbd = get_target_zoned_dev(sbi, cs_zone_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4894) if (!zbd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4895) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4896)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4897) zone_sector = (sector_t)(cs_zone_block - zbd->start_blk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4898) << log_sectors_per_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4899) err = blkdev_report_zones(zbd->bdev, zone_sector, 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4900) report_one_zone_cb, &zone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4901) if (err != 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4902) f2fs_err(sbi, "Report zone failed: %s errno=(%d)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4903) zbd->path, err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4904) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4905) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4906)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4907) if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4908) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4909)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4910) if (zone.wp != zone.start) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4911) f2fs_notice(sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4912) "New zone for curseg[%d] is not yet discarded. "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4913) "Reset the zone: curseg[0x%x,0x%x]",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4914) type, cs->segno, cs->next_blkoff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4915) err = __f2fs_issue_discard_zone(sbi, zbd->bdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4916) zone_sector >> log_sectors_per_block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4917) zone.len >> log_sectors_per_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4918) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4919) f2fs_err(sbi, "Discard zone failed: %s (errno=%d)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4920) zbd->path, err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4921) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4922) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4923) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4925) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4926) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4927)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4928) int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4929) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4930) int i, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4931)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4932) for (i = 0; i < NR_PERSISTENT_LOG; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4933) ret = fix_curseg_write_pointer(sbi, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4934) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4935) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4936) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4937)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4938) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4939) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4940)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4941) struct check_zone_write_pointer_args {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4942) struct f2fs_sb_info *sbi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4943) struct f2fs_dev_info *fdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4944) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4945)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4946) static int check_zone_write_pointer_cb(struct blk_zone *zone, unsigned int idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4947) void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4948) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4949) struct check_zone_write_pointer_args *args;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4950)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4951) args = (struct check_zone_write_pointer_args *)data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4952)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4953) return check_zone_write_pointer(args->sbi, args->fdev, zone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4954) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4955)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4956) int f2fs_check_write_pointer(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4957) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4958) int i, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4959) struct check_zone_write_pointer_args args;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4960)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4961) for (i = 0; i < sbi->s_ndevs; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4962) if (!bdev_is_zoned(FDEV(i).bdev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4963) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4964)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4965) args.sbi = sbi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4966) args.fdev = &FDEV(i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4967) ret = blkdev_report_zones(FDEV(i).bdev, 0, BLK_ALL_ZONES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4968) check_zone_write_pointer_cb, &args);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4969) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4970) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4971) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4972)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4973) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4974) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4975)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4976) static bool is_conv_zone(struct f2fs_sb_info *sbi, unsigned int zone_idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4977) unsigned int dev_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4978) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4979) if (!bdev_is_zoned(FDEV(dev_idx).bdev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4980) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4981) return !test_bit(zone_idx, FDEV(dev_idx).blkz_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4982) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4983)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4984) /* Return the zone index in the given device */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4985) static unsigned int get_zone_idx(struct f2fs_sb_info *sbi, unsigned int secno,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4986) int dev_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4987) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4988) block_t sec_start_blkaddr = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4990) return (sec_start_blkaddr - FDEV(dev_idx).start_blk) >>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4991) sbi->log_blocks_per_blkz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4992) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4993)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4994) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4995) * Return the usable segments in a section based on the zone's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4996) * corresponding zone capacity. Zone is equal to a section.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4997) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4998) static inline unsigned int f2fs_usable_zone_segs_in_sec(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4999) struct f2fs_sb_info *sbi, unsigned int segno)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5000) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5001) unsigned int dev_idx, zone_idx, unusable_segs_in_sec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5002)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5003) dev_idx = f2fs_target_device_index(sbi, START_BLOCK(sbi, segno));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5004) zone_idx = get_zone_idx(sbi, GET_SEC_FROM_SEG(sbi, segno), dev_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5005)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5006) /* Conventional zone's capacity is always equal to zone size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5007) if (is_conv_zone(sbi, zone_idx, dev_idx))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5008) return sbi->segs_per_sec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5009)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5010) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5011) * If the zone_capacity_blocks array is NULL, then zone capacity
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5012) * is equal to the zone size for all zones
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5013) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5014) if (!FDEV(dev_idx).zone_capacity_blocks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5015) return sbi->segs_per_sec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5016)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5017) /* Get the segment count beyond zone capacity block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5018) unusable_segs_in_sec = (sbi->blocks_per_blkz -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5019) FDEV(dev_idx).zone_capacity_blocks[zone_idx]) >>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5020) sbi->log_blocks_per_seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5021) return sbi->segs_per_sec - unusable_segs_in_sec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5022) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5023)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5024) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5025) * Return the number of usable blocks in a segment. The number of blocks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5026) * returned is always equal to the number of blocks in a segment for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5027) * segments fully contained within a sequential zone capacity or a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5028) * conventional zone. For segments partially contained in a sequential
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5029) * zone capacity, the number of usable blocks up to the zone capacity
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5030) * is returned. 0 is returned in all other cases.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5031) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5032) static inline unsigned int f2fs_usable_zone_blks_in_seg(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5033) struct f2fs_sb_info *sbi, unsigned int segno)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5034) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5035) block_t seg_start, sec_start_blkaddr, sec_cap_blkaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5036) unsigned int zone_idx, dev_idx, secno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5037)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5038) secno = GET_SEC_FROM_SEG(sbi, segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5039) seg_start = START_BLOCK(sbi, segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5040) dev_idx = f2fs_target_device_index(sbi, seg_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5041) zone_idx = get_zone_idx(sbi, secno, dev_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5042)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5043) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5044) * Conventional zone's capacity is always equal to zone size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5045) * so, blocks per segment is unchanged.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5046) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5047) if (is_conv_zone(sbi, zone_idx, dev_idx))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5048) return sbi->blocks_per_seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5049)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5050) if (!FDEV(dev_idx).zone_capacity_blocks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5051) return sbi->blocks_per_seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5052)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5053) sec_start_blkaddr = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5054) sec_cap_blkaddr = sec_start_blkaddr +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5055) FDEV(dev_idx).zone_capacity_blocks[zone_idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5056)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5057) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5058) * If segment starts before zone capacity and spans beyond
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5059) * zone capacity, then usable blocks are from seg start to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5060) * zone capacity. If the segment starts after the zone capacity,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5061) * then there are no usable blocks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5062) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5063) if (seg_start >= sec_cap_blkaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5064) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5065) if (seg_start + sbi->blocks_per_seg > sec_cap_blkaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5066) return sec_cap_blkaddr - seg_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5067)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5068) return sbi->blocks_per_seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5069) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5070) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5071) int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5072) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5073) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5074) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5075)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5076) int f2fs_check_write_pointer(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5077) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5078) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5079) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5080)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5081) static inline unsigned int f2fs_usable_zone_blks_in_seg(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5082) unsigned int segno)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5083) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5084) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5085) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5086)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5087) static inline unsigned int f2fs_usable_zone_segs_in_sec(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5088) unsigned int segno)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5089) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5090) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5091) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5092) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5093) unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5094) unsigned int segno)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5095) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5096) if (f2fs_sb_has_blkzoned(sbi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5097) return f2fs_usable_zone_blks_in_seg(sbi, segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5098)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5099) return sbi->blocks_per_seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5100) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5102) unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5103) unsigned int segno)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5104) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5105) if (f2fs_sb_has_blkzoned(sbi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5106) return f2fs_usable_zone_segs_in_sec(sbi, segno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5108) return sbi->segs_per_sec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5109) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5111) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5112) * Update min, max modified time for cost-benefit GC algorithm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5113) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5114) static void init_min_max_mtime(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5115) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5116) struct sit_info *sit_i = SIT_I(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5117) unsigned int segno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5119) down_write(&sit_i->sentry_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5121) sit_i->min_mtime = ULLONG_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5123) for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5124) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5125) unsigned long long mtime = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5127) for (i = 0; i < sbi->segs_per_sec; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5128) mtime += get_seg_entry(sbi, segno + i)->mtime;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5130) mtime = div_u64(mtime, sbi->segs_per_sec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5132) if (sit_i->min_mtime > mtime)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5133) sit_i->min_mtime = mtime;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5134) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5135) sit_i->max_mtime = get_mtime(sbi, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5136) sit_i->dirty_max_mtime = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5137) up_write(&sit_i->sentry_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5138) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5140) int f2fs_build_segment_manager(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5141) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5142) struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5143) struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5144) struct f2fs_sm_info *sm_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5145) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5147) sm_info = f2fs_kzalloc(sbi, sizeof(struct f2fs_sm_info), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5148) if (!sm_info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5149) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5151) /* init sm info */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5152) sbi->sm_info = sm_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5153) sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5154) sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5155) sm_info->segment_count = le32_to_cpu(raw_super->segment_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5156) sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5157) sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5158) sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5159) sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5160) sm_info->rec_prefree_segments = sm_info->main_segments *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5161) DEF_RECLAIM_PREFREE_SEGMENTS / 100;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5162) if (sm_info->rec_prefree_segments > DEF_MAX_RECLAIM_PREFREE_SEGMENTS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5163) sm_info->rec_prefree_segments = DEF_MAX_RECLAIM_PREFREE_SEGMENTS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5165) if (!f2fs_lfs_mode(sbi))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5166) sm_info->ipu_policy = 1 << F2FS_IPU_FSYNC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5167) sm_info->min_ipu_util = DEF_MIN_IPU_UTIL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5168) sm_info->min_fsync_blocks = DEF_MIN_FSYNC_BLOCKS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5169) sm_info->min_seq_blocks = sbi->blocks_per_seg * sbi->segs_per_sec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5170) sm_info->min_hot_blocks = DEF_MIN_HOT_BLOCKS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5171) sm_info->min_ssr_sections = reserved_sections(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5173) INIT_LIST_HEAD(&sm_info->sit_entry_set);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5175) init_f2fs_rwsem(&sm_info->curseg_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5177) if (!f2fs_readonly(sbi->sb)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5178) err = f2fs_create_flush_cmd_control(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5179) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5180) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5181) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5183) err = create_discard_cmd_control(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5184) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5185) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5187) err = build_sit_info(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5188) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5189) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5190) err = build_free_segmap(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5191) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5192) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5193) err = build_curseg(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5194) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5195) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5197) /* reinit free segmap based on SIT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5198) err = build_sit_entries(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5199) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5200) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5202) init_free_segmap(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5203) err = build_dirty_segmap(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5204) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5205) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5207) err = sanity_check_curseg(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5208) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5209) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5211) init_min_max_mtime(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5212) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5213) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5215) static void discard_dirty_segmap(struct f2fs_sb_info *sbi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5216) enum dirty_type dirty_type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5217) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5218) struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5220) mutex_lock(&dirty_i->seglist_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5221) kvfree(dirty_i->dirty_segmap[dirty_type]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5222) dirty_i->nr_dirty[dirty_type] = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5223) mutex_unlock(&dirty_i->seglist_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5224) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5226) static void destroy_victim_secmap(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5227) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5228) struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5230) kvfree(dirty_i->victim_secmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5231) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5233) static void destroy_dirty_segmap(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5234) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5235) struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5236) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5238) if (!dirty_i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5239) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5241) /* discard pre-free/dirty segments list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5242) for (i = 0; i < NR_DIRTY_TYPE; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5243) discard_dirty_segmap(sbi, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5245) if (__is_large_section(sbi)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5246) mutex_lock(&dirty_i->seglist_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5247) kvfree(dirty_i->dirty_secmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5248) mutex_unlock(&dirty_i->seglist_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5249) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5251) destroy_victim_secmap(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5252) SM_I(sbi)->dirty_info = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5253) kfree(dirty_i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5254) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5256) static void destroy_curseg(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5257) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5258) struct curseg_info *array = SM_I(sbi)->curseg_array;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5259) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5261) if (!array)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5262) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5263) SM_I(sbi)->curseg_array = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5264) for (i = 0; i < NR_CURSEG_TYPE; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5265) kfree(array[i].sum_blk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5266) kfree(array[i].journal);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5267) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5268) kfree(array);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5271) static void destroy_free_segmap(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5272) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5273) struct free_segmap_info *free_i = SM_I(sbi)->free_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5275) if (!free_i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5276) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5277) SM_I(sbi)->free_info = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5278) kvfree(free_i->free_segmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5279) kvfree(free_i->free_secmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5280) kfree(free_i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5281) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5283) static void destroy_sit_info(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5284) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5285) struct sit_info *sit_i = SIT_I(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5287) if (!sit_i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5288) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5290) if (sit_i->sentries)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5291) kvfree(sit_i->bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5292) kfree(sit_i->tmp_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5294) kvfree(sit_i->sentries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5295) kvfree(sit_i->sec_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5296) kvfree(sit_i->dirty_sentries_bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5298) SM_I(sbi)->sit_info = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5299) kvfree(sit_i->sit_bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5300) #ifdef CONFIG_F2FS_CHECK_FS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5301) kvfree(sit_i->sit_bitmap_mir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5302) kvfree(sit_i->invalid_segmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5303) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5304) kfree(sit_i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5305) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5307) void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5308) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5309) struct f2fs_sm_info *sm_info = SM_I(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5311) if (!sm_info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5312) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5313) f2fs_destroy_flush_cmd_control(sbi, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5314) destroy_discard_cmd_control(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5315) destroy_dirty_segmap(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5316) destroy_curseg(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5317) destroy_free_segmap(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5318) destroy_sit_info(sbi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5319) sbi->sm_info = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5320) kfree(sm_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5321) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5323) int __init f2fs_create_segment_manager_caches(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5324) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5325) discard_entry_slab = f2fs_kmem_cache_create("f2fs_discard_entry",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5326) sizeof(struct discard_entry));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5327) if (!discard_entry_slab)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5328) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5330) discard_cmd_slab = f2fs_kmem_cache_create("f2fs_discard_cmd",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5331) sizeof(struct discard_cmd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5332) if (!discard_cmd_slab)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5333) goto destroy_discard_entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5335) sit_entry_set_slab = f2fs_kmem_cache_create("f2fs_sit_entry_set",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5336) sizeof(struct sit_entry_set));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5337) if (!sit_entry_set_slab)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5338) goto destroy_discard_cmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5340) inmem_entry_slab = f2fs_kmem_cache_create("f2fs_inmem_page_entry",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5341) sizeof(struct inmem_pages));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5342) if (!inmem_entry_slab)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5343) goto destroy_sit_entry_set;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5344) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5346) destroy_sit_entry_set:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5347) kmem_cache_destroy(sit_entry_set_slab);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5348) destroy_discard_cmd:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5349) kmem_cache_destroy(discard_cmd_slab);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5350) destroy_discard_entry:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5351) kmem_cache_destroy(discard_entry_slab);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5352) fail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5353) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5354) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5356) void f2fs_destroy_segment_manager_caches(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5357) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5358) kmem_cache_destroy(sit_entry_set_slab);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5359) kmem_cache_destroy(discard_cmd_slab);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5360) kmem_cache_destroy(discard_entry_slab);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5361) kmem_cache_destroy(inmem_entry_slab);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5362) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags