^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * linux/fs/buffer.c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 1991, 1992, 2002 Linus Torvalds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * Removed a lot of unnecessary code and simplified things now that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * Speed up hash, lru, and free list operations. Use gfp() for allocating
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * hash table, use SLAB cache for buffer heads. SMP threading. -DaveM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) * Added 32k buffer block sizes - these are required older ARM systems. - RMK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/sched/signal.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/syscalls.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/iomap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <linux/percpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <linux/capability.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <linux/blkdev.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <linux/file.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <linux/quotaops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <linux/highmem.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <linux/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <linux/backing-dev.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include <linux/writeback.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #include <linux/hash.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #include <linux/suspend.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #include <linux/buffer_head.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #include <linux/task_io_accounting_ops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #include <linux/bio.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #include <linux/bitops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #include <linux/mpage.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #include <linux/bit_spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #include <linux/pagevec.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #include <linux/sched/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #include <trace/events/block.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #include <linux/fscrypt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #include "internal.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) enum rw_hint hint, struct writeback_control *wbc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) inline void touch_buffer(struct buffer_head *bh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) trace_block_touch_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) mark_page_accessed(bh->b_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) EXPORT_SYMBOL(touch_buffer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) void __lock_buffer(struct buffer_head *bh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) wait_on_bit_lock_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) EXPORT_SYMBOL(__lock_buffer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) void unlock_buffer(struct buffer_head *bh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) clear_bit_unlock(BH_Lock, &bh->b_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) smp_mb__after_atomic();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) wake_up_bit(&bh->b_state, BH_Lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) EXPORT_SYMBOL(unlock_buffer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) * Returns if the page has dirty or writeback buffers. If all the buffers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) * are unlocked and clean then the PageDirty information is stale. If
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) * any of the pages are locked, it is assumed they are locked for IO.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) void buffer_check_dirty_writeback(struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) bool *dirty, bool *writeback)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) struct buffer_head *head, *bh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) *dirty = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) *writeback = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) BUG_ON(!PageLocked(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) if (!page_has_buffers(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) if (PageWriteback(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) *writeback = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) head = page_buffers(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) bh = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) if (buffer_locked(bh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) *writeback = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) if (buffer_dirty(bh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) *dirty = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) bh = bh->b_this_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) } while (bh != head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) EXPORT_SYMBOL(buffer_check_dirty_writeback);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) * Block until a buffer comes unlocked. This doesn't stop it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) * from becoming locked again - you have to lock it yourself
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) * if you want to preserve its state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) void __wait_on_buffer(struct buffer_head * bh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) wait_on_bit_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) EXPORT_SYMBOL(__wait_on_buffer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) static void buffer_io_error(struct buffer_head *bh, char *msg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) if (!test_bit(BH_Quiet, &bh->b_state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) printk_ratelimited(KERN_ERR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) "Buffer I/O error on dev %pg, logical block %llu%s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) bh->b_bdev, (unsigned long long)bh->b_blocknr, msg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) * End-of-IO handler helper function which does not touch the bh after
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) * unlocking it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) * a race there is benign: unlock_buffer() only use the bh's address for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) * hashing after unlocking the buffer, so it doesn't actually touch the bh
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) * itself.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) if (uptodate) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) set_buffer_uptodate(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) /* This happens, due to failed read-ahead attempts. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) clear_buffer_uptodate(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) unlock_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) * Default synchronous end-of-IO handler.. Just mark it up-to-date and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) * unlock the buffer. This is what ll_rw_block uses too.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) __end_buffer_read_notouch(bh, uptodate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) put_bh(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) EXPORT_SYMBOL(end_buffer_read_sync);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) if (uptodate) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) set_buffer_uptodate(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) buffer_io_error(bh, ", lost sync page write");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) mark_buffer_write_io_error(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) clear_buffer_uptodate(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) unlock_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) put_bh(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) EXPORT_SYMBOL_NS(end_buffer_write_sync, ANDROID_GKI_VFS_EXPORT_ONLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) * Various filesystems appear to want __find_get_block to be non-blocking.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) * But it's the page lock which protects the buffers. To get around this,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) * we get exclusion from try_to_free_buffers with the blockdev mapping's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) * private_lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) * Hack idea: for the blockdev mapping, private_lock contention
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) * may be quite high. This code could TryLock the page, and if that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) * succeeds, there is no need to take private_lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) static struct buffer_head *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) __find_get_block_slow(struct block_device *bdev, sector_t block)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) struct inode *bd_inode = bdev->bd_inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) struct address_space *bd_mapping = bd_inode->i_mapping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) struct buffer_head *ret = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) pgoff_t index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) struct buffer_head *bh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) struct buffer_head *head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) int all_mapped = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) static DEFINE_RATELIMIT_STATE(last_warned, HZ, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) index = block >> (PAGE_SHIFT - bd_inode->i_blkbits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) page = find_get_page_flags(bd_mapping, index, FGP_ACCESSED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) if (!page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) spin_lock(&bd_mapping->private_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) if (!page_has_buffers(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) head = page_buffers(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) bh = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) if (!buffer_mapped(bh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) all_mapped = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) else if (bh->b_blocknr == block) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) ret = bh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) get_bh(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) bh = bh->b_this_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) } while (bh != head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) /* we might be here because some of the buffers on this page are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) * not mapped. This is due to various races between
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) * file io on the block device and getblk. It gets dealt with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) * elsewhere, don't buffer_error if we had some unmapped buffers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) ratelimit_set_flags(&last_warned, RATELIMIT_MSG_ON_RELEASE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) if (all_mapped && __ratelimit(&last_warned)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) printk("__find_get_block_slow() failed. block=%llu, "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) "b_blocknr=%llu, b_state=0x%08lx, b_size=%zu, "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) "device %pg blocksize: %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) (unsigned long long)block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) (unsigned long long)bh->b_blocknr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) bh->b_state, bh->b_size, bdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 1 << bd_inode->i_blkbits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) out_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) spin_unlock(&bd_mapping->private_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) put_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) struct buffer_head *first;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) struct buffer_head *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) int page_uptodate = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) BUG_ON(!buffer_async_read(bh));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) page = bh->b_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) if (uptodate) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) set_buffer_uptodate(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) clear_buffer_uptodate(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) buffer_io_error(bh, ", async page read");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) SetPageError(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) * Be _very_ careful from here on. Bad things can happen if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) * two buffer heads end IO at almost the same time and both
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) * decide that the page is now completely done.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) first = page_buffers(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) spin_lock_irqsave(&first->b_uptodate_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) clear_buffer_async_read(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) unlock_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) tmp = bh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) if (!buffer_uptodate(tmp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) page_uptodate = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) if (buffer_async_read(tmp)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) BUG_ON(!buffer_locked(tmp));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) goto still_busy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) tmp = tmp->b_this_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) } while (tmp != bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) * If none of the buffers had errors and they are all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) * uptodate then we can set the page uptodate.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) if (page_uptodate && !PageError(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) SetPageUptodate(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) still_busy:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) struct decrypt_bh_ctx {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) struct work_struct work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) struct buffer_head *bh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) static void decrypt_bh(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) struct decrypt_bh_ctx *ctx =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) container_of(work, struct decrypt_bh_ctx, work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) struct buffer_head *bh = ctx->bh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) err = fscrypt_decrypt_pagecache_blocks(bh->b_page, bh->b_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) bh_offset(bh));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) end_buffer_async_read(bh, err == 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) kfree(ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) * I/O completion handler for block_read_full_page() - pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) * which come unlocked at the end of I/O.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) static void end_buffer_async_read_io(struct buffer_head *bh, int uptodate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) /* Decrypt if needed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) if (uptodate &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) fscrypt_inode_uses_fs_layer_crypto(bh->b_page->mapping->host)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) struct decrypt_bh_ctx *ctx = kmalloc(sizeof(*ctx), GFP_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) if (ctx) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) INIT_WORK(&ctx->work, decrypt_bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) ctx->bh = bh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) fscrypt_enqueue_decrypt_work(&ctx->work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) uptodate = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) end_buffer_async_read(bh, uptodate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) * Completion handler for block_write_full_page() - pages which are unlocked
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) * during I/O, and which have PageWriteback cleared upon I/O completion.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) void end_buffer_async_write(struct buffer_head *bh, int uptodate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) struct buffer_head *first;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) struct buffer_head *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) BUG_ON(!buffer_async_write(bh));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) page = bh->b_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) if (uptodate) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) set_buffer_uptodate(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) buffer_io_error(bh, ", lost async page write");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) mark_buffer_write_io_error(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) clear_buffer_uptodate(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) SetPageError(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) first = page_buffers(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) spin_lock_irqsave(&first->b_uptodate_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) clear_buffer_async_write(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) unlock_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) tmp = bh->b_this_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) while (tmp != bh) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) if (buffer_async_write(tmp)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) BUG_ON(!buffer_locked(tmp));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) goto still_busy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) tmp = tmp->b_this_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) end_page_writeback(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) still_busy:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) spin_unlock_irqrestore(&first->b_uptodate_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) EXPORT_SYMBOL(end_buffer_async_write);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) * If a page's buffers are under async readin (end_buffer_async_read
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) * completion) then there is a possibility that another thread of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) * control could lock one of the buffers after it has completed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) * but while some of the other buffers have not completed. This
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) * locked buffer would confuse end_buffer_async_read() into not unlocking
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) * the page. So the absence of BH_Async_Read tells end_buffer_async_read()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) * that this buffer is not under async I/O.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) * The page comes unlocked when it has no locked buffer_async buffers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) * left.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) * PageLocked prevents anyone starting new async I/O reads any of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) * the buffers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) * PageWriteback is used to prevent simultaneous writeout of the same
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) * page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) * PageLocked prevents anyone from starting writeback of a page which is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) * under read I/O (PageWriteback is only ever set against a locked page).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) static void mark_buffer_async_read(struct buffer_head *bh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) bh->b_end_io = end_buffer_async_read_io;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) set_buffer_async_read(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) static void mark_buffer_async_write_endio(struct buffer_head *bh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) bh_end_io_t *handler)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) bh->b_end_io = handler;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) set_buffer_async_write(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) void mark_buffer_async_write(struct buffer_head *bh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) mark_buffer_async_write_endio(bh, end_buffer_async_write);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) EXPORT_SYMBOL_NS(mark_buffer_async_write, ANDROID_GKI_VFS_EXPORT_ONLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) * fs/buffer.c contains helper functions for buffer-backed address space's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) * fsync functions. A common requirement for buffer-based filesystems is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) * that certain data from the backing blockdev needs to be written out for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) * a successful fsync(). For example, ext2 indirect blocks need to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) * written back and waited upon before fsync() returns.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) * inode_has_buffers() and invalidate_inode_buffers() are provided for the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) * management of a list of dependent buffers at ->i_mapping->private_list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) * Locking is a little subtle: try_to_free_buffers() will remove buffers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) * from their controlling inode's queue when they are being freed. But
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) * try_to_free_buffers() will be operating against the *blockdev* mapping
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) * at the time, not against the S_ISREG file which depends on those buffers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) * So the locking for private_list is via the private_lock in the address_space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) * which backs the buffers. Which is different from the address_space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) * against which the buffers are listed. So for a particular address_space,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) * mapping->private_lock does *not* protect mapping->private_list! In fact,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) * mapping->private_list will always be protected by the backing blockdev's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) * ->private_lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) * Which introduces a requirement: all buffers on an address_space's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) * ->private_list must be from the same address_space: the blockdev's.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) * address_spaces which do not place buffers at ->private_list via these
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) * utility functions are free to use private_lock and private_list for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) * whatever they want. The only requirement is that list_empty(private_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) * be true at clear_inode() time.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) * FIXME: clear_inode should not call invalidate_inode_buffers(). The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) * filesystems should do that. invalidate_inode_buffers() should just go
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) * BUG_ON(!list_empty).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) * FIXME: mark_buffer_dirty_inode() is a data-plane operation. It should
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) * take an address_space, not an inode. And it should be called
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) * mark_buffer_dirty_fsync() to clearly define why those buffers are being
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) * queued up.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) * list if it is already on a list. Because if the buffer is on a list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) * it *must* already be on the right one. If not, the filesystem is being
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) * silly. This will save a ton of locking. But first we have to ensure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) * that buffers are taken *off* the old inode's list when they are freed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) * (presumably in truncate). That requires careful auditing of all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) * filesystems (do it inside bforget()). It could also be done by bringing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) * b_inode back.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) * The buffer's backing address_space's private_lock must be held
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) static void __remove_assoc_queue(struct buffer_head *bh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) list_del_init(&bh->b_assoc_buffers);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) WARN_ON(!bh->b_assoc_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) bh->b_assoc_map = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) int inode_has_buffers(struct inode *inode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) return !list_empty(&inode->i_data.private_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) * osync is designed to support O_SYNC io. It waits synchronously for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) * all already-submitted IO to complete, but does not queue any new
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) * writes to the disk.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) * you dirty the buffers, and then use osync_inode_buffers to wait for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) * completion. Any other dirty buffers which are not yet queued for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) * write will not be flushed to disk by the osync.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) struct buffer_head *bh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) struct list_head *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) spin_lock(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) repeat:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) list_for_each_prev(p, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) bh = BH_ENTRY(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) if (buffer_locked(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) get_bh(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) spin_unlock(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) wait_on_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) if (!buffer_uptodate(bh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) err = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) brelse(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) spin_lock(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) goto repeat;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) spin_unlock(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) void emergency_thaw_bdev(struct super_block *sb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) while (sb->s_bdev && !thaw_bdev(sb->s_bdev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) printk(KERN_WARNING "Emergency Thaw on %pg\n", sb->s_bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) * sync_mapping_buffers - write out & wait upon a mapping's "associated" buffers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) * @mapping: the mapping which wants those buffers written
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) * Starts I/O against the buffers at mapping->private_list, and waits upon
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) * that I/O.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) * Basically, this is a convenience function for fsync().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) * @mapping is a file or directory which needs those buffers to be written for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) * a successful fsync().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) int sync_mapping_buffers(struct address_space *mapping)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) struct address_space *buffer_mapping = mapping->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) if (buffer_mapping == NULL || list_empty(&mapping->private_list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) return fsync_buffers_list(&buffer_mapping->private_lock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) &mapping->private_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) EXPORT_SYMBOL(sync_mapping_buffers);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) * Called when we've recently written block `bblock', and it is known that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) * `bblock' was for a buffer_boundary() buffer. This means that the block at
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) * `bblock + 1' is probably a dirty indirect block. Hunt it down and, if it's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) * dirty, schedule it for IO. So that indirects merge nicely with their data.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) void write_boundary_block(struct block_device *bdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) sector_t bblock, unsigned blocksize)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) if (bh) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) if (buffer_dirty(bh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) ll_rw_block(REQ_OP_WRITE, 0, 1, &bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) put_bh(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) struct address_space *mapping = inode->i_mapping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) struct address_space *buffer_mapping = bh->b_page->mapping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) mark_buffer_dirty(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) if (!mapping->private_data) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) mapping->private_data = buffer_mapping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) BUG_ON(mapping->private_data != buffer_mapping);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) if (!bh->b_assoc_map) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) spin_lock(&buffer_mapping->private_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) list_move_tail(&bh->b_assoc_buffers,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) &mapping->private_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) bh->b_assoc_map = mapping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) spin_unlock(&buffer_mapping->private_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) EXPORT_SYMBOL(mark_buffer_dirty_inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) * Mark the page dirty, and set it dirty in the page cache, and mark the inode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) * dirty.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) * If warn is true, then emit a warning if the page is not uptodate and has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) * not been truncated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) * The caller must hold lock_page_memcg().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) void __set_page_dirty(struct page *page, struct address_space *mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) int warn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) xa_lock_irqsave(&mapping->i_pages, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) if (page->mapping) { /* Race with truncate? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) WARN_ON_ONCE(warn && !PageUptodate(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) account_page_dirtied(page, mapping);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) __xa_set_mark(&mapping->i_pages, page_index(page),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) PAGECACHE_TAG_DIRTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) xa_unlock_irqrestore(&mapping->i_pages, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) EXPORT_SYMBOL_GPL(__set_page_dirty);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) * Add a page to the dirty page list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) * It is a sad fact of life that this function is called from several places
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) * deeply under spinlocking. It may not sleep.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) * If the page has buffers, the uptodate buffers are set dirty, to preserve
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) * dirty-state coherency between the page and the buffers. It the page does
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) * not have buffers then when they are later attached they will all be set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) * dirty.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) * The buffers are dirtied before the page is dirtied. There's a small race
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) * window in which a writepage caller may see the page cleanness but not the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) * buffer dirtiness. That's fine. If this code were to set the page dirty
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) * before the buffers, a concurrent writepage caller could clear the page dirty
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) * page on the dirty page list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) * We use private_lock to lock against try_to_free_buffers while using the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) * page's buffer list. Also use this to protect against clean buffers being
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) * added to the page after it was set dirty.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) * FIXME: may need to call ->reservepage here as well. That's rather up to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) * address_space though.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) int __set_page_dirty_buffers(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) int newly_dirty;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) struct address_space *mapping = page_mapping(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) if (unlikely(!mapping))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) return !TestSetPageDirty(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) spin_lock(&mapping->private_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) if (page_has_buffers(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) struct buffer_head *head = page_buffers(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) struct buffer_head *bh = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) set_buffer_dirty(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) bh = bh->b_this_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) } while (bh != head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) * Lock out page->mem_cgroup migration to keep PageDirty
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) * synchronized with per-memcg dirty page counters.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) lock_page_memcg(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) newly_dirty = !TestSetPageDirty(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) spin_unlock(&mapping->private_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) if (newly_dirty)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) __set_page_dirty(page, mapping, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) unlock_page_memcg(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) if (newly_dirty)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) return newly_dirty;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) EXPORT_SYMBOL_NS(__set_page_dirty_buffers, ANDROID_GKI_VFS_EXPORT_ONLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) * Write out and wait upon a list of buffers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) * We have conflicting pressures: we want to make sure that all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) * initially dirty buffers get waited on, but that any subsequently
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) * dirtied buffers don't. After all, we don't want fsync to last
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) * forever if somebody is actively writing to the file.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) * Do this in two main stages: first we copy dirty buffers to a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) * temporary inode list, queueing the writes as we go. Then we clean
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) * up, waiting for those writes to complete.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) * During this second stage, any subsequent updates to the file may end
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) * up refiling the buffer on the original inode's dirty list again, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) * there is a chance we will end up with a buffer queued for write but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) * not yet completed on that list. So, as a final cleanup we go through
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) * the osync code to catch these locked, dirty buffers without requeuing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) * any newly dirty buffers for write.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) struct buffer_head *bh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) struct list_head tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) struct address_space *mapping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) int err = 0, err2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) struct blk_plug plug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) INIT_LIST_HEAD(&tmp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) blk_start_plug(&plug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) spin_lock(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) while (!list_empty(list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) bh = BH_ENTRY(list->next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) mapping = bh->b_assoc_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) __remove_assoc_queue(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) /* Avoid race with mark_buffer_dirty_inode() which does
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) * a lockless check and we rely on seeing the dirty bit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) if (buffer_dirty(bh) || buffer_locked(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) list_add(&bh->b_assoc_buffers, &tmp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) bh->b_assoc_map = mapping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) if (buffer_dirty(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) get_bh(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) spin_unlock(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) * Ensure any pending I/O completes so that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) * write_dirty_buffer() actually writes the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) * current contents - it is a noop if I/O is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) * still in flight on potentially older
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) * contents.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) write_dirty_buffer(bh, REQ_SYNC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) * Kick off IO for the previous mapping. Note
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) * that we will not run the very last mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) * wait_on_buffer() will do that for us
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) * through sync_buffer().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) brelse(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) spin_lock(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) spin_unlock(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) blk_finish_plug(&plug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) spin_lock(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) while (!list_empty(&tmp)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) bh = BH_ENTRY(tmp.prev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) get_bh(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) mapping = bh->b_assoc_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) __remove_assoc_queue(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) /* Avoid race with mark_buffer_dirty_inode() which does
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) * a lockless check and we rely on seeing the dirty bit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) if (buffer_dirty(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) list_add(&bh->b_assoc_buffers,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) &mapping->private_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) bh->b_assoc_map = mapping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) spin_unlock(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) wait_on_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) if (!buffer_uptodate(bh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) err = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) brelse(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) spin_lock(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) spin_unlock(lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) err2 = osync_buffers_list(lock, list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) return err2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) * Invalidate any and all dirty buffers on a given inode. We are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) * probably unmounting the fs, but that doesn't mean we have already
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) * done a sync(). Just drop the buffers from the inode list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) * NOTE: we take the inode's blockdev's mapping's private_lock. Which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) * assumes that all the buffers are against the blockdev. Not true
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) * for reiserfs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) void invalidate_inode_buffers(struct inode *inode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) if (inode_has_buffers(inode)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) struct address_space *mapping = &inode->i_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) struct list_head *list = &mapping->private_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) struct address_space *buffer_mapping = mapping->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) spin_lock(&buffer_mapping->private_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) while (!list_empty(list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) __remove_assoc_queue(BH_ENTRY(list->next));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) spin_unlock(&buffer_mapping->private_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) EXPORT_SYMBOL(invalidate_inode_buffers);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) * Remove any clean buffers from the inode's buffer list. This is called
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) * when we're trying to free the inode itself. Those buffers can pin it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) * Returns true if all buffers were removed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) int remove_inode_buffers(struct inode *inode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) int ret = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) if (inode_has_buffers(inode)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) struct address_space *mapping = &inode->i_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) struct list_head *list = &mapping->private_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) struct address_space *buffer_mapping = mapping->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) spin_lock(&buffer_mapping->private_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) while (!list_empty(list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) struct buffer_head *bh = BH_ENTRY(list->next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) if (buffer_dirty(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) __remove_assoc_queue(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) spin_unlock(&buffer_mapping->private_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) * Create the appropriate buffers when given a page for data area and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) * the size of each buffer.. Use the bh->b_this_page linked list to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) * follow the buffers created. Return NULL if unable to create more
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) * buffers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) * The retry flag is used to differentiate async IO (paging, swapping)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) * which may not fail from ordinary buffer allocations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) bool retry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) struct buffer_head *bh, *head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) gfp_t gfp = GFP_NOFS | __GFP_ACCOUNT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) long offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) struct mem_cgroup *memcg, *old_memcg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) if (retry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) gfp |= __GFP_NOFAIL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) memcg = get_mem_cgroup_from_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) old_memcg = set_active_memcg(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) head = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) offset = PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) while ((offset -= size) >= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) bh = alloc_buffer_head(gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) if (!bh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) goto no_grow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) bh->b_this_page = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) bh->b_blocknr = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) head = bh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) bh->b_size = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) /* Link the buffer to its page */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) set_bh_page(bh, page, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) set_active_memcg(old_memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) mem_cgroup_put(memcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) return head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) * In case anything failed, we just free everything we got.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) no_grow:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) if (head) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) bh = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) head = head->b_this_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) free_buffer_head(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) } while (head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) EXPORT_SYMBOL_GPL(alloc_page_buffers);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) link_dev_buffers(struct page *page, struct buffer_head *head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) struct buffer_head *bh, *tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) bh = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) tail = bh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) bh = bh->b_this_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) } while (bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) tail->b_this_page = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) attach_page_private(page, head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) static sector_t blkdev_max_block(struct block_device *bdev, unsigned int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) sector_t retval = ~((sector_t)0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) loff_t sz = i_size_read(bdev->bd_inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) if (sz) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) unsigned int sizebits = blksize_bits(size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) retval = (sz >> sizebits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) return retval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) * Initialise the state of a blockdev page's buffers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) static sector_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) init_page_buffers(struct page *page, struct block_device *bdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) sector_t block, int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) struct buffer_head *head = page_buffers(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) struct buffer_head *bh = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) int uptodate = PageUptodate(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) sector_t end_block = blkdev_max_block(I_BDEV(bdev->bd_inode), size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) if (!buffer_mapped(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) bh->b_end_io = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) bh->b_private = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) bh->b_bdev = bdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) bh->b_blocknr = block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) if (uptodate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) set_buffer_uptodate(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) if (block < end_block)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) set_buffer_mapped(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) block++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) bh = bh->b_this_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) } while (bh != head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) * Caller needs to validate requested block against end of device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) return end_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) * Create the page-cache page that contains the requested block.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) * This is used purely for blockdev mappings.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) grow_dev_page(struct block_device *bdev, sector_t block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) pgoff_t index, int size, int sizebits, gfp_t gfp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) struct inode *inode = bdev->bd_inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) struct buffer_head *bh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) sector_t end_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) gfp_t gfp_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) gfp_mask = mapping_gfp_constraint(inode->i_mapping, ~__GFP_FS) | gfp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) * XXX: __getblk_slow() can not really deal with failure and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) * will endlessly loop on improvised global reclaim. Prefer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) * looping in the allocator rather than here, at least that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) * code knows what it's doing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) gfp_mask |= __GFP_NOFAIL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) page = find_or_create_page(inode->i_mapping, index, gfp_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) BUG_ON(!PageLocked(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) if (page_has_buffers(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) bh = page_buffers(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) if (bh->b_size == size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) end_block = init_page_buffers(page, bdev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) (sector_t)index << sizebits,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) if (!try_to_free_buffers(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) goto failed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) * Allocate some buffers for this page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) bh = alloc_page_buffers(page, size, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) * Link the page to the buffers and initialise them. Take the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) * lock to be atomic wrt __find_get_block(), which does not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) * run under the page lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) spin_lock(&inode->i_mapping->private_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) link_dev_buffers(page, bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) end_block = init_page_buffers(page, bdev, (sector_t)index << sizebits,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) spin_unlock(&inode->i_mapping->private_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) ret = (block < end_block) ? 1 : -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) failed:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) put_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) * Create buffers for the specified block device block's page. If
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) * that page was dirty, the buffers are set dirty also.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) grow_buffers(struct block_device *bdev, sector_t block, int size, gfp_t gfp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) pgoff_t index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) int sizebits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) sizebits = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) sizebits++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) } while ((size << sizebits) < PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) index = block >> sizebits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) * Check for a block which wants to lie outside our maximum possible
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) * pagecache index. (this comparison is done using sector_t types).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) if (unlikely(index != block >> sizebits)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) printk(KERN_ERR "%s: requested out-of-range block %llu for "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) "device %pg\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) __func__, (unsigned long long)block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) /* Create a page with the proper size buffers.. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) return grow_dev_page(bdev, block, index, size, sizebits, gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) static struct buffer_head *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) __getblk_slow(struct block_device *bdev, sector_t block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) unsigned size, gfp_t gfp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) /* Size must be multiple of hard sectorsize */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) if (unlikely(size & (bdev_logical_block_size(bdev)-1) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) (size < 512 || size > PAGE_SIZE))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) printk(KERN_ERR "getblk(): invalid block size %d requested\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) printk(KERN_ERR "logical block size: %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) bdev_logical_block_size(bdev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) dump_stack();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) struct buffer_head *bh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) bh = __find_get_block(bdev, block, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) if (bh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) return bh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) ret = grow_buffers(bdev, block, size, gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) * The relationship between dirty buffers and dirty pages:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) * Whenever a page has any dirty buffers, the page's dirty bit is set, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) * the page is tagged dirty in the page cache.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) * At all times, the dirtiness of the buffers represents the dirtiness of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) * subsections of the page. If the page has buffers, the page dirty bit is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) * merely a hint about the true dirty state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) * When a page is set dirty in its entirety, all its buffers are marked dirty
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) * (if the page has buffers).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) * When a buffer is marked dirty, its page is dirtied, but the page's other
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) * buffers are not.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) * Also. When blockdev buffers are explicitly read with bread(), they
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) * individually become uptodate. But their backing page remains not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) * uptodate - even if all of its buffers are uptodate. A subsequent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) * block_read_full_page() against that page will discover all the uptodate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) * buffers, will set the page uptodate and will perform no I/O.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) * mark_buffer_dirty - mark a buffer_head as needing writeout
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) * @bh: the buffer_head to mark dirty
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) * mark_buffer_dirty() will set the dirty bit against the buffer, then set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) * its backing page dirty, then tag the page as dirty in the page cache
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) * and then attach the address_space's inode to its superblock's dirty
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) * inode list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) * mark_buffer_dirty() is atomic. It takes bh->b_page->mapping->private_lock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) * i_pages lock and mapping->host->i_lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) void mark_buffer_dirty(struct buffer_head *bh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) WARN_ON_ONCE(!buffer_uptodate(bh));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) trace_block_dirty_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) * Very *carefully* optimize the it-is-already-dirty case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) * Don't let the final "is it dirty" escape to before we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) * perhaps modified the buffer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) if (buffer_dirty(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) if (buffer_dirty(bh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) if (!test_set_buffer_dirty(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) struct page *page = bh->b_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) struct address_space *mapping = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) lock_page_memcg(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) if (!TestSetPageDirty(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) mapping = page_mapping(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) if (mapping)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) __set_page_dirty(page, mapping, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) unlock_page_memcg(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) if (mapping)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) EXPORT_SYMBOL_NS(mark_buffer_dirty, ANDROID_GKI_VFS_EXPORT_ONLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) void mark_buffer_write_io_error(struct buffer_head *bh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) struct super_block *sb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) set_buffer_write_io_error(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) /* FIXME: do we need to set this in both places? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) if (bh->b_page && bh->b_page->mapping)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) mapping_set_error(bh->b_page->mapping, -EIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) if (bh->b_assoc_map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) mapping_set_error(bh->b_assoc_map, -EIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) sb = READ_ONCE(bh->b_bdev->bd_super);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) if (sb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) errseq_set(&sb->s_wb_err, -EIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) EXPORT_SYMBOL_NS(mark_buffer_write_io_error, ANDROID_GKI_VFS_EXPORT_ONLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) * Decrement a buffer_head's reference count. If all buffers against a page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) * have zero reference count, are clean and unlocked, and if the page is clean
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) * and unlocked then try_to_free_buffers() may strip the buffers from the page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) * in preparation for freeing it (sometimes, rarely, buffers are removed from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) * a page but it ends up not being freed, and buffers may later be reattached).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) void __brelse(struct buffer_head * buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) if (atomic_read(&buf->b_count)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) put_bh(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) WARN(1, KERN_ERR "VFS: brelse: Trying to free free buffer\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) EXPORT_SYMBOL_NS(__brelse, ANDROID_GKI_VFS_EXPORT_ONLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) * bforget() is like brelse(), except it discards any
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) * potentially dirty data.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) void __bforget(struct buffer_head *bh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) clear_buffer_dirty(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) if (bh->b_assoc_map) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) struct address_space *buffer_mapping = bh->b_page->mapping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) spin_lock(&buffer_mapping->private_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) list_del_init(&bh->b_assoc_buffers);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) bh->b_assoc_map = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) spin_unlock(&buffer_mapping->private_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) __brelse(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) EXPORT_SYMBOL_NS(__bforget, ANDROID_GKI_VFS_EXPORT_ONLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) static struct buffer_head *__bread_slow(struct buffer_head *bh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) lock_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) if (buffer_uptodate(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) unlock_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) return bh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) get_bh(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) bh->b_end_io = end_buffer_read_sync;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) submit_bh(REQ_OP_READ, 0, bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) wait_on_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) if (buffer_uptodate(bh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) return bh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) brelse(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) * Per-cpu buffer LRU implementation. To reduce the cost of __find_get_block().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) * The bhs[] array is sorted - newest buffer is at bhs[0]. Buffers have their
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) * refcount elevated by one when they're in an LRU. A buffer can only appear
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) * once in a particular CPU's LRU. A single buffer can be present in multiple
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) * CPU's LRUs at the same time.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) * This is a transparent caching front-end to sb_bread(), sb_getblk() and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) * sb_find_get_block().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) * The LRUs themselves only need locking against invalidate_bh_lrus. We use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) * a local interrupt disable for that.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) #define BH_LRU_SIZE 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) struct bh_lru {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) struct buffer_head *bhs[BH_LRU_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) #ifdef CONFIG_SMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) #define bh_lru_lock() local_irq_disable()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) #define bh_lru_unlock() local_irq_enable()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) #define bh_lru_lock() preempt_disable()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) #define bh_lru_unlock() preempt_enable()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) static inline void check_irqs_on(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) #ifdef irqs_disabled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) BUG_ON(irqs_disabled());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) * Install a buffer_head into this cpu's LRU. If not already in the LRU, it is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) * inserted at the front, and the buffer_head at the back if any is evicted.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) * Or, if already in the LRU it is moved to the front.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) static void bh_lru_install(struct buffer_head *bh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) struct buffer_head *evictee = bh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) struct bh_lru *b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) check_irqs_on();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) * the refcount of buffer_head in bh_lru prevents dropping the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) * attached page(i.e., try_to_free_buffers) so it could cause
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) * failing page migration.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) * Skip putting upcoming bh into bh_lru until migration is done.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) if (lru_cache_disabled())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) bh_lru_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) b = this_cpu_ptr(&bh_lrus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) for (i = 0; i < BH_LRU_SIZE; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) swap(evictee, b->bhs[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) if (evictee == bh) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) bh_lru_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) get_bh(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) bh_lru_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) brelse(evictee);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) * Look up the bh in this cpu's LRU. If it's there, move it to the head.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) static struct buffer_head *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) lookup_bh_lru(struct block_device *bdev, sector_t block, unsigned size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) struct buffer_head *ret = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) check_irqs_on();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) bh_lru_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) for (i = 0; i < BH_LRU_SIZE; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) struct buffer_head *bh = __this_cpu_read(bh_lrus.bhs[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) if (bh && bh->b_blocknr == block && bh->b_bdev == bdev &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) bh->b_size == size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) if (i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) while (i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) __this_cpu_write(bh_lrus.bhs[i],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) __this_cpu_read(bh_lrus.bhs[i - 1]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) i--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) __this_cpu_write(bh_lrus.bhs[0], bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) get_bh(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) ret = bh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) bh_lru_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) * Perform a pagecache lookup for the matching buffer. If it's there, refresh
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) * it in the LRU and mark it as accessed. If it is not present then return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) * NULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) struct buffer_head *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) __find_get_block(struct block_device *bdev, sector_t block, unsigned size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) if (bh == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) /* __find_get_block_slow will mark the page accessed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) bh = __find_get_block_slow(bdev, block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) if (bh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) bh_lru_install(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) touch_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) return bh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) EXPORT_SYMBOL(__find_get_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) * __getblk_gfp() will locate (and, if necessary, create) the buffer_head
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) * which corresponds to the passed block_device, block and size. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) * returned buffer has its reference count incremented.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) * __getblk_gfp() will lock up the machine if grow_dev_page's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) * try_to_free_buffers() attempt is failing. FIXME, perhaps?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) struct buffer_head *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) __getblk_gfp(struct block_device *bdev, sector_t block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) unsigned size, gfp_t gfp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) struct buffer_head *bh = __find_get_block(bdev, block, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) might_sleep();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) if (bh == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) bh = __getblk_slow(bdev, block, size, gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) return bh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) EXPORT_SYMBOL(__getblk_gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) * Do async read-ahead on a buffer..
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) void __breadahead(struct block_device *bdev, sector_t block, unsigned size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) struct buffer_head *bh = __getblk(bdev, block, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) if (likely(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) ll_rw_block(REQ_OP_READ, REQ_RAHEAD, 1, &bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) brelse(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) EXPORT_SYMBOL_NS(__breadahead, ANDROID_GKI_VFS_EXPORT_ONLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) void __breadahead_gfp(struct block_device *bdev, sector_t block, unsigned size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) gfp_t gfp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) struct buffer_head *bh = __getblk_gfp(bdev, block, size, gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) if (likely(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) ll_rw_block(REQ_OP_READ, REQ_RAHEAD, 1, &bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) brelse(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) EXPORT_SYMBOL(__breadahead_gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) * __bread_gfp() - reads a specified block and returns the bh
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) * @bdev: the block_device to read from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) * @block: number of block
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) * @size: size (in bytes) to read
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) * @gfp: page allocation flag
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) * Reads a specified block, and returns buffer head that contains it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) * The page cache can be allocated from non-movable area
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) * not to prevent page migration if you set gfp to zero.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) * It returns NULL if the block was unreadable.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) struct buffer_head *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) __bread_gfp(struct block_device *bdev, sector_t block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) unsigned size, gfp_t gfp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) struct buffer_head *bh = __getblk_gfp(bdev, block, size, gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) if (likely(bh) && !buffer_uptodate(bh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) bh = __bread_slow(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) return bh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) EXPORT_SYMBOL_NS(__bread_gfp, ANDROID_GKI_VFS_EXPORT_ONLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) static void __invalidate_bh_lrus(struct bh_lru *b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) for (i = 0; i < BH_LRU_SIZE; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) brelse(b->bhs[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) b->bhs[i] = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) * invalidate_bh_lrus() is called rarely - but not only at unmount.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) * This doesn't race because it runs in each cpu either in irq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) * or with preempt disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) static void invalidate_bh_lru(void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) struct bh_lru *b = &get_cpu_var(bh_lrus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) __invalidate_bh_lrus(b);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) put_cpu_var(bh_lrus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) bool has_bh_in_lru(int cpu, void *dummy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) struct bh_lru *b = per_cpu_ptr(&bh_lrus, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) for (i = 0; i < BH_LRU_SIZE; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) if (b->bhs[i])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) void invalidate_bh_lrus(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) on_each_cpu_cond(has_bh_in_lru, invalidate_bh_lru, NULL, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) EXPORT_SYMBOL_GPL(invalidate_bh_lrus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) * It's called from workqueue context so we need a bh_lru_lock to close
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) * the race with preemption/irq.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) void invalidate_bh_lrus_cpu(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) struct bh_lru *b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) bh_lru_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) b = this_cpu_ptr(&bh_lrus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) __invalidate_bh_lrus(b);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) bh_lru_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) void set_bh_page(struct buffer_head *bh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) struct page *page, unsigned long offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) bh->b_page = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) BUG_ON(offset >= PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) if (PageHighMem(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) * This catches illegal uses and preserves the offset:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) bh->b_data = (char *)(0 + offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) bh->b_data = page_address(page) + offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) EXPORT_SYMBOL(set_bh_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) * Called when truncating a buffer on a page completely.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) /* Bits that are cleared during an invalidate */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) #define BUFFER_FLAGS_DISCARD \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) (1 << BH_Mapped | 1 << BH_New | 1 << BH_Req | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) 1 << BH_Delay | 1 << BH_Unwritten)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) static void discard_buffer(struct buffer_head * bh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) unsigned long b_state, b_state_old;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) lock_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) clear_buffer_dirty(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) bh->b_bdev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) b_state = bh->b_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) b_state_old = cmpxchg(&bh->b_state, b_state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) (b_state & ~BUFFER_FLAGS_DISCARD));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) if (b_state_old == b_state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) b_state = b_state_old;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) unlock_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) * block_invalidatepage - invalidate part or all of a buffer-backed page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) * @page: the page which is affected
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) * @offset: start of the range to invalidate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) * @length: length of the range to invalidate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) * block_invalidatepage() is called when all or part of the page has become
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) * invalidated by a truncate operation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) * block_invalidatepage() does not have to release all buffers, but it must
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) * ensure that no dirty buffer is left outside @offset and that no I/O
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) * is underway against any of the blocks which are outside the truncation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) * point. Because the caller is about to free (and possibly reuse) those
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) * blocks on-disk.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) void block_invalidatepage(struct page *page, unsigned int offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) unsigned int length)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) struct buffer_head *head, *bh, *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) unsigned int curr_off = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) unsigned int stop = length + offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) BUG_ON(!PageLocked(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) if (!page_has_buffers(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) * Check for overflow
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) BUG_ON(stop > PAGE_SIZE || stop < length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) head = page_buffers(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) bh = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) unsigned int next_off = curr_off + bh->b_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) next = bh->b_this_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) * Are we still fully in range ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) if (next_off > stop)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) * is this block fully invalidated?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) if (offset <= curr_off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) discard_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) curr_off = next_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) bh = next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) } while (bh != head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) * We release buffers only if the entire page is being invalidated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) * The get_block cached value has been unconditionally invalidated,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) * so real IO is not possible anymore.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) if (length == PAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) try_to_release_page(page, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) EXPORT_SYMBOL_NS(block_invalidatepage, ANDROID_GKI_VFS_EXPORT_ONLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) * We attach and possibly dirty the buffers atomically wrt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) * __set_page_dirty_buffers() via private_lock. try_to_free_buffers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) * is already excluded via the page lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) void create_empty_buffers(struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) unsigned long blocksize, unsigned long b_state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) struct buffer_head *bh, *head, *tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) head = alloc_page_buffers(page, blocksize, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) bh = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) bh->b_state |= b_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) tail = bh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) bh = bh->b_this_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) } while (bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) tail->b_this_page = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) spin_lock(&page->mapping->private_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) if (PageUptodate(page) || PageDirty(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) bh = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) if (PageDirty(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) set_buffer_dirty(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) if (PageUptodate(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) set_buffer_uptodate(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) bh = bh->b_this_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) } while (bh != head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) attach_page_private(page, head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) spin_unlock(&page->mapping->private_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) EXPORT_SYMBOL_NS(create_empty_buffers, ANDROID_GKI_VFS_EXPORT_ONLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) * clean_bdev_aliases: clean a range of buffers in block device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) * @bdev: Block device to clean buffers in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) * @block: Start of a range of blocks to clean
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) * @len: Number of blocks to clean
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) * We are taking a range of blocks for data and we don't want writeback of any
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) * buffer-cache aliases starting from return from this function and until the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) * moment when something will explicitly mark the buffer dirty (hopefully that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) * will not happen until we will free that block ;-) We don't even need to mark
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) * it not-uptodate - nobody can expect anything from a newly allocated buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) * anyway. We used to use unmap_buffer() for such invalidation, but that was
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) * wrong. We definitely don't want to mark the alias unmapped, for example - it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) * would confuse anyone who might pick it with bread() afterwards...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) * Also.. Note that bforget() doesn't lock the buffer. So there can be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) * writeout I/O going on against recently-freed buffers. We don't wait on that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) * I/O in bforget() - it's more efficient to wait on the I/O only if we really
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) * need to. That happens here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) void clean_bdev_aliases(struct block_device *bdev, sector_t block, sector_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) struct inode *bd_inode = bdev->bd_inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) struct address_space *bd_mapping = bd_inode->i_mapping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) struct pagevec pvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) pgoff_t index = block >> (PAGE_SHIFT - bd_inode->i_blkbits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) pgoff_t end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) int i, count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) struct buffer_head *bh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) struct buffer_head *head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) end = (block + len - 1) >> (PAGE_SHIFT - bd_inode->i_blkbits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) pagevec_init(&pvec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) while (pagevec_lookup_range(&pvec, bd_mapping, &index, end)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) count = pagevec_count(&pvec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) for (i = 0; i < count; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) struct page *page = pvec.pages[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) if (!page_has_buffers(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) * We use page lock instead of bd_mapping->private_lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) * to pin buffers here since we can afford to sleep and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) * it scales better than a global spinlock lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) lock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) /* Recheck when the page is locked which pins bhs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) if (!page_has_buffers(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) goto unlock_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) head = page_buffers(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) bh = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) if (!buffer_mapped(bh) || (bh->b_blocknr < block))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) goto next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) if (bh->b_blocknr >= block + len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) clear_buffer_dirty(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) wait_on_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) clear_buffer_req(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) next:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) bh = bh->b_this_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) } while (bh != head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) unlock_page:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) pagevec_release(&pvec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) cond_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) /* End of range already reached? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) if (index > end || !index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) EXPORT_SYMBOL_NS(clean_bdev_aliases, ANDROID_GKI_VFS_EXPORT_ONLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689) * Size is a power-of-two in the range 512..PAGE_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690) * and the case we care about most is PAGE_SIZE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) * So this *could* possibly be written with those
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) * constraints in mind (relevant mostly if some
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) * architecture has a slow bit-scan instruction)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) static inline int block_size_bits(unsigned int blocksize)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) return ilog2(blocksize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) static struct buffer_head *create_page_buffers(struct page *page, struct inode *inode, unsigned int b_state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) BUG_ON(!PageLocked(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705) if (!page_has_buffers(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) create_empty_buffers(page, 1 << READ_ONCE(inode->i_blkbits),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) b_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) return page_buffers(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712) * NOTE! All mapped/uptodate combinations are valid:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) * Mapped Uptodate Meaning
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716) * No No "unknown" - must do get_block()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) * No Yes "hole" - zero-filled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) * Yes No "allocated" - allocated on disk, not read in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719) * Yes Yes "valid" - allocated and up-to-date in memory.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721) * "Dirty" is valid only with the last case (mapped+uptodate).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725) * While block_write_full_page is writing back the dirty buffers under
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) * the page lock, whoever dirtied the buffers may decide to clean them
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) * again at any time. We handle that by only looking at the buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728) * state inside lock_buffer().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730) * If block_write_full_page() is called for regular writeback
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) * locked buffer. This only can happen if someone has written the buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733) * directly, with submit_bh(). At the address_space level PageWriteback
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734) * prevents this contention from occurring.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) * If block_write_full_page() is called with wbc->sync_mode ==
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) * WB_SYNC_ALL, the writes are posted using REQ_SYNC; this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) * causes the writes to be flagged as synchronous writes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740) int __block_write_full_page(struct inode *inode, struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) get_block_t *get_block, struct writeback_control *wbc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742) bh_end_io_t *handler)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) sector_t block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746) sector_t last_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747) struct buffer_head *bh, *head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) unsigned int blocksize, bbits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749) int nr_underway = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) int write_flags = wbc_to_write_flags(wbc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752) head = create_page_buffers(page, inode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) (1 << BH_Dirty)|(1 << BH_Uptodate));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756) * Be very careful. We have no exclusion from __set_page_dirty_buffers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757) * here, and the (potentially unmapped) buffers may become dirty at
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758) * any time. If a buffer becomes dirty here after we've inspected it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759) * then we just miss that fact, and the page stays dirty.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) * Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762) * handle that here by just cleaning them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) bh = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766) blocksize = bh->b_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) bbits = block_size_bits(blocksize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) block = (sector_t)page->index << (PAGE_SHIFT - bbits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770) last_block = (i_size_read(inode) - 1) >> bbits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) * Get all the dirty buffers mapped to disk addresses and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774) * handle any aliases from the underlying blockdev's mapping.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777) if (block > last_block) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779) * mapped buffers outside i_size will occur, because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780) * this page can be outside i_size when there is a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781) * truncate in progress.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784) * The buffer was zeroed by block_write_full_page()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786) clear_buffer_dirty(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787) set_buffer_uptodate(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) } else if ((!buffer_mapped(bh) || buffer_delay(bh)) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789) buffer_dirty(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790) WARN_ON(bh->b_size != blocksize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791) err = get_block(inode, block, bh, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793) goto recover;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794) clear_buffer_delay(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795) if (buffer_new(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796) /* blockdev mappings never come here */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797) clear_buffer_new(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798) clean_bdev_bh_alias(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801) bh = bh->b_this_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802) block++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803) } while (bh != head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806) if (!buffer_mapped(bh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809) * If it's a fully non-blocking write attempt and we cannot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810) * lock the buffer then redirty the page. Note that this can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811) * potentially cause a busy-wait loop from writeback threads
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812) * and kswapd activity, but those code paths have their own
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813) * higher-level throttling.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815) if (wbc->sync_mode != WB_SYNC_NONE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816) lock_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817) } else if (!trylock_buffer(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818) redirty_page_for_writepage(wbc, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821) if (test_clear_buffer_dirty(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822) mark_buffer_async_write_endio(bh, handler);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824) unlock_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826) } while ((bh = bh->b_this_page) != head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829) * The page and its buffers are protected by PageWriteback(), so we can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830) * drop the bh refcounts early.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832) BUG_ON(PageWriteback(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833) set_page_writeback(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836) struct buffer_head *next = bh->b_this_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837) if (buffer_async_write(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838) submit_bh_wbc(REQ_OP_WRITE, write_flags, bh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839) inode->i_write_hint, wbc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840) nr_underway++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842) bh = next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843) } while (bh != head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846) err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847) done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848) if (nr_underway == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850) * The page was marked dirty, but the buffers were
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851) * clean. Someone wrote them back by hand with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852) * ll_rw_block/submit_bh. A rare case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854) end_page_writeback(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857) * The page and buffer_heads can be released at any time from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858) * here on.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1859) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1860) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1861) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1863) recover:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1864) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1865) * ENOSPC, or some other error. We may already have added some
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1866) * blocks to the file, so we need to write these out to avoid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1867) * exposing stale data.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1868) * The page is currently locked and not marked for writeback
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1869) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1870) bh = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1871) /* Recovery: lock and submit the mapped buffers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1872) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1873) if (buffer_mapped(bh) && buffer_dirty(bh) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1874) !buffer_delay(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1875) lock_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1876) mark_buffer_async_write_endio(bh, handler);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1877) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1878) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1879) * The buffer may have been set dirty during
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1880) * attachment to a dirty page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1881) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1882) clear_buffer_dirty(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1883) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1884) } while ((bh = bh->b_this_page) != head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1885) SetPageError(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1886) BUG_ON(PageWriteback(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1887) mapping_set_error(page->mapping, err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1888) set_page_writeback(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1889) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1890) struct buffer_head *next = bh->b_this_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1891) if (buffer_async_write(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1892) clear_buffer_dirty(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1893) submit_bh_wbc(REQ_OP_WRITE, write_flags, bh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1894) inode->i_write_hint, wbc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1895) nr_underway++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1896) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1897) bh = next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1898) } while (bh != head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1899) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1900) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1901) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1902) EXPORT_SYMBOL(__block_write_full_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1904) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1905) * If a page has any new buffers, zero them out here, and mark them uptodate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1906) * and dirty so they'll be written out (in order to prevent uninitialised
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1907) * block data from leaking). And clear the new bit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1908) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1909) void page_zero_new_buffers(struct page *page, unsigned from, unsigned to)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1910) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1911) unsigned int block_start, block_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1912) struct buffer_head *head, *bh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1913)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1914) BUG_ON(!PageLocked(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1915) if (!page_has_buffers(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1916) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1917)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1918) bh = head = page_buffers(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1919) block_start = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1920) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1921) block_end = block_start + bh->b_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1922)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1923) if (buffer_new(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1924) if (block_end > from && block_start < to) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1925) if (!PageUptodate(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1926) unsigned start, size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1927)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1928) start = max(from, block_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1929) size = min(to, block_end) - start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1930)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1931) zero_user(page, start, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1932) set_buffer_uptodate(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1933) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1934)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1935) clear_buffer_new(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1936) mark_buffer_dirty(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1937) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1938) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1939)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1940) block_start = block_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1941) bh = bh->b_this_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1942) } while (bh != head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1943) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1944) EXPORT_SYMBOL_NS(page_zero_new_buffers, ANDROID_GKI_VFS_EXPORT_ONLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1945)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1946) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1947) iomap_to_bh(struct inode *inode, sector_t block, struct buffer_head *bh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1948) struct iomap *iomap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1949) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1950) loff_t offset = block << inode->i_blkbits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1951)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1952) bh->b_bdev = iomap->bdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1953)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1954) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1955) * Block points to offset in file we need to map, iomap contains
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1956) * the offset at which the map starts. If the map ends before the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1957) * current block, then do not map the buffer and let the caller
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1958) * handle it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1959) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1960) BUG_ON(offset >= iomap->offset + iomap->length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1961)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1962) switch (iomap->type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1963) case IOMAP_HOLE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1964) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1965) * If the buffer is not up to date or beyond the current EOF,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1966) * we need to mark it as new to ensure sub-block zeroing is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1967) * executed if necessary.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1968) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1969) if (!buffer_uptodate(bh) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1970) (offset >= i_size_read(inode)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1971) set_buffer_new(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1972) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1973) case IOMAP_DELALLOC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1974) if (!buffer_uptodate(bh) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1975) (offset >= i_size_read(inode)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1976) set_buffer_new(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1977) set_buffer_uptodate(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1978) set_buffer_mapped(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1979) set_buffer_delay(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1980) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1981) case IOMAP_UNWRITTEN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1982) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1983) * For unwritten regions, we always need to ensure that regions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1984) * in the block we are not writing to are zeroed. Mark the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1985) * buffer as new to ensure this.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1986) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1987) set_buffer_new(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1988) set_buffer_unwritten(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1989) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1990) case IOMAP_MAPPED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1991) if ((iomap->flags & IOMAP_F_NEW) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1992) offset >= i_size_read(inode))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1993) set_buffer_new(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1994) bh->b_blocknr = (iomap->addr + offset - iomap->offset) >>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1995) inode->i_blkbits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1996) set_buffer_mapped(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1997) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1998) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1999) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2001) int __block_write_begin_int(struct page *page, loff_t pos, unsigned len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2002) get_block_t *get_block, struct iomap *iomap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2003) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2004) unsigned from = pos & (PAGE_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2005) unsigned to = from + len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2006) struct inode *inode = page->mapping->host;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2007) unsigned block_start, block_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2008) sector_t block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2009) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2010) unsigned blocksize, bbits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2011) struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2012)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2013) BUG_ON(!PageLocked(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2014) BUG_ON(from > PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2015) BUG_ON(to > PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2016) BUG_ON(from > to);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2017)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2018) head = create_page_buffers(page, inode, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2019) blocksize = head->b_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2020) bbits = block_size_bits(blocksize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2021)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2022) block = (sector_t)page->index << (PAGE_SHIFT - bbits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2023)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2024) for(bh = head, block_start = 0; bh != head || !block_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2025) block++, block_start=block_end, bh = bh->b_this_page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2026) block_end = block_start + blocksize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2027) if (block_end <= from || block_start >= to) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2028) if (PageUptodate(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2029) if (!buffer_uptodate(bh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2030) set_buffer_uptodate(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2031) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2032) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2033) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2034) if (buffer_new(bh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2035) clear_buffer_new(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2036) if (!buffer_mapped(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2037) WARN_ON(bh->b_size != blocksize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2038) if (get_block) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2039) err = get_block(inode, block, bh, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2040) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2041) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2042) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2043) iomap_to_bh(inode, block, bh, iomap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2044) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2045)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2046) if (buffer_new(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2047) clean_bdev_bh_alias(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2048) if (PageUptodate(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2049) clear_buffer_new(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2050) set_buffer_uptodate(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2051) mark_buffer_dirty(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2052) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2053) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2054) if (block_end > to || block_start < from)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2055) zero_user_segments(page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2056) to, block_end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2057) block_start, from);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2058) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2059) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2060) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2061) if (PageUptodate(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2062) if (!buffer_uptodate(bh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2063) set_buffer_uptodate(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2064) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2065) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2066) if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2067) !buffer_unwritten(bh) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2068) (block_start < from || block_end > to)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2069) ll_rw_block(REQ_OP_READ, 0, 1, &bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2070) *wait_bh++=bh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2071) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2072) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2073) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2074) * If we issued read requests - let them complete.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2075) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2076) while(wait_bh > wait) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2077) wait_on_buffer(*--wait_bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2078) if (!buffer_uptodate(*wait_bh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2079) err = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2080) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2081) if (unlikely(err))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2082) page_zero_new_buffers(page, from, to);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2083) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2084) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2085)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2086) int __block_write_begin(struct page *page, loff_t pos, unsigned len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2087) get_block_t *get_block)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2088) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2089) return __block_write_begin_int(page, pos, len, get_block, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2090) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2091) EXPORT_SYMBOL(__block_write_begin);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2092)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2093) static int __block_commit_write(struct inode *inode, struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2094) unsigned from, unsigned to)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2095) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2096) unsigned block_start, block_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2097) int partial = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2098) unsigned blocksize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2099) struct buffer_head *bh, *head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2101) bh = head = page_buffers(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2102) blocksize = bh->b_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2104) block_start = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2105) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2106) block_end = block_start + blocksize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2107) if (block_end <= from || block_start >= to) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2108) if (!buffer_uptodate(bh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2109) partial = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2110) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2111) set_buffer_uptodate(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2112) mark_buffer_dirty(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2113) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2114) clear_buffer_new(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2116) block_start = block_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2117) bh = bh->b_this_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2118) } while (bh != head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2120) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2121) * If this is a partial write which happened to make all buffers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2122) * uptodate then we can optimize away a bogus readpage() for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2123) * the next read(). Here we 'discover' whether the page went
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2124) * uptodate as a result of this (potentially partial) write.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2125) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2126) if (!partial)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2127) SetPageUptodate(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2128) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2129) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2131) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2132) * block_write_begin takes care of the basic task of block allocation and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2133) * bringing partial write blocks uptodate first.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2134) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2135) * The filesystem needs to handle block truncation upon failure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2136) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2137) int block_write_begin(struct address_space *mapping, loff_t pos, unsigned len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2138) unsigned flags, struct page **pagep, get_block_t *get_block)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2139) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2140) pgoff_t index = pos >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2141) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2142) int status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2144) page = grab_cache_page_write_begin(mapping, index, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2145) if (!page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2146) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2148) status = __block_write_begin(page, pos, len, get_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2149) if (unlikely(status)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2150) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2151) put_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2152) page = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2153) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2155) *pagep = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2156) return status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2157) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2158) EXPORT_SYMBOL(block_write_begin);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2160) int block_write_end(struct file *file, struct address_space *mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2161) loff_t pos, unsigned len, unsigned copied,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2162) struct page *page, void *fsdata)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2163) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2164) struct inode *inode = mapping->host;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2165) unsigned start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2167) start = pos & (PAGE_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2169) if (unlikely(copied < len)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2170) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2171) * The buffers that were written will now be uptodate, so we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2172) * don't have to worry about a readpage reading them and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2173) * overwriting a partial write. However if we have encountered
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2174) * a short write and only partially written into a buffer, it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2175) * will not be marked uptodate, so a readpage might come in and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2176) * destroy our partial write.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2177) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2178) * Do the simplest thing, and just treat any short write to a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2179) * non uptodate page as a zero-length write, and force the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2180) * caller to redo the whole thing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2181) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2182) if (!PageUptodate(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2183) copied = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2185) page_zero_new_buffers(page, start+copied, start+len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2186) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2187) flush_dcache_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2189) /* This could be a short (even 0-length) commit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2190) __block_commit_write(inode, page, start, start+copied);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2192) return copied;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2193) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2194) EXPORT_SYMBOL(block_write_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2196) int generic_write_end(struct file *file, struct address_space *mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2197) loff_t pos, unsigned len, unsigned copied,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2198) struct page *page, void *fsdata)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2199) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2200) struct inode *inode = mapping->host;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2201) loff_t old_size = inode->i_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2202) bool i_size_changed = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2204) copied = block_write_end(file, mapping, pos, len, copied, page, fsdata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2206) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2207) * No need to use i_size_read() here, the i_size cannot change under us
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2208) * because we hold i_rwsem.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2209) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2210) * But it's important to update i_size while still holding page lock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2211) * page writeout could otherwise come in and zero beyond i_size.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2212) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2213) if (pos + copied > inode->i_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2214) i_size_write(inode, pos + copied);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2215) i_size_changed = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2216) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2218) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2219) put_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2221) if (old_size < pos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2222) pagecache_isize_extended(inode, old_size, pos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2223) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2224) * Don't mark the inode dirty under page lock. First, it unnecessarily
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2225) * makes the holding time of page lock longer. Second, it forces lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2226) * ordering of page lock and transaction start for journaling
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2227) * filesystems.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2228) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2229) if (i_size_changed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2230) mark_inode_dirty(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2231) return copied;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2232) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2233) EXPORT_SYMBOL(generic_write_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2235) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2236) * block_is_partially_uptodate checks whether buffers within a page are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2237) * uptodate or not.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2238) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2239) * Returns true if all buffers which correspond to a file portion
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2240) * we want to read are uptodate.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2241) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2242) int block_is_partially_uptodate(struct page *page, unsigned long from,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2243) unsigned long count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2244) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2245) unsigned block_start, block_end, blocksize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2246) unsigned to;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2247) struct buffer_head *bh, *head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2248) int ret = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2250) if (!page_has_buffers(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2251) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2253) head = page_buffers(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2254) blocksize = head->b_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2255) to = min_t(unsigned, PAGE_SIZE - from, count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2256) to = from + to;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2257) if (from < blocksize && to > PAGE_SIZE - blocksize)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2258) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2260) bh = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2261) block_start = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2262) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2263) block_end = block_start + blocksize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2264) if (block_end > from && block_start < to) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2265) if (!buffer_uptodate(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2266) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2267) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2268) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2269) if (block_end >= to)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2270) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2271) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2272) block_start = block_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2273) bh = bh->b_this_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2274) } while (bh != head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2276) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2277) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2278) EXPORT_SYMBOL_NS(block_is_partially_uptodate, ANDROID_GKI_VFS_EXPORT_ONLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2280) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2281) * Generic "read page" function for block devices that have the normal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2282) * get_block functionality. This is most of the block device filesystems.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2283) * Reads the page asynchronously --- the unlock_buffer() and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2284) * set/clear_buffer_uptodate() functions propagate buffer state into the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2285) * page struct once IO has completed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2286) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2287) int block_read_full_page(struct page *page, get_block_t *get_block)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2288) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2289) struct inode *inode = page->mapping->host;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2290) sector_t iblock, lblock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2291) struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2292) unsigned int blocksize, bbits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2293) int nr, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2294) int fully_mapped = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2296) head = create_page_buffers(page, inode, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2297) blocksize = head->b_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2298) bbits = block_size_bits(blocksize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2300) iblock = (sector_t)page->index << (PAGE_SHIFT - bbits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2301) lblock = (i_size_read(inode)+blocksize-1) >> bbits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2302) bh = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2303) nr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2304) i = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2306) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2307) if (buffer_uptodate(bh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2308) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2310) if (!buffer_mapped(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2311) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2312)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2313) fully_mapped = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2314) if (iblock < lblock) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2315) WARN_ON(bh->b_size != blocksize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2316) err = get_block(inode, iblock, bh, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2317) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2318) SetPageError(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2319) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2320) if (!buffer_mapped(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2321) zero_user(page, i * blocksize, blocksize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2322) if (!err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2323) set_buffer_uptodate(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2324) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2325) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2326) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2327) * get_block() might have updated the buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2328) * synchronously
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2329) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2330) if (buffer_uptodate(bh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2331) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2332) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2333) arr[nr++] = bh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2334) } while (i++, iblock++, (bh = bh->b_this_page) != head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2336) if (fully_mapped)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2337) SetPageMappedToDisk(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2339) if (!nr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2340) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2341) * All buffers are uptodate - we can set the page uptodate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2342) * as well. But not if get_block() returned an error.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2343) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2344) if (!PageError(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2345) SetPageUptodate(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2346) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2347) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2348) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2350) /* Stage two: lock the buffers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2351) for (i = 0; i < nr; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2352) bh = arr[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2353) lock_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2354) mark_buffer_async_read(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2355) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2357) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2358) * Stage 3: start the IO. Check for uptodateness
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2359) * inside the buffer lock in case another process reading
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2360) * the underlying blockdev brought it uptodate (the sct fix).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2361) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2362) for (i = 0; i < nr; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2363) bh = arr[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2364) if (buffer_uptodate(bh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2365) end_buffer_async_read(bh, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2366) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2367) submit_bh(REQ_OP_READ, 0, bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2368) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2369) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2370) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2371) EXPORT_SYMBOL(block_read_full_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2372)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2373) /* utility function for filesystems that need to do work on expanding
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2374) * truncates. Uses filesystem pagecache writes to allow the filesystem to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2375) * deal with the hole.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2376) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2377) int generic_cont_expand_simple(struct inode *inode, loff_t size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2378) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2379) struct address_space *mapping = inode->i_mapping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2380) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2381) void *fsdata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2382) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2384) err = inode_newsize_ok(inode, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2385) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2386) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2388) err = pagecache_write_begin(NULL, mapping, size, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2389) AOP_FLAG_CONT_EXPAND, &page, &fsdata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2390) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2391) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2393) err = pagecache_write_end(NULL, mapping, size, 0, 0, page, fsdata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2394) BUG_ON(err > 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2396) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2397) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2398) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2399) EXPORT_SYMBOL(generic_cont_expand_simple);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2401) static int cont_expand_zero(struct file *file, struct address_space *mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2402) loff_t pos, loff_t *bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2403) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2404) struct inode *inode = mapping->host;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2405) unsigned int blocksize = i_blocksize(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2406) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2407) void *fsdata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2408) pgoff_t index, curidx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2409) loff_t curpos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2410) unsigned zerofrom, offset, len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2411) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2413) index = pos >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2414) offset = pos & ~PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2416) while (index > (curidx = (curpos = *bytes)>>PAGE_SHIFT)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2417) zerofrom = curpos & ~PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2418) if (zerofrom & (blocksize-1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2419) *bytes |= (blocksize-1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2420) (*bytes)++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2421) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2422) len = PAGE_SIZE - zerofrom;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2424) err = pagecache_write_begin(file, mapping, curpos, len, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2425) &page, &fsdata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2426) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2427) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2428) zero_user(page, zerofrom, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2429) err = pagecache_write_end(file, mapping, curpos, len, len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2430) page, fsdata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2431) if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2432) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2433) BUG_ON(err != len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2434) err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2436) balance_dirty_pages_ratelimited(mapping);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2438) if (fatal_signal_pending(current)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2439) err = -EINTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2440) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2441) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2442) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2444) /* page covers the boundary, find the boundary offset */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2445) if (index == curidx) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2446) zerofrom = curpos & ~PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2447) /* if we will expand the thing last block will be filled */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2448) if (offset <= zerofrom) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2449) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2450) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2451) if (zerofrom & (blocksize-1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2452) *bytes |= (blocksize-1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2453) (*bytes)++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2454) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2455) len = offset - zerofrom;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2457) err = pagecache_write_begin(file, mapping, curpos, len, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2458) &page, &fsdata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2459) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2460) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2461) zero_user(page, zerofrom, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2462) err = pagecache_write_end(file, mapping, curpos, len, len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2463) page, fsdata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2464) if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2465) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2466) BUG_ON(err != len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2467) err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2468) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2469) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2470) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2471) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2473) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2474) * For moronic filesystems that do not allow holes in file.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2475) * We may have to extend the file.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2476) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2477) int cont_write_begin(struct file *file, struct address_space *mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2478) loff_t pos, unsigned len, unsigned flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2479) struct page **pagep, void **fsdata,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2480) get_block_t *get_block, loff_t *bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2481) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2482) struct inode *inode = mapping->host;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2483) unsigned int blocksize = i_blocksize(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2484) unsigned int zerofrom;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2485) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2487) err = cont_expand_zero(file, mapping, pos, bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2488) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2489) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2491) zerofrom = *bytes & ~PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2492) if (pos+len > *bytes && zerofrom & (blocksize-1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2493) *bytes |= (blocksize-1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2494) (*bytes)++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2495) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2496)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2497) return block_write_begin(mapping, pos, len, flags, pagep, get_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2498) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2499) EXPORT_SYMBOL(cont_write_begin);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2501) int block_commit_write(struct page *page, unsigned from, unsigned to)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2502) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2503) struct inode *inode = page->mapping->host;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2504) __block_commit_write(inode,page,from,to);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2505) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2506) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2507) EXPORT_SYMBOL(block_commit_write);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2509) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2510) * block_page_mkwrite() is not allowed to change the file size as it gets
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2511) * called from a page fault handler when a page is first dirtied. Hence we must
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2512) * be careful to check for EOF conditions here. We set the page up correctly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2513) * for a written page which means we get ENOSPC checking when writing into
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2514) * holes and correct delalloc and unwritten extent mapping on filesystems that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2515) * support these features.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2516) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2517) * We are not allowed to take the i_mutex here so we have to play games to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2518) * protect against truncate races as the page could now be beyond EOF. Because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2519) * truncate writes the inode size before removing pages, once we have the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2520) * page lock we can determine safely if the page is beyond EOF. If it is not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2521) * beyond EOF, then the page is guaranteed safe against truncation until we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2522) * unlock the page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2523) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2524) * Direct callers of this function should protect against filesystem freezing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2525) * using sb_start_pagefault() - sb_end_pagefault() functions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2526) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2527) int block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2528) get_block_t get_block)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2529) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2530) struct page *page = vmf->page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2531) struct inode *inode = file_inode(vma->vm_file);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2532) unsigned long end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2533) loff_t size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2534) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2536) lock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2537) size = i_size_read(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2538) if ((page->mapping != inode->i_mapping) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2539) (page_offset(page) > size)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2540) /* We overload EFAULT to mean page got truncated */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2541) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2542) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2543) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2545) /* page is wholly or partially inside EOF */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2546) if (((page->index + 1) << PAGE_SHIFT) > size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2547) end = size & ~PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2548) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2549) end = PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2551) ret = __block_write_begin(page, 0, end, get_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2552) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2553) ret = block_commit_write(page, 0, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2555) if (unlikely(ret < 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2556) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2557) set_page_dirty(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2558) wait_for_stable_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2559) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2560) out_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2561) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2562) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2563) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2564) EXPORT_SYMBOL(block_page_mkwrite);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2566) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2567) * nobh_write_begin()'s prereads are special: the buffer_heads are freed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2568) * immediately, while under the page lock. So it needs a special end_io
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2569) * handler which does not touch the bh after unlocking it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2570) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2571) static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2572) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2573) __end_buffer_read_notouch(bh, uptodate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2574) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2576) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2577) * Attach the singly-linked list of buffers created by nobh_write_begin, to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2578) * the page (converting it to circular linked list and taking care of page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2579) * dirty races).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2580) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2581) static void attach_nobh_buffers(struct page *page, struct buffer_head *head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2582) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2583) struct buffer_head *bh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2585) BUG_ON(!PageLocked(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2587) spin_lock(&page->mapping->private_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2588) bh = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2589) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2590) if (PageDirty(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2591) set_buffer_dirty(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2592) if (!bh->b_this_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2593) bh->b_this_page = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2594) bh = bh->b_this_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2595) } while (bh != head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2596) attach_page_private(page, head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2597) spin_unlock(&page->mapping->private_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2598) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2600) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2601) * On entry, the page is fully not uptodate.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2602) * On exit the page is fully uptodate in the areas outside (from,to)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2603) * The filesystem needs to handle block truncation upon failure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2604) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2605) int nobh_write_begin(struct address_space *mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2606) loff_t pos, unsigned len, unsigned flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2607) struct page **pagep, void **fsdata,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2608) get_block_t *get_block)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2609) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2610) struct inode *inode = mapping->host;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2611) const unsigned blkbits = inode->i_blkbits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2612) const unsigned blocksize = 1 << blkbits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2613) struct buffer_head *head, *bh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2614) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2615) pgoff_t index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2616) unsigned from, to;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2617) unsigned block_in_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2618) unsigned block_start, block_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2619) sector_t block_in_file;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2620) int nr_reads = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2621) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2622) int is_mapped_to_disk = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2624) index = pos >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2625) from = pos & (PAGE_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2626) to = from + len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2628) page = grab_cache_page_write_begin(mapping, index, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2629) if (!page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2630) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2631) *pagep = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2632) *fsdata = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2634) if (page_has_buffers(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2635) ret = __block_write_begin(page, pos, len, get_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2636) if (unlikely(ret))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2637) goto out_release;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2638) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2639) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2641) if (PageMappedToDisk(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2642) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2644) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2645) * Allocate buffers so that we can keep track of state, and potentially
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2646) * attach them to the page if an error occurs. In the common case of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2647) * no error, they will just be freed again without ever being attached
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2648) * to the page (which is all OK, because we're under the page lock).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2649) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2650) * Be careful: the buffer linked list is a NULL terminated one, rather
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2651) * than the circular one we're used to.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2652) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2653) head = alloc_page_buffers(page, blocksize, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2654) if (!head) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2655) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2656) goto out_release;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2657) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2659) block_in_file = (sector_t)page->index << (PAGE_SHIFT - blkbits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2660)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2661) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2662) * We loop across all blocks in the page, whether or not they are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2663) * part of the affected region. This is so we can discover if the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2664) * page is fully mapped-to-disk.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2665) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2666) for (block_start = 0, block_in_page = 0, bh = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2667) block_start < PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2668) block_in_page++, block_start += blocksize, bh = bh->b_this_page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2669) int create;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2670)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2671) block_end = block_start + blocksize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2672) bh->b_state = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2673) create = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2674) if (block_start >= to)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2675) create = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2676) ret = get_block(inode, block_in_file + block_in_page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2677) bh, create);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2678) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2679) goto failed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2680) if (!buffer_mapped(bh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2681) is_mapped_to_disk = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2682) if (buffer_new(bh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2683) clean_bdev_bh_alias(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2684) if (PageUptodate(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2685) set_buffer_uptodate(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2686) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2687) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2688) if (buffer_new(bh) || !buffer_mapped(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2689) zero_user_segments(page, block_start, from,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2690) to, block_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2691) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2692) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2693) if (buffer_uptodate(bh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2694) continue; /* reiserfs does this */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2695) if (block_start < from || block_end > to) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2696) lock_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2697) bh->b_end_io = end_buffer_read_nobh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2698) submit_bh(REQ_OP_READ, 0, bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2699) nr_reads++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2700) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2701) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2703) if (nr_reads) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2704) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2705) * The page is locked, so these buffers are protected from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2706) * any VM or truncate activity. Hence we don't need to care
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2707) * for the buffer_head refcounts.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2708) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2709) for (bh = head; bh; bh = bh->b_this_page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2710) wait_on_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2711) if (!buffer_uptodate(bh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2712) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2713) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2714) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2715) goto failed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2716) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2718) if (is_mapped_to_disk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2719) SetPageMappedToDisk(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2720)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2721) *fsdata = head; /* to be released by nobh_write_end */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2723) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2724)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2725) failed:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2726) BUG_ON(!ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2727) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2728) * Error recovery is a bit difficult. We need to zero out blocks that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2729) * were newly allocated, and dirty them to ensure they get written out.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2730) * Buffers need to be attached to the page at this point, otherwise
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2731) * the handling of potential IO errors during writeout would be hard
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2732) * (could try doing synchronous writeout, but what if that fails too?)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2733) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2734) attach_nobh_buffers(page, head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2735) page_zero_new_buffers(page, from, to);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2737) out_release:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2738) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2739) put_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2740) *pagep = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2741)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2742) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2743) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2744) EXPORT_SYMBOL(nobh_write_begin);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2745)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2746) int nobh_write_end(struct file *file, struct address_space *mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2747) loff_t pos, unsigned len, unsigned copied,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2748) struct page *page, void *fsdata)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2749) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2750) struct inode *inode = page->mapping->host;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2751) struct buffer_head *head = fsdata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2752) struct buffer_head *bh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2753) BUG_ON(fsdata != NULL && page_has_buffers(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2754)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2755) if (unlikely(copied < len) && head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2756) attach_nobh_buffers(page, head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2757) if (page_has_buffers(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2758) return generic_write_end(file, mapping, pos, len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2759) copied, page, fsdata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2761) SetPageUptodate(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2762) set_page_dirty(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2763) if (pos+copied > inode->i_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2764) i_size_write(inode, pos+copied);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2765) mark_inode_dirty(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2766) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2767)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2768) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2769) put_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2770)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2771) while (head) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2772) bh = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2773) head = head->b_this_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2774) free_buffer_head(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2775) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2776)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2777) return copied;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2778) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2779) EXPORT_SYMBOL(nobh_write_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2781) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2782) * nobh_writepage() - based on block_full_write_page() except
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2783) * that it tries to operate without attaching bufferheads to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2784) * the page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2785) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2786) int nobh_writepage(struct page *page, get_block_t *get_block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2787) struct writeback_control *wbc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2788) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2789) struct inode * const inode = page->mapping->host;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2790) loff_t i_size = i_size_read(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2791) const pgoff_t end_index = i_size >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2792) unsigned offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2793) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2795) /* Is the page fully inside i_size? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2796) if (page->index < end_index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2797) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2798)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2799) /* Is the page fully outside i_size? (truncate in progress) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2800) offset = i_size & (PAGE_SIZE-1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2801) if (page->index >= end_index+1 || !offset) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2802) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2803) return 0; /* don't care */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2804) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2805)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2806) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2807) * The page straddles i_size. It must be zeroed out on each and every
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2808) * writepage invocation because it may be mmapped. "A file is mapped
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2809) * in multiples of the page size. For a file that is not a multiple of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2810) * the page size, the remaining memory is zeroed when mapped, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2811) * writes to that region are not written out to the file."
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2812) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2813) zero_user_segment(page, offset, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2814) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2815) ret = mpage_writepage(page, get_block, wbc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2816) if (ret == -EAGAIN)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2817) ret = __block_write_full_page(inode, page, get_block, wbc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2818) end_buffer_async_write);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2819) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2820) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2821) EXPORT_SYMBOL(nobh_writepage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2822)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2823) int nobh_truncate_page(struct address_space *mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2824) loff_t from, get_block_t *get_block)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2825) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2826) pgoff_t index = from >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2827) unsigned offset = from & (PAGE_SIZE-1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2828) unsigned blocksize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2829) sector_t iblock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2830) unsigned length, pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2831) struct inode *inode = mapping->host;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2832) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2833) struct buffer_head map_bh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2834) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2835)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2836) blocksize = i_blocksize(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2837) length = offset & (blocksize - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2838)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2839) /* Block boundary? Nothing to do */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2840) if (!length)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2841) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2842)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2843) length = blocksize - length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2844) iblock = (sector_t)index << (PAGE_SHIFT - inode->i_blkbits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2845)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2846) page = grab_cache_page(mapping, index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2847) err = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2848) if (!page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2849) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2850)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2851) if (page_has_buffers(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2852) has_buffers:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2853) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2854) put_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2855) return block_truncate_page(mapping, from, get_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2856) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2857)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2858) /* Find the buffer that contains "offset" */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2859) pos = blocksize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2860) while (offset >= pos) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2861) iblock++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2862) pos += blocksize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2863) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2864)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2865) map_bh.b_size = blocksize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2866) map_bh.b_state = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2867) err = get_block(inode, iblock, &map_bh, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2868) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2869) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2870) /* unmapped? It's a hole - nothing to do */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2871) if (!buffer_mapped(&map_bh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2872) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2873)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2874) /* Ok, it's mapped. Make sure it's up-to-date */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2875) if (!PageUptodate(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2876) err = mapping->a_ops->readpage(NULL, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2877) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2878) put_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2879) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2880) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2881) lock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2882) if (!PageUptodate(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2883) err = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2884) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2885) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2886) if (page_has_buffers(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2887) goto has_buffers;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2888) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2889) zero_user(page, offset, length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2890) set_page_dirty(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2891) err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2892)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2893) unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2894) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2895) put_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2896) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2897) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2898) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2899) EXPORT_SYMBOL(nobh_truncate_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2900)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2901) int block_truncate_page(struct address_space *mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2902) loff_t from, get_block_t *get_block)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2903) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2904) pgoff_t index = from >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2905) unsigned offset = from & (PAGE_SIZE-1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2906) unsigned blocksize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2907) sector_t iblock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2908) unsigned length, pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2909) struct inode *inode = mapping->host;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2910) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2911) struct buffer_head *bh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2912) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2913)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2914) blocksize = i_blocksize(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2915) length = offset & (blocksize - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2916)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2917) /* Block boundary? Nothing to do */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2918) if (!length)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2919) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2920)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2921) length = blocksize - length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2922) iblock = (sector_t)index << (PAGE_SHIFT - inode->i_blkbits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2923)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2924) page = grab_cache_page(mapping, index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2925) err = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2926) if (!page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2927) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2928)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2929) if (!page_has_buffers(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2930) create_empty_buffers(page, blocksize, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2931)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2932) /* Find the buffer that contains "offset" */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2933) bh = page_buffers(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2934) pos = blocksize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2935) while (offset >= pos) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2936) bh = bh->b_this_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2937) iblock++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2938) pos += blocksize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2939) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2940)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2941) err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2942) if (!buffer_mapped(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2943) WARN_ON(bh->b_size != blocksize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2944) err = get_block(inode, iblock, bh, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2945) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2946) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2947) /* unmapped? It's a hole - nothing to do */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2948) if (!buffer_mapped(bh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2949) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2950) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2951)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2952) /* Ok, it's mapped. Make sure it's up-to-date */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2953) if (PageUptodate(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2954) set_buffer_uptodate(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2955)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2956) if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2957) err = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2958) ll_rw_block(REQ_OP_READ, 0, 1, &bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2959) wait_on_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2960) /* Uhhuh. Read error. Complain and punt. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2961) if (!buffer_uptodate(bh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2962) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2963) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2964)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2965) zero_user(page, offset, length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2966) mark_buffer_dirty(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2967) err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2968)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2969) unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2970) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2971) put_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2972) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2973) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2974) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2975) EXPORT_SYMBOL(block_truncate_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2976)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2977) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2978) * The generic ->writepage function for buffer-backed address_spaces
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2979) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2980) int block_write_full_page(struct page *page, get_block_t *get_block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2981) struct writeback_control *wbc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2982) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2983) struct inode * const inode = page->mapping->host;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2984) loff_t i_size = i_size_read(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2985) const pgoff_t end_index = i_size >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2986) unsigned offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2987)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2988) /* Is the page fully inside i_size? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2989) if (page->index < end_index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2990) return __block_write_full_page(inode, page, get_block, wbc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2991) end_buffer_async_write);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2992)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2993) /* Is the page fully outside i_size? (truncate in progress) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2994) offset = i_size & (PAGE_SIZE-1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2995) if (page->index >= end_index+1 || !offset) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2996) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2997) return 0; /* don't care */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2998) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2999)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3000) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3001) * The page straddles i_size. It must be zeroed out on each and every
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3002) * writepage invocation because it may be mmapped. "A file is mapped
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3003) * in multiples of the page size. For a file that is not a multiple of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3004) * the page size, the remaining memory is zeroed when mapped, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3005) * writes to that region are not written out to the file."
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3006) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3007) zero_user_segment(page, offset, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3008) return __block_write_full_page(inode, page, get_block, wbc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3009) end_buffer_async_write);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3010) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3011) EXPORT_SYMBOL(block_write_full_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3012)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3013) sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3014) get_block_t *get_block)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3015) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3016) struct inode *inode = mapping->host;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3017) struct buffer_head tmp = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3018) .b_size = i_blocksize(inode),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3019) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3020)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3021) get_block(inode, block, &tmp, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3022) return tmp.b_blocknr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3023) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3024) EXPORT_SYMBOL(generic_block_bmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3025)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3026) static void end_bio_bh_io_sync(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3027) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3028) struct buffer_head *bh = bio->bi_private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3029)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3030) if (unlikely(bio_flagged(bio, BIO_QUIET)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3031) set_bit(BH_Quiet, &bh->b_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3032)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3033) bh->b_end_io(bh, !bio->bi_status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3034) bio_put(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3035) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3036)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3037) static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3038) enum rw_hint write_hint, struct writeback_control *wbc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3039) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3040) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3041)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3042) BUG_ON(!buffer_locked(bh));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3043) BUG_ON(!buffer_mapped(bh));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3044) BUG_ON(!bh->b_end_io);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3045) BUG_ON(buffer_delay(bh));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3046) BUG_ON(buffer_unwritten(bh));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3047)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3048) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3049) * Only clear out a write error when rewriting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3050) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3051) if (test_set_buffer_req(bh) && (op == REQ_OP_WRITE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3052) clear_buffer_write_io_error(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3053)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3054) bio = bio_alloc(GFP_NOIO, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3055)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3056) fscrypt_set_bio_crypt_ctx_bh(bio, bh, GFP_NOIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3057)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3058) bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3059) bio_set_dev(bio, bh->b_bdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3060) bio->bi_write_hint = write_hint;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3061)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3062) bio_add_page(bio, bh->b_page, bh->b_size, bh_offset(bh));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3063) BUG_ON(bio->bi_iter.bi_size != bh->b_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3064)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3065) bio->bi_end_io = end_bio_bh_io_sync;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3066) bio->bi_private = bh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3067)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3068) if (buffer_meta(bh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3069) op_flags |= REQ_META;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3070) if (buffer_prio(bh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3071) op_flags |= REQ_PRIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3072) bio_set_op_attrs(bio, op, op_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3073)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3074) /* Take care of bh's that straddle the end of the device */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3075) guard_bio_eod(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3076)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3077) if (wbc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3078) wbc_init_bio(wbc, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3079) wbc_account_cgroup_owner(wbc, bh->b_page, bh->b_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3080) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3081)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3082) submit_bio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3083) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3084) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3085)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3086) int submit_bh(int op, int op_flags, struct buffer_head *bh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3087) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3088) return submit_bh_wbc(op, op_flags, bh, 0, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3089) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3090) EXPORT_SYMBOL(submit_bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3091)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3092) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3093) * ll_rw_block: low-level access to block devices (DEPRECATED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3094) * @op: whether to %READ or %WRITE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3095) * @op_flags: req_flag_bits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3096) * @nr: number of &struct buffer_heads in the array
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3097) * @bhs: array of pointers to &struct buffer_head
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3098) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3099) * ll_rw_block() takes an array of pointers to &struct buffer_heads, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3100) * requests an I/O operation on them, either a %REQ_OP_READ or a %REQ_OP_WRITE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3101) * @op_flags contains flags modifying the detailed I/O behavior, most notably
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3102) * %REQ_RAHEAD.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3103) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3104) * This function drops any buffer that it cannot get a lock on (with the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3105) * BH_Lock state bit), any buffer that appears to be clean when doing a write
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3106) * request, and any buffer that appears to be up-to-date when doing read
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3107) * request. Further it marks as clean buffers that are processed for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3108) * writing (the buffer cache won't assume that they are actually clean
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3109) * until the buffer gets unlocked).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3110) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3111) * ll_rw_block sets b_end_io to simple completion handler that marks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3112) * the buffer up-to-date (if appropriate), unlocks the buffer and wakes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3113) * any waiters.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3114) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3115) * All of the buffers must be for the same device, and must also be a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3116) * multiple of the current approved size for the device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3117) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3118) void ll_rw_block(int op, int op_flags, int nr, struct buffer_head *bhs[])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3119) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3120) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3122) for (i = 0; i < nr; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3123) struct buffer_head *bh = bhs[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3125) if (!trylock_buffer(bh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3126) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3127) if (op == WRITE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3128) if (test_clear_buffer_dirty(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3129) bh->b_end_io = end_buffer_write_sync;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3130) get_bh(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3131) submit_bh(op, op_flags, bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3132) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3133) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3134) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3135) if (!buffer_uptodate(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3136) bh->b_end_io = end_buffer_read_sync;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3137) get_bh(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3138) submit_bh(op, op_flags, bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3139) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3140) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3141) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3142) unlock_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3143) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3144) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3145) EXPORT_SYMBOL_NS(ll_rw_block, ANDROID_GKI_VFS_EXPORT_ONLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3147) void write_dirty_buffer(struct buffer_head *bh, int op_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3148) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3149) lock_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3150) if (!test_clear_buffer_dirty(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3151) unlock_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3152) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3153) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3154) bh->b_end_io = end_buffer_write_sync;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3155) get_bh(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3156) submit_bh(REQ_OP_WRITE, op_flags, bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3157) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3158) EXPORT_SYMBOL(write_dirty_buffer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3160) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3161) * For a data-integrity writeout, we need to wait upon any in-progress I/O
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3162) * and then start new I/O and then wait upon it. The caller must have a ref on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3163) * the buffer_head.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3164) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3165) int __sync_dirty_buffer(struct buffer_head *bh, int op_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3166) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3167) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3169) WARN_ON(atomic_read(&bh->b_count) < 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3170) lock_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3171) if (test_clear_buffer_dirty(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3172) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3173) * The bh should be mapped, but it might not be if the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3174) * device was hot-removed. Not much we can do but fail the I/O.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3175) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3176) if (!buffer_mapped(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3177) unlock_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3178) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3179) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3181) get_bh(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3182) bh->b_end_io = end_buffer_write_sync;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3183) ret = submit_bh(REQ_OP_WRITE, op_flags, bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3184) wait_on_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3185) if (!ret && !buffer_uptodate(bh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3186) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3187) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3188) unlock_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3189) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3190) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3191) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3192) EXPORT_SYMBOL(__sync_dirty_buffer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3194) int sync_dirty_buffer(struct buffer_head *bh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3195) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3196) return __sync_dirty_buffer(bh, REQ_SYNC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3197) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3198) EXPORT_SYMBOL_NS(sync_dirty_buffer, ANDROID_GKI_VFS_EXPORT_ONLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3200) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3201) * try_to_free_buffers() checks if all the buffers on this particular page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3202) * are unused, and releases them if so.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3203) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3204) * Exclusion against try_to_free_buffers may be obtained by either
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3205) * locking the page or by holding its mapping's private_lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3206) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3207) * If the page is dirty but all the buffers are clean then we need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3208) * be sure to mark the page clean as well. This is because the page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3209) * may be against a block device, and a later reattachment of buffers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3210) * to a dirty page will set *all* buffers dirty. Which would corrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3211) * filesystem data on the same device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3212) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3213) * The same applies to regular filesystem pages: if all the buffers are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3214) * clean then we set the page clean and proceed. To do that, we require
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3215) * total exclusion from __set_page_dirty_buffers(). That is obtained with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3216) * private_lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3217) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3218) * try_to_free_buffers() is non-blocking.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3219) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3220) static inline int buffer_busy(struct buffer_head *bh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3221) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3222) return atomic_read(&bh->b_count) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3223) (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3224) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3226) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3227) drop_buffers(struct page *page, struct buffer_head **buffers_to_free)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3228) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3229) struct buffer_head *head = page_buffers(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3230) struct buffer_head *bh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3232) bh = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3233) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3234) if (buffer_busy(bh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3235) goto failed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3236) bh = bh->b_this_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3237) } while (bh != head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3239) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3240) struct buffer_head *next = bh->b_this_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3242) if (bh->b_assoc_map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3243) __remove_assoc_queue(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3244) bh = next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3245) } while (bh != head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3246) *buffers_to_free = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3247) detach_page_private(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3248) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3249) failed:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3250) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3251) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3253) int try_to_free_buffers(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3254) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3255) struct address_space * const mapping = page->mapping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3256) struct buffer_head *buffers_to_free = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3257) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3259) BUG_ON(!PageLocked(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3260) if (PageWriteback(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3261) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3263) if (mapping == NULL) { /* can this still happen? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3264) ret = drop_buffers(page, &buffers_to_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3265) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3266) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3268) spin_lock(&mapping->private_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3269) ret = drop_buffers(page, &buffers_to_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3271) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3272) * If the filesystem writes its buffers by hand (eg ext3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3273) * then we can have clean buffers against a dirty page. We
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3274) * clean the page here; otherwise the VM will never notice
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3275) * that the filesystem did any IO at all.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3276) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3277) * Also, during truncate, discard_buffer will have marked all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3278) * the page's buffers clean. We discover that here and clean
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3279) * the page also.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3280) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3281) * private_lock must be held over this entire operation in order
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3282) * to synchronise against __set_page_dirty_buffers and prevent the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3283) * dirty bit from being lost.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3284) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3285) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3286) cancel_dirty_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3287) spin_unlock(&mapping->private_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3288) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3289) if (buffers_to_free) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3290) struct buffer_head *bh = buffers_to_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3292) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3293) struct buffer_head *next = bh->b_this_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3294) free_buffer_head(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3295) bh = next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3296) } while (bh != buffers_to_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3297) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3298) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3299) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3300) EXPORT_SYMBOL(try_to_free_buffers);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3302) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3303) * There are no bdflush tunables left. But distributions are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3304) * still running obsolete flush daemons, so we terminate them here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3305) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3306) * Use of bdflush() is deprecated and will be removed in a future kernel.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3307) * The `flush-X' kernel threads fully replace bdflush daemons and this call.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3308) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3309) SYSCALL_DEFINE2(bdflush, int, func, long, data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3310) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3311) static int msg_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3312)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3313) if (!capable(CAP_SYS_ADMIN))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3314) return -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3316) if (msg_count < 5) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3317) msg_count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3318) printk(KERN_INFO
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3319) "warning: process `%s' used the obsolete bdflush"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3320) " system call\n", current->comm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3321) printk(KERN_INFO "Fix your initscripts?\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3322) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3324) if (func == 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3325) do_exit(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3326) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3327) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3329) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3330) * Buffer-head allocation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3331) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3332) static struct kmem_cache *bh_cachep __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3334) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3335) * Once the number of bh's in the machine exceeds this level, we start
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3336) * stripping them in writeback.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3337) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3338) static unsigned long max_buffer_heads;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3340) int buffer_heads_over_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3342) struct bh_accounting {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3343) int nr; /* Number of live bh's */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3344) int ratelimit; /* Limit cacheline bouncing */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3345) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3347) static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3349) static void recalc_bh_state(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3350) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3351) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3352) int tot = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3354) if (__this_cpu_inc_return(bh_accounting.ratelimit) - 1 < 4096)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3355) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3356) __this_cpu_write(bh_accounting.ratelimit, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3357) for_each_online_cpu(i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3358) tot += per_cpu(bh_accounting, i).nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3359) buffer_heads_over_limit = (tot > max_buffer_heads);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3360) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3362) struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3363) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3364) struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3365) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3366) INIT_LIST_HEAD(&ret->b_assoc_buffers);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3367) spin_lock_init(&ret->b_uptodate_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3368) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3369) __this_cpu_inc(bh_accounting.nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3370) recalc_bh_state();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3371) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3372) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3373) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3374) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3375) EXPORT_SYMBOL(alloc_buffer_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3377) void free_buffer_head(struct buffer_head *bh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3378) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3379) BUG_ON(!list_empty(&bh->b_assoc_buffers));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3380) kmem_cache_free(bh_cachep, bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3381) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3382) __this_cpu_dec(bh_accounting.nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3383) recalc_bh_state();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3384) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3385) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3386) EXPORT_SYMBOL(free_buffer_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3388) static int buffer_exit_cpu_dead(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3389) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3390) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3391) struct bh_lru *b = &per_cpu(bh_lrus, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3393) for (i = 0; i < BH_LRU_SIZE; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3394) brelse(b->bhs[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3395) b->bhs[i] = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3396) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3397) this_cpu_add(bh_accounting.nr, per_cpu(bh_accounting, cpu).nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3398) per_cpu(bh_accounting, cpu).nr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3399) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3400) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3402) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3403) * bh_uptodate_or_lock - Test whether the buffer is uptodate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3404) * @bh: struct buffer_head
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3405) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3406) * Return true if the buffer is up-to-date and false,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3407) * with the buffer locked, if not.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3408) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3409) int bh_uptodate_or_lock(struct buffer_head *bh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3410) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3411) if (!buffer_uptodate(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3412) lock_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3413) if (!buffer_uptodate(bh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3414) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3415) unlock_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3416) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3417) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3418) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3419) EXPORT_SYMBOL(bh_uptodate_or_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3421) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3422) * bh_submit_read - Submit a locked buffer for reading
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3423) * @bh: struct buffer_head
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3424) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3425) * Returns zero on success and -EIO on error.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3426) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3427) int bh_submit_read(struct buffer_head *bh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3428) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3429) BUG_ON(!buffer_locked(bh));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3431) if (buffer_uptodate(bh)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3432) unlock_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3433) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3434) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3436) get_bh(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3437) bh->b_end_io = end_buffer_read_sync;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3438) submit_bh(REQ_OP_READ, 0, bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3439) wait_on_buffer(bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3440) if (buffer_uptodate(bh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3441) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3442) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3443) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3444) EXPORT_SYMBOL(bh_submit_read);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3446) void __init buffer_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3447) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3448) unsigned long nrpages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3449) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3451) bh_cachep = kmem_cache_create("buffer_head",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3452) sizeof(struct buffer_head), 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3453) (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3454) SLAB_MEM_SPREAD),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3455) NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3457) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3458) * Limit the bh occupancy to 10% of ZONE_NORMAL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3459) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3460) nrpages = (nr_free_buffer_pages() * 10) / 100;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3461) max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3462) ret = cpuhp_setup_state_nocalls(CPUHP_FS_BUFF_DEAD, "fs/buffer:dead",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3463) NULL, buffer_exit_cpu_dead);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3464) WARN_ON(ret < 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3465) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags
| Donate