b24413180f560 (Greg Kroah-Hartman 2017-11-01 15:07:57 +0100 1) // SPDX-License-Identifier: GPL-2.0
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 2) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 3) * fs/mpage.c
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 4) *
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 5) * Copyright (C) 2002, Linus Torvalds.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 6) *
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 7) * Contains functions related to preparing and submitting BIOs which contain
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 8) * multiple pagecache pages.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 9) *
e1f8e87449147 (Francois Cami 2008-10-15 22:01:59 -0700 10) * 15May2002 Andrew Morton
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 11) * Initial version
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 12) * 27Jun2002 axboe@suse.de
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 13) * use bio_add_page() to build bio's just the right size
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 14) */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 15)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 16) #include <linux/kernel.h>
630d9c47274aa (Paul Gortmaker 2011-11-16 23:57:37 -0500 17) #include <linux/export.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 18) #include <linux/mm.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 19) #include <linux/kdev_t.h>
5a0e3ad6af866 (Tejun Heo 2010-03-24 17:04:11 +0900 20) #include <linux/gfp.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 21) #include <linux/bio.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 22) #include <linux/fs.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 23) #include <linux/buffer_head.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 24) #include <linux/blkdev.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 25) #include <linux/highmem.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 26) #include <linux/prefetch.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 27) #include <linux/mpage.h>
02c43638ec46f (Andrew Morton 2016-03-15 14:55:15 -0700 28) #include <linux/mm_inline.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 29) #include <linux/writeback.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 30) #include <linux/backing-dev.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 31) #include <linux/pagevec.h>
c515e1fd361c2 (Dan Magenheimer 2011-05-26 10:01:43 -0600 32) #include <linux/cleancache.h>
4db96b71e3cae (Akinobu Mita 2014-10-09 15:26:55 -0700 33) #include "internal.h"
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 34)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 35) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 36) * I/O completion handler for multipage BIOs.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 37) *
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 38) * The mpage code never puts partial pages into a BIO (except for end-of-file).
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 39) * If a page does not map to a contiguous run of blocks then it simply falls
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 40) * back to block_read_full_page().
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 41) *
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 42) * Why is this? If a page's completion depends on a number of different BIOs
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 43) * which can complete in any order (or at the same time) then determining the
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 44) * status of that page is hard. See end_buffer_async_read() for the details.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 45) * There is no point in duplicating all that complexity.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 46) */
4246a0b63bd8f (Christoph Hellwig 2015-07-20 15:29:37 +0200 47) static void mpage_end_io(struct bio *bio)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 48) {
2c30c71bd653a (Kent Overstreet 2013-11-07 12:20:26 -0800 49) struct bio_vec *bv;
6dc4f100c175d (Ming Lei 2019-02-15 19:13:19 +0800 50) struct bvec_iter_all iter_all;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 51)
2b070cfe582b8 (Christoph Hellwig 2019-04-25 09:03:00 +0200 52) bio_for_each_segment_all(bv, bio, iter_all) {
2c30c71bd653a (Kent Overstreet 2013-11-07 12:20:26 -0800 53) struct page *page = bv->bv_page;
3f289dcb4b265 (Tejun Heo 2018-07-18 04:47:36 -0700 54) page_endio(page, bio_op(bio),
3f289dcb4b265 (Tejun Heo 2018-07-18 04:47:36 -0700 55) blk_status_to_errno(bio->bi_status));
2c30c71bd653a (Kent Overstreet 2013-11-07 12:20:26 -0800 56) }
2c30c71bd653a (Kent Overstreet 2013-11-07 12:20:26 -0800 57)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 58) bio_put(bio);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 59) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 60)
eed25cd5bbaf7 (Mike Christie 2016-06-05 14:31:59 -0500 61) static struct bio *mpage_bio_submit(int op, int op_flags, struct bio *bio)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 62) {
c32b0d4b3f19c (Hai Shan 2011-01-13 15:45:51 -0800 63) bio->bi_end_io = mpage_end_io;
eed25cd5bbaf7 (Mike Christie 2016-06-05 14:31:59 -0500 64) bio_set_op_attrs(bio, op, op_flags);
83c9c547168e8 (Ming Lei 2020-01-05 09:41:14 +0800 65) guard_bio_eod(bio);
4e49ea4a3d276 (Mike Christie 2016-06-05 14:31:41 -0500 66) submit_bio(bio);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 67) return NULL;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 68) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 69)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 70) static struct bio *
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 71) mpage_alloc(struct block_device *bdev,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 72) sector_t first_sector, int nr_vecs,
dd0fc66fb33cd (Al Viro 2005-10-07 07:46:04 +0100 73) gfp_t gfp_flags)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 74) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 75) struct bio *bio;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 76)
8a5c743e308dd (Michal Hocko 2016-07-26 15:24:53 -0700 77) /* Restrict the given (page cache) mask for slab allocations */
8a5c743e308dd (Michal Hocko 2016-07-26 15:24:53 -0700 78) gfp_flags &= GFP_KERNEL;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 79) bio = bio_alloc(gfp_flags, nr_vecs);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 80)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 81) if (bio == NULL && (current->flags & PF_MEMALLOC)) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 82) while (!bio && (nr_vecs /= 2))
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 83) bio = bio_alloc(gfp_flags, nr_vecs);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 84) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 85)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 86) if (bio) {
74d46992e0d9d (Christoph Hellwig 2017-08-23 19:10:32 +0200 87) bio_set_dev(bio, bdev);
4f024f3797c43 (Kent Overstreet 2013-10-11 15:44:27 -0700 88) bio->bi_iter.bi_sector = first_sector;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 89) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 90) return bio;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 91) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 92)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 93) /*
d4388340ae0bc (Matthew Wilcox (Oracle) 2020-06-01 21:47:02 -0700 94) * support function for mpage_readahead. The fs supplied get_block might
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 95) * return an up to date buffer. This is used to map that buffer into
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 96) * the page, which allows readpage to avoid triggering a duplicate call
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 97) * to get_block.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 98) *
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 99) * The idea is to avoid adding buffers to pages that don't already have
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 100) * them. So when the buffer is up to date and the page size == block size,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 101) * this marks the page up to date instead of adding new buffers.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 102) */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 103) static void
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 104) map_buffer_to_page(struct page *page, struct buffer_head *bh, int page_block)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 105) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 106) struct inode *inode = page->mapping->host;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 107) struct buffer_head *page_bh, *head;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 108) int block = 0;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 109)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 110) if (!page_has_buffers(page)) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 111) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 112) * don't make any buffers if there is only one buffer on
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 113) * the page and the page just needs to be set up to date
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 114) */
09cbfeaf1a5a6 (Kirill A. Shutemov 2016-04-01 15:29:47 +0300 115) if (inode->i_blkbits == PAGE_SHIFT &&
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 116) buffer_uptodate(bh)) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 117) SetPageUptodate(page);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 118) return;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 119) }
93407472a21b8 (Fabian Frederick 2017-02-27 14:28:32 -0800 120) create_empty_buffers(page, i_blocksize(inode), 0);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 121) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 122) head = page_buffers(page);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 123) page_bh = head;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 124) do {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 125) if (block == page_block) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 126) page_bh->b_state = bh->b_state;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 127) page_bh->b_bdev = bh->b_bdev;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 128) page_bh->b_blocknr = bh->b_blocknr;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 129) break;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 130) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 131) page_bh = page_bh->b_this_page;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 132) block++;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 133) } while (page_bh != head);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 134) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 135)
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 136) struct mpage_readpage_args {
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 137) struct bio *bio;
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 138) struct page *page;
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 139) unsigned int nr_pages;
74c8164e1cdb1 (Jens Axboe 2018-08-17 15:45:36 -0700 140) bool is_readahead;
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 141) sector_t last_block_in_bio;
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 142) struct buffer_head map_bh;
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 143) unsigned long first_logical_block;
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 144) get_block_t *get_block;
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 145) };
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 146)
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 147) /*
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 148) * This is the worker routine which does all the work of mapping the disk
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 149) * blocks and constructs largest possible bios, submits them for IO if the
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 150) * blocks are not contiguous on the disk.
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 151) *
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 152) * We pass a buffer_head back and forth and use its buffer_mapped() flag to
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 153) * represent the validity of its disk mapping and to decide when to do the next
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 154) * get_block() call.
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 155) */
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 156) static struct bio *do_mpage_readpage(struct mpage_readpage_args *args)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 157) {
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 158) struct page *page = args->page;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 159) struct inode *inode = page->mapping->host;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 160) const unsigned blkbits = inode->i_blkbits;
09cbfeaf1a5a6 (Kirill A. Shutemov 2016-04-01 15:29:47 +0300 161) const unsigned blocks_per_page = PAGE_SIZE >> blkbits;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 162) const unsigned blocksize = 1 << blkbits;
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 163) struct buffer_head *map_bh = &args->map_bh;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 164) sector_t block_in_file;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 165) sector_t last_block;
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 166) sector_t last_block_in_file;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 167) sector_t blocks[MAX_BUF_PER_PAGE];
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 168) unsigned page_block;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 169) unsigned first_hole = blocks_per_page;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 170) struct block_device *bdev = NULL;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 171) int length;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 172) int fully_mapped = 1;
74c8164e1cdb1 (Jens Axboe 2018-08-17 15:45:36 -0700 173) int op_flags;
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 174) unsigned nblocks;
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 175) unsigned relative_block;
74c8164e1cdb1 (Jens Axboe 2018-08-17 15:45:36 -0700 176) gfp_t gfp;
74c8164e1cdb1 (Jens Axboe 2018-08-17 15:45:36 -0700 177)
74c8164e1cdb1 (Jens Axboe 2018-08-17 15:45:36 -0700 178) if (args->is_readahead) {
74c8164e1cdb1 (Jens Axboe 2018-08-17 15:45:36 -0700 179) op_flags = REQ_RAHEAD;
74c8164e1cdb1 (Jens Axboe 2018-08-17 15:45:36 -0700 180) gfp = readahead_gfp_mask(page->mapping);
74c8164e1cdb1 (Jens Axboe 2018-08-17 15:45:36 -0700 181) } else {
74c8164e1cdb1 (Jens Axboe 2018-08-17 15:45:36 -0700 182) op_flags = 0;
74c8164e1cdb1 (Jens Axboe 2018-08-17 15:45:36 -0700 183) gfp = mapping_gfp_constraint(page->mapping, GFP_KERNEL);
74c8164e1cdb1 (Jens Axboe 2018-08-17 15:45:36 -0700 184) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 185)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 186) if (page_has_buffers(page))
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 187) goto confused;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 188)
09cbfeaf1a5a6 (Kirill A. Shutemov 2016-04-01 15:29:47 +0300 189) block_in_file = (sector_t)page->index << (PAGE_SHIFT - blkbits);
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 190) last_block = block_in_file + args->nr_pages * blocks_per_page;
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 191) last_block_in_file = (i_size_read(inode) + blocksize - 1) >> blkbits;
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 192) if (last_block > last_block_in_file)
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 193) last_block = last_block_in_file;
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 194) page_block = 0;
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 195)
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 196) /*
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 197) * Map blocks using the result from the previous get_blocks call first.
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 198) */
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 199) nblocks = map_bh->b_size >> blkbits;
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 200) if (buffer_mapped(map_bh) &&
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 201) block_in_file > args->first_logical_block &&
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 202) block_in_file < (args->first_logical_block + nblocks)) {
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 203) unsigned map_offset = block_in_file - args->first_logical_block;
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 204) unsigned last = nblocks - map_offset;
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 205)
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 206) for (relative_block = 0; ; relative_block++) {
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 207) if (relative_block == last) {
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 208) clear_buffer_mapped(map_bh);
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 209) break;
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 210) }
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 211) if (page_block == blocks_per_page)
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 212) break;
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 213) blocks[page_block] = map_bh->b_blocknr + map_offset +
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 214) relative_block;
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 215) page_block++;
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 216) block_in_file++;
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 217) }
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 218) bdev = map_bh->b_bdev;
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 219) }
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 220)
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 221) /*
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 222) * Then do more get_blocks calls until we are done with this page.
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 223) */
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 224) map_bh->b_page = page;
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 225) while (page_block < blocks_per_page) {
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 226) map_bh->b_state = 0;
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 227) map_bh->b_size = 0;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 228)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 229) if (block_in_file < last_block) {
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 230) map_bh->b_size = (last_block-block_in_file) << blkbits;
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 231) if (args->get_block(inode, block_in_file, map_bh, 0))
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 232) goto confused;
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 233) args->first_logical_block = block_in_file;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 234) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 235)
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 236) if (!buffer_mapped(map_bh)) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 237) fully_mapped = 0;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 238) if (first_hole == blocks_per_page)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 239) first_hole = page_block;
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 240) page_block++;
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 241) block_in_file++;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 242) continue;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 243) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 244)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 245) /* some filesystems will copy data into the page during
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 246) * the get_block call, in which case we don't want to
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 247) * read it again. map_buffer_to_page copies the data
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 248) * we just collected from get_block into the page's buffers
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 249) * so readpage doesn't have to repeat the get_block call
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 250) */
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 251) if (buffer_uptodate(map_bh)) {
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 252) map_buffer_to_page(page, map_bh, page_block);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 253) goto confused;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 254) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 255)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 256) if (first_hole != blocks_per_page)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 257) goto confused; /* hole -> non-hole */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 258)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 259) /* Contiguous blocks? */
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 260) if (page_block && blocks[page_block-1] != map_bh->b_blocknr-1)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 261) goto confused;
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 262) nblocks = map_bh->b_size >> blkbits;
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 263) for (relative_block = 0; ; relative_block++) {
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 264) if (relative_block == nblocks) {
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 265) clear_buffer_mapped(map_bh);
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 266) break;
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 267) } else if (page_block == blocks_per_page)
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 268) break;
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 269) blocks[page_block] = map_bh->b_blocknr+relative_block;
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 270) page_block++;
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 271) block_in_file++;
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 272) }
fa30bd058b746 (Badari Pulavarty 2006-03-26 01:38:01 -0800 273) bdev = map_bh->b_bdev;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 274) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 275)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 276) if (first_hole != blocks_per_page) {
09cbfeaf1a5a6 (Kirill A. Shutemov 2016-04-01 15:29:47 +0300 277) zero_user_segment(page, first_hole << blkbits, PAGE_SIZE);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 278) if (first_hole == 0) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 279) SetPageUptodate(page);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 280) unlock_page(page);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 281) goto out;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 282) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 283) } else if (fully_mapped) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 284) SetPageMappedToDisk(page);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 285) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 286)
c515e1fd361c2 (Dan Magenheimer 2011-05-26 10:01:43 -0600 287) if (fully_mapped && blocks_per_page == 1 && !PageUptodate(page) &&
c515e1fd361c2 (Dan Magenheimer 2011-05-26 10:01:43 -0600 288) cleancache_get_page(page) == 0) {
c515e1fd361c2 (Dan Magenheimer 2011-05-26 10:01:43 -0600 289) SetPageUptodate(page);
c515e1fd361c2 (Dan Magenheimer 2011-05-26 10:01:43 -0600 290) goto confused;
c515e1fd361c2 (Dan Magenheimer 2011-05-26 10:01:43 -0600 291) }
c515e1fd361c2 (Dan Magenheimer 2011-05-26 10:01:43 -0600 292)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 293) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 294) * This page will go to BIO. Do we need to send this BIO off first?
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 295) */
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 296) if (args->bio && (args->last_block_in_bio != blocks[0] - 1))
74c8164e1cdb1 (Jens Axboe 2018-08-17 15:45:36 -0700 297) args->bio = mpage_bio_submit(REQ_OP_READ, op_flags, args->bio);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 298)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 299) alloc_new:
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 300) if (args->bio == NULL) {
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 301) if (first_hole == blocks_per_page) {
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 302) if (!bdev_read_page(bdev, blocks[0] << (blkbits - 9),
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 303) page))
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 304) goto out;
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 305) }
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 306) args->bio = mpage_alloc(bdev, blocks[0] << (blkbits - 9),
5f7136db82996 (Matthew Wilcox (Oracle) 2021-01-29 04:38:57 +0000 307) bio_max_segs(args->nr_pages), gfp);
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 308) if (args->bio == NULL)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 309) goto confused;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 310) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 311)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 312) length = first_hole << blkbits;
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 313) if (bio_add_page(args->bio, page, length, 0) < length) {
74c8164e1cdb1 (Jens Axboe 2018-08-17 15:45:36 -0700 314) args->bio = mpage_bio_submit(REQ_OP_READ, op_flags, args->bio);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 315) goto alloc_new;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 316) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 317)
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 318) relative_block = block_in_file - args->first_logical_block;
38c8e6180939e (Miquel van Smoorenburg 2009-01-06 14:39:02 -0800 319) nblocks = map_bh->b_size >> blkbits;
38c8e6180939e (Miquel van Smoorenburg 2009-01-06 14:39:02 -0800 320) if ((buffer_boundary(map_bh) && relative_block == nblocks) ||
38c8e6180939e (Miquel van Smoorenburg 2009-01-06 14:39:02 -0800 321) (first_hole != blocks_per_page))
74c8164e1cdb1 (Jens Axboe 2018-08-17 15:45:36 -0700 322) args->bio = mpage_bio_submit(REQ_OP_READ, op_flags, args->bio);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 323) else
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 324) args->last_block_in_bio = blocks[blocks_per_page - 1];
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 325) out:
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 326) return args->bio;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 327)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 328) confused:
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 329) if (args->bio)
74c8164e1cdb1 (Jens Axboe 2018-08-17 15:45:36 -0700 330) args->bio = mpage_bio_submit(REQ_OP_READ, op_flags, args->bio);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 331) if (!PageUptodate(page))
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 332) block_read_full_page(page, args->get_block);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 333) else
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 334) unlock_page(page);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 335) goto out;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 336) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 337)
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 338) /**
d4388340ae0bc (Matthew Wilcox (Oracle) 2020-06-01 21:47:02 -0700 339) * mpage_readahead - start reads against pages
d4388340ae0bc (Matthew Wilcox (Oracle) 2020-06-01 21:47:02 -0700 340) * @rac: Describes which pages to read.
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 341) * @get_block: The filesystem's block mapper function.
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 342) *
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 343) * This function walks the pages and the blocks within each page, building and
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 344) * emitting large BIOs.
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 345) *
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 346) * If anything unusual happens, such as:
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 347) *
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 348) * - encountering a page which has buffers
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 349) * - encountering a page which has a non-hole after a hole
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 350) * - encountering a page with non-contiguous blocks
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 351) *
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 352) * then this code just gives up and calls the buffer_head-based read function.
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 353) * It does handle a page which has holes at the end - that is a common case:
ea1754a084760 (Kirill A. Shutemov 2016-04-01 15:29:48 +0300 354) * the end-of-file on blocksize < PAGE_SIZE setups.
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 355) *
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 356) * BH_Boundary explanation:
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 357) *
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 358) * There is a problem. The mpage read code assembles several pages, gets all
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 359) * their disk mappings, and then submits them all. That's fine, but obtaining
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 360) * the disk mappings may require I/O. Reads of indirect blocks, for example.
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 361) *
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 362) * So an mpage read of the first 16 blocks of an ext2 file will cause I/O to be
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 363) * submitted in the following order:
0117d4272b1ac (Mauro Carvalho Chehab 2017-05-12 07:45:42 -0300 364) *
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 365) * 12 0 1 2 3 4 5 6 7 8 9 10 11 13 14 15 16
78a4a50a86b0a (Randy Dunlap 2008-02-29 22:02:31 -0800 366) *
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 367) * because the indirect block has to be read to get the mappings of blocks
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 368) * 13,14,15,16. Obviously, this impacts performance.
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 369) *
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 370) * So what we do it to allow the filesystem's get_block() function to set
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 371) * BH_Boundary when it maps block 11. BH_Boundary says: mapping of the block
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 372) * after this one will require I/O against a block which is probably close to
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 373) * this one. So you should push what I/O you have currently accumulated.
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 374) *
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 375) * This all causes the disk requests to be issued in the correct order.
67be2dd1bace0 (Martin Waitz 2005-05-01 08:59:26 -0700 376) */
d4388340ae0bc (Matthew Wilcox (Oracle) 2020-06-01 21:47:02 -0700 377) void mpage_readahead(struct readahead_control *rac, get_block_t get_block)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 378) {
d4388340ae0bc (Matthew Wilcox (Oracle) 2020-06-01 21:47:02 -0700 379) struct page *page;
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 380) struct mpage_readpage_args args = {
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 381) .get_block = get_block,
74c8164e1cdb1 (Jens Axboe 2018-08-17 15:45:36 -0700 382) .is_readahead = true,
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 383) };
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 384)
d4388340ae0bc (Matthew Wilcox (Oracle) 2020-06-01 21:47:02 -0700 385) while ((page = readahead_page(rac))) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 386) prefetchw(&page->flags);
d4388340ae0bc (Matthew Wilcox (Oracle) 2020-06-01 21:47:02 -0700 387) args.page = page;
d4388340ae0bc (Matthew Wilcox (Oracle) 2020-06-01 21:47:02 -0700 388) args.nr_pages = readahead_count(rac);
d4388340ae0bc (Matthew Wilcox (Oracle) 2020-06-01 21:47:02 -0700 389) args.bio = do_mpage_readpage(&args);
09cbfeaf1a5a6 (Kirill A. Shutemov 2016-04-01 15:29:47 +0300 390) put_page(page);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 391) }
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 392) if (args.bio)
74c8164e1cdb1 (Jens Axboe 2018-08-17 15:45:36 -0700 393) mpage_bio_submit(REQ_OP_READ, REQ_RAHEAD, args.bio);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 394) }
d4388340ae0bc (Matthew Wilcox (Oracle) 2020-06-01 21:47:02 -0700 395) EXPORT_SYMBOL(mpage_readahead);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 396)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 397) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 398) * This isn't called much at all
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 399) */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 400) int mpage_readpage(struct page *page, get_block_t get_block)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 401) {
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 402) struct mpage_readpage_args args = {
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 403) .page = page,
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 404) .nr_pages = 1,
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 405) .get_block = get_block,
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 406) };
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 407)
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 408) args.bio = do_mpage_readpage(&args);
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 409) if (args.bio)
357c1206520da (Jens Axboe 2018-08-17 15:45:32 -0700 410) mpage_bio_submit(REQ_OP_READ, 0, args.bio);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 411) return 0;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 412) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 413) EXPORT_SYMBOL(mpage_readpage);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 414)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 415) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 416) * Writing is not so simple.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 417) *
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 418) * If the page has buffers then they will be used for obtaining the disk
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 419) * mapping. We only support pages which are fully mapped-and-dirty, with a
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 420) * special case for pages which are unmapped at the end: end-of-file.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 421) *
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 422) * If the page has no buffers (preferred) then the page is mapped here.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 423) *
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 424) * If all blocks are found to be contiguous then the page can go into the
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 425) * BIO. Otherwise fall back to the mapping's writepage().
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 426) *
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 427) * FIXME: This code wants an estimate of how many pages are still to be
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 428) * written, so it can intelligently allocate a suitably-sized BIO. For now,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 429) * just allocate full-size (16-page) BIOs.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 430) */
0ea9718016251 (Miklos Szeredi 2007-05-10 22:22:51 -0700 431)
ced117c73edc9 (Dmitri Vorobiev 2009-03-31 00:41:20 +0300 432) struct mpage_data {
ced117c73edc9 (Dmitri Vorobiev 2009-03-31 00:41:20 +0300 433) struct bio *bio;
ced117c73edc9 (Dmitri Vorobiev 2009-03-31 00:41:20 +0300 434) sector_t last_block_in_bio;
ced117c73edc9 (Dmitri Vorobiev 2009-03-31 00:41:20 +0300 435) get_block_t *get_block;
ced117c73edc9 (Dmitri Vorobiev 2009-03-31 00:41:20 +0300 436) unsigned use_writepage;
ced117c73edc9 (Dmitri Vorobiev 2009-03-31 00:41:20 +0300 437) };
ced117c73edc9 (Dmitri Vorobiev 2009-03-31 00:41:20 +0300 438)
90768eee4565a (Matthew Wilcox 2014-06-04 16:07:44 -0700 439) /*
90768eee4565a (Matthew Wilcox 2014-06-04 16:07:44 -0700 440) * We have our BIO, so we can now mark the buffers clean. Make
90768eee4565a (Matthew Wilcox 2014-06-04 16:07:44 -0700 441) * sure to only clean buffers which we know we'll be writing.
90768eee4565a (Matthew Wilcox 2014-06-04 16:07:44 -0700 442) */
90768eee4565a (Matthew Wilcox 2014-06-04 16:07:44 -0700 443) static void clean_buffers(struct page *page, unsigned first_unmapped)
90768eee4565a (Matthew Wilcox 2014-06-04 16:07:44 -0700 444) {
90768eee4565a (Matthew Wilcox 2014-06-04 16:07:44 -0700 445) unsigned buffer_counter = 0;
90768eee4565a (Matthew Wilcox 2014-06-04 16:07:44 -0700 446) struct buffer_head *bh, *head;
90768eee4565a (Matthew Wilcox 2014-06-04 16:07:44 -0700 447) if (!page_has_buffers(page))
90768eee4565a (Matthew Wilcox 2014-06-04 16:07:44 -0700 448) return;
90768eee4565a (Matthew Wilcox 2014-06-04 16:07:44 -0700 449) head = page_buffers(page);
90768eee4565a (Matthew Wilcox 2014-06-04 16:07:44 -0700 450) bh = head;
90768eee4565a (Matthew Wilcox 2014-06-04 16:07:44 -0700 451)
90768eee4565a (Matthew Wilcox 2014-06-04 16:07:44 -0700 452) do {
90768eee4565a (Matthew Wilcox 2014-06-04 16:07:44 -0700 453) if (buffer_counter++ == first_unmapped)
90768eee4565a (Matthew Wilcox 2014-06-04 16:07:44 -0700 454) break;
90768eee4565a (Matthew Wilcox 2014-06-04 16:07:44 -0700 455) clear_buffer_dirty(bh);
90768eee4565a (Matthew Wilcox 2014-06-04 16:07:44 -0700 456) bh = bh->b_this_page;
90768eee4565a (Matthew Wilcox 2014-06-04 16:07:44 -0700 457) } while (bh != head);
90768eee4565a (Matthew Wilcox 2014-06-04 16:07:44 -0700 458)
90768eee4565a (Matthew Wilcox 2014-06-04 16:07:44 -0700 459) /*
90768eee4565a (Matthew Wilcox 2014-06-04 16:07:44 -0700 460) * we cannot drop the bh if the page is not uptodate or a concurrent
90768eee4565a (Matthew Wilcox 2014-06-04 16:07:44 -0700 461) * readpage would fail to serialize with the bh and it would read from
90768eee4565a (Matthew Wilcox 2014-06-04 16:07:44 -0700 462) * disk before we reach the platter.
90768eee4565a (Matthew Wilcox 2014-06-04 16:07:44 -0700 463) */
90768eee4565a (Matthew Wilcox 2014-06-04 16:07:44 -0700 464) if (buffer_heads_over_limit && PageUptodate(page))
90768eee4565a (Matthew Wilcox 2014-06-04 16:07:44 -0700 465) try_to_free_buffers(page);
90768eee4565a (Matthew Wilcox 2014-06-04 16:07:44 -0700 466) }
90768eee4565a (Matthew Wilcox 2014-06-04 16:07:44 -0700 467)
f892760aa66a2 (Matthew Wilcox 2017-10-13 15:58:15 -0700 468) /*
f892760aa66a2 (Matthew Wilcox 2017-10-13 15:58:15 -0700 469) * For situations where we want to clean all buffers attached to a page.
f892760aa66a2 (Matthew Wilcox 2017-10-13 15:58:15 -0700 470) * We don't need to calculate how many buffers are attached to the page,
f892760aa66a2 (Matthew Wilcox 2017-10-13 15:58:15 -0700 471) * we just need to specify a number larger than the maximum number of buffers.
f892760aa66a2 (Matthew Wilcox 2017-10-13 15:58:15 -0700 472) */
f892760aa66a2 (Matthew Wilcox 2017-10-13 15:58:15 -0700 473) void clean_page_buffers(struct page *page)
f892760aa66a2 (Matthew Wilcox 2017-10-13 15:58:15 -0700 474) {
f892760aa66a2 (Matthew Wilcox 2017-10-13 15:58:15 -0700 475) clean_buffers(page, ~0U);
f892760aa66a2 (Matthew Wilcox 2017-10-13 15:58:15 -0700 476) }
f892760aa66a2 (Matthew Wilcox 2017-10-13 15:58:15 -0700 477)
ced117c73edc9 (Dmitri Vorobiev 2009-03-31 00:41:20 +0300 478) static int __mpage_writepage(struct page *page, struct writeback_control *wbc,
29a814d2ee0e4 (Alex Tomas 2008-07-11 19:27:31 -0400 479) void *data)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 480) {
0ea9718016251 (Miklos Szeredi 2007-05-10 22:22:51 -0700 481) struct mpage_data *mpd = data;
0ea9718016251 (Miklos Szeredi 2007-05-10 22:22:51 -0700 482) struct bio *bio = mpd->bio;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 483) struct address_space *mapping = page->mapping;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 484) struct inode *inode = page->mapping->host;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 485) const unsigned blkbits = inode->i_blkbits;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 486) unsigned long end_index;
09cbfeaf1a5a6 (Kirill A. Shutemov 2016-04-01 15:29:47 +0300 487) const unsigned blocks_per_page = PAGE_SIZE >> blkbits;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 488) sector_t last_block;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 489) sector_t block_in_file;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 490) sector_t blocks[MAX_BUF_PER_PAGE];
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 491) unsigned page_block;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 492) unsigned first_unmapped = blocks_per_page;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 493) struct block_device *bdev = NULL;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 494) int boundary = 0;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 495) sector_t boundary_block = 0;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 496) struct block_device *boundary_bdev = NULL;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 497) int length;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 498) struct buffer_head map_bh;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 499) loff_t i_size = i_size_read(inode);
0ea9718016251 (Miklos Szeredi 2007-05-10 22:22:51 -0700 500) int ret = 0;
7637241e651ec (Jens Axboe 2016-11-01 10:00:38 -0600 501) int op_flags = wbc_to_write_flags(wbc);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 502)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 503) if (page_has_buffers(page)) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 504) struct buffer_head *head = page_buffers(page);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 505) struct buffer_head *bh = head;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 506)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 507) /* If they're all mapped and dirty, do it */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 508) page_block = 0;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 509) do {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 510) BUG_ON(buffer_locked(bh));
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 511) if (!buffer_mapped(bh)) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 512) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 513) * unmapped dirty buffers are created by
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 514) * __set_page_dirty_buffers -> mmapped data
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 515) */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 516) if (buffer_dirty(bh))
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 517) goto confused;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 518) if (first_unmapped == blocks_per_page)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 519) first_unmapped = page_block;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 520) continue;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 521) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 522)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 523) if (first_unmapped != blocks_per_page)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 524) goto confused; /* hole -> non-hole */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 525)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 526) if (!buffer_dirty(bh) || !buffer_uptodate(bh))
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 527) goto confused;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 528) if (page_block) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 529) if (bh->b_blocknr != blocks[page_block-1] + 1)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 530) goto confused;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 531) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 532) blocks[page_block++] = bh->b_blocknr;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 533) boundary = buffer_boundary(bh);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 534) if (boundary) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 535) boundary_block = bh->b_blocknr;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 536) boundary_bdev = bh->b_bdev;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 537) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 538) bdev = bh->b_bdev;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 539) } while ((bh = bh->b_this_page) != head);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 540)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 541) if (first_unmapped)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 542) goto page_is_mapped;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 543)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 544) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 545) * Page has buffers, but they are all unmapped. The page was
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 546) * created by pagein or read over a hole which was handled by
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 547) * block_read_full_page(). If this address_space is also
d4388340ae0bc (Matthew Wilcox (Oracle) 2020-06-01 21:47:02 -0700 548) * using mpage_readahead then this can rarely happen.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 549) */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 550) goto confused;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 551) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 552)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 553) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 554) * The page has no buffers: map it to disk
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 555) */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 556) BUG_ON(!PageUptodate(page));
09cbfeaf1a5a6 (Kirill A. Shutemov 2016-04-01 15:29:47 +0300 557) block_in_file = (sector_t)page->index << (PAGE_SHIFT - blkbits);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 558) last_block = (i_size - 1) >> blkbits;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 559) map_bh.b_page = page;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 560) for (page_block = 0; page_block < blocks_per_page; ) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 561)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 562) map_bh.b_state = 0;
b0cf2321c6599 (Badari Pulavarty 2006-03-26 01:38:00 -0800 563) map_bh.b_size = 1 << blkbits;
0ea9718016251 (Miklos Szeredi 2007-05-10 22:22:51 -0700 564) if (mpd->get_block(inode, block_in_file, &map_bh, 1))
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 565) goto confused;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 566) if (buffer_new(&map_bh))
e64855c6cfaa0 (Jan Kara 2016-11-04 18:08:15 +0100 567) clean_bdev_bh_alias(&map_bh);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 568) if (buffer_boundary(&map_bh)) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 569) boundary_block = map_bh.b_blocknr;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 570) boundary_bdev = map_bh.b_bdev;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 571) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 572) if (page_block) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 573) if (map_bh.b_blocknr != blocks[page_block-1] + 1)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 574) goto confused;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 575) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 576) blocks[page_block++] = map_bh.b_blocknr;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 577) boundary = buffer_boundary(&map_bh);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 578) bdev = map_bh.b_bdev;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 579) if (block_in_file == last_block)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 580) break;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 581) block_in_file++;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 582) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 583) BUG_ON(page_block == 0);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 584)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 585) first_unmapped = page_block;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 586)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 587) page_is_mapped:
09cbfeaf1a5a6 (Kirill A. Shutemov 2016-04-01 15:29:47 +0300 588) end_index = i_size >> PAGE_SHIFT;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 589) if (page->index >= end_index) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 590) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 591) * The page straddles i_size. It must be zeroed out on each
2a61aa4016385 (Adam Buchbinder 2009-12-11 16:35:40 -0500 592) * and every writepage invocation because it may be mmapped.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 593) * "A file is mapped in multiples of the page size. For a file
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 594) * that is not a multiple of the page size, the remaining memory
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 595) * is zeroed when mapped, and writes to that region are not
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 596) * written out to the file."
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 597) */
09cbfeaf1a5a6 (Kirill A. Shutemov 2016-04-01 15:29:47 +0300 598) unsigned offset = i_size & (PAGE_SIZE - 1);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 599)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 600) if (page->index > end_index || !offset)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 601) goto confused;
09cbfeaf1a5a6 (Kirill A. Shutemov 2016-04-01 15:29:47 +0300 602) zero_user_segment(page, offset, PAGE_SIZE);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 603) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 604)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 605) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 606) * This page will go to BIO. Do we need to send this BIO off first?
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 607) */
0ea9718016251 (Miklos Szeredi 2007-05-10 22:22:51 -0700 608) if (bio && mpd->last_block_in_bio != blocks[0] - 1)
eed25cd5bbaf7 (Mike Christie 2016-06-05 14:31:59 -0500 609) bio = mpage_bio_submit(REQ_OP_WRITE, op_flags, bio);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 610)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 611) alloc_new:
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 612) if (bio == NULL) {
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 613) if (first_unmapped == blocks_per_page) {
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 614) if (!bdev_write_page(bdev, blocks[0] << (blkbits - 9),
f892760aa66a2 (Matthew Wilcox 2017-10-13 15:58:15 -0700 615) page, wbc))
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 616) goto out;
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 617) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 618) bio = mpage_alloc(bdev, blocks[0] << (blkbits - 9),
a8affc03a9b37 (Christoph Hellwig 2021-03-11 12:01:37 +0100 619) BIO_MAX_VECS, GFP_NOFS|__GFP_HIGH);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 620) if (bio == NULL)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 621) goto confused;
429b3fb027492 (Tejun Heo 2015-05-22 17:14:04 -0400 622)
b16b1deb553ad (Tejun Heo 2015-06-02 08:39:48 -0600 623) wbc_init_bio(wbc, bio);
8e8f9298818c4 (Jens Axboe 2017-06-27 09:30:05 -0600 624) bio->bi_write_hint = inode->i_write_hint;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 625) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 626)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 627) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 628) * Must try to add the page before marking the buffer clean or
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 629) * the confused fail path above (OOM) will be very confused when
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 630) * it finds all bh marked clean (i.e. it will not write anything)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 631) */
34e51a5e1a6e9 (Tejun Heo 2019-06-27 13:39:49 -0700 632) wbc_account_cgroup_owner(wbc, page, PAGE_SIZE);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 633) length = first_unmapped << blkbits;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 634) if (bio_add_page(bio, page, length, 0) < length) {
eed25cd5bbaf7 (Mike Christie 2016-06-05 14:31:59 -0500 635) bio = mpage_bio_submit(REQ_OP_WRITE, op_flags, bio);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 636) goto alloc_new;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 637) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 638)
90768eee4565a (Matthew Wilcox 2014-06-04 16:07:44 -0700 639) clean_buffers(page, first_unmapped);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 640)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 641) BUG_ON(PageWriteback(page));
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 642) set_page_writeback(page);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 643) unlock_page(page);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 644) if (boundary || (first_unmapped != blocks_per_page)) {
eed25cd5bbaf7 (Mike Christie 2016-06-05 14:31:59 -0500 645) bio = mpage_bio_submit(REQ_OP_WRITE, op_flags, bio);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 646) if (boundary_block) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 647) write_boundary_block(boundary_bdev,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 648) boundary_block, 1 << blkbits);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 649) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 650) } else {
0ea9718016251 (Miklos Szeredi 2007-05-10 22:22:51 -0700 651) mpd->last_block_in_bio = blocks[blocks_per_page - 1];
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 652) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 653) goto out;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 654)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 655) confused:
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 656) if (bio)
eed25cd5bbaf7 (Mike Christie 2016-06-05 14:31:59 -0500 657) bio = mpage_bio_submit(REQ_OP_WRITE, op_flags, bio);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 658)
0ea9718016251 (Miklos Szeredi 2007-05-10 22:22:51 -0700 659) if (mpd->use_writepage) {
0ea9718016251 (Miklos Szeredi 2007-05-10 22:22:51 -0700 660) ret = mapping->a_ops->writepage(page, wbc);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 661) } else {
0ea9718016251 (Miklos Szeredi 2007-05-10 22:22:51 -0700 662) ret = -EAGAIN;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 663) goto out;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 664) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 665) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 666) * The caller has a ref on the inode, so *mapping is stable
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 667) */
0ea9718016251 (Miklos Szeredi 2007-05-10 22:22:51 -0700 668) mapping_set_error(mapping, ret);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 669) out:
0ea9718016251 (Miklos Szeredi 2007-05-10 22:22:51 -0700 670) mpd->bio = bio;
0ea9718016251 (Miklos Szeredi 2007-05-10 22:22:51 -0700 671) return ret;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 672) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 673)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 674) /**
78a4a50a86b0a (Randy Dunlap 2008-02-29 22:02:31 -0800 675) * mpage_writepages - walk the list of dirty pages of the given address space & writepage() all of them
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 676) * @mapping: address space structure to write
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 677) * @wbc: subtract the number of written pages from *@wbc->nr_to_write
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 678) * @get_block: the filesystem's block mapper function.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 679) * If this is NULL then use a_ops->writepage. Otherwise, go
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 680) * direct-to-BIO.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 681) *
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 682) * This is a library function, which implements the writepages()
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 683) * address_space_operation.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 684) *
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 685) * If a page is already under I/O, generic_writepages() skips it, even
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 686) * if it's dirty. This is desirable behaviour for memory-cleaning writeback,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 687) * but it is INCORRECT for data-integrity system calls such as fsync(). fsync()
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 688) * and msync() need to guarantee that all the data which was dirty at the time
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 689) * the call was made get new I/O started against them. If wbc->sync_mode is
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 690) * WB_SYNC_ALL then we were called for data integrity and we must wait for
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 691) * existing IO to complete.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 692) */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 693) int
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 694) mpage_writepages(struct address_space *mapping,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 695) struct writeback_control *wbc, get_block_t get_block)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 696) {
2ed1a6bcf97a7 (Jens Axboe 2010-06-22 12:52:14 +0200 697) struct blk_plug plug;
0ea9718016251 (Miklos Szeredi 2007-05-10 22:22:51 -0700 698) int ret;
0ea9718016251 (Miklos Szeredi 2007-05-10 22:22:51 -0700 699)
2ed1a6bcf97a7 (Jens Axboe 2010-06-22 12:52:14 +0200 700) blk_start_plug(&plug);
2ed1a6bcf97a7 (Jens Axboe 2010-06-22 12:52:14 +0200 701)
0ea9718016251 (Miklos Szeredi 2007-05-10 22:22:51 -0700 702) if (!get_block)
0ea9718016251 (Miklos Szeredi 2007-05-10 22:22:51 -0700 703) ret = generic_writepages(mapping, wbc);
0ea9718016251 (Miklos Szeredi 2007-05-10 22:22:51 -0700 704) else {
0ea9718016251 (Miklos Szeredi 2007-05-10 22:22:51 -0700 705) struct mpage_data mpd = {
0ea9718016251 (Miklos Szeredi 2007-05-10 22:22:51 -0700 706) .bio = NULL,
0ea9718016251 (Miklos Szeredi 2007-05-10 22:22:51 -0700 707) .last_block_in_bio = 0,
0ea9718016251 (Miklos Szeredi 2007-05-10 22:22:51 -0700 708) .get_block = get_block,
0ea9718016251 (Miklos Szeredi 2007-05-10 22:22:51 -0700 709) .use_writepage = 1,
0ea9718016251 (Miklos Szeredi 2007-05-10 22:22:51 -0700 710) };
0ea9718016251 (Miklos Szeredi 2007-05-10 22:22:51 -0700 711)
0ea9718016251 (Miklos Szeredi 2007-05-10 22:22:51 -0700 712) ret = write_cache_pages(mapping, wbc, __mpage_writepage, &mpd);
5948edbcbf437 (Roman Pen 2015-09-15 08:27:25 -0600 713) if (mpd.bio) {
eed25cd5bbaf7 (Mike Christie 2016-06-05 14:31:59 -0500 714) int op_flags = (wbc->sync_mode == WB_SYNC_ALL ?
70fd76140a6cb (Christoph Hellwig 2016-11-01 07:40:10 -0600 715) REQ_SYNC : 0);
eed25cd5bbaf7 (Mike Christie 2016-06-05 14:31:59 -0500 716) mpage_bio_submit(REQ_OP_WRITE, op_flags, mpd.bio);
5948edbcbf437 (Roman Pen 2015-09-15 08:27:25 -0600 717) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 718) }
2ed1a6bcf97a7 (Jens Axboe 2010-06-22 12:52:14 +0200 719) blk_finish_plug(&plug);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 720) return ret;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 721) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 722) EXPORT_SYMBOL(mpage_writepages);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 723)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 724) int mpage_writepage(struct page *page, get_block_t get_block,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 725) struct writeback_control *wbc)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 726) {
0ea9718016251 (Miklos Szeredi 2007-05-10 22:22:51 -0700 727) struct mpage_data mpd = {
0ea9718016251 (Miklos Szeredi 2007-05-10 22:22:51 -0700 728) .bio = NULL,
0ea9718016251 (Miklos Szeredi 2007-05-10 22:22:51 -0700 729) .last_block_in_bio = 0,
0ea9718016251 (Miklos Szeredi 2007-05-10 22:22:51 -0700 730) .get_block = get_block,
0ea9718016251 (Miklos Szeredi 2007-05-10 22:22:51 -0700 731) .use_writepage = 0,
0ea9718016251 (Miklos Szeredi 2007-05-10 22:22:51 -0700 732) };
0ea9718016251 (Miklos Szeredi 2007-05-10 22:22:51 -0700 733) int ret = __mpage_writepage(page, wbc, &mpd);
5948edbcbf437 (Roman Pen 2015-09-15 08:27:25 -0600 734) if (mpd.bio) {
eed25cd5bbaf7 (Mike Christie 2016-06-05 14:31:59 -0500 735) int op_flags = (wbc->sync_mode == WB_SYNC_ALL ?
70fd76140a6cb (Christoph Hellwig 2016-11-01 07:40:10 -0600 736) REQ_SYNC : 0);
eed25cd5bbaf7 (Mike Christie 2016-06-05 14:31:59 -0500 737) mpage_bio_submit(REQ_OP_WRITE, op_flags, mpd.bio);
5948edbcbf437 (Roman Pen 2015-09-15 08:27:25 -0600 738) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 739) return ret;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 740) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 741) EXPORT_SYMBOL(mpage_writepage);
VisionFive2 Linux kernel
StarFive Tech Linux Kernel for VisionFive (JH7110) boards (mirror)
More than 9999 Commits
35 Branches
59 Tags