457c899653991 (Thomas Gleixner 2019-05-19 13:08:55 +0100 1) // SPDX-License-Identifier: GPL-2.0-only
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 2) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 3) * Copyright (C) 1991, 1992 Linus Torvalds
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 4) * Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE
7b51e703a89b8 (Christoph Hellwig 2020-12-10 08:55:44 +0100 5) * Copyright (C) 2016 - 2020 Christoph Hellwig
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 6) */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 7)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 8) #include <linux/init.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 9) #include <linux/mm.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 10) #include <linux/fcntl.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 11) #include <linux/slab.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 12) #include <linux/kmod.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 13) #include <linux/major.h>
7db9cfd380205 (Pavel Emelyanov 2008-06-05 22:46:27 -0700 14) #include <linux/device_cgroup.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 15) #include <linux/highmem.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 16) #include <linux/blkdev.h>
66114cad64bf7 (Tejun Heo 2015-05-22 17:13:32 -0400 17) #include <linux/backing-dev.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 18) #include <linux/module.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 19) #include <linux/blkpg.h>
b502bd1152472 (Muthu Kumar 2012-03-23 15:01:50 -0700 20) #include <linux/magic.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 21) #include <linux/buffer_head.h>
ff01bb4832651 (Al Viro 2011-09-16 02:31:11 -0400 22) #include <linux/swap.h>
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 23) #include <linux/pagevec.h>
811d736f9e801 (David Howells 2006-08-29 19:06:09 +0100 24) #include <linux/writeback.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 25) #include <linux/mpage.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 26) #include <linux/mount.h>
9030d16eb8adc (David Howells 2019-03-25 16:38:23 +0000 27) #include <linux/pseudo_fs.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 28) #include <linux/uio.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 29) #include <linux/namei.h>
1368c4f2482c9 (Vignesh Babu BM 2007-05-08 00:24:32 -0700 30) #include <linux/log2.h>
ff01bb4832651 (Al Viro 2011-09-16 02:31:11 -0400 31) #include <linux/cleancache.h>
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 32) #include <linux/task_io_accounting_ops.h>
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 33) #include <linux/falloc.h>
15e3d2c5cd532 (Christoph Hellwig 2020-11-24 09:34:00 +0100 34) #include <linux/part_stat.h>
7c0f6ba682b9c (Linus Torvalds 2016-12-24 11:46:01 -0800 35) #include <linux/uaccess.h>
56939e014a6c2 (Domenico Andreoli 2020-03-23 08:22:15 -0700 36) #include <linux/suspend.h>
07f3f05c1e305 (David Howells 2006-09-30 20:52:18 +0200 37) #include "internal.h"
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 38)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 39) struct bdev_inode {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 40) struct block_device bdev;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 41) struct inode vfs_inode;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 42) };
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 43)
4c54ac62dceec (Adrian Bunk 2008-02-18 13:48:31 +0100 44) static const struct address_space_operations def_blk_aops;
4c54ac62dceec (Adrian Bunk 2008-02-18 13:48:31 +0100 45)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 46) static inline struct bdev_inode *BDEV_I(struct inode *inode)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 47) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 48) return container_of(inode, struct bdev_inode, vfs_inode);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 49) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 50)
ff5053f66677b (Geert Uytterhoeven 2015-06-26 13:58:32 +0200 51) struct block_device *I_BDEV(struct inode *inode)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 52) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 53) return &BDEV_I(inode)->bdev;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 54) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 55) EXPORT_SYMBOL(I_BDEV);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 56)
dbd3ca50753e7 (Vivek Goyal 2015-11-09 09:23:40 -0700 57) static void bdev_write_inode(struct block_device *bdev)
564f00f6c03a9 (Christoph Hellwig 2015-01-14 10:42:33 +0100 58) {
dbd3ca50753e7 (Vivek Goyal 2015-11-09 09:23:40 -0700 59) struct inode *inode = bdev->bd_inode;
dbd3ca50753e7 (Vivek Goyal 2015-11-09 09:23:40 -0700 60) int ret;
dbd3ca50753e7 (Vivek Goyal 2015-11-09 09:23:40 -0700 61)
564f00f6c03a9 (Christoph Hellwig 2015-01-14 10:42:33 +0100 62) spin_lock(&inode->i_lock);
564f00f6c03a9 (Christoph Hellwig 2015-01-14 10:42:33 +0100 63) while (inode->i_state & I_DIRTY) {
564f00f6c03a9 (Christoph Hellwig 2015-01-14 10:42:33 +0100 64) spin_unlock(&inode->i_lock);
dbd3ca50753e7 (Vivek Goyal 2015-11-09 09:23:40 -0700 65) ret = write_inode_now(inode, true);
dbd3ca50753e7 (Vivek Goyal 2015-11-09 09:23:40 -0700 66) if (ret) {
dbd3ca50753e7 (Vivek Goyal 2015-11-09 09:23:40 -0700 67) char name[BDEVNAME_SIZE];
dbd3ca50753e7 (Vivek Goyal 2015-11-09 09:23:40 -0700 68) pr_warn_ratelimited("VFS: Dirty inode writeback failed "
dbd3ca50753e7 (Vivek Goyal 2015-11-09 09:23:40 -0700 69) "for block device %s (err=%d).\n",
dbd3ca50753e7 (Vivek Goyal 2015-11-09 09:23:40 -0700 70) bdevname(bdev, name), ret);
dbd3ca50753e7 (Vivek Goyal 2015-11-09 09:23:40 -0700 71) }
564f00f6c03a9 (Christoph Hellwig 2015-01-14 10:42:33 +0100 72) spin_lock(&inode->i_lock);
564f00f6c03a9 (Christoph Hellwig 2015-01-14 10:42:33 +0100 73) }
564f00f6c03a9 (Christoph Hellwig 2015-01-14 10:42:33 +0100 74) spin_unlock(&inode->i_lock);
564f00f6c03a9 (Christoph Hellwig 2015-01-14 10:42:33 +0100 75) }
564f00f6c03a9 (Christoph Hellwig 2015-01-14 10:42:33 +0100 76)
f9a14399aea13 (Peter Zijlstra 2007-05-06 14:49:55 -0700 77) /* Kill _all_ buffers and pagecache , dirty or not.. */
3373a3461aa15 (Zheng Bin 2020-06-18 12:21:38 +0800 78) static void kill_bdev(struct block_device *bdev)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 79) {
ff01bb4832651 (Al Viro 2011-09-16 02:31:11 -0400 80) struct address_space *mapping = bdev->bd_inode->i_mapping;
ff01bb4832651 (Al Viro 2011-09-16 02:31:11 -0400 81)
7716506adac46 (Matthew Wilcox (Oracle) 2021-05-04 18:32:45 -0700 82) if (mapping_empty(mapping))
f9a14399aea13 (Peter Zijlstra 2007-05-06 14:49:55 -0700 83) return;
ff01bb4832651 (Al Viro 2011-09-16 02:31:11 -0400 84)
f9a14399aea13 (Peter Zijlstra 2007-05-06 14:49:55 -0700 85) invalidate_bh_lrus();
ff01bb4832651 (Al Viro 2011-09-16 02:31:11 -0400 86) truncate_inode_pages(mapping, 0);
3373a3461aa15 (Zheng Bin 2020-06-18 12:21:38 +0800 87) }
ff01bb4832651 (Al Viro 2011-09-16 02:31:11 -0400 88)
ff01bb4832651 (Al Viro 2011-09-16 02:31:11 -0400 89) /* Invalidate clean unused buffers and pagecache. */
ff01bb4832651 (Al Viro 2011-09-16 02:31:11 -0400 90) void invalidate_bdev(struct block_device *bdev)
ff01bb4832651 (Al Viro 2011-09-16 02:31:11 -0400 91) {
ff01bb4832651 (Al Viro 2011-09-16 02:31:11 -0400 92) struct address_space *mapping = bdev->bd_inode->i_mapping;
ff01bb4832651 (Al Viro 2011-09-16 02:31:11 -0400 93)
a5f6a6a9c72ea (Andrey Ryabinin 2017-05-03 14:56:02 -0700 94) if (mapping->nrpages) {
a5f6a6a9c72ea (Andrey Ryabinin 2017-05-03 14:56:02 -0700 95) invalidate_bh_lrus();
a5f6a6a9c72ea (Andrey Ryabinin 2017-05-03 14:56:02 -0700 96) lru_add_drain_all(); /* make sure all lru add caches are flushed */
a5f6a6a9c72ea (Andrey Ryabinin 2017-05-03 14:56:02 -0700 97) invalidate_mapping_pages(mapping, 0, -1);
a5f6a6a9c72ea (Andrey Ryabinin 2017-05-03 14:56:02 -0700 98) }
ff01bb4832651 (Al Viro 2011-09-16 02:31:11 -0400 99) /* 99% of the time, we don't need to flush the cleancache on the bdev.
ff01bb4832651 (Al Viro 2011-09-16 02:31:11 -0400 100) * But, for the strange corners, lets be cautious
ff01bb4832651 (Al Viro 2011-09-16 02:31:11 -0400 101) */
3167760f83899 (Dan Magenheimer 2011-09-21 11:56:28 -0400 102) cleancache_invalidate_inode(mapping);
ff01bb4832651 (Al Viro 2011-09-16 02:31:11 -0400 103) }
ff01bb4832651 (Al Viro 2011-09-16 02:31:11 -0400 104) EXPORT_SYMBOL(invalidate_bdev);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 105)
384d87ef2c954 (Jan Kara 2020-09-04 10:58:52 +0200 106) /*
384d87ef2c954 (Jan Kara 2020-09-04 10:58:52 +0200 107) * Drop all buffers & page cache for given bdev range. This function bails
384d87ef2c954 (Jan Kara 2020-09-04 10:58:52 +0200 108) * with error if bdev has other exclusive owner (such as filesystem).
384d87ef2c954 (Jan Kara 2020-09-04 10:58:52 +0200 109) */
384d87ef2c954 (Jan Kara 2020-09-04 10:58:52 +0200 110) int truncate_bdev_range(struct block_device *bdev, fmode_t mode,
384d87ef2c954 (Jan Kara 2020-09-04 10:58:52 +0200 111) loff_t lstart, loff_t lend)
384d87ef2c954 (Jan Kara 2020-09-04 10:58:52 +0200 112) {
384d87ef2c954 (Jan Kara 2020-09-04 10:58:52 +0200 113) /*
384d87ef2c954 (Jan Kara 2020-09-04 10:58:52 +0200 114) * If we don't hold exclusive handle for the device, upgrade to it
384d87ef2c954 (Jan Kara 2020-09-04 10:58:52 +0200 115) * while we discard the buffer cache to avoid discarding buffers
384d87ef2c954 (Jan Kara 2020-09-04 10:58:52 +0200 116) * under live filesystem.
384d87ef2c954 (Jan Kara 2020-09-04 10:58:52 +0200 117) */
384d87ef2c954 (Jan Kara 2020-09-04 10:58:52 +0200 118) if (!(mode & FMODE_EXCL)) {
37c3fc9abb25c (Christoph Hellwig 2020-11-25 21:20:08 +0100 119) int err = bd_prepare_to_claim(bdev, truncate_bdev_range);
384d87ef2c954 (Jan Kara 2020-09-04 10:58:52 +0200 120) if (err)
56887cffe946b (Jan Kara 2021-02-22 10:48:09 +0100 121) goto invalidate;
384d87ef2c954 (Jan Kara 2020-09-04 10:58:52 +0200 122) }
37c3fc9abb25c (Christoph Hellwig 2020-11-25 21:20:08 +0100 123)
384d87ef2c954 (Jan Kara 2020-09-04 10:58:52 +0200 124) truncate_inode_pages_range(bdev->bd_inode->i_mapping, lstart, lend);
37c3fc9abb25c (Christoph Hellwig 2020-11-25 21:20:08 +0100 125) if (!(mode & FMODE_EXCL))
37c3fc9abb25c (Christoph Hellwig 2020-11-25 21:20:08 +0100 126) bd_abort_claiming(bdev, truncate_bdev_range);
384d87ef2c954 (Jan Kara 2020-09-04 10:58:52 +0200 127) return 0;
56887cffe946b (Jan Kara 2021-02-22 10:48:09 +0100 128)
56887cffe946b (Jan Kara 2021-02-22 10:48:09 +0100 129) invalidate:
56887cffe946b (Jan Kara 2021-02-22 10:48:09 +0100 130) /*
56887cffe946b (Jan Kara 2021-02-22 10:48:09 +0100 131) * Someone else has handle exclusively open. Try invalidating instead.
56887cffe946b (Jan Kara 2021-02-22 10:48:09 +0100 132) * The 'end' argument is inclusive so the rounding is safe.
56887cffe946b (Jan Kara 2021-02-22 10:48:09 +0100 133) */
56887cffe946b (Jan Kara 2021-02-22 10:48:09 +0100 134) return invalidate_inode_pages2_range(bdev->bd_inode->i_mapping,
56887cffe946b (Jan Kara 2021-02-22 10:48:09 +0100 135) lstart >> PAGE_SHIFT,
56887cffe946b (Jan Kara 2021-02-22 10:48:09 +0100 136) lend >> PAGE_SHIFT);
384d87ef2c954 (Jan Kara 2020-09-04 10:58:52 +0200 137) }
384d87ef2c954 (Jan Kara 2020-09-04 10:58:52 +0200 138)
04906b2f542c2 (Jan Kara 2019-01-14 09:48:10 +0100 139) static void set_init_blocksize(struct block_device *bdev)
04906b2f542c2 (Jan Kara 2019-01-14 09:48:10 +0100 140) {
8dc932d3e8afb (Maxim Mikityanskiy 2021-01-26 21:59:07 +0200 141) unsigned int bsize = bdev_logical_block_size(bdev);
8dc932d3e8afb (Maxim Mikityanskiy 2021-01-26 21:59:07 +0200 142) loff_t size = i_size_read(bdev->bd_inode);
8dc932d3e8afb (Maxim Mikityanskiy 2021-01-26 21:59:07 +0200 143)
8dc932d3e8afb (Maxim Mikityanskiy 2021-01-26 21:59:07 +0200 144) while (bsize < PAGE_SIZE) {
8dc932d3e8afb (Maxim Mikityanskiy 2021-01-26 21:59:07 +0200 145) if (size & bsize)
8dc932d3e8afb (Maxim Mikityanskiy 2021-01-26 21:59:07 +0200 146) break;
8dc932d3e8afb (Maxim Mikityanskiy 2021-01-26 21:59:07 +0200 147) bsize <<= 1;
8dc932d3e8afb (Maxim Mikityanskiy 2021-01-26 21:59:07 +0200 148) }
8dc932d3e8afb (Maxim Mikityanskiy 2021-01-26 21:59:07 +0200 149) bdev->bd_inode->i_blkbits = blksize_bits(bsize);
04906b2f542c2 (Jan Kara 2019-01-14 09:48:10 +0100 150) }
04906b2f542c2 (Jan Kara 2019-01-14 09:48:10 +0100 151)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 152) int set_blocksize(struct block_device *bdev, int size)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 153) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 154) /* Size must be a power of two, and between 512 and PAGE_SIZE */
1368c4f2482c9 (Vignesh Babu BM 2007-05-08 00:24:32 -0700 155) if (size > PAGE_SIZE || size < 512 || !is_power_of_2(size))
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 156) return -EINVAL;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 157)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 158) /* Size cannot be smaller than the size supported by the device */
e1defc4ff0cf5 (Martin K. Petersen 2009-05-22 17:17:49 -0400 159) if (size < bdev_logical_block_size(bdev))
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 160) return -EINVAL;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 161)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 162) /* Don't change the size if it is same as current */
6b7b181b67aa8 (Christoph Hellwig 2020-06-26 10:01:55 +0200 163) if (bdev->bd_inode->i_blkbits != blksize_bits(size)) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 164) sync_blockdev(bdev);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 165) bdev->bd_inode->i_blkbits = blksize_bits(size);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 166) kill_bdev(bdev);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 167) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 168) return 0;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 169) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 170)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 171) EXPORT_SYMBOL(set_blocksize);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 172)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 173) int sb_set_blocksize(struct super_block *sb, int size)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 174) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 175) if (set_blocksize(sb->s_bdev, size))
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 176) return 0;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 177) /* If we get here, we know size is power of two
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 178) * and it's value is between 512 and PAGE_SIZE */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 179) sb->s_blocksize = size;
38885bd4c2a4b (Coywolf Qi Hunt 2006-03-24 03:18:05 -0800 180) sb->s_blocksize_bits = blksize_bits(size);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 181) return sb->s_blocksize;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 182) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 183)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 184) EXPORT_SYMBOL(sb_set_blocksize);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 185)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 186) int sb_min_blocksize(struct super_block *sb, int size)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 187) {
e1defc4ff0cf5 (Martin K. Petersen 2009-05-22 17:17:49 -0400 188) int minsize = bdev_logical_block_size(sb->s_bdev);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 189) if (size < minsize)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 190) size = minsize;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 191) return sb_set_blocksize(sb, size);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 192) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 193)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 194) EXPORT_SYMBOL(sb_min_blocksize);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 195)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 196) static int
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 197) blkdev_get_block(struct inode *inode, sector_t iblock,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 198) struct buffer_head *bh, int create)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 199) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 200) bh->b_bdev = I_BDEV(inode);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 201) bh->b_blocknr = iblock;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 202) set_buffer_mapped(bh);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 203) return 0;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 204) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 205)
4ebb16ca9a06a (Dan Williams 2015-10-28 07:48:19 +0900 206) static struct inode *bdev_file_inode(struct file *file)
4ebb16ca9a06a (Dan Williams 2015-10-28 07:48:19 +0900 207) {
4ebb16ca9a06a (Dan Williams 2015-10-28 07:48:19 +0900 208) return file->f_mapping->host;
4ebb16ca9a06a (Dan Williams 2015-10-28 07:48:19 +0900 209) }
4ebb16ca9a06a (Dan Williams 2015-10-28 07:48:19 +0900 210)
78250c02d9739 (Jens Axboe 2016-11-17 17:50:47 +0100 211) static unsigned int dio_bio_write_op(struct kiocb *iocb)
78250c02d9739 (Jens Axboe 2016-11-17 17:50:47 +0100 212) {
78250c02d9739 (Jens Axboe 2016-11-17 17:50:47 +0100 213) unsigned int op = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE;
78250c02d9739 (Jens Axboe 2016-11-17 17:50:47 +0100 214)
78250c02d9739 (Jens Axboe 2016-11-17 17:50:47 +0100 215) /* avoid the need for a I/O completion work item */
78250c02d9739 (Jens Axboe 2016-11-17 17:50:47 +0100 216) if (iocb->ki_flags & IOCB_DSYNC)
78250c02d9739 (Jens Axboe 2016-11-17 17:50:47 +0100 217) op |= REQ_FUA;
78250c02d9739 (Jens Axboe 2016-11-17 17:50:47 +0100 218) return op;
78250c02d9739 (Jens Axboe 2016-11-17 17:50:47 +0100 219) }
78250c02d9739 (Jens Axboe 2016-11-17 17:50:47 +0100 220)
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 221) #define DIO_INLINE_BIO_VECS 4
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 222)
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 223) static void blkdev_bio_end_io_simple(struct bio *bio)
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 224) {
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 225) struct task_struct *waiter = bio->bi_private;
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 226)
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 227) WRITE_ONCE(bio->bi_private, NULL);
0619317ff8baa (Jens Axboe 2018-11-13 21:16:54 -0700 228) blk_wake_io_task(waiter);
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 229) }
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 230)
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 231) static ssize_t
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 232) __blkdev_direct_IO_simple(struct kiocb *iocb, struct iov_iter *iter,
5f7136db82996 (Matthew Wilcox (Oracle) 2021-01-29 04:38:57 +0000 233) unsigned int nr_pages)
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 234) {
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 235) struct file *file = iocb->ki_filp;
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 236) struct block_device *bdev = I_BDEV(bdev_file_inode(file));
9fec4a21886e6 (Christoph Hellwig 2019-06-26 15:49:26 +0200 237) struct bio_vec inline_vecs[DIO_INLINE_BIO_VECS], *vecs;
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 238) loff_t pos = iocb->ki_pos;
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 239) bool should_dirty = false;
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 240) struct bio bio;
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 241) ssize_t ret;
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 242) blk_qc_t qc;
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 243)
9a794fb9bdded (Jens Axboe 2016-11-22 08:12:39 -0700 244) if ((pos | iov_iter_alignment(iter)) &
9a794fb9bdded (Jens Axboe 2016-11-22 08:12:39 -0700 245) (bdev_logical_block_size(bdev) - 1))
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 246) return -EINVAL;
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 247)
72ecad22d9f19 (Jens Axboe 2016-11-16 23:11:42 -0700 248) if (nr_pages <= DIO_INLINE_BIO_VECS)
72ecad22d9f19 (Jens Axboe 2016-11-16 23:11:42 -0700 249) vecs = inline_vecs;
72ecad22d9f19 (Jens Axboe 2016-11-16 23:11:42 -0700 250) else {
6da2ec56059c3 (Kees Cook 2018-06-12 13:55:00 -0700 251) vecs = kmalloc_array(nr_pages, sizeof(struct bio_vec),
6da2ec56059c3 (Kees Cook 2018-06-12 13:55:00 -0700 252) GFP_KERNEL);
72ecad22d9f19 (Jens Axboe 2016-11-16 23:11:42 -0700 253) if (!vecs)
72ecad22d9f19 (Jens Axboe 2016-11-16 23:11:42 -0700 254) return -ENOMEM;
72ecad22d9f19 (Jens Axboe 2016-11-16 23:11:42 -0700 255) }
72ecad22d9f19 (Jens Axboe 2016-11-16 23:11:42 -0700 256)
3a83f4677539b (Ming Lei 2016-11-22 08:57:21 -0700 257) bio_init(&bio, vecs, nr_pages);
74d46992e0d9d (Christoph Hellwig 2017-08-23 19:10:32 +0200 258) bio_set_dev(&bio, bdev);
4d1a4765426fb (Damien Le Moal 2016-11-22 15:38:49 +0900 259) bio.bi_iter.bi_sector = pos >> 9;
45d06cf701a38 (Jens Axboe 2017-06-27 11:01:22 -0600 260) bio.bi_write_hint = iocb->ki_hint;
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 261) bio.bi_private = current;
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 262) bio.bi_end_io = blkdev_bio_end_io_simple;
074111ca5f713 (Adam Manzanares 2018-05-22 10:52:20 -0700 263) bio.bi_ioprio = iocb->ki_ioprio;
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 264)
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 265) ret = bio_iov_iter_get_pages(&bio, iter);
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 266) if (unlikely(ret))
9362dd1109f87 (Martin Wilck 2018-07-25 23:15:08 +0200 267) goto out;
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 268) ret = bio.bi_iter.bi_size;
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 269)
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 270) if (iov_iter_rw(iter) == READ) {
78250c02d9739 (Jens Axboe 2016-11-17 17:50:47 +0100 271) bio.bi_opf = REQ_OP_READ;
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 272) if (iter_is_iovec(iter))
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 273) should_dirty = true;
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 274) } else {
78250c02d9739 (Jens Axboe 2016-11-17 17:50:47 +0100 275) bio.bi_opf = dio_bio_write_op(iocb);
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 276) task_io_account_write(ret);
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 277) }
f8b78caf21d5b (Pavel Begunkov 2020-11-20 17:10:28 +0000 278) if (iocb->ki_flags & IOCB_NOWAIT)
f8b78caf21d5b (Pavel Begunkov 2020-11-20 17:10:28 +0000 279) bio.bi_opf |= REQ_NOWAIT;
d1e36282b0bbd (Jens Axboe 2018-08-29 10:36:56 -0600 280) if (iocb->ki_flags & IOCB_HIPRI)
0bbb280d7b767 (Jens Axboe 2018-12-21 09:10:46 -0700 281) bio_set_polled(&bio, iocb);
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 282)
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 283) qc = submit_bio(&bio);
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 284) for (;;) {
1ac5cd4978794 (Linus Torvalds 2019-01-02 10:46:03 -0800 285) set_current_state(TASK_UNINTERRUPTIBLE);
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 286) if (!READ_ONCE(bio.bi_private))
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 287) break;
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 288) if (!(iocb->ki_flags & IOCB_HIPRI) ||
0a1b8b87d064a (Jens Axboe 2018-11-26 08:24:43 -0700 289) !blk_poll(bdev_get_queue(bdev), qc, true))
e6249cdd46e43 (Ming Lei 2020-05-03 09:54:22 +0800 290) blk_io_schedule();
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 291) }
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 292) __set_current_state(TASK_RUNNING);
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 293)
9fec4a21886e6 (Christoph Hellwig 2019-06-26 15:49:26 +0200 294) bio_release_pages(&bio, should_dirty);
4e4cbee93d561 (Christoph Hellwig 2017-06-03 09:38:06 +0200 295) if (unlikely(bio.bi_status))
c6b1e36c8fa04 (Linus Torvalds 2017-07-03 10:34:51 -0700 296) ret = blk_status_to_errno(bio.bi_status);
9ae3b3f52c62d (Jens Axboe 2017-06-28 15:30:13 -0600 297)
9362dd1109f87 (Martin Wilck 2018-07-25 23:15:08 +0200 298) out:
9362dd1109f87 (Martin Wilck 2018-07-25 23:15:08 +0200 299) if (vecs != inline_vecs)
9362dd1109f87 (Martin Wilck 2018-07-25 23:15:08 +0200 300) kfree(vecs);
9362dd1109f87 (Martin Wilck 2018-07-25 23:15:08 +0200 301)
9ae3b3f52c62d (Jens Axboe 2017-06-28 15:30:13 -0600 302) bio_uninit(&bio);
9ae3b3f52c62d (Jens Axboe 2017-06-28 15:30:13 -0600 303)
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 304) return ret;
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 305) }
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 306)
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 307) struct blkdev_dio {
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 308) union {
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 309) struct kiocb *iocb;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 310) struct task_struct *waiter;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 311) };
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 312) size_t size;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 313) atomic_t ref;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 314) bool multi_bio : 1;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 315) bool should_dirty : 1;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 316) bool is_sync : 1;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 317) struct bio bio;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 318) };
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 319)
52190f8abe7f2 (Kent Overstreet 2018-05-20 18:25:55 -0400 320) static struct bio_set blkdev_dio_pool;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 321)
eae83ce10b471 (Christoph Hellwig 2018-11-30 08:31:52 -0700 322) static int blkdev_iopoll(struct kiocb *kiocb, bool wait)
eae83ce10b471 (Christoph Hellwig 2018-11-30 08:31:52 -0700 323) {
eae83ce10b471 (Christoph Hellwig 2018-11-30 08:31:52 -0700 324) struct block_device *bdev = I_BDEV(kiocb->ki_filp->f_mapping->host);
eae83ce10b471 (Christoph Hellwig 2018-11-30 08:31:52 -0700 325) struct request_queue *q = bdev_get_queue(bdev);
eae83ce10b471 (Christoph Hellwig 2018-11-30 08:31:52 -0700 326)
eae83ce10b471 (Christoph Hellwig 2018-11-30 08:31:52 -0700 327) return blk_poll(q, READ_ONCE(kiocb->ki_cookie), wait);
eae83ce10b471 (Christoph Hellwig 2018-11-30 08:31:52 -0700 328) }
eae83ce10b471 (Christoph Hellwig 2018-11-30 08:31:52 -0700 329)
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 330) static void blkdev_bio_end_io(struct bio *bio)
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 331) {
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 332) struct blkdev_dio *dio = bio->bi_private;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 333) bool should_dirty = dio->should_dirty;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 334)
a89afe58f1a74 (Jason Yan 2019-04-12 10:09:16 +0800 335) if (bio->bi_status && !dio->bio.bi_status)
a89afe58f1a74 (Jason Yan 2019-04-12 10:09:16 +0800 336) dio->bio.bi_status = bio->bi_status;
a89afe58f1a74 (Jason Yan 2019-04-12 10:09:16 +0800 337)
a89afe58f1a74 (Jason Yan 2019-04-12 10:09:16 +0800 338) if (!dio->multi_bio || atomic_dec_and_test(&dio->ref)) {
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 339) if (!dio->is_sync) {
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 340) struct kiocb *iocb = dio->iocb;
4e4cbee93d561 (Christoph Hellwig 2017-06-03 09:38:06 +0200 341) ssize_t ret;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 342)
4e4cbee93d561 (Christoph Hellwig 2017-06-03 09:38:06 +0200 343) if (likely(!dio->bio.bi_status)) {
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 344) ret = dio->size;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 345) iocb->ki_pos += ret;
4e4cbee93d561 (Christoph Hellwig 2017-06-03 09:38:06 +0200 346) } else {
4e4cbee93d561 (Christoph Hellwig 2017-06-03 09:38:06 +0200 347) ret = blk_status_to_errno(dio->bio.bi_status);
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 348) }
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 349)
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 350) dio->iocb->ki_complete(iocb, ret, 0);
531724abc3bfb (Christoph Hellwig 2018-11-30 09:23:48 +0100 351) if (dio->multi_bio)
531724abc3bfb (Christoph Hellwig 2018-11-30 09:23:48 +0100 352) bio_put(&dio->bio);
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 353) } else {
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 354) struct task_struct *waiter = dio->waiter;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 355)
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 356) WRITE_ONCE(dio->waiter, NULL);
0619317ff8baa (Jens Axboe 2018-11-13 21:16:54 -0700 357) blk_wake_io_task(waiter);
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 358) }
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 359) }
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 360)
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 361) if (should_dirty) {
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 362) bio_check_pages_dirty(bio);
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 363) } else {
57dfe3ce10e5c (Christoph Hellwig 2019-06-26 15:49:25 +0200 364) bio_release_pages(bio, false);
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 365) bio_put(bio);
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 366) }
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 367) }
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 368)
5f7136db82996 (Matthew Wilcox (Oracle) 2021-01-29 04:38:57 +0000 369) static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter,
5f7136db82996 (Matthew Wilcox (Oracle) 2021-01-29 04:38:57 +0000 370) unsigned int nr_pages)
b2e895dbd80c4 (Andrew Morton 2007-02-03 01:14:01 -0800 371) {
b2e895dbd80c4 (Andrew Morton 2007-02-03 01:14:01 -0800 372) struct file *file = iocb->ki_filp;
4ebb16ca9a06a (Dan Williams 2015-10-28 07:48:19 +0900 373) struct inode *inode = bdev_file_inode(file);
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 374) struct block_device *bdev = I_BDEV(inode);
64d656a162d7b (Christoph Hellwig 2016-12-22 19:20:45 +0100 375) struct blk_plug plug;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 376) struct blkdev_dio *dio;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 377) struct bio *bio;
cb700eb3faa48 (Jens Axboe 2018-11-15 19:56:53 -0700 378) bool is_poll = (iocb->ki_flags & IOCB_HIPRI) != 0;
690e5325b8c7d (Christoph Hellwig 2017-01-24 14:50:19 +0100 379) bool is_read = (iov_iter_rw(iter) == READ), is_sync;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 380) loff_t pos = iocb->ki_pos;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 381) blk_qc_t qc = BLK_QC_T_NONE;
7b6620d7db566 (Jens Axboe 2019-08-15 11:09:16 -0600 382) int ret = 0;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 383)
9a794fb9bdded (Jens Axboe 2016-11-22 08:12:39 -0700 384) if ((pos | iov_iter_alignment(iter)) &
9a794fb9bdded (Jens Axboe 2016-11-22 08:12:39 -0700 385) (bdev_logical_block_size(bdev) - 1))
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 386) return -EINVAL;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 387)
7b6620d7db566 (Jens Axboe 2019-08-15 11:09:16 -0600 388) bio = bio_alloc_bioset(GFP_KERNEL, nr_pages, &blkdev_dio_pool);
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 389)
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 390) dio = container_of(bio, struct blkdev_dio, bio);
690e5325b8c7d (Christoph Hellwig 2017-01-24 14:50:19 +0100 391) dio->is_sync = is_sync = is_sync_kiocb(iocb);
531724abc3bfb (Christoph Hellwig 2018-11-30 09:23:48 +0100 392) if (dio->is_sync) {
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 393) dio->waiter = current;
531724abc3bfb (Christoph Hellwig 2018-11-30 09:23:48 +0100 394) bio_get(bio);
531724abc3bfb (Christoph Hellwig 2018-11-30 09:23:48 +0100 395) } else {
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 396) dio->iocb = iocb;
531724abc3bfb (Christoph Hellwig 2018-11-30 09:23:48 +0100 397) }
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 398)
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 399) dio->size = 0;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 400) dio->multi_bio = false;
00e23707442a7 (David Howells 2018-10-22 13:07:28 +0100 401) dio->should_dirty = is_read && iter_is_iovec(iter);
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 402)
cb700eb3faa48 (Jens Axboe 2018-11-15 19:56:53 -0700 403) /*
cb700eb3faa48 (Jens Axboe 2018-11-15 19:56:53 -0700 404) * Don't plug for HIPRI/polled IO, as those should go straight
cb700eb3faa48 (Jens Axboe 2018-11-15 19:56:53 -0700 405) * to issue
cb700eb3faa48 (Jens Axboe 2018-11-15 19:56:53 -0700 406) */
cb700eb3faa48 (Jens Axboe 2018-11-15 19:56:53 -0700 407) if (!is_poll)
cb700eb3faa48 (Jens Axboe 2018-11-15 19:56:53 -0700 408) blk_start_plug(&plug);
cb700eb3faa48 (Jens Axboe 2018-11-15 19:56:53 -0700 409)
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 410) for (;;) {
74d46992e0d9d (Christoph Hellwig 2017-08-23 19:10:32 +0200 411) bio_set_dev(bio, bdev);
4d1a4765426fb (Damien Le Moal 2016-11-22 15:38:49 +0900 412) bio->bi_iter.bi_sector = pos >> 9;
45d06cf701a38 (Jens Axboe 2017-06-27 11:01:22 -0600 413) bio->bi_write_hint = iocb->ki_hint;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 414) bio->bi_private = dio;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 415) bio->bi_end_io = blkdev_bio_end_io;
074111ca5f713 (Adam Manzanares 2018-05-22 10:52:20 -0700 416) bio->bi_ioprio = iocb->ki_ioprio;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 417)
e15c2ffa1091c (Jens Axboe 2019-08-06 13:34:31 -0600 418) ret = bio_iov_iter_get_pages(bio, iter);
e15c2ffa1091c (Jens Axboe 2019-08-06 13:34:31 -0600 419) if (unlikely(ret)) {
4e4cbee93d561 (Christoph Hellwig 2017-06-03 09:38:06 +0200 420) bio->bi_status = BLK_STS_IOERR;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 421) bio_endio(bio);
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 422) break;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 423) }
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 424)
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 425) if (is_read) {
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 426) bio->bi_opf = REQ_OP_READ;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 427) if (dio->should_dirty)
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 428) bio_set_pages_dirty(bio);
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 429) } else {
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 430) bio->bi_opf = dio_bio_write_op(iocb);
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 431) task_io_account_write(bio->bi_iter.bi_size);
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 432) }
f8b78caf21d5b (Pavel Begunkov 2020-11-20 17:10:28 +0000 433) if (iocb->ki_flags & IOCB_NOWAIT)
f8b78caf21d5b (Pavel Begunkov 2020-11-20 17:10:28 +0000 434) bio->bi_opf |= REQ_NOWAIT;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 435)
7b6620d7db566 (Jens Axboe 2019-08-15 11:09:16 -0600 436) dio->size += bio->bi_iter.bi_size;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 437) pos += bio->bi_iter.bi_size;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 438)
a8affc03a9b37 (Christoph Hellwig 2021-03-11 12:01:37 +0100 439) nr_pages = bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS);
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 440) if (!nr_pages) {
eae83ce10b471 (Christoph Hellwig 2018-11-30 08:31:52 -0700 441) bool polled = false;
eae83ce10b471 (Christoph Hellwig 2018-11-30 08:31:52 -0700 442)
eae83ce10b471 (Christoph Hellwig 2018-11-30 08:31:52 -0700 443) if (iocb->ki_flags & IOCB_HIPRI) {
0bbb280d7b767 (Jens Axboe 2018-12-21 09:10:46 -0700 444) bio_set_polled(bio, iocb);
eae83ce10b471 (Christoph Hellwig 2018-11-30 08:31:52 -0700 445) polled = true;
eae83ce10b471 (Christoph Hellwig 2018-11-30 08:31:52 -0700 446) }
d34513d384487 (Jens Axboe 2018-11-06 14:29:11 -0700 447)
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 448) qc = submit_bio(bio);
eae83ce10b471 (Christoph Hellwig 2018-11-30 08:31:52 -0700 449)
eae83ce10b471 (Christoph Hellwig 2018-11-30 08:31:52 -0700 450) if (polled)
eae83ce10b471 (Christoph Hellwig 2018-11-30 08:31:52 -0700 451) WRITE_ONCE(iocb->ki_cookie, qc);
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 452) break;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 453) }
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 454)
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 455) if (!dio->multi_bio) {
531724abc3bfb (Christoph Hellwig 2018-11-30 09:23:48 +0100 456) /*
531724abc3bfb (Christoph Hellwig 2018-11-30 09:23:48 +0100 457) * AIO needs an extra reference to ensure the dio
531724abc3bfb (Christoph Hellwig 2018-11-30 09:23:48 +0100 458) * structure which is embedded into the first bio
531724abc3bfb (Christoph Hellwig 2018-11-30 09:23:48 +0100 459) * stays around.
531724abc3bfb (Christoph Hellwig 2018-11-30 09:23:48 +0100 460) */
531724abc3bfb (Christoph Hellwig 2018-11-30 09:23:48 +0100 461) if (!is_sync)
531724abc3bfb (Christoph Hellwig 2018-11-30 09:23:48 +0100 462) bio_get(bio);
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 463) dio->multi_bio = true;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 464) atomic_set(&dio->ref, 2);
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 465) } else {
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 466) atomic_inc(&dio->ref);
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 467) }
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 468)
7b6620d7db566 (Jens Axboe 2019-08-15 11:09:16 -0600 469) submit_bio(bio);
7b6620d7db566 (Jens Axboe 2019-08-15 11:09:16 -0600 470) bio = bio_alloc(GFP_KERNEL, nr_pages);
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 471) }
cb700eb3faa48 (Jens Axboe 2018-11-15 19:56:53 -0700 472)
cb700eb3faa48 (Jens Axboe 2018-11-15 19:56:53 -0700 473) if (!is_poll)
cb700eb3faa48 (Jens Axboe 2018-11-15 19:56:53 -0700 474) blk_finish_plug(&plug);
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 475)
690e5325b8c7d (Christoph Hellwig 2017-01-24 14:50:19 +0100 476) if (!is_sync)
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 477) return -EIOCBQUEUED;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 478)
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 479) for (;;) {
1ac5cd4978794 (Linus Torvalds 2019-01-02 10:46:03 -0800 480) set_current_state(TASK_UNINTERRUPTIBLE);
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 481) if (!READ_ONCE(dio->waiter))
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 482) break;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 483)
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 484) if (!(iocb->ki_flags & IOCB_HIPRI) ||
0a1b8b87d064a (Jens Axboe 2018-11-26 08:24:43 -0700 485) !blk_poll(bdev_get_queue(bdev), qc, true))
e6249cdd46e43 (Ming Lei 2020-05-03 09:54:22 +0800 486) blk_io_schedule();
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 487) }
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 488) __set_current_state(TASK_RUNNING);
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 489)
36ffc6c1c0e67 (Christoph Hellwig 2017-06-03 09:38:00 +0200 490) if (!ret)
4e4cbee93d561 (Christoph Hellwig 2017-06-03 09:38:06 +0200 491) ret = blk_status_to_errno(dio->bio.bi_status);
e15c2ffa1091c (Jens Axboe 2019-08-06 13:34:31 -0600 492) if (likely(!ret))
e15c2ffa1091c (Jens Axboe 2019-08-06 13:34:31 -0600 493) ret = dio->size;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 494)
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 495) bio_put(&dio->bio);
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 496) return ret;
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 497) }
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 498)
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 499) static ssize_t
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 500) blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 501) {
5f7136db82996 (Matthew Wilcox (Oracle) 2021-01-29 04:38:57 +0000 502) unsigned int nr_pages;
b2e895dbd80c4 (Andrew Morton 2007-02-03 01:14:01 -0800 503)
3e1a88ec96259 (Pavel Begunkov 2021-01-09 16:03:02 +0000 504) if (!iov_iter_count(iter))
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 505) return 0;
3e1a88ec96259 (Pavel Begunkov 2021-01-09 16:03:02 +0000 506)
a8affc03a9b37 (Christoph Hellwig 2021-03-11 12:01:37 +0100 507) nr_pages = bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS + 1);
a8affc03a9b37 (Christoph Hellwig 2021-03-11 12:01:37 +0100 508) if (is_sync_kiocb(iocb) && nr_pages <= BIO_MAX_VECS)
189ce2b9dcc34 (Christoph Hellwig 2016-10-31 11:59:25 -0600 509) return __blkdev_direct_IO_simple(iocb, iter, nr_pages);
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 510)
5f7136db82996 (Matthew Wilcox (Oracle) 2021-01-29 04:38:57 +0000 511) return __blkdev_direct_IO(iocb, iter, bio_max_segs(nr_pages));
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 512) }
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 513)
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 514) static __init int blkdev_init(void)
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 515) {
52190f8abe7f2 (Kent Overstreet 2018-05-20 18:25:55 -0400 516) return bioset_init(&blkdev_dio_pool, 4, offsetof(struct blkdev_dio, bio), BIOSET_NEED_BVECS);
b2e895dbd80c4 (Andrew Morton 2007-02-03 01:14:01 -0800 517) }
542ff7bf18c63 (Christoph Hellwig 2016-11-16 23:14:22 -0700 518) module_init(blkdev_init);
b2e895dbd80c4 (Andrew Morton 2007-02-03 01:14:01 -0800 519)
5cee5815d1564 (Jan Kara 2009-04-27 16:43:51 +0200 520) int __sync_blockdev(struct block_device *bdev, int wait)
5cee5815d1564 (Jan Kara 2009-04-27 16:43:51 +0200 521) {
5cee5815d1564 (Jan Kara 2009-04-27 16:43:51 +0200 522) if (!bdev)
5cee5815d1564 (Jan Kara 2009-04-27 16:43:51 +0200 523) return 0;
5cee5815d1564 (Jan Kara 2009-04-27 16:43:51 +0200 524) if (!wait)
5cee5815d1564 (Jan Kara 2009-04-27 16:43:51 +0200 525) return filemap_flush(bdev->bd_inode->i_mapping);
5cee5815d1564 (Jan Kara 2009-04-27 16:43:51 +0200 526) return filemap_write_and_wait(bdev->bd_inode->i_mapping);
5cee5815d1564 (Jan Kara 2009-04-27 16:43:51 +0200 527) }
5cee5815d1564 (Jan Kara 2009-04-27 16:43:51 +0200 528)
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 529) /*
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 530) * Write out and wait upon all the dirty data associated with a block
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 531) * device via its mapping. Does not take the superblock lock.
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 532) */
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 533) int sync_blockdev(struct block_device *bdev)
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 534) {
5cee5815d1564 (Jan Kara 2009-04-27 16:43:51 +0200 535) return __sync_blockdev(bdev, 1);
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 536) }
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 537) EXPORT_SYMBOL(sync_blockdev);
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 538)
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 539) /*
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 540) * Write out and wait upon all dirty data associated with this
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 541) * device. Filesystem data as well as the underlying block
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 542) * device. Takes the superblock lock.
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 543) */
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 544) int fsync_bdev(struct block_device *bdev)
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 545) {
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 546) struct super_block *sb = get_super(bdev);
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 547) if (sb) {
60b0680fa236a (Jan Kara 2009-04-27 16:43:53 +0200 548) int res = sync_filesystem(sb);
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 549) drop_super(sb);
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 550) return res;
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 551) }
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 552) return sync_blockdev(bdev);
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 553) }
47e4491b40df7 (Al Viro 2009-04-01 07:07:16 -0400 554) EXPORT_SYMBOL(fsync_bdev);
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 555)
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 556) /**
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 557) * freeze_bdev -- lock a filesystem and force it into a consistent state
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 558) * @bdev: blockdevice to lock
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 559) *
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 560) * If a superblock is found on this device, we take the s_umount semaphore
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 561) * on it to make sure nobody unmounts until the snapshot creation is done.
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 562) * The reference counter (bd_fsfreeze_count) guarantees that only the last
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 563) * unfreeze process can unfreeze the frozen filesystem actually when multiple
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 564) * freeze requests arrive simultaneously. It counts up in freeze_bdev() and
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 565) * count down in thaw_bdev(). When it becomes 0, thaw_bdev() will unfreeze
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 566) * actually.
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 567) */
040f04bd2e825 (Christoph Hellwig 2020-11-24 11:54:06 +0100 568) int freeze_bdev(struct block_device *bdev)
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 569) {
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 570) struct super_block *sb;
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 571) int error = 0;
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 572)
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 573) mutex_lock(&bdev->bd_fsfreeze_mutex);
040f04bd2e825 (Christoph Hellwig 2020-11-24 11:54:06 +0100 574) if (++bdev->bd_fsfreeze_count > 1)
040f04bd2e825 (Christoph Hellwig 2020-11-24 11:54:06 +0100 575) goto done;
4504230a71566 (Christoph Hellwig 2009-08-03 23:28:35 +0200 576)
4504230a71566 (Christoph Hellwig 2009-08-03 23:28:35 +0200 577) sb = get_active_super(bdev);
4504230a71566 (Christoph Hellwig 2009-08-03 23:28:35 +0200 578) if (!sb)
040f04bd2e825 (Christoph Hellwig 2020-11-24 11:54:06 +0100 579) goto sync;
48b6bca6b7b83 (Benjamin Marzinski 2014-11-13 20:42:03 -0600 580) if (sb->s_op->freeze_super)
48b6bca6b7b83 (Benjamin Marzinski 2014-11-13 20:42:03 -0600 581) error = sb->s_op->freeze_super(sb);
48b6bca6b7b83 (Benjamin Marzinski 2014-11-13 20:42:03 -0600 582) else
48b6bca6b7b83 (Benjamin Marzinski 2014-11-13 20:42:03 -0600 583) error = freeze_super(sb);
040f04bd2e825 (Christoph Hellwig 2020-11-24 11:54:06 +0100 584) deactivate_super(sb);
040f04bd2e825 (Christoph Hellwig 2020-11-24 11:54:06 +0100 585)
18e9e5104fcd9 (Josef Bacik 2010-03-23 10:34:56 -0400 586) if (error) {
18e9e5104fcd9 (Josef Bacik 2010-03-23 10:34:56 -0400 587) bdev->bd_fsfreeze_count--;
040f04bd2e825 (Christoph Hellwig 2020-11-24 11:54:06 +0100 588) goto done;
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 589) }
040f04bd2e825 (Christoph Hellwig 2020-11-24 11:54:06 +0100 590) bdev->bd_fsfreeze_sb = sb;
040f04bd2e825 (Christoph Hellwig 2020-11-24 11:54:06 +0100 591)
040f04bd2e825 (Christoph Hellwig 2020-11-24 11:54:06 +0100 592) sync:
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 593) sync_blockdev(bdev);
040f04bd2e825 (Christoph Hellwig 2020-11-24 11:54:06 +0100 594) done:
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 595) mutex_unlock(&bdev->bd_fsfreeze_mutex);
040f04bd2e825 (Christoph Hellwig 2020-11-24 11:54:06 +0100 596) return error;
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 597) }
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 598) EXPORT_SYMBOL(freeze_bdev);
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 599)
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 600) /**
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 601) * thaw_bdev -- unlock filesystem
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 602) * @bdev: blockdevice to unlock
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 603) *
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 604) * Unlocks the filesystem and marks it writeable again after freeze_bdev().
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 605) */
040f04bd2e825 (Christoph Hellwig 2020-11-24 11:54:06 +0100 606) int thaw_bdev(struct block_device *bdev)
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 607) {
040f04bd2e825 (Christoph Hellwig 2020-11-24 11:54:06 +0100 608) struct super_block *sb;
4504230a71566 (Christoph Hellwig 2009-08-03 23:28:35 +0200 609) int error = -EINVAL;
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 610)
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 611) mutex_lock(&bdev->bd_fsfreeze_mutex);
4504230a71566 (Christoph Hellwig 2009-08-03 23:28:35 +0200 612) if (!bdev->bd_fsfreeze_count)
18e9e5104fcd9 (Josef Bacik 2010-03-23 10:34:56 -0400 613) goto out;
4504230a71566 (Christoph Hellwig 2009-08-03 23:28:35 +0200 614)
4504230a71566 (Christoph Hellwig 2009-08-03 23:28:35 +0200 615) error = 0;
4504230a71566 (Christoph Hellwig 2009-08-03 23:28:35 +0200 616) if (--bdev->bd_fsfreeze_count > 0)
18e9e5104fcd9 (Josef Bacik 2010-03-23 10:34:56 -0400 617) goto out;
4504230a71566 (Christoph Hellwig 2009-08-03 23:28:35 +0200 618)
040f04bd2e825 (Christoph Hellwig 2020-11-24 11:54:06 +0100 619) sb = bdev->bd_fsfreeze_sb;
4504230a71566 (Christoph Hellwig 2009-08-03 23:28:35 +0200 620) if (!sb)
18e9e5104fcd9 (Josef Bacik 2010-03-23 10:34:56 -0400 621) goto out;
4504230a71566 (Christoph Hellwig 2009-08-03 23:28:35 +0200 622)
48b6bca6b7b83 (Benjamin Marzinski 2014-11-13 20:42:03 -0600 623) if (sb->s_op->thaw_super)
48b6bca6b7b83 (Benjamin Marzinski 2014-11-13 20:42:03 -0600 624) error = sb->s_op->thaw_super(sb);
48b6bca6b7b83 (Benjamin Marzinski 2014-11-13 20:42:03 -0600 625) else
48b6bca6b7b83 (Benjamin Marzinski 2014-11-13 20:42:03 -0600 626) error = thaw_super(sb);
997198ba1ed69 (Pierre Morel 2016-10-04 10:53:40 +0200 627) if (error)
18e9e5104fcd9 (Josef Bacik 2010-03-23 10:34:56 -0400 628) bdev->bd_fsfreeze_count++;
04a6a536bc3fd (Satya Tangirala 2020-12-24 04:49:54 +0000 629) else
04a6a536bc3fd (Satya Tangirala 2020-12-24 04:49:54 +0000 630) bdev->bd_fsfreeze_sb = NULL;
18e9e5104fcd9 (Josef Bacik 2010-03-23 10:34:56 -0400 631) out:
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 632) mutex_unlock(&bdev->bd_fsfreeze_mutex);
997198ba1ed69 (Pierre Morel 2016-10-04 10:53:40 +0200 633) return error;
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 634) }
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 635) EXPORT_SYMBOL(thaw_bdev);
585d3bc06f4ca (Nicholas Piggin 2009-02-25 10:44:19 +0100 636)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 637) static int blkdev_writepage(struct page *page, struct writeback_control *wbc)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 638) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 639) return block_write_full_page(page, blkdev_get_block, wbc);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 640) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 641)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 642) static int blkdev_readpage(struct file * file, struct page * page)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 643) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 644) return block_read_full_page(page, blkdev_get_block);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 645) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 646)
d4388340ae0bc (Matthew Wilcox (Oracle) 2020-06-01 21:47:02 -0700 647) static void blkdev_readahead(struct readahead_control *rac)
447f05bb488bf (Akinobu Mita 2014-10-09 15:26:58 -0700 648) {
d4388340ae0bc (Matthew Wilcox (Oracle) 2020-06-01 21:47:02 -0700 649) mpage_readahead(rac, blkdev_get_block);
447f05bb488bf (Akinobu Mita 2014-10-09 15:26:58 -0700 650) }
447f05bb488bf (Akinobu Mita 2014-10-09 15:26:58 -0700 651)
6272b5a5868f3 (Nicholas Piggin 2007-10-16 01:25:04 -0700 652) static int blkdev_write_begin(struct file *file, struct address_space *mapping,
6272b5a5868f3 (Nicholas Piggin 2007-10-16 01:25:04 -0700 653) loff_t pos, unsigned len, unsigned flags,
6272b5a5868f3 (Nicholas Piggin 2007-10-16 01:25:04 -0700 654) struct page **pagep, void **fsdata)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 655) {
155130a4f7848 (Christoph Hellwig 2010-06-04 11:29:58 +0200 656) return block_write_begin(mapping, pos, len, flags, pagep,
155130a4f7848 (Christoph Hellwig 2010-06-04 11:29:58 +0200 657) blkdev_get_block);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 658) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 659)
6272b5a5868f3 (Nicholas Piggin 2007-10-16 01:25:04 -0700 660) static int blkdev_write_end(struct file *file, struct address_space *mapping,
6272b5a5868f3 (Nicholas Piggin 2007-10-16 01:25:04 -0700 661) loff_t pos, unsigned len, unsigned copied,
6272b5a5868f3 (Nicholas Piggin 2007-10-16 01:25:04 -0700 662) struct page *page, void *fsdata)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 663) {
6272b5a5868f3 (Nicholas Piggin 2007-10-16 01:25:04 -0700 664) int ret;
6272b5a5868f3 (Nicholas Piggin 2007-10-16 01:25:04 -0700 665) ret = block_write_end(file, mapping, pos, len, copied, page, fsdata);
6272b5a5868f3 (Nicholas Piggin 2007-10-16 01:25:04 -0700 666)
6272b5a5868f3 (Nicholas Piggin 2007-10-16 01:25:04 -0700 667) unlock_page(page);
09cbfeaf1a5a6 (Kirill A. Shutemov 2016-04-01 15:29:47 +0300 668) put_page(page);
6272b5a5868f3 (Nicholas Piggin 2007-10-16 01:25:04 -0700 669)
6272b5a5868f3 (Nicholas Piggin 2007-10-16 01:25:04 -0700 670) return ret;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 671) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 672)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 673) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 674) * private llseek:
496ad9aa8ef44 (Al Viro 2013-01-23 17:07:38 -0500 675) * for a block special file file_inode(file)->i_size is zero
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 676) * so we compute the size by hand (just as in block_read/write above)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 677) */
965c8e59cfcf8 (Andrew Morton 2012-12-17 15:59:39 -0800 678) static loff_t block_llseek(struct file *file, loff_t offset, int whence)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 679) {
4ebb16ca9a06a (Dan Williams 2015-10-28 07:48:19 +0900 680) struct inode *bd_inode = bdev_file_inode(file);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 681) loff_t retval;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 682)
5955102c9984f (Al Viro 2016-01-22 15:40:57 -0500 683) inode_lock(bd_inode);
5d48f3a2de568 (Al Viro 2013-06-23 21:34:45 +0400 684) retval = fixed_size_llseek(file, offset, whence, i_size_read(bd_inode));
5955102c9984f (Al Viro 2016-01-22 15:40:57 -0500 685) inode_unlock(bd_inode);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 686) return retval;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 687) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 688)
02c24a82187d5 (Josef Bacik 2011-07-16 20:44:56 -0400 689) int blkdev_fsync(struct file *filp, loff_t start, loff_t end, int datasync)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 690) {
4ebb16ca9a06a (Dan Williams 2015-10-28 07:48:19 +0900 691) struct inode *bd_inode = bdev_file_inode(filp);
b8af67e2681c6 (Anton Blanchard 2010-04-23 13:18:06 -0400 692) struct block_device *bdev = I_BDEV(bd_inode);
ab0a9735e0691 (Christoph Hellwig 2009-10-29 14:14:04 +0100 693) int error;
da5aa861bea09 (Rafael J. Wysocki 2011-08-02 02:17:48 +0200 694)
372cf243ea9a3 (Jeff Layton 2017-07-06 07:02:28 -0400 695) error = file_write_and_wait_range(filp, start, end);
da5aa861bea09 (Rafael J. Wysocki 2011-08-02 02:17:48 +0200 696) if (error)
da5aa861bea09 (Rafael J. Wysocki 2011-08-02 02:17:48 +0200 697) return error;
ab0a9735e0691 (Christoph Hellwig 2009-10-29 14:14:04 +0100 698)
b8af67e2681c6 (Anton Blanchard 2010-04-23 13:18:06 -0400 699) /*
b8af67e2681c6 (Anton Blanchard 2010-04-23 13:18:06 -0400 700) * There is no need to serialise calls to blkdev_issue_flush with
b8af67e2681c6 (Anton Blanchard 2010-04-23 13:18:06 -0400 701) * i_mutex and doing so causes performance issues with concurrent
b8af67e2681c6 (Anton Blanchard 2010-04-23 13:18:06 -0400 702) * O_SYNC writers to a block device.
b8af67e2681c6 (Anton Blanchard 2010-04-23 13:18:06 -0400 703) */
c6bf3f0e25f4c (Christoph Hellwig 2021-01-26 15:52:35 +0100 704) error = blkdev_issue_flush(bdev);
ab0a9735e0691 (Christoph Hellwig 2009-10-29 14:14:04 +0100 705) if (error == -EOPNOTSUPP)
ab0a9735e0691 (Christoph Hellwig 2009-10-29 14:14:04 +0100 706) error = 0;
b8af67e2681c6 (Anton Blanchard 2010-04-23 13:18:06 -0400 707)
ab0a9735e0691 (Christoph Hellwig 2009-10-29 14:14:04 +0100 708) return error;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 709) }
b1dd3b2843b3b (Andrew Morton 2010-04-06 14:35:00 -0700 710) EXPORT_SYMBOL(blkdev_fsync);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 711)
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 712) /**
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 713) * bdev_read_page() - Start reading a page from a block device
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 714) * @bdev: The device to read the page from
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 715) * @sector: The offset on the device to read the page to (need not be aligned)
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 716) * @page: The page to read
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 717) *
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 718) * On entry, the page should be locked. It will be unlocked when the page
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 719) * has been read. If the block driver implements rw_page synchronously,
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 720) * that will be true on exit from this function, but it need not be.
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 721) *
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 722) * Errors returned by this function are usually "soft", eg out of memory, or
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 723) * queue full; callers should try a different route to read this page rather
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 724) * than propagate an error back up the stack.
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 725) *
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 726) * Return: negative errno if an error occurs, 0 if submission was successful.
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 727) */
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 728) int bdev_read_page(struct block_device *bdev, sector_t sector,
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 729) struct page *page)
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 730) {
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 731) const struct block_device_operations *ops = bdev->bd_disk->fops;
2e6edc95382cc (Dan Williams 2015-11-19 13:29:28 -0800 732) int result = -EOPNOTSUPP;
2e6edc95382cc (Dan Williams 2015-11-19 13:29:28 -0800 733)
f68eb1e71a927 (Vishal Verma 2015-05-12 13:48:53 -0400 734) if (!ops->rw_page || bdev_get_integrity(bdev))
2e6edc95382cc (Dan Williams 2015-11-19 13:29:28 -0800 735) return result;
2e6edc95382cc (Dan Williams 2015-11-19 13:29:28 -0800 736)
e556f6ba10f0f (Christoph Hellwig 2020-06-26 10:01:56 +0200 737) result = blk_queue_enter(bdev->bd_disk->queue, 0);
2e6edc95382cc (Dan Williams 2015-11-19 13:29:28 -0800 738) if (result)
2e6edc95382cc (Dan Williams 2015-11-19 13:29:28 -0800 739) return result;
3f289dcb4b265 (Tejun Heo 2018-07-18 04:47:36 -0700 740) result = ops->rw_page(bdev, sector + get_start_sect(bdev), page,
3f289dcb4b265 (Tejun Heo 2018-07-18 04:47:36 -0700 741) REQ_OP_READ);
e556f6ba10f0f (Christoph Hellwig 2020-06-26 10:01:56 +0200 742) blk_queue_exit(bdev->bd_disk->queue);
2e6edc95382cc (Dan Williams 2015-11-19 13:29:28 -0800 743) return result;
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 744) }
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 745)
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 746) /**
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 747) * bdev_write_page() - Start writing a page to a block device
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 748) * @bdev: The device to write the page to
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 749) * @sector: The offset on the device to write the page to (need not be aligned)
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 750) * @page: The page to write
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 751) * @wbc: The writeback_control for the write
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 752) *
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 753) * On entry, the page should be locked and not currently under writeback.
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 754) * On exit, if the write started successfully, the page will be unlocked and
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 755) * under writeback. If the write failed already (eg the driver failed to
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 756) * queue the page to the device), the page will still be locked. If the
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 757) * caller is a ->writepage implementation, it will need to unlock the page.
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 758) *
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 759) * Errors returned by this function are usually "soft", eg out of memory, or
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 760) * queue full; callers should try a different route to write this page rather
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 761) * than propagate an error back up the stack.
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 762) *
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 763) * Return: negative errno if an error occurs, 0 if submission was successful.
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 764) */
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 765) int bdev_write_page(struct block_device *bdev, sector_t sector,
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 766) struct page *page, struct writeback_control *wbc)
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 767) {
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 768) int result;
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 769) const struct block_device_operations *ops = bdev->bd_disk->fops;
2e6edc95382cc (Dan Williams 2015-11-19 13:29:28 -0800 770)
f68eb1e71a927 (Vishal Verma 2015-05-12 13:48:53 -0400 771) if (!ops->rw_page || bdev_get_integrity(bdev))
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 772) return -EOPNOTSUPP;
e556f6ba10f0f (Christoph Hellwig 2020-06-26 10:01:56 +0200 773) result = blk_queue_enter(bdev->bd_disk->queue, 0);
2e6edc95382cc (Dan Williams 2015-11-19 13:29:28 -0800 774) if (result)
2e6edc95382cc (Dan Williams 2015-11-19 13:29:28 -0800 775) return result;
2e6edc95382cc (Dan Williams 2015-11-19 13:29:28 -0800 776)
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 777) set_page_writeback(page);
3f289dcb4b265 (Tejun Heo 2018-07-18 04:47:36 -0700 778) result = ops->rw_page(bdev, sector + get_start_sect(bdev), page,
3f289dcb4b265 (Tejun Heo 2018-07-18 04:47:36 -0700 779) REQ_OP_WRITE);
f892760aa66a2 (Matthew Wilcox 2017-10-13 15:58:15 -0700 780) if (result) {
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 781) end_page_writeback(page);
f892760aa66a2 (Matthew Wilcox 2017-10-13 15:58:15 -0700 782) } else {
f892760aa66a2 (Matthew Wilcox 2017-10-13 15:58:15 -0700 783) clean_page_buffers(page);
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 784) unlock_page(page);
f892760aa66a2 (Matthew Wilcox 2017-10-13 15:58:15 -0700 785) }
e556f6ba10f0f (Christoph Hellwig 2020-06-26 10:01:56 +0200 786) blk_queue_exit(bdev->bd_disk->queue);
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 787) return result;
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 788) }
47a191fd38ebd (Matthew Wilcox 2014-06-04 16:07:46 -0700 789)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 790) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 791) * pseudo-fs
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 792) */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 793)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 794) static __cacheline_aligned_in_smp DEFINE_SPINLOCK(bdev_lock);
e18b890bb0881 (Christoph Lameter 2006-12-06 20:33:20 -0800 795) static struct kmem_cache * bdev_cachep __read_mostly;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 796)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 797) static struct inode *bdev_alloc_inode(struct super_block *sb)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 798) {
e94b1766097d5 (Christoph Lameter 2006-12-06 20:33:17 -0800 799) struct bdev_inode *ei = kmem_cache_alloc(bdev_cachep, GFP_KERNEL);
2d2f6f1b47994 (Christoph Hellwig 2021-01-07 19:36:40 +0100 800)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 801) if (!ei)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 802) return NULL;
2d2f6f1b47994 (Christoph Hellwig 2021-01-07 19:36:40 +0100 803) memset(&ei->bdev, 0, sizeof(ei->bdev));
2d2f6f1b47994 (Christoph Hellwig 2021-01-07 19:36:40 +0100 804) ei->bdev.bd_bdi = &noop_backing_dev_info;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 805) return &ei->vfs_inode;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 806) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 807)
41149cb08aebd (Al Viro 2019-04-10 15:12:38 -0400 808) static void bdev_free_inode(struct inode *inode)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 809) {
15e3d2c5cd532 (Christoph Hellwig 2020-11-24 09:34:00 +0100 810) struct block_device *bdev = I_BDEV(inode);
15e3d2c5cd532 (Christoph Hellwig 2020-11-24 09:34:00 +0100 811)
15e3d2c5cd532 (Christoph Hellwig 2020-11-24 09:34:00 +0100 812) free_percpu(bdev->bd_stats);
231926dbf0f08 (Christoph Hellwig 2020-11-24 12:01:45 +0100 813) kfree(bdev->bd_meta_info);
15e3d2c5cd532 (Christoph Hellwig 2020-11-24 09:34:00 +0100 814)
fc553003e3619 (Christoph Hellwig 2021-07-22 09:53:54 +0200 815) if (!bdev_is_partition(bdev))
fc553003e3619 (Christoph Hellwig 2021-07-22 09:53:54 +0200 816) kfree(bdev->bd_disk);
41149cb08aebd (Al Viro 2019-04-10 15:12:38 -0400 817) kmem_cache_free(bdev_cachep, BDEV_I(inode));
fa0d7e3de6d6f (Nicholas Piggin 2011-01-07 17:49:49 +1100 818) }
fa0d7e3de6d6f (Nicholas Piggin 2011-01-07 17:49:49 +1100 819)
e6cb53827ed60 (Christoph Hellwig 2020-11-23 15:41:40 +0100 820) static void init_once(void *data)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 821) {
e6cb53827ed60 (Christoph Hellwig 2020-11-23 15:41:40 +0100 822) struct bdev_inode *ei = data;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 823)
a35afb830f8d7 (Christoph Lameter 2007-05-16 22:10:57 -0700 824) inode_init_once(&ei->vfs_inode);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 825) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 826)
b57922d97fd6f (Al Viro 2010-06-07 14:34:48 -0400 827) static void bdev_evict_inode(struct inode *inode)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 828) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 829) struct block_device *bdev = &BDEV_I(inode)->bdev;
91b0abe36a7b2 (Johannes Weiner 2014-04-03 14:47:49 -0700 830) truncate_inode_pages_final(&inode->i_data);
b57922d97fd6f (Al Viro 2010-06-07 14:34:48 -0400 831) invalidate_inode_buffers(inode); /* is it needed here? */
dbd5768f87ff6 (Jan Kara 2012-05-03 14:48:02 +0200 832) clear_inode(inode);
f759741d9d913 (Jan Kara 2017-03-23 01:37:00 +0100 833) /* Detach inode from wb early as bdi_put() may free bdi->wb */
f759741d9d913 (Jan Kara 2017-03-23 01:37:00 +0100 834) inode_detach_wb(inode);
a5a79d00017c9 (Jan Kara 2017-03-02 16:50:13 +0100 835) if (bdev->bd_bdi != &noop_backing_dev_info) {
b1d2dc5659b41 (Jan Kara 2017-02-02 15:56:52 +0100 836) bdi_put(bdev->bd_bdi);
a5a79d00017c9 (Jan Kara 2017-03-02 16:50:13 +0100 837) bdev->bd_bdi = &noop_backing_dev_info;
a5a79d00017c9 (Jan Kara 2017-03-02 16:50:13 +0100 838) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 839) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 840)
ee9b6d61a2a43 (Josef 'Jeff' Sipek 2007-02-12 00:55:41 -0800 841) static const struct super_operations bdev_sops = {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 842) .statfs = simple_statfs,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 843) .alloc_inode = bdev_alloc_inode,
41149cb08aebd (Al Viro 2019-04-10 15:12:38 -0400 844) .free_inode = bdev_free_inode,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 845) .drop_inode = generic_delete_inode,
b57922d97fd6f (Al Viro 2010-06-07 14:34:48 -0400 846) .evict_inode = bdev_evict_inode,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 847) };
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 848)
9030d16eb8adc (David Howells 2019-03-25 16:38:23 +0000 849) static int bd_init_fs_context(struct fs_context *fc)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 850) {
9030d16eb8adc (David Howells 2019-03-25 16:38:23 +0000 851) struct pseudo_fs_context *ctx = init_pseudo(fc, BDEVFS_MAGIC);
9030d16eb8adc (David Howells 2019-03-25 16:38:23 +0000 852) if (!ctx)
9030d16eb8adc (David Howells 2019-03-25 16:38:23 +0000 853) return -ENOMEM;
9030d16eb8adc (David Howells 2019-03-25 16:38:23 +0000 854) fc->s_iflags |= SB_I_CGROUPWB;
9030d16eb8adc (David Howells 2019-03-25 16:38:23 +0000 855) ctx->ops = &bdev_sops;
9030d16eb8adc (David Howells 2019-03-25 16:38:23 +0000 856) return 0;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 857) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 858)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 859) static struct file_system_type bd_type = {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 860) .name = "bdev",
9030d16eb8adc (David Howells 2019-03-25 16:38:23 +0000 861) .init_fs_context = bd_init_fs_context,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 862) .kill_sb = kill_anon_super,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 863) };
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 864)
a212b105b07d7 (Tejun Heo 2015-05-22 17:13:33 -0400 865) struct super_block *blockdev_superblock __read_mostly;
a212b105b07d7 (Tejun Heo 2015-05-22 17:13:33 -0400 866) EXPORT_SYMBOL_GPL(blockdev_superblock);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 867)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 868) void __init bdev_cache_init(void)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 869) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 870) int err;
ace8577aeb438 (Sergey Senozhatsky 2012-01-10 02:43:59 +0300 871) static struct vfsmount *bd_mnt;
c2acf7b908217 (Denis ChengRq 2008-12-01 14:34:56 -0800 872)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 873) bdev_cachep = kmem_cache_create("bdev_cache", sizeof(struct bdev_inode),
fffb60f93ce58 (Paul Jackson 2006-03-24 03:16:06 -0800 874) 0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT|
5d097056c9a01 (Vladimir Davydov 2016-01-14 15:18:21 -0800 875) SLAB_MEM_SPREAD|SLAB_ACCOUNT|SLAB_PANIC),
20c2df83d25c6 (Paul Mundt 2007-07-20 10:11:58 +0900 876) init_once);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 877) err = register_filesystem(&bd_type);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 878) if (err)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 879) panic("Cannot register bdev pseudo-fs");
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 880) bd_mnt = kern_mount(&bd_type);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 881) if (IS_ERR(bd_mnt))
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 882) panic("Cannot create bdev pseudo-fs");
ace8577aeb438 (Sergey Senozhatsky 2012-01-10 02:43:59 +0300 883) blockdev_superblock = bd_mnt->mnt_sb; /* For writeback */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 884) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 885)
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 886) struct block_device *bdev_alloc(struct gendisk *disk, u8 partno)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 887) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 888) struct block_device *bdev;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 889) struct inode *inode;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 890)
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 891) inode = new_inode(blockdev_superblock);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 892) if (!inode)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 893) return NULL;
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 894) inode->i_mode = S_IFBLK;
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 895) inode->i_rdev = 0;
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 896) inode->i_data.a_ops = &def_blk_aops;
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 897) mapping_set_gfp_mask(&inode->i_data, GFP_USER);
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 898)
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 899) bdev = I_BDEV(inode);
e6cb53827ed60 (Christoph Hellwig 2020-11-23 15:41:40 +0100 900) mutex_init(&bdev->bd_mutex);
e6cb53827ed60 (Christoph Hellwig 2020-11-23 15:41:40 +0100 901) mutex_init(&bdev->bd_fsfreeze_mutex);
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 902) spin_lock_init(&bdev->bd_size_lock);
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 903) bdev->bd_disk = disk;
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 904) bdev->bd_partno = partno;
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 905) bdev->bd_inode = inode;
e6cb53827ed60 (Christoph Hellwig 2020-11-23 15:41:40 +0100 906) #ifdef CONFIG_SYSFS
e6cb53827ed60 (Christoph Hellwig 2020-11-23 15:41:40 +0100 907) INIT_LIST_HEAD(&bdev->bd_holder_disks);
e6cb53827ed60 (Christoph Hellwig 2020-11-23 15:41:40 +0100 908) #endif
15e3d2c5cd532 (Christoph Hellwig 2020-11-24 09:34:00 +0100 909) bdev->bd_stats = alloc_percpu(struct disk_stats);
15e3d2c5cd532 (Christoph Hellwig 2020-11-24 09:34:00 +0100 910) if (!bdev->bd_stats) {
15e3d2c5cd532 (Christoph Hellwig 2020-11-24 09:34:00 +0100 911) iput(inode);
15e3d2c5cd532 (Christoph Hellwig 2020-11-24 09:34:00 +0100 912) return NULL;
15e3d2c5cd532 (Christoph Hellwig 2020-11-24 09:34:00 +0100 913) }
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 914) return bdev;
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 915) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 916)
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 917) void bdev_add(struct block_device *bdev, dev_t dev)
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 918) {
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 919) bdev->bd_dev = dev;
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 920) bdev->bd_inode->i_rdev = dev;
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 921) bdev->bd_inode->i_ino = dev;
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 922) insert_inode_hash(bdev->bd_inode);
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 923) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 924)
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 925) static struct block_device *bdget(dev_t dev)
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 926) {
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 927) struct inode *inode;
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 928)
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 929) inode = ilookup(blockdev_superblock, dev);
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 930) if (!inode)
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 931) return NULL;
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 932) return &BDEV_I(inode)->bdev;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 933) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 934)
dddac6a7b445d (Alan Jenkins 2009-07-29 21:07:55 +0200 935) /**
dddac6a7b445d (Alan Jenkins 2009-07-29 21:07:55 +0200 936) * bdgrab -- Grab a reference to an already referenced block device
dddac6a7b445d (Alan Jenkins 2009-07-29 21:07:55 +0200 937) * @bdev: Block device to grab a reference to.
3a4174e68684e (Christoph Hellwig 2020-11-27 16:45:27 +0100 938) *
3a4174e68684e (Christoph Hellwig 2020-11-27 16:45:27 +0100 939) * Returns the block_device with an additional reference when successful,
3a4174e68684e (Christoph Hellwig 2020-11-27 16:45:27 +0100 940) * or NULL if the inode is already beeing freed.
dddac6a7b445d (Alan Jenkins 2009-07-29 21:07:55 +0200 941) */
dddac6a7b445d (Alan Jenkins 2009-07-29 21:07:55 +0200 942) struct block_device *bdgrab(struct block_device *bdev)
dddac6a7b445d (Alan Jenkins 2009-07-29 21:07:55 +0200 943) {
3a4174e68684e (Christoph Hellwig 2020-11-27 16:45:27 +0100 944) if (!igrab(bdev->bd_inode))
3a4174e68684e (Christoph Hellwig 2020-11-27 16:45:27 +0100 945) return NULL;
dddac6a7b445d (Alan Jenkins 2009-07-29 21:07:55 +0200 946) return bdev;
dddac6a7b445d (Alan Jenkins 2009-07-29 21:07:55 +0200 947) }
c1681bf8a7b1b (Anatol Pomozov 2013-04-01 09:47:56 -0700 948) EXPORT_SYMBOL(bdgrab);
dddac6a7b445d (Alan Jenkins 2009-07-29 21:07:55 +0200 949)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 950) long nr_blockdev_pages(void)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 951) {
1008fe6dc36dd (Christoph Hellwig 2020-06-26 10:01:58 +0200 952) struct inode *inode;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 953) long ret = 0;
1008fe6dc36dd (Christoph Hellwig 2020-06-26 10:01:58 +0200 954)
1008fe6dc36dd (Christoph Hellwig 2020-06-26 10:01:58 +0200 955) spin_lock(&blockdev_superblock->s_inode_list_lock);
1008fe6dc36dd (Christoph Hellwig 2020-06-26 10:01:58 +0200 956) list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list)
1008fe6dc36dd (Christoph Hellwig 2020-06-26 10:01:58 +0200 957) ret += inode->i_mapping->nrpages;
1008fe6dc36dd (Christoph Hellwig 2020-06-26 10:01:58 +0200 958) spin_unlock(&blockdev_superblock->s_inode_list_lock);
1008fe6dc36dd (Christoph Hellwig 2020-06-26 10:01:58 +0200 959)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 960) return ret;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 961) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 962)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 963) void bdput(struct block_device *bdev)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 964) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 965) iput(bdev->bd_inode);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 966) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 967) EXPORT_SYMBOL(bdput);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 968)
1a3cbbc5a5e8a (Tejun Heo 2010-04-07 18:52:29 +0900 969) /**
1a3cbbc5a5e8a (Tejun Heo 2010-04-07 18:52:29 +0900 970) * bd_may_claim - test whether a block device can be claimed
1a3cbbc5a5e8a (Tejun Heo 2010-04-07 18:52:29 +0900 971) * @bdev: block device of interest
1a3cbbc5a5e8a (Tejun Heo 2010-04-07 18:52:29 +0900 972) * @whole: whole block device containing @bdev, may equal @bdev
1a3cbbc5a5e8a (Tejun Heo 2010-04-07 18:52:29 +0900 973) * @holder: holder trying to claim @bdev
1a3cbbc5a5e8a (Tejun Heo 2010-04-07 18:52:29 +0900 974) *
25985edcedea6 (Lucas De Marchi 2011-03-30 22:57:33 -0300 975) * Test whether @bdev can be claimed by @holder.
1a3cbbc5a5e8a (Tejun Heo 2010-04-07 18:52:29 +0900 976) *
1a3cbbc5a5e8a (Tejun Heo 2010-04-07 18:52:29 +0900 977) * CONTEXT:
1a3cbbc5a5e8a (Tejun Heo 2010-04-07 18:52:29 +0900 978) * spin_lock(&bdev_lock).
1a3cbbc5a5e8a (Tejun Heo 2010-04-07 18:52:29 +0900 979) *
1a3cbbc5a5e8a (Tejun Heo 2010-04-07 18:52:29 +0900 980) * RETURNS:
1a3cbbc5a5e8a (Tejun Heo 2010-04-07 18:52:29 +0900 981) * %true if @bdev can be claimed, %false otherwise.
1a3cbbc5a5e8a (Tejun Heo 2010-04-07 18:52:29 +0900 982) */
1a3cbbc5a5e8a (Tejun Heo 2010-04-07 18:52:29 +0900 983) static bool bd_may_claim(struct block_device *bdev, struct block_device *whole,
1a3cbbc5a5e8a (Tejun Heo 2010-04-07 18:52:29 +0900 984) void *holder)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 985) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 986) if (bdev->bd_holder == holder)
1a3cbbc5a5e8a (Tejun Heo 2010-04-07 18:52:29 +0900 987) return true; /* already a holder */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 988) else if (bdev->bd_holder != NULL)
1a3cbbc5a5e8a (Tejun Heo 2010-04-07 18:52:29 +0900 989) return false; /* held by someone else */
bcc7f5b4bee8e (NeilBrown 2016-12-12 08:21:51 -0700 990) else if (whole == bdev)
1a3cbbc5a5e8a (Tejun Heo 2010-04-07 18:52:29 +0900 991) return true; /* is a whole device which isn't held */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 992)
e525fd89d380c (Tejun Heo 2010-11-13 11:55:17 +0100 993) else if (whole->bd_holder == bd_may_claim)
1a3cbbc5a5e8a (Tejun Heo 2010-04-07 18:52:29 +0900 994) return true; /* is a partition of a device that is being partitioned */
1a3cbbc5a5e8a (Tejun Heo 2010-04-07 18:52:29 +0900 995) else if (whole->bd_holder != NULL)
1a3cbbc5a5e8a (Tejun Heo 2010-04-07 18:52:29 +0900 996) return false; /* is a partition of a held device */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 997) else
1a3cbbc5a5e8a (Tejun Heo 2010-04-07 18:52:29 +0900 998) return true; /* is a partition of an un-held device */
1a3cbbc5a5e8a (Tejun Heo 2010-04-07 18:52:29 +0900 999) }
1a3cbbc5a5e8a (Tejun Heo 2010-04-07 18:52:29 +0900 1000)
6b4517a7913a0 (Tejun Heo 2010-04-07 18:53:59 +0900 1001) /**
58e46ed9cc634 (Christoph Hellwig 2020-07-16 16:33:08 +0200 1002) * bd_prepare_to_claim - claim a block device
6b4517a7913a0 (Tejun Heo 2010-04-07 18:53:59 +0900 1003) * @bdev: block device of interest
6b4517a7913a0 (Tejun Heo 2010-04-07 18:53:59 +0900 1004) * @holder: holder trying to claim @bdev
6b4517a7913a0 (Tejun Heo 2010-04-07 18:53:59 +0900 1005) *
58e46ed9cc634 (Christoph Hellwig 2020-07-16 16:33:08 +0200 1006) * Claim @bdev. This function fails if @bdev is already claimed by another
58e46ed9cc634 (Christoph Hellwig 2020-07-16 16:33:08 +0200 1007) * holder and waits if another claiming is in progress. return, the caller
58e46ed9cc634 (Christoph Hellwig 2020-07-16 16:33:08 +0200 1008) * has ownership of bd_claiming and bd_holder[s].
6b4517a7913a0 (Tejun Heo 2010-04-07 18:53:59 +0900 1009) *
6b4517a7913a0 (Tejun Heo 2010-04-07 18:53:59 +0900 1010) * RETURNS:
6b4517a7913a0 (Tejun Heo 2010-04-07 18:53:59 +0900 1011) * 0 if @bdev can be claimed, -EBUSY otherwise.
6b4517a7913a0 (Tejun Heo 2010-04-07 18:53:59 +0900 1012) */
37c3fc9abb25c (Christoph Hellwig 2020-11-25 21:20:08 +0100 1013) int bd_prepare_to_claim(struct block_device *bdev, void *holder)
6b4517a7913a0 (Tejun Heo 2010-04-07 18:53:59 +0900 1014) {
37c3fc9abb25c (Christoph Hellwig 2020-11-25 21:20:08 +0100 1015) struct block_device *whole = bdev_whole(bdev);
37c3fc9abb25c (Christoph Hellwig 2020-11-25 21:20:08 +0100 1016)
37c3fc9abb25c (Christoph Hellwig 2020-11-25 21:20:08 +0100 1017) if (WARN_ON_ONCE(!holder))
37c3fc9abb25c (Christoph Hellwig 2020-11-25 21:20:08 +0100 1018) return -EINVAL;
6b4517a7913a0 (Tejun Heo 2010-04-07 18:53:59 +0900 1019) retry:
58e46ed9cc634 (Christoph Hellwig 2020-07-16 16:33:08 +0200 1020) spin_lock(&bdev_lock);
6b4517a7913a0 (Tejun Heo 2010-04-07 18:53:59 +0900 1021) /* if someone else claimed, fail */
58e46ed9cc634 (Christoph Hellwig 2020-07-16 16:33:08 +0200 1022) if (!bd_may_claim(bdev, whole, holder)) {
58e46ed9cc634 (Christoph Hellwig 2020-07-16 16:33:08 +0200 1023) spin_unlock(&bdev_lock);
6b4517a7913a0 (Tejun Heo 2010-04-07 18:53:59 +0900 1024) return -EBUSY;
58e46ed9cc634 (Christoph Hellwig 2020-07-16 16:33:08 +0200 1025) }
6b4517a7913a0 (Tejun Heo 2010-04-07 18:53:59 +0900 1026)
e75aa85892b2e (Tejun Heo 2010-08-04 17:59:39 +0200 1027) /* if claiming is already in progress, wait for it to finish */
e75aa85892b2e (Tejun Heo 2010-08-04 17:59:39 +0200 1028) if (whole->bd_claiming) {
6b4517a7913a0 (Tejun Heo 2010-04-07 18:53:59 +0900 1029) wait_queue_head_t *wq = bit_waitqueue(&whole->bd_claiming, 0);
6b4517a7913a0 (Tejun Heo 2010-04-07 18:53:59 +0900 1030) DEFINE_WAIT(wait);
6b4517a7913a0 (Tejun Heo 2010-04-07 18:53:59 +0900 1031)
6b4517a7913a0 (Tejun Heo 2010-04-07 18:53:59 +0900 1032) prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE);
6b4517a7913a0 (Tejun Heo 2010-04-07 18:53:59 +0900 1033) spin_unlock(&bdev_lock);
6b4517a7913a0 (Tejun Heo 2010-04-07 18:53:59 +0900 1034) schedule();
6b4517a7913a0 (Tejun Heo 2010-04-07 18:53:59 +0900 1035) finish_wait(wq, &wait);
6b4517a7913a0 (Tejun Heo 2010-04-07 18:53:59 +0900 1036) goto retry;
6b4517a7913a0 (Tejun Heo 2010-04-07 18:53:59 +0900 1037) }
6b4517a7913a0 (Tejun Heo 2010-04-07 18:53:59 +0900 1038)
6b4517a7913a0 (Tejun Heo 2010-04-07 18:53:59 +0900 1039) /* yay, all mine */
58e46ed9cc634 (Christoph Hellwig 2020-07-16 16:33:08 +0200 1040) whole->bd_claiming = holder;
58e46ed9cc634 (Christoph Hellwig 2020-07-16 16:33:08 +0200 1041) spin_unlock(&bdev_lock);
6b4517a7913a0 (Tejun Heo 2010-04-07 18:53:59 +0900 1042) return 0;
6b4517a7913a0 (Tejun Heo 2010-04-07 18:53:59 +0900 1043) }
ecbe6bc0003bf (Christoph Hellwig 2020-07-16 16:33:09 +0200 1044) EXPORT_SYMBOL_GPL(bd_prepare_to_claim); /* only for the loop driver */
6b4517a7913a0 (Tejun Heo 2010-04-07 18:53:59 +0900 1045)
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1046) static void bd_clear_claiming(struct block_device *whole, void *holder)
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1047) {
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1048) lockdep_assert_held(&bdev_lock);
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1049) /* tell others that we're done */
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1050) BUG_ON(whole->bd_claiming != holder);
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1051) whole->bd_claiming = NULL;
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1052) wake_up_bit(&whole->bd_claiming, 0);
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1053) }
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1054)
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1055) /**
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1056) * bd_finish_claiming - finish claiming of a block device
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1057) * @bdev: block device of interest
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1058) * @holder: holder that has claimed @bdev
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1059) *
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1060) * Finish exclusive open of a block device. Mark the device as exlusively
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1061) * open by the holder and wake up all waiters for exclusive open to finish.
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1062) */
37c3fc9abb25c (Christoph Hellwig 2020-11-25 21:20:08 +0100 1063) static void bd_finish_claiming(struct block_device *bdev, void *holder)
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1064) {
37c3fc9abb25c (Christoph Hellwig 2020-11-25 21:20:08 +0100 1065) struct block_device *whole = bdev_whole(bdev);
37c3fc9abb25c (Christoph Hellwig 2020-11-25 21:20:08 +0100 1066)
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1067) spin_lock(&bdev_lock);
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1068) BUG_ON(!bd_may_claim(bdev, whole, holder));
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1069) /*
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1070) * Note that for a whole device bd_holders will be incremented twice,
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1071) * and bd_holder will be set to bd_may_claim before being set to holder
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1072) */
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1073) whole->bd_holders++;
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1074) whole->bd_holder = bd_may_claim;
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1075) bdev->bd_holders++;
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1076) bdev->bd_holder = holder;
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1077) bd_clear_claiming(whole, holder);
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1078) spin_unlock(&bdev_lock);
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1079) }
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1080)
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1081) /**
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1082) * bd_abort_claiming - abort claiming of a block device
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1083) * @bdev: block device of interest
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1084) * @holder: holder that has claimed @bdev
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1085) *
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1086) * Abort claiming of a block device when the exclusive open failed. This can be
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1087) * also used when exclusive open is not actually desired and we just needed
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1088) * to block other exclusive openers for a while.
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1089) */
37c3fc9abb25c (Christoph Hellwig 2020-11-25 21:20:08 +0100 1090) void bd_abort_claiming(struct block_device *bdev, void *holder)
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1091) {
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1092) spin_lock(&bdev_lock);
37c3fc9abb25c (Christoph Hellwig 2020-11-25 21:20:08 +0100 1093) bd_clear_claiming(bdev_whole(bdev), holder);
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1094) spin_unlock(&bdev_lock);
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1095) }
89e524c04fa96 (Jan Kara 2019-07-30 13:10:14 +0200 1096) EXPORT_SYMBOL(bd_abort_claiming);
6b4517a7913a0 (Tejun Heo 2010-04-07 18:53:59 +0900 1097)
641dc636b0475 (Jun'ichi Nomura 2006-03-27 01:17:57 -0800 1098) #ifdef CONFIG_SYSFS
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1099) struct bd_holder_disk {
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1100) struct list_head list;
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1101) struct gendisk *disk;
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1102) int refcnt;
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1103) };
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1104)
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1105) static struct bd_holder_disk *bd_find_holder_disk(struct block_device *bdev,
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1106) struct gendisk *disk)
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1107) {
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1108) struct bd_holder_disk *holder;
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1109)
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1110) list_for_each_entry(holder, &bdev->bd_holder_disks, list)
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1111) if (holder->disk == disk)
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1112) return holder;
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1113) return NULL;
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1114) }
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1115)
4d7dd8fd95578 (Andrew Morton 2006-09-29 01:58:56 -0700 1116) static int add_symlink(struct kobject *from, struct kobject *to)
641dc636b0475 (Jun'ichi Nomura 2006-03-27 01:17:57 -0800 1117) {
4d7dd8fd95578 (Andrew Morton 2006-09-29 01:58:56 -0700 1118) return sysfs_create_link(from, to, kobject_name(to));
641dc636b0475 (Jun'ichi Nomura 2006-03-27 01:17:57 -0800 1119) }
641dc636b0475 (Jun'ichi Nomura 2006-03-27 01:17:57 -0800 1120)
641dc636b0475 (Jun'ichi Nomura 2006-03-27 01:17:57 -0800 1121) static void del_symlink(struct kobject *from, struct kobject *to)
641dc636b0475 (Jun'ichi Nomura 2006-03-27 01:17:57 -0800 1122) {
641dc636b0475 (Jun'ichi Nomura 2006-03-27 01:17:57 -0800 1123) sysfs_remove_link(from, kobject_name(to));
641dc636b0475 (Jun'ichi Nomura 2006-03-27 01:17:57 -0800 1124) }
641dc636b0475 (Jun'ichi Nomura 2006-03-27 01:17:57 -0800 1125)
df6c0cd9a872e (Jun'ichi Nomura 2006-10-30 16:23:56 -0500 1126) /**
e09b457bdb7e8 (Tejun Heo 2010-11-13 11:55:17 +0100 1127) * bd_link_disk_holder - create symlinks between holding disk and slave bdev
e09b457bdb7e8 (Tejun Heo 2010-11-13 11:55:17 +0100 1128) * @bdev: the claimed slave bdev
e09b457bdb7e8 (Tejun Heo 2010-11-13 11:55:17 +0100 1129) * @disk: the holding disk
df6c0cd9a872e (Jun'ichi Nomura 2006-10-30 16:23:56 -0500 1130) *
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1131) * DON'T USE THIS UNLESS YOU'RE ALREADY USING IT.
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1132) *
e09b457bdb7e8 (Tejun Heo 2010-11-13 11:55:17 +0100 1133) * This functions creates the following sysfs symlinks.
641dc636b0475 (Jun'ichi Nomura 2006-03-27 01:17:57 -0800 1134) *
e09b457bdb7e8 (Tejun Heo 2010-11-13 11:55:17 +0100 1135) * - from "slaves" directory of the holder @disk to the claimed @bdev
e09b457bdb7e8 (Tejun Heo 2010-11-13 11:55:17 +0100 1136) * - from "holders" directory of the @bdev to the holder @disk
641dc636b0475 (Jun'ichi Nomura 2006-03-27 01:17:57 -0800 1137) *
e09b457bdb7e8 (Tejun Heo 2010-11-13 11:55:17 +0100 1138) * For example, if /dev/dm-0 maps to /dev/sda and disk for dm-0 is
e09b457bdb7e8 (Tejun Heo 2010-11-13 11:55:17 +0100 1139) * passed to bd_link_disk_holder(), then:
641dc636b0475 (Jun'ichi Nomura 2006-03-27 01:17:57 -0800 1140) *
e09b457bdb7e8 (Tejun Heo 2010-11-13 11:55:17 +0100 1141) * /sys/block/dm-0/slaves/sda --> /sys/block/sda
e09b457bdb7e8 (Tejun Heo 2010-11-13 11:55:17 +0100 1142) * /sys/block/sda/holders/dm-0 --> /sys/block/dm-0
641dc636b0475 (Jun'ichi Nomura 2006-03-27 01:17:57 -0800 1143) *
e09b457bdb7e8 (Tejun Heo 2010-11-13 11:55:17 +0100 1144) * The caller must have claimed @bdev before calling this function and
e09b457bdb7e8 (Tejun Heo 2010-11-13 11:55:17 +0100 1145) * ensure that both @bdev and @disk are valid during the creation and
e09b457bdb7e8 (Tejun Heo 2010-11-13 11:55:17 +0100 1146) * lifetime of these symlinks.
641dc636b0475 (Jun'ichi Nomura 2006-03-27 01:17:57 -0800 1147) *
e09b457bdb7e8 (Tejun Heo 2010-11-13 11:55:17 +0100 1148) * CONTEXT:
e09b457bdb7e8 (Tejun Heo 2010-11-13 11:55:17 +0100 1149) * Might sleep.
641dc636b0475 (Jun'ichi Nomura 2006-03-27 01:17:57 -0800 1150) *
e09b457bdb7e8 (Tejun Heo 2010-11-13 11:55:17 +0100 1151) * RETURNS:
e09b457bdb7e8 (Tejun Heo 2010-11-13 11:55:17 +0100 1152) * 0 on success, -errno on failure.
641dc636b0475 (Jun'ichi Nomura 2006-03-27 01:17:57 -0800 1153) */
e09b457bdb7e8 (Tejun Heo 2010-11-13 11:55:17 +0100 1154) int bd_link_disk_holder(struct block_device *bdev, struct gendisk *disk)
641dc636b0475 (Jun'ichi Nomura 2006-03-27 01:17:57 -0800 1155) {
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1156) struct bd_holder_disk *holder;
e09b457bdb7e8 (Tejun Heo 2010-11-13 11:55:17 +0100 1157) int ret = 0;
641dc636b0475 (Jun'ichi Nomura 2006-03-27 01:17:57 -0800 1158)
2e7b651df113c (Peter Zijlstra 2006-12-08 02:36:13 -0800 1159) mutex_lock(&bdev->bd_mutex);
df6c0cd9a872e (Jun'ichi Nomura 2006-10-30 16:23:56 -0500 1160)
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1161) WARN_ON_ONCE(!bdev->bd_holder);
4e91672c76319 (Johannes Weiner 2007-07-15 23:41:25 -0700 1162)
e09b457bdb7e8 (Tejun Heo 2010-11-13 11:55:17 +0100 1163) /* FIXME: remove the following once add_disk() handles errors */
1bdd5ae0251d6 (Christoph Hellwig 2020-11-23 19:00:13 +0100 1164) if (WARN_ON(!disk->slave_dir || !bdev->bd_holder_dir))
e09b457bdb7e8 (Tejun Heo 2010-11-13 11:55:17 +0100 1165) goto out_unlock;
4e91672c76319 (Johannes Weiner 2007-07-15 23:41:25 -0700 1166)
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1167) holder = bd_find_holder_disk(bdev, disk);
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1168) if (holder) {
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1169) holder->refcnt++;
e09b457bdb7e8 (Tejun Heo 2010-11-13 11:55:17 +0100 1170) goto out_unlock;
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1171) }
641dc636b0475 (Jun'ichi Nomura 2006-03-27 01:17:57 -0800 1172)
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1173) holder = kzalloc(sizeof(*holder), GFP_KERNEL);
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1174) if (!holder) {
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1175) ret = -ENOMEM;
e09b457bdb7e8 (Tejun Heo 2010-11-13 11:55:17 +0100 1176) goto out_unlock;
e09b457bdb7e8 (Tejun Heo 2010-11-13 11:55:17 +0100 1177) }
641dc636b0475 (Jun'ichi Nomura 2006-03-27 01:17:57 -0800 1178)
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1179) INIT_LIST_HEAD(&holder->list);
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1180) holder->disk = disk;
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1181) holder->refcnt = 1;
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1182)
8d65269fe8065 (Christoph Hellwig 2020-11-17 08:18:55 +0100 1183) ret = add_symlink(disk->slave_dir, bdev_kobj(bdev));
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1184) if (ret)
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1185) goto out_free;
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1186)
1bdd5ae0251d6 (Christoph Hellwig 2020-11-23 19:00:13 +0100 1187) ret = add_symlink(bdev->bd_holder_dir, &disk_to_dev(disk)->kobj);
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1188) if (ret)
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1189) goto out_del;
e7407d1619713 (Tejun Heo 2011-02-24 09:56:32 +0100 1190) /*
e7407d1619713 (Tejun Heo 2011-02-24 09:56:32 +0100 1191) * bdev could be deleted beneath us which would implicitly destroy
e7407d1619713 (Tejun Heo 2011-02-24 09:56:32 +0100 1192) * the holder directory. Hold on to it.
e7407d1619713 (Tejun Heo 2011-02-24 09:56:32 +0100 1193) */
1bdd5ae0251d6 (Christoph Hellwig 2020-11-23 19:00:13 +0100 1194) kobject_get(bdev->bd_holder_dir);
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1195)
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1196) list_add(&holder->list, &bdev->bd_holder_disks);
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1197) goto out_unlock;
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1198)
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1199) out_del:
8d65269fe8065 (Christoph Hellwig 2020-11-17 08:18:55 +0100 1200) del_symlink(disk->slave_dir, bdev_kobj(bdev));
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1201) out_free:
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1202) kfree(holder);
e09b457bdb7e8 (Tejun Heo 2010-11-13 11:55:17 +0100 1203) out_unlock:
b4cf1b72eec0e (Jun'ichi Nomura 2006-03-27 01:18:00 -0800 1204) mutex_unlock(&bdev->bd_mutex);
e09b457bdb7e8 (Tejun Heo 2010-11-13 11:55:17 +0100 1205) return ret;
641dc636b0475 (Jun'ichi Nomura 2006-03-27 01:17:57 -0800 1206) }
e09b457bdb7e8 (Tejun Heo 2010-11-13 11:55:17 +0100 1207) EXPORT_SYMBOL_GPL(bd_link_disk_holder);
641dc636b0475 (Jun'ichi Nomura 2006-03-27 01:17:57 -0800 1208)
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1209) /**
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1210) * bd_unlink_disk_holder - destroy symlinks created by bd_link_disk_holder()
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1211) * @bdev: the calimed slave bdev
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1212) * @disk: the holding disk
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1213) *
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1214) * DON'T USE THIS UNLESS YOU'RE ALREADY USING IT.
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1215) *
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1216) * CONTEXT:
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1217) * Might sleep.
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1218) */
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1219) void bd_unlink_disk_holder(struct block_device *bdev, struct gendisk *disk)
641dc636b0475 (Jun'ichi Nomura 2006-03-27 01:17:57 -0800 1220) {
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1221) struct bd_holder_disk *holder;
641dc636b0475 (Jun'ichi Nomura 2006-03-27 01:17:57 -0800 1222)
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1223) mutex_lock(&bdev->bd_mutex);
641dc636b0475 (Jun'ichi Nomura 2006-03-27 01:17:57 -0800 1224)
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1225) holder = bd_find_holder_disk(bdev, disk);
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1226)
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1227) if (!WARN_ON_ONCE(holder == NULL) && !--holder->refcnt) {
8d65269fe8065 (Christoph Hellwig 2020-11-17 08:18:55 +0100 1228) del_symlink(disk->slave_dir, bdev_kobj(bdev));
1bdd5ae0251d6 (Christoph Hellwig 2020-11-23 19:00:13 +0100 1229) del_symlink(bdev->bd_holder_dir, &disk_to_dev(disk)->kobj);
1bdd5ae0251d6 (Christoph Hellwig 2020-11-23 19:00:13 +0100 1230) kobject_put(bdev->bd_holder_dir);
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1231) list_del_init(&holder->list);
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1232) kfree(holder);
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1233) }
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1234)
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1235) mutex_unlock(&bdev->bd_mutex);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1236) }
49731baa41df4 (Tejun Heo 2011-01-14 18:43:57 +0100 1237) EXPORT_SYMBOL_GPL(bd_unlink_disk_holder);
641dc636b0475 (Jun'ichi Nomura 2006-03-27 01:17:57 -0800 1238) #endif
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1239)
4385bab128911 (Al Viro 2013-05-05 22:11:03 -0400 1240) static void __blkdev_put(struct block_device *bdev, fmode_t mode, int for_part);
37be41241f431 (NeilBrown 2006-12-08 02:36:16 -0800 1241)
142fe8f4bb169 (Christoph Hellwig 2019-11-14 15:34:35 +0100 1242) int bdev_disk_changed(struct block_device *bdev, bool invalidate)
a1548b674403c (Christoph Hellwig 2019-11-14 15:34:34 +0100 1243) {
142fe8f4bb169 (Christoph Hellwig 2019-11-14 15:34:35 +0100 1244) struct gendisk *disk = bdev->bd_disk;
d173b65aa75fe (Christoph Hellwig 2021-04-08 21:41:40 +0200 1245) int ret = 0;
a1548b674403c (Christoph Hellwig 2019-11-14 15:34:34 +0100 1246)
f0b870df80bc7 (Christoph Hellwig 2019-11-14 15:34:36 +0100 1247) lockdep_assert_held(&bdev->bd_mutex);
f0b870df80bc7 (Christoph Hellwig 2019-11-14 15:34:36 +0100 1248)
bc6a385132601 (Gulam Mohamed 2021-05-14 15:18:42 +0200 1249) if (!(disk->flags & GENHD_FL_UP))
bc6a385132601 (Gulam Mohamed 2021-05-14 15:18:42 +0200 1250) return -ENXIO;
bc6a385132601 (Gulam Mohamed 2021-05-14 15:18:42 +0200 1251)
a1548b674403c (Christoph Hellwig 2019-11-14 15:34:34 +0100 1252) rescan:
d3c4a43d92912 (Christoph Hellwig 2021-04-06 08:22:55 +0200 1253) if (bdev->bd_part_count)
d3c4a43d92912 (Christoph Hellwig 2021-04-06 08:22:55 +0200 1254) return -EBUSY;
d3c4a43d92912 (Christoph Hellwig 2021-04-06 08:22:55 +0200 1255) sync_blockdev(bdev);
d3c4a43d92912 (Christoph Hellwig 2021-04-06 08:22:55 +0200 1256) invalidate_bdev(bdev);
d3c4a43d92912 (Christoph Hellwig 2021-04-06 08:22:55 +0200 1257) blk_drop_partitions(disk);
a1548b674403c (Christoph Hellwig 2019-11-14 15:34:34 +0100 1258)
5116784039f04 (Chris Chiu 2021-03-23 16:52:19 +0800 1259) clear_bit(GD_NEED_PART_SCAN, &disk->state);
5116784039f04 (Chris Chiu 2021-03-23 16:52:19 +0800 1260)
d981cb5b9fa0d (Christoph Hellwig 2020-03-18 09:12:06 +0100 1261) /*
d981cb5b9fa0d (Christoph Hellwig 2020-03-18 09:12:06 +0100 1262) * Historically we only set the capacity to zero for devices that
d981cb5b9fa0d (Christoph Hellwig 2020-03-18 09:12:06 +0100 1263) * support partitions (independ of actually having partitions created).
d981cb5b9fa0d (Christoph Hellwig 2020-03-18 09:12:06 +0100 1264) * Doing that is rather inconsistent, but changing it broke legacy
d981cb5b9fa0d (Christoph Hellwig 2020-03-18 09:12:06 +0100 1265) * udisks polling for legacy ide-cdrom devices. Use the crude check
d981cb5b9fa0d (Christoph Hellwig 2020-03-18 09:12:06 +0100 1266) * below to get the sane behavior for most device while not breaking
d981cb5b9fa0d (Christoph Hellwig 2020-03-18 09:12:06 +0100 1267) * userspace for this particular setup.
d981cb5b9fa0d (Christoph Hellwig 2020-03-18 09:12:06 +0100 1268) */
d981cb5b9fa0d (Christoph Hellwig 2020-03-18 09:12:06 +0100 1269) if (invalidate) {
d981cb5b9fa0d (Christoph Hellwig 2020-03-18 09:12:06 +0100 1270) if (disk_part_scan_enabled(disk) ||
d981cb5b9fa0d (Christoph Hellwig 2020-03-18 09:12:06 +0100 1271) !(disk->flags & GENHD_FL_REMOVABLE))
d981cb5b9fa0d (Christoph Hellwig 2020-03-18 09:12:06 +0100 1272) set_capacity(disk, 0);
d981cb5b9fa0d (Christoph Hellwig 2020-03-18 09:12:06 +0100 1273) }
a1548b674403c (Christoph Hellwig 2019-11-14 15:34:34 +0100 1274)
142fe8f4bb169 (Christoph Hellwig 2019-11-14 15:34:35 +0100 1275) if (get_capacity(disk)) {
142fe8f4bb169 (Christoph Hellwig 2019-11-14 15:34:35 +0100 1276) ret = blk_add_partitions(disk, bdev);
142fe8f4bb169 (Christoph Hellwig 2019-11-14 15:34:35 +0100 1277) if (ret == -EAGAIN)
142fe8f4bb169 (Christoph Hellwig 2019-11-14 15:34:35 +0100 1278) goto rescan;
490547ca2df66 (Eric Biggers 2019-12-02 10:21:34 -0800 1279) } else if (invalidate) {
a1548b674403c (Christoph Hellwig 2019-11-14 15:34:34 +0100 1280) /*
a1548b674403c (Christoph Hellwig 2019-11-14 15:34:34 +0100 1281) * Tell userspace that the media / partition table may have
a1548b674403c (Christoph Hellwig 2019-11-14 15:34:34 +0100 1282) * changed.
a1548b674403c (Christoph Hellwig 2019-11-14 15:34:34 +0100 1283) */
a1548b674403c (Christoph Hellwig 2019-11-14 15:34:34 +0100 1284) kobject_uevent(&disk_to_dev(disk)->kobj, KOBJ_CHANGE);
a1548b674403c (Christoph Hellwig 2019-11-14 15:34:34 +0100 1285) }
a1548b674403c (Christoph Hellwig 2019-11-14 15:34:34 +0100 1286)
a1548b674403c (Christoph Hellwig 2019-11-14 15:34:34 +0100 1287) return ret;
a1548b674403c (Christoph Hellwig 2019-11-14 15:34:34 +0100 1288) }
f0b870df80bc7 (Christoph Hellwig 2019-11-14 15:34:36 +0100 1289) /*
3d742d4b6ebb3 (Randy Dunlap 2021-02-24 12:00:48 -0800 1290) * Only exported for loop and dasd for historic reasons. Don't use in new
f0b870df80bc7 (Christoph Hellwig 2019-11-14 15:34:36 +0100 1291) * code!
f0b870df80bc7 (Christoph Hellwig 2019-11-14 15:34:36 +0100 1292) */
f0b870df80bc7 (Christoph Hellwig 2019-11-14 15:34:36 +0100 1293) EXPORT_SYMBOL_GPL(bdev_disk_changed);
a1548b674403c (Christoph Hellwig 2019-11-14 15:34:34 +0100 1294)
6d740cd5b18bc (Peter Zijlstra 2007-02-20 13:58:18 -0800 1295) /*
6d740cd5b18bc (Peter Zijlstra 2007-02-20 13:58:18 -0800 1296) * bd_mutex locking:
6d740cd5b18bc (Peter Zijlstra 2007-02-20 13:58:18 -0800 1297) *
6d740cd5b18bc (Peter Zijlstra 2007-02-20 13:58:18 -0800 1298) * mutex_lock(part->bd_mutex)
6d740cd5b18bc (Peter Zijlstra 2007-02-20 13:58:18 -0800 1299) * mutex_lock_nested(whole->bd_mutex, 1)
6d740cd5b18bc (Peter Zijlstra 2007-02-20 13:58:18 -0800 1300) */
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1301) static int __blkdev_get(struct block_device *bdev, fmode_t mode)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1302) {
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1303) struct gendisk *disk = bdev->bd_disk;
cb8432d650fe3 (Christoph Hellwig 2020-11-26 18:47:17 +0100 1304) int ret = 0;
5b642d8b9f2ef (Christoph Hellwig 2020-07-16 16:33:10 +0200 1305)
6c60ff048ca1e (Christoph Hellwig 2021-05-14 15:18:41 +0200 1306) if (!(disk->flags & GENHD_FL_UP))
6c60ff048ca1e (Christoph Hellwig 2021-05-14 15:18:41 +0200 1307) return -ENXIO;
6c60ff048ca1e (Christoph Hellwig 2021-05-14 15:18:41 +0200 1308)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1309) if (!bdev->bd_openers) {
a954ea812018a (Christoph Hellwig 2020-11-23 13:29:55 +0100 1310) if (!bdev_is_partition(bdev)) {
1196f8b814f32 (Tejun Heo 2011-04-21 20:54:45 +0200 1311) ret = 0;
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1312) if (disk->fops->open)
572c48921574d (Al Viro 2007-10-08 13:24:05 -0400 1313) ret = disk->fops->open(bdev, mode);
7e69723fef877 (Tejun Heo 2011-05-23 13:26:07 +0200 1314)
a782483cc1f87 (Christoph Hellwig 2020-11-26 18:43:37 +0100 1315) if (!ret)
04906b2f542c2 (Jan Kara 2019-01-14 09:48:10 +0100 1316) set_init_blocksize(bdev);
7e69723fef877 (Tejun Heo 2011-05-23 13:26:07 +0200 1317)
1196f8b814f32 (Tejun Heo 2011-04-21 20:54:45 +0200 1318) /*
1196f8b814f32 (Tejun Heo 2011-04-21 20:54:45 +0200 1319) * If the device is invalidated, rescan partition
1196f8b814f32 (Tejun Heo 2011-04-21 20:54:45 +0200 1320) * if open succeeded or failed with -ENOMEDIUM.
1196f8b814f32 (Tejun Heo 2011-04-21 20:54:45 +0200 1321) * The latter is necessary to prevent ghost
1196f8b814f32 (Tejun Heo 2011-04-21 20:54:45 +0200 1322) * partitions on a removed medium.
1196f8b814f32 (Tejun Heo 2011-04-21 20:54:45 +0200 1323) */
38430f0876fa8 (Christoph Hellwig 2020-09-21 09:19:45 +0200 1324) if (test_bit(GD_NEED_PART_SCAN, &disk->state) &&
731dc4868311e (Jan Kara 2019-10-21 10:37:59 +0200 1325) (!ret || ret == -ENOMEDIUM))
731dc4868311e (Jan Kara 2019-10-21 10:37:59 +0200 1326) bdev_disk_changed(bdev, ret == -ENOMEDIUM);
5a023cdba50c5 (Dan Williams 2015-11-30 10:20:29 -0800 1327)
1196f8b814f32 (Tejun Heo 2011-04-21 20:54:45 +0200 1328) if (ret)
cb8432d650fe3 (Christoph Hellwig 2020-11-26 18:47:17 +0100 1329) return ret;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1330) } else {
977115c0f664e (Christoph Hellwig 2020-11-26 10:41:07 +0100 1331) struct block_device *whole = bdgrab(disk->part0);
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1332)
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1333) mutex_lock_nested(&whole->bd_mutex, 1);
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1334) ret = __blkdev_get(whole, mode);
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1335) if (ret) {
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1336) mutex_unlock(&whole->bd_mutex);
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1337) bdput(whole);
cb8432d650fe3 (Christoph Hellwig 2020-11-26 18:47:17 +0100 1338) return ret;
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1339) }
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1340) whole->bd_part_count++;
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1341) mutex_unlock(&whole->bd_mutex);
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1342)
6c60ff048ca1e (Christoph Hellwig 2021-05-14 15:18:41 +0200 1343) if (!bdev_nr_sectors(bdev)) {
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1344) __blkdev_put(whole, mode, 1);
63d9932caecee (Christoph Hellwig 2020-11-26 09:22:18 +0100 1345) bdput(whole);
cb8432d650fe3 (Christoph Hellwig 2020-11-26 18:47:17 +0100 1346) return -ENXIO;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1347) }
04906b2f542c2 (Jan Kara 2019-01-14 09:48:10 +0100 1348) set_init_blocksize(bdev);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1349) }
03e262798884b (Jan Kara 2017-03-23 01:36:53 +0100 1350)
03e262798884b (Jan Kara 2017-03-23 01:36:53 +0100 1351) if (bdev->bd_bdi == &noop_backing_dev_info)
03e262798884b (Jan Kara 2017-03-23 01:36:53 +0100 1352) bdev->bd_bdi = bdi_get(disk->queue->backing_dev_info);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1353) } else {
a954ea812018a (Christoph Hellwig 2020-11-23 13:29:55 +0100 1354) if (!bdev_is_partition(bdev)) {
1196f8b814f32 (Tejun Heo 2011-04-21 20:54:45 +0200 1355) if (bdev->bd_disk->fops->open)
572c48921574d (Al Viro 2007-10-08 13:24:05 -0400 1356) ret = bdev->bd_disk->fops->open(bdev, mode);
1196f8b814f32 (Tejun Heo 2011-04-21 20:54:45 +0200 1357) /* the same as first opener case, read comment there */
38430f0876fa8 (Christoph Hellwig 2020-09-21 09:19:45 +0200 1358) if (test_bit(GD_NEED_PART_SCAN, &disk->state) &&
731dc4868311e (Jan Kara 2019-10-21 10:37:59 +0200 1359) (!ret || ret == -ENOMEDIUM))
731dc4868311e (Jan Kara 2019-10-21 10:37:59 +0200 1360) bdev_disk_changed(bdev, ret == -ENOMEDIUM);
1196f8b814f32 (Tejun Heo 2011-04-21 20:54:45 +0200 1361) if (ret)
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1362) return ret;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1363) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1364) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1365) bdev->bd_openers++;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1366) return 0;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1367) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1368)
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1369) struct block_device *blkdev_get_no_open(dev_t dev)
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1370) {
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1371) struct block_device *bdev;
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1372) struct gendisk *disk;
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1373)
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1374) bdev = bdget(dev);
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1375) if (!bdev) {
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1376) blk_request_module(dev);
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1377) bdev = bdget(dev);
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1378) if (!bdev)
6c60ff048ca1e (Christoph Hellwig 2021-05-14 15:18:41 +0200 1379) return NULL;
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1380) }
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1381)
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1382) disk = bdev->bd_disk;
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1383) if (!kobject_get_unless_zero(&disk_to_dev(disk)->kobj))
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1384) goto bdput;
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1385) if ((disk->flags & (GENHD_FL_UP | GENHD_FL_HIDDEN)) != GENHD_FL_UP)
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1386) goto put_disk;
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1387) if (!try_module_get(bdev->bd_disk->fops->owner))
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1388) goto put_disk;
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1389) return bdev;
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1390) put_disk:
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1391) put_disk(disk);
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1392) bdput:
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1393) bdput(bdev);
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1394) return NULL;
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1395) }
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1396)
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1397) void blkdev_put_no_open(struct block_device *bdev)
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1398) {
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1399) module_put(bdev->bd_disk->fops->owner);
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1400) put_disk(bdev->bd_disk);
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1401) bdput(bdev);
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1402) }
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1403)
d4d77629953ea (Tejun Heo 2010-11-13 11:55:18 +0100 1404) /**
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1405) * blkdev_get_by_dev - open a block device by device number
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1406) * @dev: device number of block device to open
d4d77629953ea (Tejun Heo 2010-11-13 11:55:18 +0100 1407) * @mode: FMODE_* mask
d4d77629953ea (Tejun Heo 2010-11-13 11:55:18 +0100 1408) * @holder: exclusive holder identifier
d4d77629953ea (Tejun Heo 2010-11-13 11:55:18 +0100 1409) *
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1410) * Open the block device described by device number @dev. If @mode includes
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1411) * %FMODE_EXCL, the block device is opened with exclusive access. Specifying
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1412) * %FMODE_EXCL with a %NULL @holder is invalid. Exclusive opens may nest for
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1413) * the same @holder.
d4d77629953ea (Tejun Heo 2010-11-13 11:55:18 +0100 1414) *
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1415) * Use this interface ONLY if you really do not have anything better - i.e. when
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1416) * you are behind a truly sucky interface and all you are given is a device
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1417) * number. Everything else should use blkdev_get_by_path().
d4d77629953ea (Tejun Heo 2010-11-13 11:55:18 +0100 1418) *
d4d77629953ea (Tejun Heo 2010-11-13 11:55:18 +0100 1419) * CONTEXT:
d4d77629953ea (Tejun Heo 2010-11-13 11:55:18 +0100 1420) * Might sleep.
d4d77629953ea (Tejun Heo 2010-11-13 11:55:18 +0100 1421) *
d4d77629953ea (Tejun Heo 2010-11-13 11:55:18 +0100 1422) * RETURNS:
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1423) * Reference to the block_device on success, ERR_PTR(-errno) on failure.
d4d77629953ea (Tejun Heo 2010-11-13 11:55:18 +0100 1424) */
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1425) struct block_device *blkdev_get_by_dev(dev_t dev, fmode_t mode, void *holder)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1426) {
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1427) bool unblock_events = true;
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1428) struct block_device *bdev;
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1429) struct gendisk *disk;
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1430) int ret;
e525fd89d380c (Tejun Heo 2010-11-13 11:55:17 +0100 1431)
7918f0f6fdafa (Christoph Hellwig 2020-11-23 13:44:44 +0100 1432) ret = devcgroup_check_permission(DEVCG_DEV_BLOCK,
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1433) MAJOR(dev), MINOR(dev),
7918f0f6fdafa (Christoph Hellwig 2020-11-23 13:44:44 +0100 1434) ((mode & FMODE_READ) ? DEVCG_ACC_READ : 0) |
7918f0f6fdafa (Christoph Hellwig 2020-11-23 13:44:44 +0100 1435) ((mode & FMODE_WRITE) ? DEVCG_ACC_WRITE : 0));
e5c7fb400227d (Christoph Hellwig 2020-08-31 20:02:36 +0200 1436) if (ret)
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1437) return ERR_PTR(ret);
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1438)
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1439) bdev = blkdev_get_no_open(dev);
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1440) if (!bdev)
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1441) return ERR_PTR(-ENXIO);
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1442) disk = bdev->bd_disk;
e5c7fb400227d (Christoph Hellwig 2020-08-31 20:02:36 +0200 1443)
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1444) if (mode & FMODE_EXCL) {
37c3fc9abb25c (Christoph Hellwig 2020-11-25 21:20:08 +0100 1445) ret = bd_prepare_to_claim(bdev, holder);
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1446) if (ret)
37c3fc9abb25c (Christoph Hellwig 2020-11-25 21:20:08 +0100 1447) goto put_blkdev;
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1448) }
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1449)
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1450) disk_block_events(disk);
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1451)
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1452) mutex_lock(&bdev->bd_mutex);
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1453) ret =__blkdev_get(bdev, mode);
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1454) if (ret)
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1455) goto abort_claiming;
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1456) if (mode & FMODE_EXCL) {
37c3fc9abb25c (Christoph Hellwig 2020-11-25 21:20:08 +0100 1457) bd_finish_claiming(bdev, holder);
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1458)
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1459) /*
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1460) * Block event polling for write claims if requested. Any write
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1461) * holder makes the write_holder state stick until all are
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1462) * released. This is good enough and tracking individual
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1463) * writeable reference is too fragile given the way @mode is
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1464) * used in blkdev_get/put().
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1465) */
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1466) if ((mode & FMODE_WRITE) && !bdev->bd_write_holder &&
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1467) (disk->flags & GENHD_FL_BLOCK_EVENTS_ON_EXCL_WRITE)) {
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1468) bdev->bd_write_holder = true;
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1469) unblock_events = false;
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1470) }
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1471) }
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1472) mutex_unlock(&bdev->bd_mutex);
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1473)
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1474) if (unblock_events)
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1475) disk_unblock_events(disk);
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1476) return bdev;
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1477)
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1478) abort_claiming:
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1479) if (mode & FMODE_EXCL)
37c3fc9abb25c (Christoph Hellwig 2020-11-25 21:20:08 +0100 1480) bd_abort_claiming(bdev, holder);
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1481) mutex_unlock(&bdev->bd_mutex);
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1482) disk_unblock_events(disk);
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1483) put_blkdev:
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1484) blkdev_put_no_open(bdev);
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1485) return ERR_PTR(ret);
37be41241f431 (NeilBrown 2006-12-08 02:36:16 -0800 1486) }
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1487) EXPORT_SYMBOL(blkdev_get_by_dev);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1488)
d4d77629953ea (Tejun Heo 2010-11-13 11:55:18 +0100 1489) /**
d4d77629953ea (Tejun Heo 2010-11-13 11:55:18 +0100 1490) * blkdev_get_by_path - open a block device by name
d4d77629953ea (Tejun Heo 2010-11-13 11:55:18 +0100 1491) * @path: path to the block device to open
d4d77629953ea (Tejun Heo 2010-11-13 11:55:18 +0100 1492) * @mode: FMODE_* mask
d4d77629953ea (Tejun Heo 2010-11-13 11:55:18 +0100 1493) * @holder: exclusive holder identifier
d4d77629953ea (Tejun Heo 2010-11-13 11:55:18 +0100 1494) *
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1495) * Open the block device described by the device file at @path. If @mode
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1496) * includes %FMODE_EXCL, the block device is opened with exclusive access.
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1497) * Specifying %FMODE_EXCL with a %NULL @holder is invalid. Exclusive opens may
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1498) * nest for the same @holder.
d4d77629953ea (Tejun Heo 2010-11-13 11:55:18 +0100 1499) *
d4d77629953ea (Tejun Heo 2010-11-13 11:55:18 +0100 1500) * CONTEXT:
d4d77629953ea (Tejun Heo 2010-11-13 11:55:18 +0100 1501) * Might sleep.
d4d77629953ea (Tejun Heo 2010-11-13 11:55:18 +0100 1502) *
d4d77629953ea (Tejun Heo 2010-11-13 11:55:18 +0100 1503) * RETURNS:
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1504) * Reference to the block_device on success, ERR_PTR(-errno) on failure.
d4d77629953ea (Tejun Heo 2010-11-13 11:55:18 +0100 1505) */
d4d77629953ea (Tejun Heo 2010-11-13 11:55:18 +0100 1506) struct block_device *blkdev_get_by_path(const char *path, fmode_t mode,
d4d77629953ea (Tejun Heo 2010-11-13 11:55:18 +0100 1507) void *holder)
d4d77629953ea (Tejun Heo 2010-11-13 11:55:18 +0100 1508) {
d4d77629953ea (Tejun Heo 2010-11-13 11:55:18 +0100 1509) struct block_device *bdev;
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1510) dev_t dev;
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1511) int error;
d4d77629953ea (Tejun Heo 2010-11-13 11:55:18 +0100 1512)
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1513) error = lookup_bdev(path, &dev);
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1514) if (error)
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1515) return ERR_PTR(error);
d4d77629953ea (Tejun Heo 2010-11-13 11:55:18 +0100 1516)
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1517) bdev = blkdev_get_by_dev(dev, mode, holder);
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1518) if (!IS_ERR(bdev) && (mode & FMODE_WRITE) && bdev_read_only(bdev)) {
e51900f7d38cb (Chuck Ebbert 2011-02-16 18:11:53 -0500 1519) blkdev_put(bdev, mode);
e51900f7d38cb (Chuck Ebbert 2011-02-16 18:11:53 -0500 1520) return ERR_PTR(-EACCES);
e51900f7d38cb (Chuck Ebbert 2011-02-16 18:11:53 -0500 1521) }
e51900f7d38cb (Chuck Ebbert 2011-02-16 18:11:53 -0500 1522)
d4d77629953ea (Tejun Heo 2010-11-13 11:55:18 +0100 1523) return bdev;
d4d77629953ea (Tejun Heo 2010-11-13 11:55:18 +0100 1524) }
d4d77629953ea (Tejun Heo 2010-11-13 11:55:18 +0100 1525) EXPORT_SYMBOL(blkdev_get_by_path);
d4d77629953ea (Tejun Heo 2010-11-13 11:55:18 +0100 1526)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1527) static int blkdev_open(struct inode * inode, struct file * filp)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1528) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1529) struct block_device *bdev;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1530)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1531) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1532) * Preserve backwards compatibility and allow large file access
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1533) * even if userspace doesn't ask for it explicitly. Some mkfs
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1534) * binary needs it. We might want to drop this workaround
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1535) * during an unstable branch.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1536) */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1537) filp->f_flags |= O_LARGEFILE;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1538)
a304f0744824f (Jens Axboe 2020-05-22 09:14:08 -0600 1539) filp->f_mode |= FMODE_NOWAIT | FMODE_BUF_RASYNC;
c35fc7a5abae9 (Christoph Hellwig 2017-08-29 16:13:21 +0200 1540)
572c48921574d (Al Viro 2007-10-08 13:24:05 -0400 1541) if (filp->f_flags & O_NDELAY)
572c48921574d (Al Viro 2007-10-08 13:24:05 -0400 1542) filp->f_mode |= FMODE_NDELAY;
572c48921574d (Al Viro 2007-10-08 13:24:05 -0400 1543) if (filp->f_flags & O_EXCL)
572c48921574d (Al Viro 2007-10-08 13:24:05 -0400 1544) filp->f_mode |= FMODE_EXCL;
572c48921574d (Al Viro 2007-10-08 13:24:05 -0400 1545) if ((filp->f_flags & O_ACCMODE) == 3)
572c48921574d (Al Viro 2007-10-08 13:24:05 -0400 1546) filp->f_mode |= FMODE_WRITE_IOCTL;
572c48921574d (Al Viro 2007-10-08 13:24:05 -0400 1547)
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1548) bdev = blkdev_get_by_dev(inode->i_rdev, filp->f_mode, filp);
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1549) if (IS_ERR(bdev))
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1550) return PTR_ERR(bdev);
572c48921574d (Al Viro 2007-10-08 13:24:05 -0400 1551) filp->f_mapping = bdev->bd_inode->i_mapping;
5660e13d2fd6a (Jeff Layton 2017-07-06 07:02:25 -0400 1552) filp->f_wb_err = filemap_sample_wb_err(filp->f_mapping);
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1553) return 0;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1554) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1555)
4385bab128911 (Al Viro 2013-05-05 22:11:03 -0400 1556) static void __blkdev_put(struct block_device *bdev, fmode_t mode, int for_part)
2e7b651df113c (Peter Zijlstra 2006-12-08 02:36:13 -0800 1557) {
2e7b651df113c (Peter Zijlstra 2006-12-08 02:36:13 -0800 1558) struct gendisk *disk = bdev->bd_disk;
37be41241f431 (NeilBrown 2006-12-08 02:36:16 -0800 1559) struct block_device *victim = NULL;
2e7b651df113c (Peter Zijlstra 2006-12-08 02:36:13 -0800 1560)
b849dd84b6ccf (Douglas Anderson 2020-03-24 14:48:27 -0700 1561) /*
b849dd84b6ccf (Douglas Anderson 2020-03-24 14:48:27 -0700 1562) * Sync early if it looks like we're the last one. If someone else
b849dd84b6ccf (Douglas Anderson 2020-03-24 14:48:27 -0700 1563) * opens the block device between now and the decrement of bd_openers
b849dd84b6ccf (Douglas Anderson 2020-03-24 14:48:27 -0700 1564) * then we did a sync that we didn't need to, but that's not the end
b849dd84b6ccf (Douglas Anderson 2020-03-24 14:48:27 -0700 1565) * of the world and we want to avoid long (could be several minute)
b849dd84b6ccf (Douglas Anderson 2020-03-24 14:48:27 -0700 1566) * syncs while holding the mutex.
b849dd84b6ccf (Douglas Anderson 2020-03-24 14:48:27 -0700 1567) */
b849dd84b6ccf (Douglas Anderson 2020-03-24 14:48:27 -0700 1568) if (bdev->bd_openers == 1)
b849dd84b6ccf (Douglas Anderson 2020-03-24 14:48:27 -0700 1569) sync_blockdev(bdev);
b849dd84b6ccf (Douglas Anderson 2020-03-24 14:48:27 -0700 1570)
6796bf54a64df (NeilBrown 2006-12-08 02:36:16 -0800 1571) mutex_lock_nested(&bdev->bd_mutex, for_part);
37be41241f431 (NeilBrown 2006-12-08 02:36:16 -0800 1572) if (for_part)
37be41241f431 (NeilBrown 2006-12-08 02:36:16 -0800 1573) bdev->bd_part_count--;
37be41241f431 (NeilBrown 2006-12-08 02:36:16 -0800 1574)
2e7b651df113c (Peter Zijlstra 2006-12-08 02:36:13 -0800 1575) if (!--bdev->bd_openers) {
6a027eff62f6a (Tejun Heo 2010-11-13 11:55:17 +0100 1576) WARN_ON_ONCE(bdev->bd_holders);
2e7b651df113c (Peter Zijlstra 2006-12-08 02:36:13 -0800 1577) sync_blockdev(bdev);
2e7b651df113c (Peter Zijlstra 2006-12-08 02:36:13 -0800 1578) kill_bdev(bdev);
43d1c0eb7e119 (Ilya Dryomov 2015-11-20 22:22:34 +0100 1579) bdev_write_inode(bdev);
ec5d451438a2f (Christoph Hellwig 2020-11-17 12:01:26 +0100 1580) if (bdev_is_partition(bdev))
a954ea812018a (Christoph Hellwig 2020-11-23 13:29:55 +0100 1581) victim = bdev_whole(bdev);
2e7b651df113c (Peter Zijlstra 2006-12-08 02:36:13 -0800 1582) }
cb8432d650fe3 (Christoph Hellwig 2020-11-26 18:47:17 +0100 1583)
cb8432d650fe3 (Christoph Hellwig 2020-11-26 18:47:17 +0100 1584) if (!bdev_is_partition(bdev) && disk->fops->release)
cb8432d650fe3 (Christoph Hellwig 2020-11-26 18:47:17 +0100 1585) disk->fops->release(disk, mode);
2e7b651df113c (Peter Zijlstra 2006-12-08 02:36:13 -0800 1586) mutex_unlock(&bdev->bd_mutex);
63d9932caecee (Christoph Hellwig 2020-11-26 09:22:18 +0100 1587) if (victim) {
9a1c3542768b5 (Al Viro 2008-02-22 20:40:24 -0500 1588) __blkdev_put(victim, mode, 1);
63d9932caecee (Christoph Hellwig 2020-11-26 09:22:18 +0100 1589) bdput(victim);
63d9932caecee (Christoph Hellwig 2020-11-26 09:22:18 +0100 1590) }
2e7b651df113c (Peter Zijlstra 2006-12-08 02:36:13 -0800 1591) }
2e7b651df113c (Peter Zijlstra 2006-12-08 02:36:13 -0800 1592)
4385bab128911 (Al Viro 2013-05-05 22:11:03 -0400 1593) void blkdev_put(struct block_device *bdev, fmode_t mode)
37be41241f431 (NeilBrown 2006-12-08 02:36:16 -0800 1594) {
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1595) struct gendisk *disk = bdev->bd_disk;
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1596)
85ef06d1d252f (Tejun Heo 2011-07-01 16:17:47 +0200 1597) mutex_lock(&bdev->bd_mutex);
85ef06d1d252f (Tejun Heo 2011-07-01 16:17:47 +0200 1598)
e525fd89d380c (Tejun Heo 2010-11-13 11:55:17 +0100 1599) if (mode & FMODE_EXCL) {
a954ea812018a (Christoph Hellwig 2020-11-23 13:29:55 +0100 1600) struct block_device *whole = bdev_whole(bdev);
6a027eff62f6a (Tejun Heo 2010-11-13 11:55:17 +0100 1601) bool bdev_free;
6a027eff62f6a (Tejun Heo 2010-11-13 11:55:17 +0100 1602)
6a027eff62f6a (Tejun Heo 2010-11-13 11:55:17 +0100 1603) /*
6a027eff62f6a (Tejun Heo 2010-11-13 11:55:17 +0100 1604) * Release a claim on the device. The holder fields
6a027eff62f6a (Tejun Heo 2010-11-13 11:55:17 +0100 1605) * are protected with bdev_lock. bd_mutex is to
6a027eff62f6a (Tejun Heo 2010-11-13 11:55:17 +0100 1606) * synchronize disk_holder unlinking.
6a027eff62f6a (Tejun Heo 2010-11-13 11:55:17 +0100 1607) */
6a027eff62f6a (Tejun Heo 2010-11-13 11:55:17 +0100 1608) spin_lock(&bdev_lock);
6a027eff62f6a (Tejun Heo 2010-11-13 11:55:17 +0100 1609)
6a027eff62f6a (Tejun Heo 2010-11-13 11:55:17 +0100 1610) WARN_ON_ONCE(--bdev->bd_holders < 0);
a954ea812018a (Christoph Hellwig 2020-11-23 13:29:55 +0100 1611) WARN_ON_ONCE(--whole->bd_holders < 0);
6a027eff62f6a (Tejun Heo 2010-11-13 11:55:17 +0100 1612)
6a027eff62f6a (Tejun Heo 2010-11-13 11:55:17 +0100 1613) if ((bdev_free = !bdev->bd_holders))
6a027eff62f6a (Tejun Heo 2010-11-13 11:55:17 +0100 1614) bdev->bd_holder = NULL;
a954ea812018a (Christoph Hellwig 2020-11-23 13:29:55 +0100 1615) if (!whole->bd_holders)
a954ea812018a (Christoph Hellwig 2020-11-23 13:29:55 +0100 1616) whole->bd_holder = NULL;
6a027eff62f6a (Tejun Heo 2010-11-13 11:55:17 +0100 1617)
6a027eff62f6a (Tejun Heo 2010-11-13 11:55:17 +0100 1618) spin_unlock(&bdev_lock);
6a027eff62f6a (Tejun Heo 2010-11-13 11:55:17 +0100 1619)
77ea887e433ad (Tejun Heo 2010-12-08 20:57:37 +0100 1620) /*
77ea887e433ad (Tejun Heo 2010-12-08 20:57:37 +0100 1621) * If this was the last claim, remove holder link and
77ea887e433ad (Tejun Heo 2010-12-08 20:57:37 +0100 1622) * unblock evpoll if it was a write holder.
77ea887e433ad (Tejun Heo 2010-12-08 20:57:37 +0100 1623) */
85ef06d1d252f (Tejun Heo 2011-07-01 16:17:47 +0200 1624) if (bdev_free && bdev->bd_write_holder) {
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1625) disk_unblock_events(disk);
85ef06d1d252f (Tejun Heo 2011-07-01 16:17:47 +0200 1626) bdev->bd_write_holder = false;
77ea887e433ad (Tejun Heo 2010-12-08 20:57:37 +0100 1627) }
6936217cc7e58 (Tejun Heo 2011-03-09 19:54:27 +0100 1628) }
77ea887e433ad (Tejun Heo 2010-12-08 20:57:37 +0100 1629)
85ef06d1d252f (Tejun Heo 2011-07-01 16:17:47 +0200 1630) /*
85ef06d1d252f (Tejun Heo 2011-07-01 16:17:47 +0200 1631) * Trigger event checking and tell drivers to flush MEDIA_CHANGE
85ef06d1d252f (Tejun Heo 2011-07-01 16:17:47 +0200 1632) * event. This is to ensure detection of media removal commanded
85ef06d1d252f (Tejun Heo 2011-07-01 16:17:47 +0200 1633) * from userland - e.g. eject(1).
85ef06d1d252f (Tejun Heo 2011-07-01 16:17:47 +0200 1634) */
5b56b6ed574b5 (Christoph Hellwig 2020-11-23 10:19:22 +0100 1635) disk_flush_events(disk, DISK_EVENT_MEDIA_CHANGE);
85ef06d1d252f (Tejun Heo 2011-07-01 16:17:47 +0200 1636) mutex_unlock(&bdev->bd_mutex);
85ef06d1d252f (Tejun Heo 2011-07-01 16:17:47 +0200 1637)
4385bab128911 (Al Viro 2013-05-05 22:11:03 -0400 1638) __blkdev_put(bdev, mode, 0);
22ae8ce8b8924 (Christoph Hellwig 2020-11-26 09:23:26 +0100 1639) blkdev_put_no_open(bdev);
37be41241f431 (NeilBrown 2006-12-08 02:36:16 -0800 1640) }
2e7b651df113c (Peter Zijlstra 2006-12-08 02:36:13 -0800 1641) EXPORT_SYMBOL(blkdev_put);
2e7b651df113c (Peter Zijlstra 2006-12-08 02:36:13 -0800 1642)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1643) static int blkdev_close(struct inode * inode, struct file * filp)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1644) {
4ebb16ca9a06a (Dan Williams 2015-10-28 07:48:19 +0900 1645) struct block_device *bdev = I_BDEV(bdev_file_inode(filp));
4385bab128911 (Al Viro 2013-05-05 22:11:03 -0400 1646) blkdev_put(bdev, filp->f_mode);
4385bab128911 (Al Viro 2013-05-05 22:11:03 -0400 1647) return 0;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1648) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1649)
bb93e3a52f8db (Arnd Bergmann 2005-06-23 00:10:15 -0700 1650) static long block_ioctl(struct file *file, unsigned cmd, unsigned long arg)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1651) {
4ebb16ca9a06a (Dan Williams 2015-10-28 07:48:19 +0900 1652) struct block_device *bdev = I_BDEV(bdev_file_inode(file));
56b26add02b4b (Al Viro 2008-09-19 03:17:36 -0400 1653) fmode_t mode = file->f_mode;
fd4ce1acd0f85 (Christoph Hellwig 2008-11-05 14:58:42 +0100 1654)
fd4ce1acd0f85 (Christoph Hellwig 2008-11-05 14:58:42 +0100 1655) /*
fd4ce1acd0f85 (Christoph Hellwig 2008-11-05 14:58:42 +0100 1656) * O_NDELAY can be altered using fcntl(.., F_SETFL, ..), so we have
fd4ce1acd0f85 (Christoph Hellwig 2008-11-05 14:58:42 +0100 1657) * to updated it before every ioctl.
fd4ce1acd0f85 (Christoph Hellwig 2008-11-05 14:58:42 +0100 1658) */
56b26add02b4b (Al Viro 2008-09-19 03:17:36 -0400 1659) if (file->f_flags & O_NDELAY)
fd4ce1acd0f85 (Christoph Hellwig 2008-11-05 14:58:42 +0100 1660) mode |= FMODE_NDELAY;
fd4ce1acd0f85 (Christoph Hellwig 2008-11-05 14:58:42 +0100 1661) else
fd4ce1acd0f85 (Christoph Hellwig 2008-11-05 14:58:42 +0100 1662) mode &= ~FMODE_NDELAY;
fd4ce1acd0f85 (Christoph Hellwig 2008-11-05 14:58:42 +0100 1663)
56b26add02b4b (Al Viro 2008-09-19 03:17:36 -0400 1664) return blkdev_ioctl(bdev, mode, cmd, arg);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1665) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1666)
eef99380679e2 (Christoph Hellwig 2009-08-20 17:43:41 +0200 1667) /*
eef99380679e2 (Christoph Hellwig 2009-08-20 17:43:41 +0200 1668) * Write data to the block device. Only intended for the block device itself
eef99380679e2 (Christoph Hellwig 2009-08-20 17:43:41 +0200 1669) * and the raw driver which basically is a fake block device.
eef99380679e2 (Christoph Hellwig 2009-08-20 17:43:41 +0200 1670) *
eef99380679e2 (Christoph Hellwig 2009-08-20 17:43:41 +0200 1671) * Does not take i_mutex for the write and thus is not for general purpose
eef99380679e2 (Christoph Hellwig 2009-08-20 17:43:41 +0200 1672) * use.
eef99380679e2 (Christoph Hellwig 2009-08-20 17:43:41 +0200 1673) */
1456c0a87c424 (Al Viro 2014-04-03 03:21:50 -0400 1674) ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from)
eef99380679e2 (Christoph Hellwig 2009-08-20 17:43:41 +0200 1675) {
eef99380679e2 (Christoph Hellwig 2009-08-20 17:43:41 +0200 1676) struct file *file = iocb->ki_filp;
4ebb16ca9a06a (Dan Williams 2015-10-28 07:48:19 +0900 1677) struct inode *bd_inode = bdev_file_inode(file);
7ec7b94a33397 (Al Viro 2015-04-07 11:35:14 -0400 1678) loff_t size = i_size_read(bd_inode);
53362a05ae683 (Jianpeng Ma 2012-08-02 09:50:39 +0200 1679) struct blk_plug plug;
cf7b39a0cbf6b (yangerkun 2021-04-01 15:18:07 +0800 1680) size_t shorted = 0;
eef99380679e2 (Christoph Hellwig 2009-08-20 17:43:41 +0200 1681) ssize_t ret;
5f380c7fa7e01 (Al Viro 2015-04-07 11:28:12 -0400 1682)
7ec7b94a33397 (Al Viro 2015-04-07 11:35:14 -0400 1683) if (bdev_read_only(I_BDEV(bd_inode)))
7ec7b94a33397 (Al Viro 2015-04-07 11:35:14 -0400 1684) return -EPERM;
5f380c7fa7e01 (Al Viro 2015-04-07 11:28:12 -0400 1685)
bb3247a399801 (Christoph Hellwig 2020-09-21 09:19:55 +0200 1686) if (IS_SWAPFILE(bd_inode) && !is_hibernate_resume_dev(bd_inode->i_rdev))
dc617f29dbe5e (Darrick J. Wong 2019-08-20 07:55:16 -0700 1687) return -ETXTBSY;
dc617f29dbe5e (Darrick J. Wong 2019-08-20 07:55:16 -0700 1688)
7ec7b94a33397 (Al Viro 2015-04-07 11:35:14 -0400 1689) if (!iov_iter_count(from))
5f380c7fa7e01 (Al Viro 2015-04-07 11:28:12 -0400 1690) return 0;
5f380c7fa7e01 (Al Viro 2015-04-07 11:28:12 -0400 1691)
7ec7b94a33397 (Al Viro 2015-04-07 11:35:14 -0400 1692) if (iocb->ki_pos >= size)
7ec7b94a33397 (Al Viro 2015-04-07 11:35:14 -0400 1693) return -ENOSPC;
7ec7b94a33397 (Al Viro 2015-04-07 11:35:14 -0400 1694)
c35fc7a5abae9 (Christoph Hellwig 2017-08-29 16:13:21 +0200 1695) if ((iocb->ki_flags & (IOCB_NOWAIT | IOCB_DIRECT)) == IOCB_NOWAIT)
c35fc7a5abae9 (Christoph Hellwig 2017-08-29 16:13:21 +0200 1696) return -EOPNOTSUPP;
c35fc7a5abae9 (Christoph Hellwig 2017-08-29 16:13:21 +0200 1697)
cf7b39a0cbf6b (yangerkun 2021-04-01 15:18:07 +0800 1698) size -= iocb->ki_pos;
cf7b39a0cbf6b (yangerkun 2021-04-01 15:18:07 +0800 1699) if (iov_iter_count(from) > size) {
cf7b39a0cbf6b (yangerkun 2021-04-01 15:18:07 +0800 1700) shorted = iov_iter_count(from) - size;
cf7b39a0cbf6b (yangerkun 2021-04-01 15:18:07 +0800 1701) iov_iter_truncate(from, size);
cf7b39a0cbf6b (yangerkun 2021-04-01 15:18:07 +0800 1702) }
eef99380679e2 (Christoph Hellwig 2009-08-20 17:43:41 +0200 1703)
53362a05ae683 (Jianpeng Ma 2012-08-02 09:50:39 +0200 1704) blk_start_plug(&plug);
1456c0a87c424 (Al Viro 2014-04-03 03:21:50 -0400 1705) ret = __generic_file_write_iter(iocb, from);
e259221763a40 (Christoph Hellwig 2016-04-07 08:52:01 -0700 1706) if (ret > 0)
e259221763a40 (Christoph Hellwig 2016-04-07 08:52:01 -0700 1707) ret = generic_write_sync(iocb, ret);
cf7b39a0cbf6b (yangerkun 2021-04-01 15:18:07 +0800 1708) iov_iter_reexpand(from, iov_iter_count(from) + shorted);
53362a05ae683 (Jianpeng Ma 2012-08-02 09:50:39 +0200 1709) blk_finish_plug(&plug);
eef99380679e2 (Christoph Hellwig 2009-08-20 17:43:41 +0200 1710) return ret;
eef99380679e2 (Christoph Hellwig 2009-08-20 17:43:41 +0200 1711) }
1456c0a87c424 (Al Viro 2014-04-03 03:21:50 -0400 1712) EXPORT_SYMBOL_GPL(blkdev_write_iter);
eef99380679e2 (Christoph Hellwig 2009-08-20 17:43:41 +0200 1713)
b2de525f09570 (David Jeffery 2014-09-29 10:21:10 -0400 1714) ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to)
684c9aaebbb0e (Linus Torvalds 2012-12-07 16:48:39 -0800 1715) {
684c9aaebbb0e (Linus Torvalds 2012-12-07 16:48:39 -0800 1716) struct file *file = iocb->ki_filp;
4ebb16ca9a06a (Dan Williams 2015-10-28 07:48:19 +0900 1717) struct inode *bd_inode = bdev_file_inode(file);
684c9aaebbb0e (Linus Torvalds 2012-12-07 16:48:39 -0800 1718) loff_t size = i_size_read(bd_inode);
a886038baa48a (Al Viro 2014-04-02 20:02:21 -0400 1719) loff_t pos = iocb->ki_pos;
cf7b39a0cbf6b (yangerkun 2021-04-01 15:18:07 +0800 1720) size_t shorted = 0;
cf7b39a0cbf6b (yangerkun 2021-04-01 15:18:07 +0800 1721) ssize_t ret;
684c9aaebbb0e (Linus Torvalds 2012-12-07 16:48:39 -0800 1722)
684c9aaebbb0e (Linus Torvalds 2012-12-07 16:48:39 -0800 1723) if (pos >= size)
684c9aaebbb0e (Linus Torvalds 2012-12-07 16:48:39 -0800 1724) return 0;
684c9aaebbb0e (Linus Torvalds 2012-12-07 16:48:39 -0800 1725)
684c9aaebbb0e (Linus Torvalds 2012-12-07 16:48:39 -0800 1726) size -= pos;
cf7b39a0cbf6b (yangerkun 2021-04-01 15:18:07 +0800 1727) if (iov_iter_count(to) > size) {
cf7b39a0cbf6b (yangerkun 2021-04-01 15:18:07 +0800 1728) shorted = iov_iter_count(to) - size;
cf7b39a0cbf6b (yangerkun 2021-04-01 15:18:07 +0800 1729) iov_iter_truncate(to, size);
cf7b39a0cbf6b (yangerkun 2021-04-01 15:18:07 +0800 1730) }
cf7b39a0cbf6b (yangerkun 2021-04-01 15:18:07 +0800 1731)
cf7b39a0cbf6b (yangerkun 2021-04-01 15:18:07 +0800 1732) ret = generic_file_read_iter(iocb, to);
cf7b39a0cbf6b (yangerkun 2021-04-01 15:18:07 +0800 1733) iov_iter_reexpand(to, iov_iter_count(to) + shorted);
cf7b39a0cbf6b (yangerkun 2021-04-01 15:18:07 +0800 1734) return ret;
684c9aaebbb0e (Linus Torvalds 2012-12-07 16:48:39 -0800 1735) }
b2de525f09570 (David Jeffery 2014-09-29 10:21:10 -0400 1736) EXPORT_SYMBOL_GPL(blkdev_read_iter);
684c9aaebbb0e (Linus Torvalds 2012-12-07 16:48:39 -0800 1737)
87d8fe1ee6b8d (Theodore Ts'o 2009-01-03 09:47:09 -0500 1738) /*
87d8fe1ee6b8d (Theodore Ts'o 2009-01-03 09:47:09 -0500 1739) * Try to release a page associated with block device when the system
87d8fe1ee6b8d (Theodore Ts'o 2009-01-03 09:47:09 -0500 1740) * is under memory pressure.
87d8fe1ee6b8d (Theodore Ts'o 2009-01-03 09:47:09 -0500 1741) */
87d8fe1ee6b8d (Theodore Ts'o 2009-01-03 09:47:09 -0500 1742) static int blkdev_releasepage(struct page *page, gfp_t wait)
87d8fe1ee6b8d (Theodore Ts'o 2009-01-03 09:47:09 -0500 1743) {
87d8fe1ee6b8d (Theodore Ts'o 2009-01-03 09:47:09 -0500 1744) struct super_block *super = BDEV_I(page->mapping->host)->bdev.bd_super;
87d8fe1ee6b8d (Theodore Ts'o 2009-01-03 09:47:09 -0500 1745)
87d8fe1ee6b8d (Theodore Ts'o 2009-01-03 09:47:09 -0500 1746) if (super && super->s_op->bdev_try_to_free_page)
87d8fe1ee6b8d (Theodore Ts'o 2009-01-03 09:47:09 -0500 1747) return super->s_op->bdev_try_to_free_page(super, page, wait);
87d8fe1ee6b8d (Theodore Ts'o 2009-01-03 09:47:09 -0500 1748)
87d8fe1ee6b8d (Theodore Ts'o 2009-01-03 09:47:09 -0500 1749) return try_to_free_buffers(page);
87d8fe1ee6b8d (Theodore Ts'o 2009-01-03 09:47:09 -0500 1750) }
87d8fe1ee6b8d (Theodore Ts'o 2009-01-03 09:47:09 -0500 1751)
7f6d5b529b7df (Ross Zwisler 2016-02-26 15:19:55 -0800 1752) static int blkdev_writepages(struct address_space *mapping,
7f6d5b529b7df (Ross Zwisler 2016-02-26 15:19:55 -0800 1753) struct writeback_control *wbc)
7f6d5b529b7df (Ross Zwisler 2016-02-26 15:19:55 -0800 1754) {
7f6d5b529b7df (Ross Zwisler 2016-02-26 15:19:55 -0800 1755) return generic_writepages(mapping, wbc);
7f6d5b529b7df (Ross Zwisler 2016-02-26 15:19:55 -0800 1756) }
7f6d5b529b7df (Ross Zwisler 2016-02-26 15:19:55 -0800 1757)
4c54ac62dceec (Adrian Bunk 2008-02-18 13:48:31 +0100 1758) static const struct address_space_operations def_blk_aops = {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1759) .readpage = blkdev_readpage,
d4388340ae0bc (Matthew Wilcox (Oracle) 2020-06-01 21:47:02 -0700 1760) .readahead = blkdev_readahead,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1761) .writepage = blkdev_writepage,
6272b5a5868f3 (Nicholas Piggin 2007-10-16 01:25:04 -0700 1762) .write_begin = blkdev_write_begin,
6272b5a5868f3 (Nicholas Piggin 2007-10-16 01:25:04 -0700 1763) .write_end = blkdev_write_end,
7f6d5b529b7df (Ross Zwisler 2016-02-26 15:19:55 -0800 1764) .writepages = blkdev_writepages,
87d8fe1ee6b8d (Theodore Ts'o 2009-01-03 09:47:09 -0500 1765) .releasepage = blkdev_releasepage,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1766) .direct_IO = blkdev_direct_IO,
88dbcbb3a4847 (Jan Kara 2018-12-28 00:39:16 -0800 1767) .migratepage = buffer_migrate_page_norefs,
b45972265f823 (Mel Gorman 2013-07-03 15:02:05 -0700 1768) .is_dirty_writeback = buffer_check_dirty_writeback,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1769) };
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1770)
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1771) #define BLKDEV_FALLOC_FL_SUPPORTED \
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1772) (FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE | \
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1773) FALLOC_FL_ZERO_RANGE | FALLOC_FL_NO_HIDE_STALE)
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1774)
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1775) static long blkdev_fallocate(struct file *file, int mode, loff_t start,
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1776) loff_t len)
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1777) {
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1778) struct block_device *bdev = I_BDEV(bdev_file_inode(file));
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1779) loff_t end = start + len - 1;
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1780) loff_t isize;
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1781) int error;
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1782)
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1783) /* Fail if we don't recognize the flags. */
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1784) if (mode & ~BLKDEV_FALLOC_FL_SUPPORTED)
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1785) return -EOPNOTSUPP;
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1786)
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1787) /* Don't go off the end of the device. */
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1788) isize = i_size_read(bdev->bd_inode);
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1789) if (start >= isize)
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1790) return -EINVAL;
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1791) if (end >= isize) {
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1792) if (mode & FALLOC_FL_KEEP_SIZE) {
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1793) len = isize - start;
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1794) end = start + len - 1;
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1795) } else
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1796) return -EINVAL;
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1797) }
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1798)
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1799) /*
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1800) * Don't allow IO that isn't aligned to logical block size.
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1801) */
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1802) if ((start | len) & (bdev_logical_block_size(bdev) - 1))
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1803) return -EINVAL;
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1804)
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1805) /* Invalidate the page cache, including dirty pages. */
384d87ef2c954 (Jan Kara 2020-09-04 10:58:52 +0200 1806) error = truncate_bdev_range(bdev, file->f_mode, start, end);
384d87ef2c954 (Jan Kara 2020-09-04 10:58:52 +0200 1807) if (error)
384d87ef2c954 (Jan Kara 2020-09-04 10:58:52 +0200 1808) return error;
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1809)
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1810) switch (mode) {
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1811) case FALLOC_FL_ZERO_RANGE:
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1812) case FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE:
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1813) error = blkdev_issue_zeroout(bdev, start >> 9, len >> 9,
ee472d835c264 (Christoph Hellwig 2017-04-05 19:21:08 +0200 1814) GFP_KERNEL, BLKDEV_ZERO_NOUNMAP);
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1815) break;
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1816) case FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE:
34045129b128f (Christoph Hellwig 2017-04-05 19:21:11 +0200 1817) error = blkdev_issue_zeroout(bdev, start >> 9, len >> 9,
34045129b128f (Christoph Hellwig 2017-04-05 19:21:11 +0200 1818) GFP_KERNEL, BLKDEV_ZERO_NOFALLBACK);
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1819) break;
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1820) case FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE | FALLOC_FL_NO_HIDE_STALE:
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1821) error = blkdev_issue_discard(bdev, start >> 9, len >> 9,
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1822) GFP_KERNEL, 0);
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1823) break;
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1824) default:
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1825) return -EOPNOTSUPP;
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1826) }
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1827) if (error)
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1828) return error;
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1829)
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1830) /*
767630c63bb23 (Jan Kara 2021-01-07 16:40:34 +0100 1831) * Invalidate the page cache again; if someone wandered in and dirtied
767630c63bb23 (Jan Kara 2021-01-07 16:40:34 +0100 1832) * a page, we just discard it - userspace has no way of knowing whether
767630c63bb23 (Jan Kara 2021-01-07 16:40:34 +0100 1833) * the write happened before or after discard completing...
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1834) */
767630c63bb23 (Jan Kara 2021-01-07 16:40:34 +0100 1835) return truncate_bdev_range(bdev, file->f_mode, start, end);
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1836) }
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1837)
4b6f5d20b04dc (Arjan van de Ven 2006-03-28 01:56:42 -0800 1838) const struct file_operations def_blk_fops = {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1839) .open = blkdev_open,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1840) .release = blkdev_close,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1841) .llseek = block_llseek,
a886038baa48a (Al Viro 2014-04-02 20:02:21 -0400 1842) .read_iter = blkdev_read_iter,
1456c0a87c424 (Al Viro 2014-04-03 03:21:50 -0400 1843) .write_iter = blkdev_write_iter,
eae83ce10b471 (Christoph Hellwig 2018-11-30 08:31:52 -0700 1844) .iopoll = blkdev_iopoll,
acc93d30d7d43 (Dan Williams 2016-05-07 11:40:28 -0700 1845) .mmap = generic_file_mmap,
b1dd3b2843b3b (Andrew Morton 2010-04-06 14:35:00 -0700 1846) .fsync = blkdev_fsync,
bb93e3a52f8db (Arnd Bergmann 2005-06-23 00:10:15 -0700 1847) .unlocked_ioctl = block_ioctl,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1848) #ifdef CONFIG_COMPAT
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1849) .compat_ioctl = compat_blkdev_ioctl,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1850) #endif
1e8b33328a540 (Linus Torvalds 2012-11-29 10:49:50 -0800 1851) .splice_read = generic_file_splice_read,
8d0207652cbe2 (Al Viro 2014-04-05 04:27:08 -0400 1852) .splice_write = iter_file_splice_write,
25f4c41415e51 (Darrick J. Wong 2016-10-11 13:51:11 -0700 1853) .fallocate = blkdev_fallocate,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1854) };
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1855)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1856) /**
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1857) * lookup_bdev - lookup a struct block_device by name
94e2959e7a6a4 (Randy Dunlap 2009-01-06 14:41:15 -0800 1858) * @pathname: special file representing the block device
875b2376fd663 (Randy Dunlap 2020-12-28 19:47:06 -0800 1859) * @dev: return value of the block device's dev_t
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1860) *
57d1b5366f46f (Randy Dunlap 2008-10-09 10:42:38 +0200 1861) * Get a reference to the blockdevice at @pathname in the current
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1862) * namespace if possible and return it. Return ERR_PTR(error)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1863) * otherwise.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1864) */
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1865) int lookup_bdev(const char *pathname, dev_t *dev)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1866) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1867) struct inode *inode;
421748ecde8e6 (Al Viro 2008-08-02 01:04:36 -0400 1868) struct path path;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1869) int error;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1870)
421748ecde8e6 (Al Viro 2008-08-02 01:04:36 -0400 1871) if (!pathname || !*pathname)
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1872) return -EINVAL;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1873)
421748ecde8e6 (Al Viro 2008-08-02 01:04:36 -0400 1874) error = kern_path(pathname, LOOKUP_FOLLOW, &path);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1875) if (error)
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1876) return error;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1877)
bb668734c4c96 (David Howells 2015-03-17 22:26:21 +0000 1878) inode = d_backing_inode(path.dentry);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1879) error = -ENOTBLK;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1880) if (!S_ISBLK(inode->i_mode))
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1881) goto out_path_put;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1882) error = -EACCES;
a2982cc922c30 (Eric W. Biederman 2016-06-09 15:34:02 -0500 1883) if (!may_open_dev(&path))
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1884) goto out_path_put;
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1885)
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1886) *dev = inode->i_rdev;
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1887) error = 0;
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1888) out_path_put:
421748ecde8e6 (Al Viro 2008-08-02 01:04:36 -0400 1889) path_put(&path);
4e7b5671c6a88 (Christoph Hellwig 2020-11-23 13:38:40 +0100 1890) return error;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1891) }
d5686b444ff3f (Al Viro 2008-08-01 05:00:11 -0400 1892) EXPORT_SYMBOL(lookup_bdev);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1893)
93b270f76e7ef (NeilBrown 2011-02-24 17:25:47 +1100 1894) int __invalidate_device(struct block_device *bdev, bool kill_dirty)
b71e8a4ce03b3 (David Howells 2006-08-29 19:06:11 +0100 1895) {
b71e8a4ce03b3 (David Howells 2006-08-29 19:06:11 +0100 1896) struct super_block *sb = get_super(bdev);
b71e8a4ce03b3 (David Howells 2006-08-29 19:06:11 +0100 1897) int res = 0;
b71e8a4ce03b3 (David Howells 2006-08-29 19:06:11 +0100 1898)
b71e8a4ce03b3 (David Howells 2006-08-29 19:06:11 +0100 1899) if (sb) {
b71e8a4ce03b3 (David Howells 2006-08-29 19:06:11 +0100 1900) /*
b71e8a4ce03b3 (David Howells 2006-08-29 19:06:11 +0100 1901) * no need to lock the super, get_super holds the
b71e8a4ce03b3 (David Howells 2006-08-29 19:06:11 +0100 1902) * read mutex so the filesystem cannot go away
b71e8a4ce03b3 (David Howells 2006-08-29 19:06:11 +0100 1903) * under us (->put_super runs with the write lock
b71e8a4ce03b3 (David Howells 2006-08-29 19:06:11 +0100 1904) * hold).
b71e8a4ce03b3 (David Howells 2006-08-29 19:06:11 +0100 1905) */
b71e8a4ce03b3 (David Howells 2006-08-29 19:06:11 +0100 1906) shrink_dcache_sb(sb);
93b270f76e7ef (NeilBrown 2011-02-24 17:25:47 +1100 1907) res = invalidate_inodes(sb, kill_dirty);
b71e8a4ce03b3 (David Howells 2006-08-29 19:06:11 +0100 1908) drop_super(sb);
b71e8a4ce03b3 (David Howells 2006-08-29 19:06:11 +0100 1909) }
f98393a64ca13 (Peter Zijlstra 2007-05-06 14:49:54 -0700 1910) invalidate_bdev(bdev);
b71e8a4ce03b3 (David Howells 2006-08-29 19:06:11 +0100 1911) return res;
b71e8a4ce03b3 (David Howells 2006-08-29 19:06:11 +0100 1912) }
b71e8a4ce03b3 (David Howells 2006-08-29 19:06:11 +0100 1913) EXPORT_SYMBOL(__invalidate_device);
5c0d6b60a0ba4 (Jan Kara 2012-07-03 16:45:31 +0200 1914)
5c0d6b60a0ba4 (Jan Kara 2012-07-03 16:45:31 +0200 1915) void iterate_bdevs(void (*func)(struct block_device *, void *), void *arg)
5c0d6b60a0ba4 (Jan Kara 2012-07-03 16:45:31 +0200 1916) {
5c0d6b60a0ba4 (Jan Kara 2012-07-03 16:45:31 +0200 1917) struct inode *inode, *old_inode = NULL;
5c0d6b60a0ba4 (Jan Kara 2012-07-03 16:45:31 +0200 1918)
74278da9f70d8 (Dave Chinner 2015-03-04 12:37:22 -0500 1919) spin_lock(&blockdev_superblock->s_inode_list_lock);
5c0d6b60a0ba4 (Jan Kara 2012-07-03 16:45:31 +0200 1920) list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list) {
5c0d6b60a0ba4 (Jan Kara 2012-07-03 16:45:31 +0200 1921) struct address_space *mapping = inode->i_mapping;
af309226db916 (Rabin Vincent 2016-12-01 09:18:28 +0100 1922) struct block_device *bdev;
5c0d6b60a0ba4 (Jan Kara 2012-07-03 16:45:31 +0200 1923)
5c0d6b60a0ba4 (Jan Kara 2012-07-03 16:45:31 +0200 1924) spin_lock(&inode->i_lock);
5c0d6b60a0ba4 (Jan Kara 2012-07-03 16:45:31 +0200 1925) if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW) ||
5c0d6b60a0ba4 (Jan Kara 2012-07-03 16:45:31 +0200 1926) mapping->nrpages == 0) {
5c0d6b60a0ba4 (Jan Kara 2012-07-03 16:45:31 +0200 1927) spin_unlock(&inode->i_lock);
5c0d6b60a0ba4 (Jan Kara 2012-07-03 16:45:31 +0200 1928) continue;
5c0d6b60a0ba4 (Jan Kara 2012-07-03 16:45:31 +0200 1929) }
5c0d6b60a0ba4 (Jan Kara 2012-07-03 16:45:31 +0200 1930) __iget(inode);
5c0d6b60a0ba4 (Jan Kara 2012-07-03 16:45:31 +0200 1931) spin_unlock(&inode->i_lock);
74278da9f70d8 (Dave Chinner 2015-03-04 12:37:22 -0500 1932) spin_unlock(&blockdev_superblock->s_inode_list_lock);
5c0d6b60a0ba4 (Jan Kara 2012-07-03 16:45:31 +0200 1933) /*
5c0d6b60a0ba4 (Jan Kara 2012-07-03 16:45:31 +0200 1934) * We hold a reference to 'inode' so it couldn't have been
5c0d6b60a0ba4 (Jan Kara 2012-07-03 16:45:31 +0200 1935) * removed from s_inodes list while we dropped the
74278da9f70d8 (Dave Chinner 2015-03-04 12:37:22 -0500 1936) * s_inode_list_lock We cannot iput the inode now as we can
5c0d6b60a0ba4 (Jan Kara 2012-07-03 16:45:31 +0200 1937) * be holding the last reference and we cannot iput it under
74278da9f70d8 (Dave Chinner 2015-03-04 12:37:22 -0500 1938) * s_inode_list_lock. So we keep the reference and iput it
5c0d6b60a0ba4 (Jan Kara 2012-07-03 16:45:31 +0200 1939) * later.
5c0d6b60a0ba4 (Jan Kara 2012-07-03 16:45:31 +0200 1940) */
5c0d6b60a0ba4 (Jan Kara 2012-07-03 16:45:31 +0200 1941) iput(old_inode);
5c0d6b60a0ba4 (Jan Kara 2012-07-03 16:45:31 +0200 1942) old_inode = inode;
af309226db916 (Rabin Vincent 2016-12-01 09:18:28 +0100 1943) bdev = I_BDEV(inode);
5c0d6b60a0ba4 (Jan Kara 2012-07-03 16:45:31 +0200 1944)
af309226db916 (Rabin Vincent 2016-12-01 09:18:28 +0100 1945) mutex_lock(&bdev->bd_mutex);
af309226db916 (Rabin Vincent 2016-12-01 09:18:28 +0100 1946) if (bdev->bd_openers)
af309226db916 (Rabin Vincent 2016-12-01 09:18:28 +0100 1947) func(bdev, arg);
af309226db916 (Rabin Vincent 2016-12-01 09:18:28 +0100 1948) mutex_unlock(&bdev->bd_mutex);
5c0d6b60a0ba4 (Jan Kara 2012-07-03 16:45:31 +0200 1949)
74278da9f70d8 (Dave Chinner 2015-03-04 12:37:22 -0500 1950) spin_lock(&blockdev_superblock->s_inode_list_lock);
5c0d6b60a0ba4 (Jan Kara 2012-07-03 16:45:31 +0200 1951) }
74278da9f70d8 (Dave Chinner 2015-03-04 12:37:22 -0500 1952) spin_unlock(&blockdev_superblock->s_inode_list_lock);
5c0d6b60a0ba4 (Jan Kara 2012-07-03 16:45:31 +0200 1953) iput(old_inode);
5c0d6b60a0ba4 (Jan Kara 2012-07-03 16:45:31 +0200 1954) }
VisionFive2 Linux kernel
StarFive Tech Linux Kernel for VisionFive (JH7110) boards (mirror)
More than 9999 Commits
35 Branches
59 Tags