^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (C) 1991, 1992 Linus Torvalds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Copyright (C) 1994, Karl Keyte: Added support for disk statistics
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Elevator latency, (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Queue request tables / lock, selectable elevator, Jens Axboe <axboe@suse.de>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * kernel-doc documentation started by NeilBrown <neilb@cse.unsw.edu.au>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * - July2000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * bio rewrite, highmem i/o, etc, Jens Axboe <axboe@suse.de> - may 2001
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * This handles all read/write requests to block devices
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/backing-dev.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/bio.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/blkdev.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/blk-mq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/highmem.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/pagemap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/kernel_stat.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/string.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/completion.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <linux/swap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <linux/writeback.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <linux/task_io_accounting_ops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <linux/fault-inject.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <linux/list_sort.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <linux/delay.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <linux/ratelimit.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <linux/pm_runtime.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include <linux/blk-cgroup.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #include <linux/t10-pi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #include <linux/debugfs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #include <linux/bpf.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #include <linux/psi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #include <linux/sched/sysctl.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #include <linux/blk-crypto.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #define CREATE_TRACE_POINTS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #include <trace/events/block.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #include "blk.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #include "blk-mq.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #include "blk-mq-sched.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #include "blk-pm.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #include "blk-rq-qos.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) struct dentry *blk_debugfs_root;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) EXPORT_TRACEPOINT_SYMBOL_GPL(block_split);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) EXPORT_TRACEPOINT_SYMBOL_GPL(block_unplug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) EXPORT_TRACEPOINT_SYMBOL_GPL(block_getrq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_insert);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_issue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_merge);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_requeue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_complete);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) DEFINE_IDA(blk_queue_ida);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) * For queue allocation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) struct kmem_cache *blk_requestq_cachep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) * Controlling structure to kblockd
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) static struct workqueue_struct *kblockd_workqueue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) * blk_queue_flag_set - atomically set a queue flag
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) * @flag: flag to be set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) * @q: request queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) void blk_queue_flag_set(unsigned int flag, struct request_queue *q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) set_bit(flag, &q->queue_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) EXPORT_SYMBOL(blk_queue_flag_set);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) * blk_queue_flag_clear - atomically clear a queue flag
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) * @flag: flag to be cleared
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) * @q: request queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) void blk_queue_flag_clear(unsigned int flag, struct request_queue *q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) clear_bit(flag, &q->queue_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) EXPORT_SYMBOL(blk_queue_flag_clear);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) * blk_queue_flag_test_and_set - atomically test and set a queue flag
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) * @flag: flag to be set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) * @q: request queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) * Returns the previous value of @flag - 0 if the flag was not set and 1 if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) * the flag was already set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) return test_and_set_bit(flag, &q->queue_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) EXPORT_SYMBOL_GPL(blk_queue_flag_test_and_set);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) void blk_rq_init(struct request_queue *q, struct request *rq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) memset(rq, 0, sizeof(*rq));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) INIT_LIST_HEAD(&rq->queuelist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) rq->q = q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) rq->__sector = (sector_t) -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) INIT_HLIST_NODE(&rq->hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) RB_CLEAR_NODE(&rq->rb_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) rq->tag = BLK_MQ_NO_TAG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) rq->internal_tag = BLK_MQ_NO_TAG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) rq->start_time_ns = ktime_get_ns();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) rq->part = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) blk_crypto_rq_set_defaults(rq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) EXPORT_SYMBOL(blk_rq_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) #define REQ_OP_NAME(name) [REQ_OP_##name] = #name
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) static const char *const blk_op_name[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) REQ_OP_NAME(READ),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) REQ_OP_NAME(WRITE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) REQ_OP_NAME(FLUSH),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) REQ_OP_NAME(DISCARD),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) REQ_OP_NAME(SECURE_ERASE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) REQ_OP_NAME(ZONE_RESET),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) REQ_OP_NAME(ZONE_RESET_ALL),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) REQ_OP_NAME(ZONE_OPEN),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) REQ_OP_NAME(ZONE_CLOSE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) REQ_OP_NAME(ZONE_FINISH),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) REQ_OP_NAME(ZONE_APPEND),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) REQ_OP_NAME(WRITE_SAME),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) REQ_OP_NAME(WRITE_ZEROES),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) REQ_OP_NAME(SCSI_IN),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) REQ_OP_NAME(SCSI_OUT),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) REQ_OP_NAME(DRV_IN),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) REQ_OP_NAME(DRV_OUT),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) #undef REQ_OP_NAME
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) * blk_op_str - Return string XXX in the REQ_OP_XXX.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) * @op: REQ_OP_XXX.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) * Description: Centralize block layer function to convert REQ_OP_XXX into
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) * string format. Useful in the debugging and tracing bio or request. For
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) * invalid REQ_OP_XXX it returns string "UNKNOWN".
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) inline const char *blk_op_str(unsigned int op)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) const char *op_str = "UNKNOWN";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) if (op < ARRAY_SIZE(blk_op_name) && blk_op_name[op])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) op_str = blk_op_name[op];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) return op_str;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) EXPORT_SYMBOL_GPL(blk_op_str);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) static const struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) int errno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) const char *name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) } blk_errors[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) [BLK_STS_OK] = { 0, "" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) [BLK_STS_NOTSUPP] = { -EOPNOTSUPP, "operation not supported" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) [BLK_STS_TIMEOUT] = { -ETIMEDOUT, "timeout" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) [BLK_STS_NOSPC] = { -ENOSPC, "critical space allocation" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) [BLK_STS_TRANSPORT] = { -ENOLINK, "recoverable transport" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) [BLK_STS_TARGET] = { -EREMOTEIO, "critical target" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) [BLK_STS_NEXUS] = { -EBADE, "critical nexus" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) [BLK_STS_MEDIUM] = { -ENODATA, "critical medium" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) [BLK_STS_PROTECTION] = { -EILSEQ, "protection" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) [BLK_STS_RESOURCE] = { -ENOMEM, "kernel resource" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) [BLK_STS_DEV_RESOURCE] = { -EBUSY, "device resource" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) [BLK_STS_AGAIN] = { -EAGAIN, "nonblocking retry" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) /* device mapper special case, should not leak out: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) [BLK_STS_DM_REQUEUE] = { -EREMCHG, "dm internal retry" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) /* zone device specific errors */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) [BLK_STS_ZONE_OPEN_RESOURCE] = { -ETOOMANYREFS, "open zones exceeded" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) [BLK_STS_ZONE_ACTIVE_RESOURCE] = { -EOVERFLOW, "active zones exceeded" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) /* everything else not covered above: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) [BLK_STS_IOERR] = { -EIO, "I/O" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) blk_status_t errno_to_blk_status(int errno)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) for (i = 0; i < ARRAY_SIZE(blk_errors); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) if (blk_errors[i].errno == errno)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) return (__force blk_status_t)i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) return BLK_STS_IOERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) EXPORT_SYMBOL_GPL(errno_to_blk_status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) int blk_status_to_errno(blk_status_t status)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) int idx = (__force int)status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) return blk_errors[idx].errno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) EXPORT_SYMBOL_GPL(blk_status_to_errno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) static void print_req_error(struct request *req, blk_status_t status,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) const char *caller)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) int idx = (__force int)status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) printk_ratelimited(KERN_ERR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) "%s: %s error, dev %s, sector %llu op 0x%x:(%s) flags 0x%x "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) "phys_seg %u prio class %u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) caller, blk_errors[idx].name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) req->rq_disk ? req->rq_disk->disk_name : "?",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) blk_rq_pos(req), req_op(req), blk_op_str(req_op(req)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) req->cmd_flags & ~REQ_OP_MASK,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) req->nr_phys_segments,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) IOPRIO_PRIO_CLASS(req->ioprio));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) static void req_bio_endio(struct request *rq, struct bio *bio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) unsigned int nbytes, blk_status_t error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) if (error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) bio->bi_status = error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) if (unlikely(rq->rq_flags & RQF_QUIET))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) bio_set_flag(bio, BIO_QUIET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) bio_advance(bio, nbytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) if (req_op(rq) == REQ_OP_ZONE_APPEND && error == BLK_STS_OK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) * Partial zone append completions cannot be supported as the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) * BIO fragments may end up not being written sequentially.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) if (bio->bi_iter.bi_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) bio->bi_status = BLK_STS_IOERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) bio->bi_iter.bi_sector = rq->__sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) /* don't actually finish bio if it's part of flush sequence */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) if (bio->bi_iter.bi_size == 0 && !(rq->rq_flags & RQF_FLUSH_SEQ))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) bio_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) void blk_dump_rq_flags(struct request *rq, char *msg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) printk(KERN_INFO "%s: dev %s: flags=%llx\n", msg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) rq->rq_disk ? rq->rq_disk->disk_name : "?",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) (unsigned long long) rq->cmd_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) printk(KERN_INFO " sector %llu, nr/cnr %u/%u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) (unsigned long long)blk_rq_pos(rq),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) blk_rq_sectors(rq), blk_rq_cur_sectors(rq));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) printk(KERN_INFO " bio %p, biotail %p, len %u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) rq->bio, rq->biotail, blk_rq_bytes(rq));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) EXPORT_SYMBOL(blk_dump_rq_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) * blk_sync_queue - cancel any pending callbacks on a queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) * @q: the queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) * The block layer may perform asynchronous callback activity
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) * on a queue, such as calling the unplug function after a timeout.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) * A block device may call blk_sync_queue to ensure that any
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) * such activity is cancelled, thus allowing it to release resources
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) * that the callbacks might use. The caller must already have made sure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) * that its ->submit_bio will not re-add plugging prior to calling
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) * this function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) * This function does not cancel any asynchronous activity arising
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) * out of elevator or throttling code. That would require elevator_exit()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) * and blkcg_exit_queue() to be called with queue lock initialized.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) void blk_sync_queue(struct request_queue *q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) del_timer_sync(&q->timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) cancel_work_sync(&q->timeout_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) EXPORT_SYMBOL(blk_sync_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) * blk_set_pm_only - increment pm_only counter
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) * @q: request queue pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) void blk_set_pm_only(struct request_queue *q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) atomic_inc(&q->pm_only);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) EXPORT_SYMBOL_GPL(blk_set_pm_only);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) void blk_clear_pm_only(struct request_queue *q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) int pm_only;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) pm_only = atomic_dec_return(&q->pm_only);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) WARN_ON_ONCE(pm_only < 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) if (pm_only == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) wake_up_all(&q->mq_freeze_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) EXPORT_SYMBOL_GPL(blk_clear_pm_only);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) * blk_put_queue - decrement the request_queue refcount
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) * @q: the request_queue structure to decrement the refcount for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) * Decrements the refcount of the request_queue kobject. When this reaches 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) * we'll have blk_release_queue() called.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) * Context: Any context, but the last reference must not be dropped from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) * atomic context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) void blk_put_queue(struct request_queue *q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) kobject_put(&q->kobj);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) EXPORT_SYMBOL(blk_put_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) void blk_set_queue_dying(struct request_queue *q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) blk_queue_flag_set(QUEUE_FLAG_DYING, q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) * When queue DYING flag is set, we need to block new req
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) * entering queue, so we call blk_freeze_queue_start() to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) * prevent I/O from crossing blk_queue_enter().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) blk_freeze_queue_start(q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) if (queue_is_mq(q))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) blk_mq_wake_waiters(q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) /* Make blk_queue_enter() reexamine the DYING flag. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) wake_up_all(&q->mq_freeze_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) EXPORT_SYMBOL_GPL(blk_set_queue_dying);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) * blk_cleanup_queue - shutdown a request queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) * @q: request queue to shutdown
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) * Mark @q DYING, drain all pending requests, mark @q DEAD, destroy and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) * put it. All future requests will be failed immediately with -ENODEV.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) * Context: can sleep
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) void blk_cleanup_queue(struct request_queue *q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) /* cannot be called from atomic context */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) might_sleep();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) WARN_ON_ONCE(blk_queue_registered(q));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) /* mark @q DYING, no new request or merges will be allowed afterwards */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) blk_set_queue_dying(q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) blk_queue_flag_set(QUEUE_FLAG_NOMERGES, q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) blk_queue_flag_set(QUEUE_FLAG_NOXMERGES, q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) * Drain all requests queued before DYING marking. Set DEAD flag to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) * prevent that blk_mq_run_hw_queues() accesses the hardware queues
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) * after draining finished.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) blk_freeze_queue(q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) rq_qos_exit(q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) blk_queue_flag_set(QUEUE_FLAG_DEAD, q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) /* for synchronous bio-based driver finish in-flight integrity i/o */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) blk_flush_integrity();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) /* @q won't process any more request, flush async actions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) del_timer_sync(&q->backing_dev_info->laptop_mode_wb_timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) blk_sync_queue(q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) if (queue_is_mq(q))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) blk_mq_exit_queue(q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) * In theory, request pool of sched_tags belongs to request queue.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) * However, the current implementation requires tag_set for freeing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) * requests, so free the pool now.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) * Queue has become frozen, there can't be any in-queue requests, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) * it is safe to free requests now.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) mutex_lock(&q->sysfs_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) if (q->elevator)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) blk_mq_sched_free_requests(q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) mutex_unlock(&q->sysfs_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) percpu_ref_exit(&q->q_usage_counter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) /* @q is and will stay empty, shutdown and put */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) blk_put_queue(q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) EXPORT_SYMBOL(blk_cleanup_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) * blk_queue_enter() - try to increase q->q_usage_counter
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) * @q: request queue pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) * @flags: BLK_MQ_REQ_NOWAIT and/or BLK_MQ_REQ_PM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) const bool pm = flags & BLK_MQ_REQ_PM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) while (true) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) bool success = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) if (percpu_ref_tryget_live(&q->q_usage_counter)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) * The code that increments the pm_only counter is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) * responsible for ensuring that that counter is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) * globally visible before the queue is unfrozen.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) if (pm || !blk_queue_pm_only(q)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) success = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) percpu_ref_put(&q->q_usage_counter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) if (success)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) if (flags & BLK_MQ_REQ_NOWAIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) * read pair of barrier in blk_freeze_queue_start(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) * we need to order reading __PERCPU_REF_DEAD flag of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) * .q_usage_counter and reading .mq_freeze_depth or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) * queue dying flag, otherwise the following wait may
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) * never return if the two reads are reordered.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) smp_rmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) wait_event(q->mq_freeze_wq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) (!q->mq_freeze_depth &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) (pm || (blk_pm_request_resume(q),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) !blk_queue_pm_only(q)))) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) blk_queue_dying(q));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) if (blk_queue_dying(q))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) static inline int bio_queue_enter(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) struct request_queue *q = bio->bi_disk->queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) bool nowait = bio->bi_opf & REQ_NOWAIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) ret = blk_queue_enter(q, nowait ? BLK_MQ_REQ_NOWAIT : 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) if (unlikely(ret)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) if (nowait && !blk_queue_dying(q))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) bio_wouldblock_error(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) bio_io_error(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) void blk_queue_exit(struct request_queue *q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) percpu_ref_put(&q->q_usage_counter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) static void blk_queue_usage_counter_release(struct percpu_ref *ref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) struct request_queue *q =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) container_of(ref, struct request_queue, q_usage_counter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) wake_up_all(&q->mq_freeze_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) static void blk_rq_timed_out_timer(struct timer_list *t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) struct request_queue *q = from_timer(q, t, timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) kblockd_schedule_work(&q->timeout_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) static void blk_timeout_work(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) struct request_queue *blk_alloc_queue(int node_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) struct request_queue *q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) q = kmem_cache_alloc_node(blk_requestq_cachep,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) GFP_KERNEL | __GFP_ZERO, node_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) if (!q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) q->last_merge = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) q->id = ida_simple_get(&blk_queue_ida, 0, 0, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) if (q->id < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) goto fail_q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) ret = bioset_init(&q->bio_split, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) goto fail_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) q->backing_dev_info = bdi_alloc(node_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) if (!q->backing_dev_info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) goto fail_split;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) q->stats = blk_alloc_queue_stats();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) if (!q->stats)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) goto fail_stats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) q->node = node_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) atomic_set(&q->nr_active_requests_shared_sbitmap, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) timer_setup(&q->backing_dev_info->laptop_mode_wb_timer,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) laptop_mode_timer_fn, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) timer_setup(&q->timeout, blk_rq_timed_out_timer, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) INIT_WORK(&q->timeout_work, blk_timeout_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) INIT_LIST_HEAD(&q->icq_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) #ifdef CONFIG_BLK_CGROUP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) INIT_LIST_HEAD(&q->blkg_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) kobject_init(&q->kobj, &blk_queue_ktype);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) mutex_init(&q->debugfs_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) mutex_init(&q->sysfs_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) mutex_init(&q->sysfs_dir_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) spin_lock_init(&q->queue_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) init_waitqueue_head(&q->mq_freeze_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) mutex_init(&q->mq_freeze_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) * Init percpu_ref in atomic mode so that it's faster to shutdown.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) * See blk_register_queue() for details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) if (percpu_ref_init(&q->q_usage_counter,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) blk_queue_usage_counter_release,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) PERCPU_REF_INIT_ATOMIC, GFP_KERNEL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) goto fail_bdi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) if (blkcg_init_queue(q))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) goto fail_ref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) blk_queue_dma_alignment(q, 511);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) blk_set_default_limits(&q->limits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) q->nr_requests = BLKDEV_MAX_RQ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) return q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) fail_ref:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) percpu_ref_exit(&q->q_usage_counter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) fail_bdi:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) blk_free_queue_stats(q->stats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) fail_stats:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) bdi_put(q->backing_dev_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) fail_split:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) bioset_exit(&q->bio_split);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) fail_id:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) ida_simple_remove(&blk_queue_ida, q->id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) fail_q:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) kmem_cache_free(blk_requestq_cachep, q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) EXPORT_SYMBOL(blk_alloc_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) * blk_get_queue - increment the request_queue refcount
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) * @q: the request_queue structure to increment the refcount for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) * Increment the refcount of the request_queue kobject.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) * Context: Any context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) bool blk_get_queue(struct request_queue *q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) if (likely(!blk_queue_dying(q))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) __blk_get_queue(q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) EXPORT_SYMBOL(blk_get_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) * blk_get_request - allocate a request
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) * @q: request queue to allocate a request for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) * @op: operation (REQ_OP_*) and REQ_* flags, e.g. REQ_SYNC.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) * @flags: BLK_MQ_REQ_* flags, e.g. BLK_MQ_REQ_NOWAIT.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) struct request *blk_get_request(struct request_queue *q, unsigned int op,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) blk_mq_req_flags_t flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) struct request *req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) WARN_ON_ONCE(op & REQ_NOWAIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) WARN_ON_ONCE(flags & ~(BLK_MQ_REQ_NOWAIT | BLK_MQ_REQ_PM));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) req = blk_mq_alloc_request(q, op, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) if (!IS_ERR(req) && q->mq_ops->initialize_rq_fn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) q->mq_ops->initialize_rq_fn(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) return req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) EXPORT_SYMBOL(blk_get_request);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) void blk_put_request(struct request *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) blk_mq_free_request(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) EXPORT_SYMBOL(blk_put_request);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) static void handle_bad_sector(struct bio *bio, sector_t maxsector)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) char b[BDEVNAME_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) pr_info_ratelimited("attempt to access beyond end of device\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) "%s: rw=%d, want=%llu, limit=%llu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) bio_devname(bio, b), bio->bi_opf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) bio_end_sector(bio), maxsector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) #ifdef CONFIG_FAIL_MAKE_REQUEST
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) static DECLARE_FAULT_ATTR(fail_make_request);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) static int __init setup_fail_make_request(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) return setup_fault_attr(&fail_make_request, str);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) __setup("fail_make_request=", setup_fail_make_request);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) static bool should_fail_request(struct hd_struct *part, unsigned int bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) return part->make_it_fail && should_fail(&fail_make_request, bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) static int __init fail_make_request_debugfs(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) struct dentry *dir = fault_create_debugfs_attr("fail_make_request",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) NULL, &fail_make_request);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) return PTR_ERR_OR_ZERO(dir);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) late_initcall(fail_make_request_debugfs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) #else /* CONFIG_FAIL_MAKE_REQUEST */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) static inline bool should_fail_request(struct hd_struct *part,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) unsigned int bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) #endif /* CONFIG_FAIL_MAKE_REQUEST */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) static inline bool bio_check_ro(struct bio *bio, struct hd_struct *part)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) const int op = bio_op(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) if (part->policy && op_is_write(op)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) char b[BDEVNAME_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) if (op_is_flush(bio->bi_opf) && !bio_sectors(bio))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) WARN_ONCE(1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) "Trying to write to read-only block-device %s (partno %d)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) bio_devname(bio, b), part->partno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) /* Older lvm-tools actually trigger this */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) static noinline int should_fail_bio(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) if (should_fail_request(&bio->bi_disk->part0, bio->bi_iter.bi_size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) ALLOW_ERROR_INJECTION(should_fail_bio, ERRNO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) * Check whether this bio extends beyond the end of the device or partition.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) * This may well happen - the kernel calls bread() without checking the size of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) * the device, e.g., when mounting a file system.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) static inline int bio_check_eod(struct bio *bio, sector_t maxsector)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) unsigned int nr_sectors = bio_sectors(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) if (nr_sectors && maxsector &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) (nr_sectors > maxsector ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) bio->bi_iter.bi_sector > maxsector - nr_sectors)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) handle_bad_sector(bio, maxsector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) * Remap block n of partition p to block n+start(p) of the disk.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) static inline int blk_partition_remap(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) struct hd_struct *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) int ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) p = __disk_get_part(bio->bi_disk, bio->bi_partno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) if (unlikely(!p))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) if (unlikely(should_fail_request(p, bio->bi_iter.bi_size)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) if (unlikely(bio_check_ro(bio, p)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) if (bio_sectors(bio)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) if (bio_check_eod(bio, part_nr_sects_read(p)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) bio->bi_iter.bi_sector += p->start_sect;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) trace_block_bio_remap(bio->bi_disk->queue, bio, part_devt(p),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) bio->bi_iter.bi_sector - p->start_sect);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) bio->bi_partno = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) * Check write append to a zoned block device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) static inline blk_status_t blk_check_zone_append(struct request_queue *q,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) sector_t pos = bio->bi_iter.bi_sector;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) int nr_sectors = bio_sectors(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) /* Only applicable to zoned block devices */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) if (!blk_queue_is_zoned(q))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) return BLK_STS_NOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) /* The bio sector must point to the start of a sequential zone */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) if (pos & (blk_queue_zone_sectors(q) - 1) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) !blk_queue_zone_is_seq(q, pos))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) return BLK_STS_IOERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) * Not allowed to cross zone boundaries. Otherwise, the BIO will be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) * split and could result in non-contiguous sectors being written in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) * different zones.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) if (nr_sectors > q->limits.chunk_sectors)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) return BLK_STS_IOERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) /* Make sure the BIO is small enough and will not get split */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) if (nr_sectors > q->limits.max_zone_append_sectors)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) return BLK_STS_IOERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) bio->bi_opf |= REQ_NOMERGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) return BLK_STS_OK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) static noinline_for_stack bool submit_bio_checks(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) struct request_queue *q = bio->bi_disk->queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) blk_status_t status = BLK_STS_IOERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) struct blk_plug *plug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) might_sleep();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) plug = blk_mq_plug(q, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) if (plug && plug->nowait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) bio->bi_opf |= REQ_NOWAIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) * For a REQ_NOWAIT based request, return -EOPNOTSUPP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) * if queue does not support NOWAIT.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) if ((bio->bi_opf & REQ_NOWAIT) && !blk_queue_nowait(q))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) goto not_supported;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) if (should_fail_bio(bio))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) goto end_io;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) if (bio->bi_partno) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) if (unlikely(blk_partition_remap(bio)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) goto end_io;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) if (unlikely(bio_check_ro(bio, &bio->bi_disk->part0)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) goto end_io;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) if (unlikely(bio_check_eod(bio, get_capacity(bio->bi_disk))))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) goto end_io;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) * Filter flush bio's early so that bio based drivers without flush
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) * support don't have to worry about them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) if (op_is_flush(bio->bi_opf) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) !test_bit(QUEUE_FLAG_WC, &q->queue_flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) bio->bi_opf &= ~(REQ_PREFLUSH | REQ_FUA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) if (!bio_sectors(bio)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) status = BLK_STS_OK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) goto end_io;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) if (!test_bit(QUEUE_FLAG_POLL, &q->queue_flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) bio->bi_opf &= ~REQ_HIPRI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) switch (bio_op(bio)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) case REQ_OP_DISCARD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) if (!blk_queue_discard(q))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) goto not_supported;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) case REQ_OP_SECURE_ERASE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) if (!blk_queue_secure_erase(q))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) goto not_supported;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) case REQ_OP_WRITE_SAME:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) if (!q->limits.max_write_same_sectors)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) goto not_supported;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) case REQ_OP_ZONE_APPEND:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) status = blk_check_zone_append(q, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) if (status != BLK_STS_OK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) goto end_io;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) case REQ_OP_ZONE_RESET:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) case REQ_OP_ZONE_OPEN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) case REQ_OP_ZONE_CLOSE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) case REQ_OP_ZONE_FINISH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) if (!blk_queue_is_zoned(q))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) goto not_supported;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) case REQ_OP_ZONE_RESET_ALL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) if (!blk_queue_is_zoned(q) || !blk_queue_zone_resetall(q))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) goto not_supported;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) case REQ_OP_WRITE_ZEROES:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) if (!q->limits.max_write_zeroes_sectors)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) goto not_supported;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) * Various block parts want %current->io_context, so allocate it up
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) * front rather than dealing with lots of pain to allocate it only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) * where needed. This may fail and the block layer knows how to live
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) * with it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) if (unlikely(!current->io_context))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) create_task_io_context(current, GFP_ATOMIC, q->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) if (blk_throtl_bio(bio))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) blk_cgroup_bio_start(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) blkcg_bio_issue_init(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) if (!bio_flagged(bio, BIO_TRACE_COMPLETION)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) trace_block_bio_queue(q, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) /* Now that enqueuing has been traced, we need to trace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) * completion as well.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) bio_set_flag(bio, BIO_TRACE_COMPLETION);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) not_supported:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) status = BLK_STS_NOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) end_io:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) bio->bi_status = status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) bio_endio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) static blk_qc_t __submit_bio(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) struct gendisk *disk = bio->bi_disk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) blk_qc_t ret = BLK_QC_T_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) if (blk_crypto_bio_prep(&bio)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) if (!disk->fops->submit_bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) return blk_mq_submit_bio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) ret = disk->fops->submit_bio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) blk_queue_exit(disk->queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) * The loop in this function may be a bit non-obvious, and so deserves some
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) * explanation:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) * - Before entering the loop, bio->bi_next is NULL (as all callers ensure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) * that), so we have a list with a single bio.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) * - We pretend that we have just taken it off a longer list, so we assign
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) * bio_list to a pointer to the bio_list_on_stack, thus initialising the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) * bio_list of new bios to be added. ->submit_bio() may indeed add some more
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) * bios through a recursive call to submit_bio_noacct. If it did, we find a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) * non-NULL value in bio_list and re-enter the loop from the top.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) * - In this case we really did just take the bio of the top of the list (no
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) * pretending) and so remove it from bio_list, and call into ->submit_bio()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) * again.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) * bio_list_on_stack[0] contains bios submitted by the current ->submit_bio.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) * bio_list_on_stack[1] contains bios that were submitted before the current
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) * ->submit_bio_bio, but that haven't been processed yet.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) static blk_qc_t __submit_bio_noacct(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) struct bio_list bio_list_on_stack[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) blk_qc_t ret = BLK_QC_T_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) BUG_ON(bio->bi_next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) bio_list_init(&bio_list_on_stack[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) current->bio_list = bio_list_on_stack;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) struct request_queue *q = bio->bi_disk->queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) struct bio_list lower, same;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) if (unlikely(bio_queue_enter(bio) != 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) * Create a fresh bio_list for all subordinate requests.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) bio_list_on_stack[1] = bio_list_on_stack[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) bio_list_init(&bio_list_on_stack[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) ret = __submit_bio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) * Sort new bios into those for a lower level and those for the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) * same level.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) bio_list_init(&lower);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) bio_list_init(&same);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) while ((bio = bio_list_pop(&bio_list_on_stack[0])) != NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) if (q == bio->bi_disk->queue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) bio_list_add(&same, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) bio_list_add(&lower, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) * Now assemble so we handle the lowest level first.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) bio_list_merge(&bio_list_on_stack[0], &lower);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) bio_list_merge(&bio_list_on_stack[0], &same);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) bio_list_merge(&bio_list_on_stack[0], &bio_list_on_stack[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) } while ((bio = bio_list_pop(&bio_list_on_stack[0])));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) current->bio_list = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) static blk_qc_t __submit_bio_noacct_mq(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) struct bio_list bio_list[2] = { };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) blk_qc_t ret = BLK_QC_T_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) current->bio_list = bio_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) struct gendisk *disk = bio->bi_disk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) if (unlikely(bio_queue_enter(bio) != 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) if (!blk_crypto_bio_prep(&bio)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) blk_queue_exit(disk->queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) ret = BLK_QC_T_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) ret = blk_mq_submit_bio(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) } while ((bio = bio_list_pop(&bio_list[0])));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) current->bio_list = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) * submit_bio_noacct - re-submit a bio to the block device layer for I/O
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) * @bio: The bio describing the location in memory and on the device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) * This is a version of submit_bio() that shall only be used for I/O that is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) * resubmitted to lower level drivers by stacking block drivers. All file
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) * systems and other upper level users of the block layer should use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) * submit_bio() instead.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) blk_qc_t submit_bio_noacct(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) if (!submit_bio_checks(bio))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) return BLK_QC_T_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) * We only want one ->submit_bio to be active at a time, else stack
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) * usage with stacked devices could be a problem. Use current->bio_list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) * to collect a list of requests submited by a ->submit_bio method while
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) * it is active, and then process them after it returned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) if (current->bio_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) bio_list_add(¤t->bio_list[0], bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) return BLK_QC_T_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) if (!bio->bi_disk->fops->submit_bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) return __submit_bio_noacct_mq(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) return __submit_bio_noacct(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) EXPORT_SYMBOL(submit_bio_noacct);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) * submit_bio - submit a bio to the block device layer for I/O
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) * @bio: The &struct bio which describes the I/O
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) * submit_bio() is used to submit I/O requests to block devices. It is passed a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) * fully set up &struct bio that describes the I/O that needs to be done. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) * bio will be send to the device described by the bi_disk and bi_partno fields.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) * The success/failure status of the request, along with notification of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) * completion, is delivered asynchronously through the ->bi_end_io() callback
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) * in @bio. The bio must NOT be touched by thecaller until ->bi_end_io() has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) * been called.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) blk_qc_t submit_bio(struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) if (blkcg_punt_bio_submit(bio))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) return BLK_QC_T_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) * If it's a regular read/write or a barrier with data attached,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) * go through the normal accounting stuff before submission.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) if (bio_has_data(bio)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) unsigned int count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) if (unlikely(bio_op(bio) == REQ_OP_WRITE_SAME))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) count = queue_logical_block_size(bio->bi_disk->queue) >> 9;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) count = bio_sectors(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) if (op_is_write(bio_op(bio))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) count_vm_events(PGPGOUT, count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) task_io_account_read(bio->bi_iter.bi_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) count_vm_events(PGPGIN, count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) if (unlikely(block_dump)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) char b[BDEVNAME_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) printk(KERN_DEBUG "%s(%d): %s block %Lu on %s (%u sectors)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) current->comm, task_pid_nr(current),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) op_is_write(bio_op(bio)) ? "WRITE" : "READ",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) (unsigned long long)bio->bi_iter.bi_sector,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) bio_devname(bio, b), count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) * If we're reading data that is part of the userspace workingset, count
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) * submission time as memory stall. When the device is congested, or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) * the submitting cgroup IO-throttled, submission can be a significant
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) * part of overall IO time.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) if (unlikely(bio_op(bio) == REQ_OP_READ &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) bio_flagged(bio, BIO_WORKINGSET))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) unsigned long pflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) blk_qc_t ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) psi_memstall_enter(&pflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) ret = submit_bio_noacct(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) psi_memstall_leave(&pflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) return submit_bio_noacct(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) EXPORT_SYMBOL(submit_bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) * blk_cloned_rq_check_limits - Helper function to check a cloned request
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) * for the new queue limits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) * @q: the queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) * @rq: the request being checked
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) * @rq may have been made based on weaker limitations of upper-level queues
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) * in request stacking drivers, and it may violate the limitation of @q.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) * Since the block layer and the underlying device driver trust @rq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) * after it is inserted to @q, it should be checked against @q before
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) * the insertion using this generic function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) * Request stacking drivers like request-based dm may change the queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) * limits when retrying requests on other queues. Those requests need
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) * to be checked against the new queue limits again during dispatch.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) static blk_status_t blk_cloned_rq_check_limits(struct request_queue *q,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) struct request *rq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) unsigned int max_sectors = blk_queue_get_max_sectors(q, req_op(rq));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) if (blk_rq_sectors(rq) > max_sectors) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) * SCSI device does not have a good way to return if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) * Write Same/Zero is actually supported. If a device rejects
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) * a non-read/write command (discard, write same,etc.) the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) * low-level device driver will set the relevant queue limit to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) * 0 to prevent blk-lib from issuing more of the offending
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) * operations. Commands queued prior to the queue limit being
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) * reset need to be completed with BLK_STS_NOTSUPP to avoid I/O
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) * errors being propagated to upper layers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) if (max_sectors == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) return BLK_STS_NOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) printk(KERN_ERR "%s: over max size limit. (%u > %u)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) __func__, blk_rq_sectors(rq), max_sectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) return BLK_STS_IOERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) * queue's settings related to segment counting like q->bounce_pfn
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) * may differ from that of other stacking queues.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) * Recalculate it to check the request correctly on this queue's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) * limitation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) rq->nr_phys_segments = blk_recalc_rq_segments(rq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) if (rq->nr_phys_segments > queue_max_segments(q)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) printk(KERN_ERR "%s: over max segments limit. (%hu > %hu)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) __func__, rq->nr_phys_segments, queue_max_segments(q));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) return BLK_STS_IOERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) return BLK_STS_OK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) * blk_insert_cloned_request - Helper for stacking drivers to submit a request
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) * @q: the queue to submit the request
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) * @rq: the request being queued
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) blk_status_t blk_insert_cloned_request(struct request_queue *q, struct request *rq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) blk_status_t ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) ret = blk_cloned_rq_check_limits(q, rq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) if (ret != BLK_STS_OK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) if (rq->rq_disk &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) should_fail_request(&rq->rq_disk->part0, blk_rq_bytes(rq)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) return BLK_STS_IOERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) if (blk_crypto_insert_cloned_request(rq))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) return BLK_STS_IOERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) if (blk_queue_io_stat(q))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) blk_account_io_start(rq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) * Since we have a scheduler attached on the top device,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) * bypass a potential scheduler on the bottom device for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) * insert.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) return blk_mq_request_issue_directly(rq, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) EXPORT_SYMBOL_GPL(blk_insert_cloned_request);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) * blk_rq_err_bytes - determine number of bytes till the next failure boundary
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) * @rq: request to examine
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) * A request could be merge of IOs which require different failure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) * handling. This function determines the number of bytes which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) * can be failed from the beginning of the request without
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) * crossing into area which need to be retried further.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) * Return:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) * The number of bytes to fail.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) unsigned int blk_rq_err_bytes(const struct request *rq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) unsigned int bytes = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) if (!(rq->rq_flags & RQF_MIXED_MERGE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) return blk_rq_bytes(rq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) * Currently the only 'mixing' which can happen is between
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) * different fastfail types. We can safely fail portions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) * which have all the failfast bits that the first one has -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) * the ones which are at least as eager to fail as the first
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) * one.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) for (bio = rq->bio; bio; bio = bio->bi_next) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) if ((bio->bi_opf & ff) != ff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) bytes += bio->bi_iter.bi_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) /* this could lead to infinite loop */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) BUG_ON(blk_rq_bytes(rq) && !bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) return bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) EXPORT_SYMBOL_GPL(blk_rq_err_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) static void update_io_ticks(struct hd_struct *part, unsigned long now, bool end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) unsigned long stamp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) stamp = READ_ONCE(part->stamp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) if (unlikely(stamp != now)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) if (likely(cmpxchg(&part->stamp, stamp, now) == stamp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) __part_stat_add(part, io_ticks, end ? now - stamp : 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) if (part->partno) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) part = &part_to_disk(part)->part0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) static void blk_account_io_completion(struct request *req, unsigned int bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) if (req->part && blk_do_io_stat(req)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) const int sgrp = op_stat_group(req_op(req));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) struct hd_struct *part;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) part_stat_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) part = req->part;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) part_stat_add(part, sectors[sgrp], bytes >> 9);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) part_stat_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) void blk_account_io_done(struct request *req, u64 now)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) * Account IO completion. flush_rq isn't accounted as a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) * normal IO on queueing nor completion. Accounting the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) * containing request is enough.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) if (req->part && blk_do_io_stat(req) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) !(req->rq_flags & RQF_FLUSH_SEQ)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) const int sgrp = op_stat_group(req_op(req));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) struct hd_struct *part;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) part_stat_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) part = req->part;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) update_io_ticks(part, jiffies, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) part_stat_inc(part, ios[sgrp]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) part_stat_add(part, nsecs[sgrp], now - req->start_time_ns);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) part_stat_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) hd_struct_put(part);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) void blk_account_io_start(struct request *rq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) if (!blk_do_io_stat(rq))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) rq->part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) part_stat_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) update_io_ticks(rq->part, jiffies, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) part_stat_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) static unsigned long __part_start_io_acct(struct hd_struct *part,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) unsigned int sectors, unsigned int op)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) const int sgrp = op_stat_group(op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) unsigned long now = READ_ONCE(jiffies);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) part_stat_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) update_io_ticks(part, now, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) part_stat_inc(part, ios[sgrp]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) part_stat_add(part, sectors[sgrp], sectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) part_stat_local_inc(part, in_flight[op_is_write(op)]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) part_stat_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) return now;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) unsigned long part_start_io_acct(struct gendisk *disk, struct hd_struct **part,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) *part = disk_map_sector_rcu(disk, bio->bi_iter.bi_sector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) return __part_start_io_acct(*part, bio_sectors(bio), bio_op(bio));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) EXPORT_SYMBOL_GPL(part_start_io_acct);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) unsigned long disk_start_io_acct(struct gendisk *disk, unsigned int sectors,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) unsigned int op)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) return __part_start_io_acct(&disk->part0, sectors, op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) EXPORT_SYMBOL(disk_start_io_acct);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) static void __part_end_io_acct(struct hd_struct *part, unsigned int op,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) unsigned long start_time)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) const int sgrp = op_stat_group(op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) unsigned long now = READ_ONCE(jiffies);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) unsigned long duration = now - start_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) part_stat_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) update_io_ticks(part, now, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) part_stat_add(part, nsecs[sgrp], jiffies_to_nsecs(duration));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) part_stat_local_dec(part, in_flight[op_is_write(op)]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) part_stat_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) void part_end_io_acct(struct hd_struct *part, struct bio *bio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) unsigned long start_time)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) __part_end_io_acct(part, bio_op(bio), start_time);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) hd_struct_put(part);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) EXPORT_SYMBOL_GPL(part_end_io_acct);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) void disk_end_io_acct(struct gendisk *disk, unsigned int op,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) unsigned long start_time)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) __part_end_io_acct(&disk->part0, op, start_time);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) EXPORT_SYMBOL(disk_end_io_acct);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) * Steal bios from a request and add them to a bio list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) * The request must not have been partially completed before.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) void blk_steal_bios(struct bio_list *list, struct request *rq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) if (rq->bio) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) if (list->tail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) list->tail->bi_next = rq->bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) list->head = rq->bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) list->tail = rq->biotail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) rq->bio = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) rq->biotail = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) rq->__data_len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) EXPORT_SYMBOL_GPL(blk_steal_bios);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) * blk_update_request - Special helper function for request stacking drivers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) * @req: the request being processed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) * @error: block status code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) * @nr_bytes: number of bytes to complete @req
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) * Ends I/O on a number of bytes attached to @req, but doesn't complete
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) * the request structure even if @req doesn't have leftover.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) * If @req has leftover, sets it up for the next range of segments.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) * This special helper function is only for request stacking drivers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) * (e.g. request-based dm) so that they can handle partial completion.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) * Actual device drivers should use blk_mq_end_request instead.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) * Passing the result of blk_rq_bytes() as @nr_bytes guarantees
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) * %false return from this function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) * Note:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) * The RQF_SPECIAL_PAYLOAD flag is ignored on purpose in both
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) * blk_rq_bytes() and in blk_update_request().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) * Return:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) * %false - this request doesn't have any more data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) * %true - this request has more data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) **/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) bool blk_update_request(struct request *req, blk_status_t error,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) unsigned int nr_bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) int total_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) trace_block_rq_complete(req, blk_status_to_errno(error), nr_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) if (!req->bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) #ifdef CONFIG_BLK_DEV_INTEGRITY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) error == BLK_STS_OK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) req->q->integrity.profile->complete_fn(req, nr_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) if (unlikely(error && !blk_rq_is_passthrough(req) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) !(req->rq_flags & RQF_QUIET)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) print_req_error(req, error, __func__);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) blk_account_io_completion(req, nr_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) total_bytes = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) while (req->bio) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) struct bio *bio = req->bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) if (bio_bytes == bio->bi_iter.bi_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) req->bio = bio->bi_next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) /* Completion has already been traced */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) bio_clear_flag(bio, BIO_TRACE_COMPLETION);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) req_bio_endio(req, bio, bio_bytes, error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) total_bytes += bio_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) nr_bytes -= bio_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) if (!nr_bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) * completely done
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) if (!req->bio) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) * Reset counters so that the request stacking driver
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) * can find how many bytes remain in the request
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) * later.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) req->__data_len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) req->__data_len -= total_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) /* update sector only for requests with clear definition of sector */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) if (!blk_rq_is_passthrough(req))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) req->__sector += total_bytes >> 9;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) /* mixed attributes always follow the first bio */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) if (req->rq_flags & RQF_MIXED_MERGE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) req->cmd_flags &= ~REQ_FAILFAST_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) req->cmd_flags |= req->bio->bi_opf & REQ_FAILFAST_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) if (!(req->rq_flags & RQF_SPECIAL_PAYLOAD)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) * If total number of sectors is less than the first segment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) * size, something has gone terribly wrong.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) blk_dump_rq_flags(req, "request botched");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) req->__data_len = blk_rq_cur_bytes(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) /* recalculate the number of segments */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) req->nr_phys_segments = blk_recalc_rq_segments(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) EXPORT_SYMBOL_GPL(blk_update_request);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) * rq_flush_dcache_pages - Helper function to flush all pages in a request
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) * @rq: the request to be flushed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) * Flush all pages in @rq.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) void rq_flush_dcache_pages(struct request *rq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) struct req_iterator iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) struct bio_vec bvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) rq_for_each_segment(bvec, rq, iter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) flush_dcache_page(bvec.bv_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) EXPORT_SYMBOL_GPL(rq_flush_dcache_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) * blk_lld_busy - Check if underlying low-level drivers of a device are busy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) * @q : the queue of the device being checked
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) * Check if underlying low-level drivers of a device are busy.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) * If the drivers want to export their busy state, they must set own
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) * exporting function using blk_queue_lld_busy() first.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) * Basically, this function is used only by request stacking drivers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) * to stop dispatching requests to underlying devices when underlying
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) * devices are busy. This behavior helps more I/O merging on the queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) * of the request stacking driver and prevents I/O throughput regression
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) * on burst I/O load.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) * Return:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) * 0 - Not busy (The request stacking driver should dispatch request)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) * 1 - Busy (The request stacking driver should stop dispatching request)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) int blk_lld_busy(struct request_queue *q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) if (queue_is_mq(q) && q->mq_ops->busy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) return q->mq_ops->busy(q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) EXPORT_SYMBOL_GPL(blk_lld_busy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) * blk_rq_unprep_clone - Helper function to free all bios in a cloned request
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) * @rq: the clone request to be cleaned up
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) * Free all bios in @rq for a cloned request.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) void blk_rq_unprep_clone(struct request *rq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) struct bio *bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) while ((bio = rq->bio) != NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) rq->bio = bio->bi_next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) bio_put(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) EXPORT_SYMBOL_GPL(blk_rq_unprep_clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) * blk_rq_prep_clone - Helper function to setup clone request
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) * @rq: the request to be setup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) * @rq_src: original request to be cloned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) * @bs: bio_set that bios for clone are allocated from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) * @gfp_mask: memory allocation mask for bio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) * @bio_ctr: setup function to be called for each clone bio.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) * Returns %0 for success, non %0 for failure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) * @data: private data to be passed to @bio_ctr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) * Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) * Also, pages which the original bios are pointing to are not copied
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) * and the cloned bios just point same pages.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) * So cloned bios must be completed before original bios, which means
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) * the caller must complete @rq before @rq_src.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) struct bio_set *bs, gfp_t gfp_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) int (*bio_ctr)(struct bio *, struct bio *, void *),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) struct bio *bio, *bio_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) if (!bs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) bs = &fs_bio_set;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) __rq_for_each_bio(bio_src, rq_src) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) bio = bio_clone_fast(bio_src, gfp_mask, bs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) if (!bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) goto free_and_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) if (bio_ctr && bio_ctr(bio, bio_src, data))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) goto free_and_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) if (rq->bio) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) rq->biotail->bi_next = bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) rq->biotail = bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) rq->bio = rq->biotail = bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) bio = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) /* Copy attributes of the original request to the clone request. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) rq->__sector = blk_rq_pos(rq_src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) rq->__data_len = blk_rq_bytes(rq_src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) if (rq_src->rq_flags & RQF_SPECIAL_PAYLOAD) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) rq->rq_flags |= RQF_SPECIAL_PAYLOAD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) rq->special_vec = rq_src->special_vec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) rq->nr_phys_segments = rq_src->nr_phys_segments;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) rq->ioprio = rq_src->ioprio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) if (rq->bio && blk_crypto_rq_bio_prep(rq, rq->bio, gfp_mask) < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) goto free_and_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) free_and_out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) if (bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) bio_put(bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) blk_rq_unprep_clone(rq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) EXPORT_SYMBOL_GPL(blk_rq_prep_clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) int kblockd_schedule_work(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) return queue_work(kblockd_workqueue, work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) EXPORT_SYMBOL(kblockd_schedule_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) unsigned long delay)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) return mod_delayed_work_on(cpu, kblockd_workqueue, dwork, delay);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) EXPORT_SYMBOL(kblockd_mod_delayed_work_on);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) * blk_start_plug - initialize blk_plug and track it inside the task_struct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) * @plug: The &struct blk_plug that needs to be initialized
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) * blk_start_plug() indicates to the block layer an intent by the caller
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) * to submit multiple I/O requests in a batch. The block layer may use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) * this hint to defer submitting I/Os from the caller until blk_finish_plug()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) * is called. However, the block layer may choose to submit requests
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) * before a call to blk_finish_plug() if the number of queued I/Os
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) * exceeds %BLK_MAX_REQUEST_COUNT, or if the size of the I/O is larger than
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) * %BLK_PLUG_FLUSH_SIZE. The queued I/Os may also be submitted early if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) * the task schedules (see below).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) * Tracking blk_plug inside the task_struct will help with auto-flushing the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) * pending I/O should the task end up blocking between blk_start_plug() and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) * blk_finish_plug(). This is important from a performance perspective, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) * also ensures that we don't deadlock. For instance, if the task is blocking
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) * for a memory allocation, memory reclaim could end up wanting to free a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689) * page belonging to that request that is currently residing in our private
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690) * plug. By flushing the pending I/O when the process goes to sleep, we avoid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) * this kind of deadlock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) void blk_start_plug(struct blk_plug *plug)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) struct task_struct *tsk = current;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) * If this is a nested plug, don't actually assign it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700) if (tsk->plug)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) INIT_LIST_HEAD(&plug->mq_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) INIT_LIST_HEAD(&plug->cb_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705) plug->rq_count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) plug->multiple_queues = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) plug->nowait = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) * Store ordering should not be needed here, since a potential
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) * preempt will imply a full memory barrier
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) tsk->plug = plug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) EXPORT_SYMBOL(blk_start_plug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) static void flush_plug_callbacks(struct blk_plug *plug, bool from_schedule)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719) LIST_HEAD(callbacks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721) while (!list_empty(&plug->cb_list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) list_splice_init(&plug->cb_list, &callbacks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) while (!list_empty(&callbacks)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725) struct blk_plug_cb *cb = list_first_entry(&callbacks,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) struct blk_plug_cb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728) list_del(&cb->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) cb->callback(cb, from_schedule);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734) struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, void *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) struct blk_plug *plug = current->plug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) struct blk_plug_cb *cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740) if (!plug)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) list_for_each_entry(cb, &plug->cb_list, list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) if (cb->callback == unplug && cb->data == data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) return cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747) /* Not currently on the callback list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) BUG_ON(size < sizeof(*cb));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749) cb = kzalloc(size, GFP_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) if (cb) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751) cb->data = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752) cb->callback = unplug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) list_add(&cb->list, &plug->cb_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755) return cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757) EXPORT_SYMBOL(blk_check_plugged);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759) void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) flush_plug_callbacks(plug, from_schedule);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) if (!list_empty(&plug->mq_list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764) blk_mq_flush_plug_list(plug, from_schedule);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768) * blk_finish_plug - mark the end of a batch of submitted I/O
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) * @plug: The &struct blk_plug passed to blk_start_plug()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772) * Indicate that a batch of I/O submissions is complete. This function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) * must be paired with an initial call to blk_start_plug(). The intent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774) * is to allow the block layer to optimize I/O submission. See the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775) * documentation for blk_start_plug() for more information.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777) void blk_finish_plug(struct blk_plug *plug)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779) if (plug != current->plug)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781) blk_flush_plug_list(plug, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783) current->plug = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785) EXPORT_SYMBOL(blk_finish_plug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787) void blk_io_schedule(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789) /* Prevent hang_check timer from firing at us during very long I/O */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790) unsigned long timeout = sysctl_hung_task_timeout_secs * HZ / 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792) if (timeout)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793) io_schedule_timeout(timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795) io_schedule();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797) EXPORT_SYMBOL_GPL(blk_io_schedule);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799) int __init blk_dev_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801) BUILD_BUG_ON(REQ_OP_LAST >= (1 << REQ_OP_BITS));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802) BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803) sizeof_field(struct request, cmd_flags));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804) BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805) sizeof_field(struct bio, bi_opf));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807) /* used for unplugging and affects IO latency/throughput - HIGHPRI */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808) kblockd_workqueue = alloc_workqueue("kblockd",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809) WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810) if (!kblockd_workqueue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811) panic("Failed to create kblockd\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813) blk_requestq_cachep = kmem_cache_create("request_queue",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814) sizeof(struct request_queue), 0, SLAB_PANIC, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816) blk_debugfs_root = debugfs_create_dir("block", NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags