^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: GPL-2.0-or-later */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Header file for the BFQ I/O scheduler: data structures and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * prototypes of interface functions among BFQ components.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #ifndef _BFQ_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #define _BFQ_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/blktrace_api.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/hrtimer.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/blk-cgroup.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include "blk-cgroup-rwstat.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #define BFQ_IOPRIO_CLASSES 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #define BFQ_CL_IDLE_TIMEOUT (HZ/5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #define BFQ_MIN_WEIGHT 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #define BFQ_MAX_WEIGHT 1000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #define BFQ_WEIGHT_CONVERSION_COEFF 10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #define BFQ_DEFAULT_QUEUE_IOPRIO 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #define BFQ_WEIGHT_LEGACY_DFL 100
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #define BFQ_DEFAULT_GRP_IOPRIO 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #define BFQ_DEFAULT_GRP_CLASS IOPRIO_CLASS_BE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #define MAX_PID_STR_LENGTH 12
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * Soft real-time applications are extremely more latency sensitive
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * than interactive ones. Over-raise the weight of the former to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * privilege them against the latter.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #define BFQ_SOFTRT_WEIGHT_FACTOR 100
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) struct bfq_entity;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) * struct bfq_service_tree - per ioprio_class service tree.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) * Each service tree represents a B-WF2Q+ scheduler on its own. Each
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) * ioprio_class has its own independent scheduler, and so its own
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) * bfq_service_tree. All the fields are protected by the queue lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) * of the containing bfqd.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) struct bfq_service_tree {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) /* tree for active entities (i.e., those backlogged) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) struct rb_root active;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) /* tree for idle entities (i.e., not backlogged, with V < F_i)*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) struct rb_root idle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) /* idle entity with minimum F_i */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) struct bfq_entity *first_idle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) /* idle entity with maximum F_i */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) struct bfq_entity *last_idle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) /* scheduler virtual time */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) u64 vtime;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) /* scheduler weight sum; active and idle entities contribute to it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) unsigned long wsum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) * struct bfq_sched_data - multi-class scheduler.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) * bfq_sched_data is the basic scheduler queue. It supports three
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) * ioprio_classes, and can be used either as a toplevel queue or as an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) * intermediate queue in a hierarchical setup.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) * The supported ioprio_classes are the same as in CFQ, in descending
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) * priority order, IOPRIO_CLASS_RT, IOPRIO_CLASS_BE, IOPRIO_CLASS_IDLE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) * Requests from higher priority queues are served before all the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) * requests from lower priority queues; among requests of the same
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) * queue requests are served according to B-WF2Q+.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) * The schedule is implemented by the service trees, plus the field
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) * @next_in_service, which points to the entity on the active trees
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) * that will be served next, if 1) no changes in the schedule occurs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) * before the current in-service entity is expired, 2) the in-service
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) * queue becomes idle when it expires, and 3) if the entity pointed by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) * in_service_entity is not a queue, then the in-service child entity
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) * of the entity pointed by in_service_entity becomes idle on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) * expiration. This peculiar definition allows for the following
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) * optimization, not yet exploited: while a given entity is still in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) * service, we already know which is the best candidate for next
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) * service among the other active entities in the same parent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) * entity. We can then quickly compare the timestamps of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) * in-service entity with those of such best candidate.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) * All fields are protected by the lock of the containing bfqd.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) struct bfq_sched_data {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) /* entity in service */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) struct bfq_entity *in_service_entity;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) /* head-of-line entity (see comments above) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) struct bfq_entity *next_in_service;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) /* array of service trees, one per ioprio_class */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) struct bfq_service_tree service_tree[BFQ_IOPRIO_CLASSES];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) /* last time CLASS_IDLE was served */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) unsigned long bfq_class_idle_last_service;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) * struct bfq_weight_counter - counter of the number of all active queues
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) * with a given weight.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) struct bfq_weight_counter {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) unsigned int weight; /* weight of the queues this counter refers to */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) unsigned int num_active; /* nr of active queues with this weight */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) * Weights tree member (see bfq_data's @queue_weights_tree)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) struct rb_node weights_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) * struct bfq_entity - schedulable entity.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) * A bfq_entity is used to represent either a bfq_queue (leaf node in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) * cgroup hierarchy) or a bfq_group into the upper level scheduler. Each
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) * entity belongs to the sched_data of the parent group in the cgroup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) * hierarchy. Non-leaf entities have also their own sched_data, stored
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) * in @my_sched_data.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) * Each entity stores independently its priority values; this would
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) * allow different weights on different devices, but this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) * functionality is not exported to userspace by now. Priorities and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) * weights are updated lazily, first storing the new values into the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) * new_* fields, then setting the @prio_changed flag. As soon as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) * there is a transition in the entity state that allows the priority
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) * update to take place the effective and the requested priority
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) * values are synchronized.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) * Unless cgroups are used, the weight value is calculated from the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) * ioprio to export the same interface as CFQ. When dealing with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) * "well-behaved" queues (i.e., queues that do not spend too much
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) * time to consume their budget and have true sequential behavior, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) * when there are no external factors breaking anticipation) the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) * relative weights at each level of the cgroups hierarchy should be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) * guaranteed. All the fields are protected by the queue lock of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) * containing bfqd.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) struct bfq_entity {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) /* service_tree member */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) struct rb_node rb_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) * Flag, true if the entity is on a tree (either the active or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) * the idle one of its service_tree) or is in service.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) bool on_st_or_in_serv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) /* B-WF2Q+ start and finish timestamps [sectors/weight] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) u64 start, finish;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) /* tree the entity is enqueued into; %NULL if not on a tree */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) struct rb_root *tree;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) * minimum start time of the (active) subtree rooted at this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) * entity; used for O(log N) lookups into active trees
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) u64 min_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) /* amount of service received during the last service slot */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) int service;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) /* budget, used also to calculate F_i: F_i = S_i + @budget / @weight */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) int budget;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) /* device weight, if non-zero, it overrides the default weight of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) * bfq_group_data */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) int dev_weight;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) /* weight of the queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) int weight;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) /* next weight if a change is in progress */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) int new_weight;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) /* original weight, used to implement weight boosting */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) int orig_weight;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) /* parent entity, for hierarchical scheduling */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) struct bfq_entity *parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) * For non-leaf nodes in the hierarchy, the associated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) * scheduler queue, %NULL on leaf nodes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) struct bfq_sched_data *my_sched_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) /* the scheduler queue this entity belongs to */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) struct bfq_sched_data *sched_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) /* flag, set to request a weight, ioprio or ioprio_class change */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) int prio_changed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) /* flag, set if the entity is counted in groups_with_pending_reqs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) bool in_groups_with_pending_reqs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) struct bfq_group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) * struct bfq_ttime - per process thinktime stats.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) struct bfq_ttime {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) /* completion time of the last request */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) u64 last_end_request;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) /* total process thinktime */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) u64 ttime_total;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) /* number of thinktime samples */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) unsigned long ttime_samples;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) /* average process thinktime */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) u64 ttime_mean;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) * struct bfq_queue - leaf schedulable entity.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) * A bfq_queue is a leaf request queue; it can be associated with an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) * io_context or more, if it is async or shared between cooperating
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) * processes. @cgroup holds a reference to the cgroup, to be sure that it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) * does not disappear while a bfqq still references it (mostly to avoid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) * races between request issuing and task migration followed by cgroup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) * destruction).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) * All the fields are protected by the queue lock of the containing bfqd.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) struct bfq_queue {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) /* reference counter */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) int ref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) /* parent bfq_data */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) struct bfq_data *bfqd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) /* current ioprio and ioprio class */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) unsigned short ioprio, ioprio_class;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) /* next ioprio and ioprio class if a change is in progress */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) unsigned short new_ioprio, new_ioprio_class;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) /* last total-service-time sample, see bfq_update_inject_limit() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) u64 last_serv_time_ns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) /* limit for request injection */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) unsigned int inject_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) /* last time the inject limit has been decreased, in jiffies */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) unsigned long decrease_time_jif;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) * Shared bfq_queue if queue is cooperating with one or more
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) * other queues.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) struct bfq_queue *new_bfqq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) /* request-position tree member (see bfq_group's @rq_pos_tree) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) struct rb_node pos_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) /* request-position tree root (see bfq_group's @rq_pos_tree) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) struct rb_root *pos_root;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) /* sorted list of pending requests */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) struct rb_root sort_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) /* if fifo isn't expired, next request to serve */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) struct request *next_rq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) /* number of sync and async requests queued */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) int queued[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) /* number of requests currently allocated */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) int allocated;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) /* number of pending metadata requests */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) int meta_pending;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) /* fifo list of requests in sort_list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) struct list_head fifo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) /* entity representing this queue in the scheduler */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) struct bfq_entity entity;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) /* pointer to the weight counter associated with this entity */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) struct bfq_weight_counter *weight_counter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) /* maximum budget allowed from the feedback mechanism */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) int max_budget;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) /* budget expiration (in jiffies) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) unsigned long budget_timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) /* number of requests on the dispatch list or inside driver */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) int dispatched;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) /* status flags */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) /* node for active/idle bfqq list inside parent bfqd */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) struct list_head bfqq_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) /* associated @bfq_ttime struct */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) struct bfq_ttime ttime;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) /* bit vector: a 1 for each seeky requests in history */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) u32 seek_history;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) /* node for the device's burst list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) struct hlist_node burst_list_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) /* position of the last request enqueued */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) sector_t last_request_pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) /* Number of consecutive pairs of request completion and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) * arrival, such that the queue becomes idle after the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) * completion, but the next request arrives within an idle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) * time slice; used only if the queue's IO_bound flag has been
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) * cleared.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) unsigned int requests_within_timer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) /* pid of the process owning the queue, used for logging purposes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) pid_t pid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) * Pointer to the bfq_io_cq owning the bfq_queue, set to %NULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) * if the queue is shared.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) struct bfq_io_cq *bic;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) /* current maximum weight-raising time for this queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) unsigned long wr_cur_max_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) * Minimum time instant such that, only if a new request is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) * enqueued after this time instant in an idle @bfq_queue with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) * no outstanding requests, then the task associated with the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) * queue it is deemed as soft real-time (see the comments on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) * the function bfq_bfqq_softrt_next_start())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) unsigned long soft_rt_next_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) * Start time of the current weight-raising period if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) * the @bfq-queue is being weight-raised, otherwise
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) * finish time of the last weight-raising period.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) unsigned long last_wr_start_finish;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) /* factor by which the weight of this queue is multiplied */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) unsigned int wr_coeff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) * Time of the last transition of the @bfq_queue from idle to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) * backlogged.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) unsigned long last_idle_bklogged;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) * Cumulative service received from the @bfq_queue since the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) * last transition from idle to backlogged.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) unsigned long service_from_backlogged;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) * Cumulative service received from the @bfq_queue since its
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) * last transition to weight-raised state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) unsigned long service_from_wr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) * Value of wr start time when switching to soft rt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) unsigned long wr_start_at_switch_to_srt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) unsigned long split_time; /* time of last split */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) unsigned long first_IO_time; /* time of first I/O for this queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) /* max service rate measured so far */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) u32 max_service_rate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) * Pointer to the waker queue for this queue, i.e., to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) * queue Q such that this queue happens to get new I/O right
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) * after some I/O request of Q is completed. For details, see
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) * the comments on the choice of the queue for injection in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) * bfq_select_queue().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) struct bfq_queue *waker_bfqq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) /* node for woken_list, see below */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) struct hlist_node woken_list_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) * Head of the list of the woken queues for this queue, i.e.,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) * of the list of the queues for which this queue is a waker
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) * queue. This list is used to reset the waker_bfqq pointer in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) * the woken queues when this queue exits.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) struct hlist_head woken_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) * struct bfq_io_cq - per (request_queue, io_context) structure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) struct bfq_io_cq {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) /* associated io_cq structure */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) struct io_cq icq; /* must be the first member */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) /* array of two process queues, the sync and the async */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) struct bfq_queue *bfqq[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) /* per (request_queue, blkcg) ioprio */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) int ioprio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) #ifdef CONFIG_BFQ_GROUP_IOSCHED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) uint64_t blkcg_serial_nr; /* the current blkcg serial */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) * Snapshot of the has_short_time flag before merging; taken
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) * to remember its value while the queue is merged, so as to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) * be able to restore it in case of split.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) bool saved_has_short_ttime;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) * Same purpose as the previous two fields for the I/O bound
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) * classification of a queue.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) bool saved_IO_bound;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) * Same purpose as the previous fields for the value of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) * field keeping the queue's belonging to a large burst
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) bool saved_in_large_burst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) * True if the queue belonged to a burst list before its merge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) * with another cooperating queue.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) bool was_in_burst_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) * Save the weight when a merge occurs, to be able
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) * to restore it in case of split. If the weight is not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) * correctly resumed when the queue is recycled,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) * then the weight of the recycled queue could differ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) * from the weight of the original queue.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) unsigned int saved_weight;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) * Similar to previous fields: save wr information.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) unsigned long saved_wr_coeff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) unsigned long saved_last_wr_start_finish;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) unsigned long saved_wr_start_at_switch_to_srt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) unsigned int saved_wr_cur_max_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) struct bfq_ttime saved_ttime;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) * struct bfq_data - per-device data structure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) * All the fields are protected by @lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) struct bfq_data {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) /* device request queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) struct request_queue *queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) /* dispatch queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) struct list_head dispatch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) /* root bfq_group for the device */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) struct bfq_group *root_group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) * rbtree of weight counters of @bfq_queues, sorted by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) * weight. Used to keep track of whether all @bfq_queues have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) * the same weight. The tree contains one counter for each
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) * distinct weight associated to some active and not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) * weight-raised @bfq_queue (see the comments to the functions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) * bfq_weights_tree_[add|remove] for further details).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) struct rb_root_cached queue_weights_tree;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) * Number of groups with at least one descendant process that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) * has at least one request waiting for completion. Note that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) * this accounts for also requests already dispatched, but not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) * yet completed. Therefore this number of groups may differ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) * (be larger) than the number of active groups, as a group is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) * considered active only if its corresponding entity has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) * descendant queues with at least one request queued. This
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) * number is used to decide whether a scenario is symmetric.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) * For a detailed explanation see comments on the computation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) * of the variable asymmetric_scenario in the function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) * bfq_better_to_idle().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) * However, it is hard to compute this number exactly, for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) * groups with multiple descendant processes. Consider a group
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) * that is inactive, i.e., that has no descendant process with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) * pending I/O inside BFQ queues. Then suppose that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) * num_groups_with_pending_reqs is still accounting for this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) * group, because the group has descendant processes with some
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) * I/O request still in flight. num_groups_with_pending_reqs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) * should be decremented when the in-flight request of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) * last descendant process is finally completed (assuming that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) * nothing else has changed for the group in the meantime, in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) * terms of composition of the group and active/inactive state of child
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) * groups and processes). To accomplish this, an additional
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) * pending-request counter must be added to entities, and must
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) * be updated correctly. To avoid this additional field and operations,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) * we resort to the following tradeoff between simplicity and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) * accuracy: for an inactive group that is still counted in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) * num_groups_with_pending_reqs, we decrement
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) * num_groups_with_pending_reqs when the first descendant
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) * process of the group remains with no request waiting for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) * completion.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) * Even this simpler decrement strategy requires a little
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) * carefulness: to avoid multiple decrements, we flag a group,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) * more precisely an entity representing a group, as still
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) * counted in num_groups_with_pending_reqs when it becomes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) * inactive. Then, when the first descendant queue of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) * entity remains with no request waiting for completion,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) * num_groups_with_pending_reqs is decremented, and this flag
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) * is reset. After this flag is reset for the entity,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) * num_groups_with_pending_reqs won't be decremented any
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) * longer in case a new descendant queue of the entity remains
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) * with no request waiting for completion.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) unsigned int num_groups_with_pending_reqs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) * Per-class (RT, BE, IDLE) number of bfq_queues containing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) * requests (including the queue in service, even if it is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) * idling).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) unsigned int busy_queues[3];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) /* number of weight-raised busy @bfq_queues */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) int wr_busy_queues;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) /* number of queued requests */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) int queued;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) /* number of requests dispatched and waiting for completion */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) int rq_in_driver;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) /* true if the device is non rotational and performs queueing */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) bool nonrot_with_queueing;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) * Maximum number of requests in driver in the last
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) * @hw_tag_samples completed requests.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) int max_rq_in_driver;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) /* number of samples used to calculate hw_tag */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) int hw_tag_samples;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) /* flag set to one if the driver is showing a queueing behavior */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) int hw_tag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) /* number of budgets assigned */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) int budgets_assigned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) * Timer set when idling (waiting) for the next request from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) * the queue in service.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) struct hrtimer idle_slice_timer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) /* bfq_queue in service */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) struct bfq_queue *in_service_queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) /* on-disk position of the last served request */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) sector_t last_position;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) /* position of the last served request for the in-service queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) sector_t in_serv_last_pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) /* time of last request completion (ns) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) u64 last_completion;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) /* bfqq owning the last completed rq */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) struct bfq_queue *last_completed_rq_bfqq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) /* time of last transition from empty to non-empty (ns) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) u64 last_empty_occupied_ns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) * Flag set to activate the sampling of the total service time
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) * of a just-arrived first I/O request (see
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) * bfq_update_inject_limit()). This will cause the setting of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) * waited_rq when the request is finally dispatched.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) bool wait_dispatch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) * If set, then bfq_update_inject_limit() is invoked when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) * waited_rq is eventually completed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) struct request *waited_rq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) * True if some request has been injected during the last service hole.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) bool rqs_injected;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) /* time of first rq dispatch in current observation interval (ns) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) u64 first_dispatch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) /* time of last rq dispatch in current observation interval (ns) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) u64 last_dispatch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) /* beginning of the last budget */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) ktime_t last_budget_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) /* beginning of the last idle slice */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) ktime_t last_idling_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) unsigned long last_idling_start_jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) /* number of samples in current observation interval */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) int peak_rate_samples;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) /* num of samples of seq dispatches in current observation interval */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) u32 sequential_samples;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) /* total num of sectors transferred in current observation interval */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) u64 tot_sectors_dispatched;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) /* max rq size seen during current observation interval (sectors) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) u32 last_rq_max_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) /* time elapsed from first dispatch in current observ. interval (us) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) u64 delta_from_first;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) * Current estimate of the device peak rate, measured in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) * [(sectors/usec) / 2^BFQ_RATE_SHIFT]. The left-shift by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) * BFQ_RATE_SHIFT is performed to increase precision in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) * fixed-point calculations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) u32 peak_rate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) /* maximum budget allotted to a bfq_queue before rescheduling */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) int bfq_max_budget;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) /* list of all the bfq_queues active on the device */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) struct list_head active_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) /* list of all the bfq_queues idle on the device */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) struct list_head idle_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) * Timeout for async/sync requests; when it fires, requests
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) * are served in fifo order.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) u64 bfq_fifo_expire[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) /* weight of backward seeks wrt forward ones */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) unsigned int bfq_back_penalty;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) /* maximum allowed backward seek */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) unsigned int bfq_back_max;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) /* maximum idling time */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) u32 bfq_slice_idle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) /* user-configured max budget value (0 for auto-tuning) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) int bfq_user_max_budget;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) * Timeout for bfq_queues to consume their budget; used to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) * prevent seeky queues from imposing long latencies to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) * sequential or quasi-sequential ones (this also implies that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) * seeky queues cannot receive guarantees in the service
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) * domain; after a timeout they are charged for the time they
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) * have been in service, to preserve fairness among them, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) * without service-domain guarantees).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) unsigned int bfq_timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) * Number of consecutive requests that must be issued within
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) * the idle time slice to set again idling to a queue which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) * was marked as non-I/O-bound (see the definition of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) * IO_bound flag for further details).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) unsigned int bfq_requests_within_timer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) * Force device idling whenever needed to provide accurate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) * service guarantees, without caring about throughput
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) * issues. CAVEAT: this may even increase latencies, in case
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) * of useless idling for processes that did stop doing I/O.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) bool strict_guarantees;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) * Last time at which a queue entered the current burst of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) * queues being activated shortly after each other; for more
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) * details about this and the following parameters related to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) * a burst of activations, see the comments on the function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) * bfq_handle_burst.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) unsigned long last_ins_in_burst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) * Reference time interval used to decide whether a queue has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) * been activated shortly after @last_ins_in_burst.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) unsigned long bfq_burst_interval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) /* number of queues in the current burst of queue activations */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) int burst_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) /* common parent entity for the queues in the burst */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) struct bfq_entity *burst_parent_entity;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) /* Maximum burst size above which the current queue-activation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) * burst is deemed as 'large'.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) unsigned long bfq_large_burst_thresh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) /* true if a large queue-activation burst is in progress */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) bool large_burst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) * Head of the burst list (as for the above fields, more
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) * details in the comments on the function bfq_handle_burst).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) struct hlist_head burst_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) /* if set to true, low-latency heuristics are enabled */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) bool low_latency;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) * Maximum factor by which the weight of a weight-raised queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) * is multiplied.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) unsigned int bfq_wr_coeff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) /* maximum duration of a weight-raising period (jiffies) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) unsigned int bfq_wr_max_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) /* Maximum weight-raising duration for soft real-time processes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) unsigned int bfq_wr_rt_max_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) * Minimum idle period after which weight-raising may be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) * reactivated for a queue (in jiffies).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) unsigned int bfq_wr_min_idle_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) * Minimum period between request arrivals after which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) * weight-raising may be reactivated for an already busy async
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) * queue (in jiffies).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) unsigned long bfq_wr_min_inter_arr_async;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) /* Max service-rate for a soft real-time queue, in sectors/sec */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) unsigned int bfq_wr_max_softrt_rate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) * Cached value of the product ref_rate*ref_wr_duration, used
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) * for computing the maximum duration of weight raising
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) * automatically.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) u64 rate_dur_prod;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) /* fallback dummy bfqq for extreme OOM conditions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) struct bfq_queue oom_bfqq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) spinlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) * bic associated with the task issuing current bio for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) * merging. This and the next field are used as a support to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) * be able to perform the bic lookup, needed by bio-merge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) * functions, before the scheduler lock is taken, and thus
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) * avoid taking the request-queue lock while the scheduler
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) * lock is being held.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) struct bfq_io_cq *bio_bic;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) /* bfqq associated with the task issuing current bio for merging */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) struct bfq_queue *bio_bfqq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) * Depth limits used in bfq_limit_depth (see comments on the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) * function)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) unsigned int word_depths[2][2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) enum bfqq_state_flags {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) BFQQF_just_created = 0, /* queue just allocated */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) BFQQF_busy, /* has requests or is in service */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) BFQQF_wait_request, /* waiting for a request */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) BFQQF_non_blocking_wait_rq, /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) * waiting for a request
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) * without idling the device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) BFQQF_fifo_expire, /* FIFO checked in this slice */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) BFQQF_has_short_ttime, /* queue has a short think time */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) BFQQF_sync, /* synchronous queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) BFQQF_IO_bound, /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) * bfqq has timed-out at least once
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) * having consumed at most 2/10 of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) * its budget
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) BFQQF_in_large_burst, /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) * bfqq activated in a large burst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) * see comments to bfq_handle_burst.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) BFQQF_softrt_update, /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) * may need softrt-next-start
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) * update
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) BFQQF_coop, /* bfqq is shared */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) BFQQF_split_coop, /* shared bfqq will be split */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) BFQQF_has_waker /* bfqq has a waker queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) #define BFQ_BFQQ_FNS(name) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) void bfq_mark_bfqq_##name(struct bfq_queue *bfqq); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) void bfq_clear_bfqq_##name(struct bfq_queue *bfqq); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) int bfq_bfqq_##name(const struct bfq_queue *bfqq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) BFQ_BFQQ_FNS(just_created);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) BFQ_BFQQ_FNS(busy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) BFQ_BFQQ_FNS(wait_request);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) BFQ_BFQQ_FNS(non_blocking_wait_rq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) BFQ_BFQQ_FNS(fifo_expire);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) BFQ_BFQQ_FNS(has_short_ttime);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) BFQ_BFQQ_FNS(sync);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) BFQ_BFQQ_FNS(IO_bound);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) BFQ_BFQQ_FNS(in_large_burst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) BFQ_BFQQ_FNS(coop);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) BFQ_BFQQ_FNS(split_coop);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) BFQ_BFQQ_FNS(softrt_update);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) BFQ_BFQQ_FNS(has_waker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) #undef BFQ_BFQQ_FNS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) /* Expiration reasons. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) enum bfqq_expiration {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) BFQQE_TOO_IDLE = 0, /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) * queue has been idling for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) * too long
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) BFQQE_BUDGET_TIMEOUT, /* budget took too long to be used */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) BFQQE_BUDGET_EXHAUSTED, /* budget consumed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) BFQQE_NO_MORE_REQUESTS, /* the queue has no more requests */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) BFQQE_PREEMPTED /* preemption in progress */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) struct bfq_stat {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) struct percpu_counter cpu_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) atomic64_t aux_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) struct bfqg_stats {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) /* basic stats */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) struct blkg_rwstat bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) struct blkg_rwstat ios;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) #ifdef CONFIG_BFQ_CGROUP_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) /* number of ios merged */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) struct blkg_rwstat merged;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) /* total time spent on device in ns, may not be accurate w/ queueing */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) struct blkg_rwstat service_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) /* total time spent waiting in scheduler queue in ns */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) struct blkg_rwstat wait_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) /* number of IOs queued up */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) struct blkg_rwstat queued;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) /* total disk time and nr sectors dispatched by this group */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) struct bfq_stat time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) /* sum of number of ios queued across all samples */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) struct bfq_stat avg_queue_size_sum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) /* count of samples taken for average */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) struct bfq_stat avg_queue_size_samples;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) /* how many times this group has been removed from service tree */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) struct bfq_stat dequeue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) /* total time spent waiting for it to be assigned a timeslice. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) struct bfq_stat group_wait_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) /* time spent idling for this blkcg_gq */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) struct bfq_stat idle_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) /* total time with empty current active q with other requests queued */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) struct bfq_stat empty_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) /* fields after this shouldn't be cleared on stat reset */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) u64 start_group_wait_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) u64 start_idle_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) u64 start_empty_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) uint16_t flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) #endif /* CONFIG_BFQ_CGROUP_DEBUG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) #ifdef CONFIG_BFQ_GROUP_IOSCHED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) * struct bfq_group_data - per-blkcg storage for the blkio subsystem.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) * @ps: @blkcg_policy_storage that this structure inherits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) * @weight: weight of the bfq_group
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) struct bfq_group_data {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) /* must be the first member */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) struct blkcg_policy_data pd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) unsigned int weight;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) * struct bfq_group - per (device, cgroup) data structure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) * @entity: schedulable entity to insert into the parent group sched_data.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) * @sched_data: own sched_data, to contain child entities (they may be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) * both bfq_queues and bfq_groups).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) * @bfqd: the bfq_data for the device this group acts upon.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) * @async_bfqq: array of async queues for all the tasks belonging to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) * the group, one queue per ioprio value per ioprio_class,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) * except for the idle class that has only one queue.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) * @async_idle_bfqq: async queue for the idle class (ioprio is ignored).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) * @my_entity: pointer to @entity, %NULL for the toplevel group; used
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) * to avoid too many special cases during group creation/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) * migration.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) * @stats: stats for this bfqg.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) * @active_entities: number of active entities belonging to the group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) * unused for the root group. Used to know whether there
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) * are groups with more than one active @bfq_entity
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) * (see the comments to the function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) * bfq_bfqq_may_idle()).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) * @rq_pos_tree: rbtree sorted by next_request position, used when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) * determining if two or more queues have interleaving
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) * requests (see bfq_find_close_cooperator()).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) * Each (device, cgroup) pair has its own bfq_group, i.e., for each cgroup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) * there is a set of bfq_groups, each one collecting the lower-level
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) * entities belonging to the group that are acting on the same device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) * Locking works as follows:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) * o @bfqd is protected by the queue lock, RCU is used to access it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) * from the readers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) * o All the other fields are protected by the @bfqd queue lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) struct bfq_group {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) /* must be the first member */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) struct blkg_policy_data pd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) /* cached path for this blkg (see comments in bfq_bic_update_cgroup) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) char blkg_path[128];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) /* reference counter (see comments in bfq_bic_update_cgroup) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) int ref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) struct bfq_entity entity;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) struct bfq_sched_data sched_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) void *bfqd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) struct bfq_queue *async_bfqq[2][IOPRIO_BE_NR];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) struct bfq_queue *async_idle_bfqq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) struct bfq_entity *my_entity;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) int active_entities;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) struct rb_root rq_pos_tree;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) struct bfqg_stats stats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) struct bfq_group {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) struct bfq_entity entity;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) struct bfq_sched_data sched_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) struct bfq_queue *async_bfqq[2][IOPRIO_BE_NR];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) struct bfq_queue *async_idle_bfqq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) struct rb_root rq_pos_tree;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) struct bfq_queue *bfq_entity_to_bfqq(struct bfq_entity *entity);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) /* --------------- main algorithm interface ----------------- */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) #define BFQ_SERVICE_TREE_INIT ((struct bfq_service_tree) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) { RB_ROOT, RB_ROOT, NULL, NULL, 0, 0 })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) extern const int bfq_timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) struct bfq_queue *bic_to_bfqq(struct bfq_io_cq *bic, bool is_sync);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) void bic_set_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq, bool is_sync);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) struct bfq_data *bic_to_bfqd(struct bfq_io_cq *bic);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) void bfq_pos_tree_add_move(struct bfq_data *bfqd, struct bfq_queue *bfqq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) void bfq_weights_tree_add(struct bfq_data *bfqd, struct bfq_queue *bfqq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) struct rb_root_cached *root);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) void __bfq_weights_tree_remove(struct bfq_data *bfqd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) struct bfq_queue *bfqq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) struct rb_root_cached *root);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) void bfq_weights_tree_remove(struct bfq_data *bfqd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) struct bfq_queue *bfqq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) void bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) bool compensate, enum bfqq_expiration reason);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) void bfq_put_queue(struct bfq_queue *bfqq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) void bfq_end_wr_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) void bfq_release_process_ref(struct bfq_data *bfqd, struct bfq_queue *bfqq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) void bfq_schedule_dispatch(struct bfq_data *bfqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) void bfq_put_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) /* ------------ end of main algorithm interface -------------- */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) /* ---------------- cgroups-support interface ---------------- */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) void bfqg_stats_update_legacy_io(struct request_queue *q, struct request *rq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) unsigned int op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) void bfqg_stats_update_io_remove(struct bfq_group *bfqg, unsigned int op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) void bfqg_stats_update_io_merged(struct bfq_group *bfqg, unsigned int op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) u64 io_start_time_ns, unsigned int op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) void bfqg_stats_update_dequeue(struct bfq_group *bfqg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) void bfqg_stats_update_idle_time(struct bfq_group *bfqg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) struct bfq_group *bfqg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) void bfq_bic_update_cgroup(struct bfq_io_cq *bic, struct bio *bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) void bfq_end_wr_async(struct bfq_data *bfqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) struct bfq_group *bfq_find_set_group(struct bfq_data *bfqd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) struct blkcg *blkcg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) struct blkcg_gq *bfqg_to_blkg(struct bfq_group *bfqg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) struct bfq_group *bfqq_group(struct bfq_queue *bfqq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) struct bfq_group *bfq_create_group_hierarchy(struct bfq_data *bfqd, int node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) void bfqg_and_blkg_put(struct bfq_group *bfqg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) #ifdef CONFIG_BFQ_GROUP_IOSCHED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) extern struct cftype bfq_blkcg_legacy_files[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) extern struct cftype bfq_blkg_files[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) extern struct blkcg_policy blkcg_policy_bfq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) /* ------------- end of cgroups-support interface ------------- */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) /* - interface of the internal hierarchical B-WF2Q+ scheduler - */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) #ifdef CONFIG_BFQ_GROUP_IOSCHED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) /* both next loops stop at one of the child entities of the root group */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) #define for_each_entity(entity) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) for (; entity ; entity = entity->parent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) * For each iteration, compute parent in advance, so as to be safe if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) * entity is deallocated during the iteration. Such a deallocation may
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) * happen as a consequence of a bfq_put_queue that frees the bfq_queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) * containing entity.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) #define for_each_entity_safe(entity, parent) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) for (; entity && ({ parent = entity->parent; 1; }); entity = parent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) #else /* CONFIG_BFQ_GROUP_IOSCHED */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) * Next two macros are fake loops when cgroups support is not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) * enabled. I fact, in such a case, there is only one level to go up
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) * (to reach the root group).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) #define for_each_entity(entity) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) for (; entity ; entity = NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) #define for_each_entity_safe(entity, parent) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) for (parent = NULL; entity ; entity = parent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) #endif /* CONFIG_BFQ_GROUP_IOSCHED */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) struct bfq_group *bfq_bfqq_to_bfqg(struct bfq_queue *bfqq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) struct bfq_queue *bfq_entity_to_bfqq(struct bfq_entity *entity);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) unsigned int bfq_tot_busy_queues(struct bfq_data *bfqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) struct bfq_service_tree *bfq_entity_service_tree(struct bfq_entity *entity);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) struct bfq_entity *bfq_entity_of(struct rb_node *node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) unsigned short bfq_ioprio_to_weight(int ioprio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) void bfq_put_idle_entity(struct bfq_service_tree *st,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) struct bfq_entity *entity);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) struct bfq_service_tree *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) __bfq_entity_update_weight_prio(struct bfq_service_tree *old_st,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) struct bfq_entity *entity,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) bool update_class_too);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) void bfq_bfqq_served(struct bfq_queue *bfqq, int served);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) void bfq_bfqq_charge_time(struct bfq_data *bfqd, struct bfq_queue *bfqq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) unsigned long time_ms);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) bool __bfq_deactivate_entity(struct bfq_entity *entity,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) bool ins_into_idle_tree);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) bool next_queue_may_preempt(struct bfq_data *bfqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) struct bfq_queue *bfq_get_next_queue(struct bfq_data *bfqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) bool __bfq_bfqd_reset_in_service(struct bfq_data *bfqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) void bfq_deactivate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) bool ins_into_idle_tree, bool expiration);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) void bfq_activate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) void bfq_requeue_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) bool expiration);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) void bfq_del_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) bool expiration);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) void bfq_add_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) /* --------------- end of interface of B-WF2Q+ ---------------- */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) /* Logging facilities. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) static inline void bfq_pid_to_str(int pid, char *str, int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) if (pid != -1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) snprintf(str, len, "%d", pid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) snprintf(str, len, "SHARED-");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) #ifdef CONFIG_BFQ_GROUP_IOSCHED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) struct bfq_group *bfqq_group(struct bfq_queue *bfqq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) #define bfq_log_bfqq(bfqd, bfqq, fmt, args...) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) char pid_str[MAX_PID_STR_LENGTH]; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) if (likely(!blk_trace_note_message_enabled((bfqd)->queue))) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) break; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) bfq_pid_to_str((bfqq)->pid, pid_str, MAX_PID_STR_LENGTH); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) blk_add_cgroup_trace_msg((bfqd)->queue, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) bfqg_to_blkg(bfqq_group(bfqq))->blkcg, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) "bfq%s%c " fmt, pid_str, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) bfq_bfqq_sync((bfqq)) ? 'S' : 'A', ##args); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) #define bfq_log_bfqg(bfqd, bfqg, fmt, args...) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) blk_add_cgroup_trace_msg((bfqd)->queue, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) bfqg_to_blkg(bfqg)->blkcg, fmt, ##args); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) #else /* CONFIG_BFQ_GROUP_IOSCHED */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) #define bfq_log_bfqq(bfqd, bfqq, fmt, args...) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) char pid_str[MAX_PID_STR_LENGTH]; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) if (likely(!blk_trace_note_message_enabled((bfqd)->queue))) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) break; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) bfq_pid_to_str((bfqq)->pid, pid_str, MAX_PID_STR_LENGTH); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) blk_add_trace_msg((bfqd)->queue, "bfq%s%c " fmt, pid_str, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) bfq_bfqq_sync((bfqq)) ? 'S' : 'A', \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) ##args); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) #define bfq_log_bfqg(bfqd, bfqg, fmt, args...) do {} while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) #endif /* CONFIG_BFQ_GROUP_IOSCHED */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) #define bfq_log(bfqd, fmt, args...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) blk_add_trace_msg((bfqd)->queue, "bfq " fmt, ##args)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) #endif /* _BFQ_H */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags