Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3) #include "blk-rq-qos.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  * Increment 'v', if 'v' is below 'below'. Returns true if we succeeded,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  * false if 'v' + 1 would be bigger than 'below'.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) static bool atomic_inc_below(atomic_t *v, unsigned int below)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) 	unsigned int cur = atomic_read(v);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) 	for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) 		unsigned int old;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) 		if (cur >= below)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) 			return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) 		old = atomic_cmpxchg(v, cur, cur + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) 		if (old == cur)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) 			break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) 		cur = old;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) 	return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) bool rq_wait_inc_below(struct rq_wait *rq_wait, unsigned int limit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) 	return atomic_inc_below(&rq_wait->inflight, limit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) void __rq_qos_cleanup(struct rq_qos *rqos, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) 	do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 		if (rqos->ops->cleanup)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 			rqos->ops->cleanup(rqos, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) 		rqos = rqos->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) 	} while (rqos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) void __rq_qos_done(struct rq_qos *rqos, struct request *rq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) 	do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 		if (rqos->ops->done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) 			rqos->ops->done(rqos, rq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) 		rqos = rqos->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 	} while (rqos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) void __rq_qos_issue(struct rq_qos *rqos, struct request *rq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 	do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) 		if (rqos->ops->issue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 			rqos->ops->issue(rqos, rq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 		rqos = rqos->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 	} while (rqos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) void __rq_qos_requeue(struct rq_qos *rqos, struct request *rq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 	do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 		if (rqos->ops->requeue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 			rqos->ops->requeue(rqos, rq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 		rqos = rqos->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 	} while (rqos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) void __rq_qos_throttle(struct rq_qos *rqos, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 	do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 		if (rqos->ops->throttle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 			rqos->ops->throttle(rqos, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 		rqos = rqos->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 	} while (rqos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) void __rq_qos_track(struct rq_qos *rqos, struct request *rq, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 	do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 		if (rqos->ops->track)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 			rqos->ops->track(rqos, rq, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 		rqos = rqos->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 	} while (rqos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) void __rq_qos_merge(struct rq_qos *rqos, struct request *rq, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 	do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 		if (rqos->ops->merge)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 			rqos->ops->merge(rqos, rq, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 		rqos = rqos->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 	} while (rqos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) void __rq_qos_done_bio(struct rq_qos *rqos, struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 	do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 		if (rqos->ops->done_bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 			rqos->ops->done_bio(rqos, bio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 		rqos = rqos->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 	} while (rqos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) void __rq_qos_queue_depth_changed(struct rq_qos *rqos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 	do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 		if (rqos->ops->queue_depth_changed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 			rqos->ops->queue_depth_changed(rqos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 		rqos = rqos->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 	} while (rqos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)  * Return true, if we can't increase the depth further by scaling
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) bool rq_depth_calc_max_depth(struct rq_depth *rqd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 	unsigned int depth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 	bool ret = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 	 * For QD=1 devices, this is a special case. It's important for those
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	 * to have one request ready when one completes, so force a depth of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 	 * 2 for those devices. On the backend, it'll be a depth of 1 anyway,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 	 * since the device can't have more than that in flight. If we're
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 	 * scaling down, then keep a setting of 1/1/1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 	if (rqd->queue_depth == 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 		if (rqd->scale_step > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 			rqd->max_depth = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 		else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 			rqd->max_depth = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 			ret = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 	} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 		 * scale_step == 0 is our default state. If we have suffered
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 		 * latency spikes, step will be > 0, and we shrink the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 		 * allowed write depths. If step is < 0, we're only doing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 		 * writes, and we allow a temporarily higher depth to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 		 * increase performance.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 		depth = min_t(unsigned int, rqd->default_depth,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 			      rqd->queue_depth);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 		if (rqd->scale_step > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 			depth = 1 + ((depth - 1) >> min(31, rqd->scale_step));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 		else if (rqd->scale_step < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 			unsigned int maxd = 3 * rqd->queue_depth / 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 			depth = 1 + ((depth - 1) << -rqd->scale_step);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 			if (depth > maxd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 				depth = maxd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 				ret = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 			}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 		rqd->max_depth = depth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 	return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) /* Returns true on success and false if scaling up wasn't possible */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) bool rq_depth_scale_up(struct rq_depth *rqd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 	 * Hit max in previous round, stop here
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 	if (rqd->scaled_max)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 		return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 	rqd->scale_step--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 	rqd->scaled_max = rq_depth_calc_max_depth(rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 	return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)  * Scale rwb down. If 'hard_throttle' is set, do it quicker, since we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180)  * had a latency violation. Returns true on success and returns false if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181)  * scaling down wasn't possible.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) bool rq_depth_scale_down(struct rq_depth *rqd, bool hard_throttle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 	 * Stop scaling down when we've hit the limit. This also prevents
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 	 * ->scale_step from going to crazy values, if the device can't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 	 * keep up.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 	if (rqd->max_depth == 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 		return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 	if (rqd->scale_step < 0 && hard_throttle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 		rqd->scale_step = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 	else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 		rqd->scale_step++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 	rqd->scaled_max = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 	rq_depth_calc_max_depth(rqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) 	return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) struct rq_qos_wait_data {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 	struct wait_queue_entry wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 	struct task_struct *task;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 	struct rq_wait *rqw;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 	acquire_inflight_cb_t *cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 	void *private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 	bool got_token;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) static int rq_qos_wake_function(struct wait_queue_entry *curr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 				unsigned int mode, int wake_flags, void *key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 	struct rq_qos_wait_data *data = container_of(curr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 						     struct rq_qos_wait_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 						     wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 	 * If we fail to get a budget, return -1 to interrupt the wake up loop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 	 * in __wake_up_common.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 	if (!data->cb(data->rqw, data->private_data))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 		return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 	data->got_token = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 	smp_wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 	list_del_init(&curr->entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 	wake_up_process(data->task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 	return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234)  * rq_qos_wait - throttle on a rqw if we need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)  * @rqw: rqw to throttle on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236)  * @private_data: caller provided specific data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)  * @acquire_inflight_cb: inc the rqw->inflight counter if we can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238)  * @cleanup_cb: the callback to cleanup in case we race with a waker
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240)  * This provides a uniform place for the rq_qos users to do their throttling.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)  * Since you can end up with a lot of things sleeping at once, this manages the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242)  * waking up based on the resources available.  The acquire_inflight_cb should
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)  * inc the rqw->inflight if we have the ability to do so, or return false if not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244)  * and then we will sleep until the room becomes available.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246)  * cleanup_cb is in case that we race with a waker and need to cleanup the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)  * inflight count accordingly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) void rq_qos_wait(struct rq_wait *rqw, void *private_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 		 acquire_inflight_cb_t *acquire_inflight_cb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 		 cleanup_cb_t *cleanup_cb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) 	struct rq_qos_wait_data data = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) 		.wq = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 			.func	= rq_qos_wake_function,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 			.entry	= LIST_HEAD_INIT(data.wq.entry),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 		},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 		.task = current,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 		.rqw = rqw,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 		.cb = acquire_inflight_cb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) 		.private_data = private_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 	};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 	bool has_sleeper;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 	has_sleeper = wq_has_sleeper(&rqw->wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 	if (!has_sleeper && acquire_inflight_cb(rqw, private_data))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 	has_sleeper = !prepare_to_wait_exclusive(&rqw->wait, &data.wq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 						 TASK_UNINTERRUPTIBLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 	do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 		/* The memory barrier in set_task_state saves us here. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 		if (data.got_token)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 			break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 		if (!has_sleeper && acquire_inflight_cb(rqw, private_data)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 			finish_wait(&rqw->wait, &data.wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 			/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) 			 * We raced with wbt_wake_function() getting a token,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) 			 * which means we now have two. Put our local token
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) 			 * and wake anyone else potentially waiting for one.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) 			 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 			smp_rmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 			if (data.got_token)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 				cleanup_cb(rqw, private_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) 			break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) 		io_schedule();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) 		has_sleeper = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) 		set_current_state(TASK_UNINTERRUPTIBLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) 	} while (1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) 	finish_wait(&rqw->wait, &data.wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) void rq_qos_exit(struct request_queue *q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) 	blk_mq_debugfs_unregister_queue_rqos(q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) 	while (q->rq_qos) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 		struct rq_qos *rqos = q->rq_qos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) 		q->rq_qos = rqos->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) 		rqos->ops->exit(rqos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) }