^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Block stat tracking code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2016 Jens Axboe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/rculist.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/blk-mq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include "blk-stat.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include "blk-mq.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include "blk.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) struct blk_queue_stats {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) struct list_head callbacks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) spinlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) bool enable_accounting;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) void blk_rq_stat_init(struct blk_rq_stat *stat)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) stat->min = -1ULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) stat->max = stat->nr_samples = stat->mean = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) stat->batch = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) /* src is a per-cpu stat, mean isn't initialized */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) void blk_rq_stat_sum(struct blk_rq_stat *dst, struct blk_rq_stat *src)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) if (!src->nr_samples)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) dst->min = min(dst->min, src->min);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) dst->max = max(dst->max, src->max);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) dst->mean = div_u64(src->batch + dst->mean * dst->nr_samples,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) dst->nr_samples + src->nr_samples);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) dst->nr_samples += src->nr_samples;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) void blk_rq_stat_add(struct blk_rq_stat *stat, u64 value)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) stat->min = min(stat->min, value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) stat->max = max(stat->max, value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) stat->batch += value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) stat->nr_samples++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) void blk_stat_add(struct request *rq, u64 now)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) struct request_queue *q = rq->q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) struct blk_stat_callback *cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) struct blk_rq_stat *stat;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) int bucket, cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) u64 value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) value = (now >= rq->io_start_time_ns) ? now - rq->io_start_time_ns : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) blk_throtl_stat_add(rq, value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) cpu = get_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) list_for_each_entry_rcu(cb, &q->stats->callbacks, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) if (!blk_stat_is_active(cb))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) bucket = cb->bucket_fn(rq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) if (bucket < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) stat = &per_cpu_ptr(cb->cpu_stat, cpu)[bucket];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) blk_rq_stat_add(stat, value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) put_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) static void blk_stat_timer_fn(struct timer_list *t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) struct blk_stat_callback *cb = from_timer(cb, t, timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) unsigned int bucket;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) for (bucket = 0; bucket < cb->buckets; bucket++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) blk_rq_stat_init(&cb->stat[bucket]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) for_each_online_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) struct blk_rq_stat *cpu_stat;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) for (bucket = 0; bucket < cb->buckets; bucket++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) blk_rq_stat_sum(&cb->stat[bucket], &cpu_stat[bucket]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) blk_rq_stat_init(&cpu_stat[bucket]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) cb->timer_fn(cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) struct blk_stat_callback *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) blk_stat_alloc_callback(void (*timer_fn)(struct blk_stat_callback *),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) int (*bucket_fn)(const struct request *),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) unsigned int buckets, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) struct blk_stat_callback *cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) cb = kmalloc(sizeof(*cb), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) if (!cb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) cb->stat = kmalloc_array(buckets, sizeof(struct blk_rq_stat),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) if (!cb->stat) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) kfree(cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) cb->cpu_stat = __alloc_percpu(buckets * sizeof(struct blk_rq_stat),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) __alignof__(struct blk_rq_stat));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) if (!cb->cpu_stat) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) kfree(cb->stat);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) kfree(cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) cb->timer_fn = timer_fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) cb->bucket_fn = bucket_fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) cb->data = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) cb->buckets = buckets;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) timer_setup(&cb->timer, blk_stat_timer_fn, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) return cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) void blk_stat_add_callback(struct request_queue *q,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) struct blk_stat_callback *cb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) unsigned int bucket;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) struct blk_rq_stat *cpu_stat;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) for (bucket = 0; bucket < cb->buckets; bucket++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) blk_rq_stat_init(&cpu_stat[bucket]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) spin_lock_irqsave(&q->stats->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) list_add_tail_rcu(&cb->list, &q->stats->callbacks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) blk_queue_flag_set(QUEUE_FLAG_STATS, q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) spin_unlock_irqrestore(&q->stats->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) void blk_stat_remove_callback(struct request_queue *q,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) struct blk_stat_callback *cb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) spin_lock_irqsave(&q->stats->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) list_del_rcu(&cb->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) if (list_empty(&q->stats->callbacks) && !q->stats->enable_accounting)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) blk_queue_flag_clear(QUEUE_FLAG_STATS, q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) spin_unlock_irqrestore(&q->stats->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) del_timer_sync(&cb->timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) static void blk_stat_free_callback_rcu(struct rcu_head *head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) struct blk_stat_callback *cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) cb = container_of(head, struct blk_stat_callback, rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) free_percpu(cb->cpu_stat);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) kfree(cb->stat);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) kfree(cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) void blk_stat_free_callback(struct blk_stat_callback *cb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) if (cb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) call_rcu(&cb->rcu, blk_stat_free_callback_rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) void blk_stat_enable_accounting(struct request_queue *q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) spin_lock_irqsave(&q->stats->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) q->stats->enable_accounting = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) blk_queue_flag_set(QUEUE_FLAG_STATS, q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) spin_unlock_irqrestore(&q->stats->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) EXPORT_SYMBOL_GPL(blk_stat_enable_accounting);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) struct blk_queue_stats *blk_alloc_queue_stats(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) struct blk_queue_stats *stats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) stats = kmalloc(sizeof(*stats), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) if (!stats)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) INIT_LIST_HEAD(&stats->callbacks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) spin_lock_init(&stats->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) stats->enable_accounting = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) return stats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) void blk_free_queue_stats(struct blk_queue_stats *stats)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) if (!stats)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) WARN_ON(!list_empty(&stats->callbacks));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) kfree(stats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags