^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Functions related to interrupt-poll handling in the block layer. This
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * is similar to NAPI for network devices.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/bio.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/irq_poll.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/delay.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) static unsigned int irq_poll_budget __read_mostly = 256;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) static DEFINE_PER_CPU(struct list_head, blk_cpu_iopoll);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * irq_poll_sched - Schedule a run of the iopoll handler
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * @iop: The parent iopoll structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * Add this irq_poll structure to the pending poll list and trigger the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * raise of the blk iopoll softirq.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) **/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) void irq_poll_sched(struct irq_poll *iop)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) if (test_bit(IRQ_POLL_F_DISABLE, &iop->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) if (test_and_set_bit(IRQ_POLL_F_SCHED, &iop->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) list_add_tail(&iop->list, this_cpu_ptr(&blk_cpu_iopoll));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) raise_softirq_irqoff(IRQ_POLL_SOFTIRQ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) EXPORT_SYMBOL(irq_poll_sched);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) * __irq_poll_complete - Mark this @iop as un-polled again
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) * @iop: The parent iopoll structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) * See irq_poll_complete(). This function must be called with interrupts
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) * disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) **/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) static void __irq_poll_complete(struct irq_poll *iop)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) list_del(&iop->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) smp_mb__before_atomic();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) clear_bit_unlock(IRQ_POLL_F_SCHED, &iop->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) * irq_poll_complete - Mark this @iop as un-polled again
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) * @iop: The parent iopoll structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) * If a driver consumes less than the assigned budget in its run of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) * iopoll handler, it'll end the polled mode by calling this function. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) * iopoll handler will not be invoked again before irq_poll_sched()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) * is called.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) **/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) void irq_poll_complete(struct irq_poll *iop)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) __irq_poll_complete(iop);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) EXPORT_SYMBOL(irq_poll_complete);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) static void __latent_entropy irq_poll_softirq(struct softirq_action *h)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) struct list_head *list = this_cpu_ptr(&blk_cpu_iopoll);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) int rearm = 0, budget = irq_poll_budget;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) unsigned long start_time = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) local_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) while (!list_empty(list)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) struct irq_poll *iop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) int work, weight;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) * If softirq window is exhausted then punt.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) if (budget <= 0 || time_after(jiffies, start_time)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) rearm = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) local_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) /* Even though interrupts have been re-enabled, this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) * access is safe because interrupts can only add new
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) * entries to the tail of this list, and only ->poll()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) * calls can remove this head entry from the list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) iop = list_entry(list->next, struct irq_poll, list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) weight = iop->weight;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) work = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) if (test_bit(IRQ_POLL_F_SCHED, &iop->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) work = iop->poll(iop, weight);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) budget -= work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) local_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) * Drivers must not modify the iopoll state, if they
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) * consume their assigned weight (or more, some drivers can't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) * easily just stop processing, they have to complete an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) * entire mask of commands).In such cases this code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) * still "owns" the iopoll instance and therefore can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) * move the instance around on the list at-will.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) if (work >= weight) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) if (test_bit(IRQ_POLL_F_DISABLE, &iop->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) __irq_poll_complete(iop);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) list_move_tail(&iop->list, list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) if (rearm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) __raise_softirq_irqoff(IRQ_POLL_SOFTIRQ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) local_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) * irq_poll_disable - Disable iopoll on this @iop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) * @iop: The parent iopoll structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) * Disable io polling and wait for any pending callbacks to have completed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) **/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) void irq_poll_disable(struct irq_poll *iop)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) set_bit(IRQ_POLL_F_DISABLE, &iop->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) while (test_and_set_bit(IRQ_POLL_F_SCHED, &iop->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) msleep(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) clear_bit(IRQ_POLL_F_DISABLE, &iop->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) EXPORT_SYMBOL(irq_poll_disable);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) * irq_poll_enable - Enable iopoll on this @iop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) * @iop: The parent iopoll structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) * Enable iopoll on this @iop. Note that the handler run will not be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) * scheduled, it will only mark it as active.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) **/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) void irq_poll_enable(struct irq_poll *iop)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) BUG_ON(!test_bit(IRQ_POLL_F_SCHED, &iop->state));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) smp_mb__before_atomic();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) clear_bit_unlock(IRQ_POLL_F_SCHED, &iop->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) EXPORT_SYMBOL(irq_poll_enable);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) * irq_poll_init - Initialize this @iop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) * @iop: The parent iopoll structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) * @weight: The default weight (or command completion budget)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) * @poll_fn: The handler to invoke
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) * Initialize and enable this irq_poll structure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) **/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) void irq_poll_init(struct irq_poll *iop, int weight, irq_poll_fn *poll_fn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) memset(iop, 0, sizeof(*iop));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) INIT_LIST_HEAD(&iop->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) iop->weight = weight;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) iop->poll = poll_fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) EXPORT_SYMBOL(irq_poll_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) static int irq_poll_cpu_dead(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) * If a CPU goes away, splice its entries to the current CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) * and trigger a run of the softirq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) local_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) list_splice_init(&per_cpu(blk_cpu_iopoll, cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) this_cpu_ptr(&blk_cpu_iopoll));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) __raise_softirq_irqoff(IRQ_POLL_SOFTIRQ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) local_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) static __init int irq_poll_setup(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) for_each_possible_cpu(i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) INIT_LIST_HEAD(&per_cpu(blk_cpu_iopoll, i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) open_softirq(IRQ_POLL_SOFTIRQ, irq_poll_softirq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) cpuhp_setup_state_nocalls(CPUHP_IRQ_POLL_DEAD, "irq_poll:dead", NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) irq_poll_cpu_dead);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) subsys_initcall(irq_poll_setup);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags