^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/pci.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/io-64-nonatomic-lo-hi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/dmaengine.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <uapi/linux/idxd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include "../dmaengine.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include "idxd.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include "registers.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) static void idxd_device_reinit(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) struct idxd_device *idxd = container_of(work, struct idxd_device, work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) struct device *dev = &idxd->pdev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) int rc, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) idxd_device_reset(idxd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) rc = idxd_device_config(idxd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) if (rc < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) rc = idxd_device_enable(idxd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) if (rc < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) for (i = 0; i < idxd->max_wqs; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) struct idxd_wq *wq = &idxd->wqs[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) if (wq->state == IDXD_WQ_ENABLED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) rc = idxd_wq_enable(wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) if (rc < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) dev_warn(dev, "Unable to re-enable wq %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) dev_name(&wq->conf_dev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) idxd_device_wqs_clear_state(idxd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) irqreturn_t idxd_irq_handler(int vec, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) struct idxd_irq_entry *irq_entry = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) struct idxd_device *idxd = irq_entry->idxd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) idxd_mask_msix_vector(idxd, irq_entry->id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) return IRQ_WAKE_THREAD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) static int process_misc_interrupts(struct idxd_device *idxd, u32 cause)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) struct device *dev = &idxd->pdev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) union gensts_reg gensts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) u32 val = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) bool err = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) if (cause & IDXD_INTC_ERR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) spin_lock_bh(&idxd->dev_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) for (i = 0; i < 4; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) idxd->sw_err.bits[i] = ioread64(idxd->reg_base +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) IDXD_SWERR_OFFSET + i * sizeof(u64));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) iowrite64(idxd->sw_err.bits[0] & IDXD_SWERR_ACK,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) idxd->reg_base + IDXD_SWERR_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) if (idxd->sw_err.valid && idxd->sw_err.wq_idx_valid) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) int id = idxd->sw_err.wq_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) struct idxd_wq *wq = &idxd->wqs[id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) if (wq->type == IDXD_WQT_USER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) wake_up_interruptible(&wq->err_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) for (i = 0; i < idxd->max_wqs; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) struct idxd_wq *wq = &idxd->wqs[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) if (wq->type == IDXD_WQT_USER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) wake_up_interruptible(&wq->err_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) spin_unlock_bh(&idxd->dev_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) val |= IDXD_INTC_ERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) for (i = 0; i < 4; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) dev_warn(dev, "err[%d]: %#16.16llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) i, idxd->sw_err.bits[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) err = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) if (cause & IDXD_INTC_CMD) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) val |= IDXD_INTC_CMD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) complete(idxd->cmd_done);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) if (cause & IDXD_INTC_OCCUPY) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) /* Driver does not utilize occupancy interrupt */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) val |= IDXD_INTC_OCCUPY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) if (cause & IDXD_INTC_PERFMON_OVFL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) * Driver does not utilize perfmon counter overflow interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) * yet.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) val |= IDXD_INTC_PERFMON_OVFL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) val ^= cause;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) if (val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) dev_warn_once(dev, "Unexpected interrupt cause bits set: %#x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) if (!err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) if (gensts.state == IDXD_DEVICE_STATE_HALT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) idxd->state = IDXD_DEV_HALTED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) if (gensts.reset_type == IDXD_DEVICE_RESET_SOFTWARE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) * If we need a software reset, we will throw the work
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) * on a system workqueue in order to allow interrupts
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) * for the device command completions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) INIT_WORK(&idxd->work, idxd_device_reinit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) queue_work(idxd->wq, &idxd->work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) spin_lock_bh(&idxd->dev_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) idxd_device_wqs_clear_state(idxd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) dev_err(&idxd->pdev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) "idxd halted, need %s.\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) gensts.reset_type == IDXD_DEVICE_RESET_FLR ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) "FLR" : "system reset");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) spin_unlock_bh(&idxd->dev_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) irqreturn_t idxd_misc_thread(int vec, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) struct idxd_irq_entry *irq_entry = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) struct idxd_device *idxd = irq_entry->idxd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) u32 cause;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) cause = ioread32(idxd->reg_base + IDXD_INTCAUSE_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) if (cause)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) iowrite32(cause, idxd->reg_base + IDXD_INTCAUSE_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) while (cause) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) rc = process_misc_interrupts(idxd, cause);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) if (rc < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) cause = ioread32(idxd->reg_base + IDXD_INTCAUSE_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) if (cause)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) iowrite32(cause, idxd->reg_base + IDXD_INTCAUSE_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) idxd_unmask_msix_vector(idxd, irq_entry->id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) return IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) int *processed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) struct idxd_desc *desc, *t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) struct llist_node *head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) int queued = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) *processed = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) head = llist_del_all(&irq_entry->pending_llist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) if (!head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) llist_for_each_entry_safe(desc, t, head, llnode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) if (desc->completion->status) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) idxd_dma_complete_txd(desc, IDXD_COMPLETE_NORMAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) idxd_free_desc(desc->wq, desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) (*processed)++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) list_add_tail(&desc->list, &irq_entry->work_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) queued++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) return queued;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) static int irq_process_work_list(struct idxd_irq_entry *irq_entry,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) int *processed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) struct list_head *node, *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) int queued = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) *processed = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) if (list_empty(&irq_entry->work_list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) list_for_each_safe(node, next, &irq_entry->work_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) struct idxd_desc *desc =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) container_of(node, struct idxd_desc, list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) if (desc->completion->status) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) list_del(&desc->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) /* process and callback */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) idxd_dma_complete_txd(desc, IDXD_COMPLETE_NORMAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) idxd_free_desc(desc->wq, desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) (*processed)++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) queued++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) return queued;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) static int idxd_desc_process(struct idxd_irq_entry *irq_entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) int rc, processed, total = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) * There are two lists we are processing. The pending_llist is where
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) * submmiter adds all the submitted descriptor after sending it to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) * the workqueue. It's a lockless singly linked list. The work_list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) * is the common linux double linked list. We are in a scenario of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) * multiple producers and a single consumer. The producers are all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) * the kernel submitters of descriptors, and the consumer is the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) * kernel irq handler thread for the msix vector when using threaded
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) * irq. To work with the restrictions of llist to remain lockless,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) * we are doing the following steps:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) * 1. Iterate through the work_list and process any completed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) * descriptor. Delete the completed entries during iteration.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) * 2. llist_del_all() from the pending list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) * 3. Iterate through the llist that was deleted from the pending list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) * and process the completed entries.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) * 4. If the entry is still waiting on hardware, list_add_tail() to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) * the work_list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) * 5. Repeat until no more descriptors.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) rc = irq_process_work_list(irq_entry, &processed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) total += processed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) if (rc != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) rc = irq_process_pending_llist(irq_entry, &processed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) total += processed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) } while (rc != 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) return total;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) irqreturn_t idxd_wq_thread(int irq, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) struct idxd_irq_entry *irq_entry = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) int processed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) processed = idxd_desc_process(irq_entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) idxd_unmask_msix_vector(irq_entry->idxd, irq_entry->id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) if (processed == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) return IRQ_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) return IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags