^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/pci.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/io-64-nonatomic-lo-hi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/dmaengine.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/irq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/msi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <uapi/linux/idxd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include "../dmaengine.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include "idxd.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include "registers.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) u32 *status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) /* Interrupt control bits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) void idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) struct irq_data *data = irq_get_irq_data(idxd->msix_entries[vec_id].vector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) pci_msi_mask_irq(data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) void idxd_mask_msix_vectors(struct idxd_device *idxd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) struct pci_dev *pdev = idxd->pdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) int msixcnt = pci_msix_vec_count(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) for (i = 0; i < msixcnt; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) idxd_mask_msix_vector(idxd, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) void idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) struct irq_data *data = irq_get_irq_data(idxd->msix_entries[vec_id].vector);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) pci_msi_unmask_irq(data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) void idxd_unmask_error_interrupts(struct idxd_device *idxd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) union genctrl_reg genctrl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) genctrl.bits = ioread32(idxd->reg_base + IDXD_GENCTRL_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) genctrl.softerr_int_en = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) iowrite32(genctrl.bits, idxd->reg_base + IDXD_GENCTRL_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) void idxd_mask_error_interrupts(struct idxd_device *idxd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) union genctrl_reg genctrl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) genctrl.bits = ioread32(idxd->reg_base + IDXD_GENCTRL_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) genctrl.softerr_int_en = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) iowrite32(genctrl.bits, idxd->reg_base + IDXD_GENCTRL_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) static void free_hw_descs(struct idxd_wq *wq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) for (i = 0; i < wq->num_descs; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) kfree(wq->hw_descs[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) kfree(wq->hw_descs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) static int alloc_hw_descs(struct idxd_wq *wq, int num)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) struct device *dev = &wq->idxd->pdev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) int node = dev_to_node(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) wq->hw_descs = kcalloc_node(num, sizeof(struct dsa_hw_desc *),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) GFP_KERNEL, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) if (!wq->hw_descs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) for (i = 0; i < num; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) wq->hw_descs[i] = kzalloc_node(sizeof(*wq->hw_descs[i]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) GFP_KERNEL, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) if (!wq->hw_descs[i]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) free_hw_descs(wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) static void free_descs(struct idxd_wq *wq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) for (i = 0; i < wq->num_descs; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) kfree(wq->descs[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) kfree(wq->descs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) static int alloc_descs(struct idxd_wq *wq, int num)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) struct device *dev = &wq->idxd->pdev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) int node = dev_to_node(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) wq->descs = kcalloc_node(num, sizeof(struct idxd_desc *),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) GFP_KERNEL, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) if (!wq->descs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) for (i = 0; i < num; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) wq->descs[i] = kzalloc_node(sizeof(*wq->descs[i]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) GFP_KERNEL, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) if (!wq->descs[i]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) free_descs(wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) /* WQ control bits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) int idxd_wq_alloc_resources(struct idxd_wq *wq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) struct idxd_device *idxd = wq->idxd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) struct device *dev = &idxd->pdev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) int rc, num_descs, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) if (wq->type != IDXD_WQT_KERNEL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) wq->num_descs = wq->size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) num_descs = wq->size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) rc = alloc_hw_descs(wq, num_descs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) if (rc < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) wq->compls_size = num_descs * sizeof(struct dsa_completion_record);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) wq->compls = dma_alloc_coherent(dev, wq->compls_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) &wq->compls_addr, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) if (!wq->compls) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) rc = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) goto fail_alloc_compls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) rc = alloc_descs(wq, num_descs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) if (rc < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) goto fail_alloc_descs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) rc = sbitmap_queue_init_node(&wq->sbq, num_descs, -1, false, GFP_KERNEL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) dev_to_node(dev));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) if (rc < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) goto fail_sbitmap_init;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) for (i = 0; i < num_descs; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) struct idxd_desc *desc = wq->descs[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) desc->hw = wq->hw_descs[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) desc->completion = &wq->compls[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) desc->compl_dma = wq->compls_addr +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) sizeof(struct dsa_completion_record) * i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) desc->id = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) desc->wq = wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) desc->cpu = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) fail_sbitmap_init:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) free_descs(wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) fail_alloc_descs:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) fail_alloc_compls:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) free_hw_descs(wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) void idxd_wq_free_resources(struct idxd_wq *wq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) struct device *dev = &wq->idxd->pdev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) if (wq->type != IDXD_WQT_KERNEL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) free_hw_descs(wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) free_descs(wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) sbitmap_queue_free(&wq->sbq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) int idxd_wq_enable(struct idxd_wq *wq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) struct idxd_device *idxd = wq->idxd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) struct device *dev = &idxd->pdev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) u32 status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) if (wq->state == IDXD_WQ_ENABLED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) dev_dbg(dev, "WQ %d already enabled\n", wq->id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) idxd_cmd_exec(idxd, IDXD_CMD_ENABLE_WQ, wq->id, &status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) if (status != IDXD_CMDSTS_SUCCESS &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) status != IDXD_CMDSTS_ERR_WQ_ENABLED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) dev_dbg(dev, "WQ enable failed: %#x\n", status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) wq->state = IDXD_WQ_ENABLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) dev_dbg(dev, "WQ %d enabled\n", wq->id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) int idxd_wq_disable(struct idxd_wq *wq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) struct idxd_device *idxd = wq->idxd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) struct device *dev = &idxd->pdev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) u32 status, operand;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) dev_dbg(dev, "Disabling WQ %d\n", wq->id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) if (wq->state != IDXD_WQ_ENABLED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) operand = BIT(wq->id % 16) | ((wq->id / 16) << 16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) idxd_cmd_exec(idxd, IDXD_CMD_DISABLE_WQ, operand, &status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) if (status != IDXD_CMDSTS_SUCCESS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) dev_dbg(dev, "WQ disable failed: %#x\n", status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) wq->state = IDXD_WQ_DISABLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) dev_dbg(dev, "WQ %d disabled\n", wq->id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) void idxd_wq_drain(struct idxd_wq *wq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) struct idxd_device *idxd = wq->idxd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) struct device *dev = &idxd->pdev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) u32 operand;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) if (wq->state != IDXD_WQ_ENABLED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) dev_dbg(dev, "Draining WQ %d\n", wq->id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) operand = BIT(wq->id % 16) | ((wq->id / 16) << 16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) idxd_cmd_exec(idxd, IDXD_CMD_DRAIN_WQ, operand, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) void idxd_wq_reset(struct idxd_wq *wq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) struct idxd_device *idxd = wq->idxd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) struct device *dev = &idxd->pdev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) u32 operand;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) if (wq->state != IDXD_WQ_ENABLED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) operand = BIT(wq->id % 16) | ((wq->id / 16) << 16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) idxd_cmd_exec(idxd, IDXD_CMD_RESET_WQ, operand, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) wq->state = IDXD_WQ_DISABLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) int idxd_wq_map_portal(struct idxd_wq *wq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) struct idxd_device *idxd = wq->idxd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) struct pci_dev *pdev = idxd->pdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) struct device *dev = &pdev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) resource_size_t start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) start = pci_resource_start(pdev, IDXD_WQ_BAR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) start += idxd_get_wq_portal_full_offset(wq->id, IDXD_PORTAL_LIMITED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) wq->dportal = devm_ioremap(dev, start, IDXD_PORTAL_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) if (!wq->dportal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) dev_dbg(dev, "wq %d portal mapped at %p\n", wq->id, wq->dportal);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) void idxd_wq_unmap_portal(struct idxd_wq *wq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) struct device *dev = &wq->idxd->pdev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) devm_iounmap(dev, wq->dportal);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) void idxd_wq_disable_cleanup(struct idxd_wq *wq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) struct idxd_device *idxd = wq->idxd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) lockdep_assert_held(&idxd->dev_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) memset(wq->wqcfg, 0, idxd->wqcfg_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) wq->type = IDXD_WQT_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) wq->size = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) wq->group = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) wq->threshold = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) wq->priority = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) clear_bit(WQ_FLAG_DEDICATED, &wq->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) memset(wq->name, 0, WQ_NAME_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) /* Device control bits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) static inline bool idxd_is_enabled(struct idxd_device *idxd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) union gensts_reg gensts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) if (gensts.state == IDXD_DEVICE_STATE_ENABLED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) static inline bool idxd_device_is_halted(struct idxd_device *idxd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) union gensts_reg gensts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) return (gensts.state == IDXD_DEVICE_STATE_HALT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) * This is function is only used for reset during probe and will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) * poll for completion. Once the device is setup with interrupts,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) * all commands will be done via interrupt completion.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) int idxd_device_init_reset(struct idxd_device *idxd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) struct device *dev = &idxd->pdev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) union idxd_command_reg cmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) if (idxd_device_is_halted(idxd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) dev_warn(&idxd->pdev->dev, "Device is HALTED!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) memset(&cmd, 0, sizeof(cmd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) cmd.cmd = IDXD_CMD_RESET_DEVICE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) dev_dbg(dev, "%s: sending reset for init.\n", __func__);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) spin_lock_irqsave(&idxd->dev_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) iowrite32(cmd.bits, idxd->reg_base + IDXD_CMD_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) while (ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET) &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) IDXD_CMDSTS_ACTIVE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) cpu_relax();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) spin_unlock_irqrestore(&idxd->dev_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) u32 *status)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) union idxd_command_reg cmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) DECLARE_COMPLETION_ONSTACK(done);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) if (idxd_device_is_halted(idxd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) dev_warn(&idxd->pdev->dev, "Device is HALTED!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) if (status)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) *status = IDXD_CMDSTS_HW_ERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) memset(&cmd, 0, sizeof(cmd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) cmd.cmd = cmd_code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) cmd.operand = operand;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) cmd.int_req = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) spin_lock_irqsave(&idxd->dev_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) wait_event_lock_irq(idxd->cmd_waitq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) !test_bit(IDXD_FLAG_CMD_RUNNING, &idxd->flags),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) idxd->dev_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) dev_dbg(&idxd->pdev->dev, "%s: sending cmd: %#x op: %#x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) __func__, cmd_code, operand);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) idxd->cmd_status = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) __set_bit(IDXD_FLAG_CMD_RUNNING, &idxd->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) idxd->cmd_done = &done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) iowrite32(cmd.bits, idxd->reg_base + IDXD_CMD_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) * After command submitted, release lock and go to sleep until
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) * the command completes via interrupt.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) spin_unlock_irqrestore(&idxd->dev_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) wait_for_completion(&done);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) spin_lock_irqsave(&idxd->dev_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) if (status) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) *status = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) idxd->cmd_status = *status & GENMASK(7, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) __clear_bit(IDXD_FLAG_CMD_RUNNING, &idxd->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) /* Wake up other pending commands */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) wake_up(&idxd->cmd_waitq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) spin_unlock_irqrestore(&idxd->dev_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) int idxd_device_enable(struct idxd_device *idxd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) struct device *dev = &idxd->pdev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) u32 status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) if (idxd_is_enabled(idxd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) dev_dbg(dev, "Device already enabled\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) idxd_cmd_exec(idxd, IDXD_CMD_ENABLE_DEVICE, 0, &status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) /* If the command is successful or if the device was enabled */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) if (status != IDXD_CMDSTS_SUCCESS &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) status != IDXD_CMDSTS_ERR_DEV_ENABLED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) dev_dbg(dev, "%s: err_code: %#x\n", __func__, status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) idxd->state = IDXD_DEV_ENABLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) void idxd_device_wqs_clear_state(struct idxd_device *idxd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) lockdep_assert_held(&idxd->dev_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) for (i = 0; i < idxd->max_wqs; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) struct idxd_wq *wq = &idxd->wqs[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) if (wq->state == IDXD_WQ_ENABLED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) idxd_wq_disable_cleanup(wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) wq->state = IDXD_WQ_DISABLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) int idxd_device_disable(struct idxd_device *idxd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) struct device *dev = &idxd->pdev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) u32 status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) if (!idxd_is_enabled(idxd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) dev_dbg(dev, "Device is not enabled\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) idxd_cmd_exec(idxd, IDXD_CMD_DISABLE_DEVICE, 0, &status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) /* If the command is successful or if the device was disabled */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) if (status != IDXD_CMDSTS_SUCCESS &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) !(status & IDXD_CMDSTS_ERR_DIS_DEV_EN)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) dev_dbg(dev, "%s: err_code: %#x\n", __func__, status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) spin_lock_irqsave(&idxd->dev_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) idxd_device_wqs_clear_state(idxd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) idxd->state = IDXD_DEV_CONF_READY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) spin_unlock_irqrestore(&idxd->dev_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) void idxd_device_reset(struct idxd_device *idxd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) idxd_cmd_exec(idxd, IDXD_CMD_RESET_DEVICE, 0, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) spin_lock_irqsave(&idxd->dev_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) idxd_device_wqs_clear_state(idxd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) idxd->state = IDXD_DEV_CONF_READY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) spin_unlock_irqrestore(&idxd->dev_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) /* Device configuration bits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) static void idxd_group_config_write(struct idxd_group *group)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) struct idxd_device *idxd = group->idxd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) struct device *dev = &idxd->pdev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) u32 grpcfg_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) dev_dbg(dev, "Writing group %d cfg registers\n", group->id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) /* setup GRPWQCFG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) for (i = 0; i < 4; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) grpcfg_offset = idxd->grpcfg_offset +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) group->id * 64 + i * sizeof(u64);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) iowrite64(group->grpcfg.wqs[i],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) idxd->reg_base + grpcfg_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) dev_dbg(dev, "GRPCFG wq[%d:%d: %#x]: %#llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) group->id, i, grpcfg_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) ioread64(idxd->reg_base + grpcfg_offset));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) /* setup GRPENGCFG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) grpcfg_offset = idxd->grpcfg_offset + group->id * 64 + 32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) iowrite64(group->grpcfg.engines, idxd->reg_base + grpcfg_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) dev_dbg(dev, "GRPCFG engs[%d: %#x]: %#llx\n", group->id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) grpcfg_offset, ioread64(idxd->reg_base + grpcfg_offset));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) /* setup GRPFLAGS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) grpcfg_offset = idxd->grpcfg_offset + group->id * 64 + 40;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) iowrite32(group->grpcfg.flags.bits, idxd->reg_base + grpcfg_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) dev_dbg(dev, "GRPFLAGS flags[%d: %#x]: %#x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) group->id, grpcfg_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) ioread32(idxd->reg_base + grpcfg_offset));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) static int idxd_groups_config_write(struct idxd_device *idxd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) union gencfg_reg reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) struct device *dev = &idxd->pdev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) /* Setup bandwidth token limit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) if (idxd->token_limit) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) reg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) reg.token_limit = idxd->token_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) iowrite32(reg.bits, idxd->reg_base + IDXD_GENCFG_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) dev_dbg(dev, "GENCFG(%#x): %#x\n", IDXD_GENCFG_OFFSET,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) for (i = 0; i < idxd->max_groups; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) struct idxd_group *group = &idxd->groups[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) idxd_group_config_write(group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) static int idxd_wq_config_write(struct idxd_wq *wq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) struct idxd_device *idxd = wq->idxd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) struct device *dev = &idxd->pdev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) u32 wq_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) if (!wq->group)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) * Instead of memset the entire shadow copy of WQCFG, copy from the hardware after
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) * wq reset. This will copy back the sticky values that are present on some devices.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) for (i = 0; i < WQCFG_STRIDES(idxd); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) wq_offset = WQCFG_OFFSET(idxd, wq->id, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) wq->wqcfg->bits[i] = ioread32(idxd->reg_base + wq_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) /* byte 0-3 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) wq->wqcfg->wq_size = wq->size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) if (wq->size == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) dev_warn(dev, "Incorrect work queue size: 0\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) /* bytes 4-7 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) wq->wqcfg->wq_thresh = wq->threshold;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) /* byte 8-11 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) wq->wqcfg->priv = !!(wq->type == IDXD_WQT_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) wq->wqcfg->mode = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) wq->wqcfg->priority = wq->priority;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) /* bytes 12-15 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) wq->wqcfg->max_xfer_shift = ilog2(wq->max_xfer_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) wq->wqcfg->max_batch_shift = ilog2(wq->max_batch_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) dev_dbg(dev, "WQ %d CFGs\n", wq->id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) for (i = 0; i < WQCFG_STRIDES(idxd); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) wq_offset = WQCFG_OFFSET(idxd, wq->id, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) iowrite32(wq->wqcfg->bits[i], idxd->reg_base + wq_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) dev_dbg(dev, "WQ[%d][%d][%#x]: %#x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) wq->id, i, wq_offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) ioread32(idxd->reg_base + wq_offset));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) static int idxd_wqs_config_write(struct idxd_device *idxd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) int i, rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) for (i = 0; i < idxd->max_wqs; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) struct idxd_wq *wq = &idxd->wqs[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) rc = idxd_wq_config_write(wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) if (rc < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) static void idxd_group_flags_setup(struct idxd_device *idxd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) /* TC-A 0 and TC-B 1 should be defaults */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) for (i = 0; i < idxd->max_groups; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) struct idxd_group *group = &idxd->groups[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) if (group->tc_a == -1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) group->tc_a = group->grpcfg.flags.tc_a = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) group->grpcfg.flags.tc_a = group->tc_a;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) if (group->tc_b == -1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) group->tc_b = group->grpcfg.flags.tc_b = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) group->grpcfg.flags.tc_b = group->tc_b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) group->grpcfg.flags.use_token_limit = group->use_token_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) group->grpcfg.flags.tokens_reserved = group->tokens_reserved;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) if (group->tokens_allowed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) group->grpcfg.flags.tokens_allowed =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) group->tokens_allowed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) group->grpcfg.flags.tokens_allowed = idxd->max_tokens;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) static int idxd_engines_setup(struct idxd_device *idxd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) int i, engines = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) struct idxd_engine *eng;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) struct idxd_group *group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) for (i = 0; i < idxd->max_groups; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) group = &idxd->groups[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) group->grpcfg.engines = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) for (i = 0; i < idxd->max_engines; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) eng = &idxd->engines[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) group = eng->group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) if (!group)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) group->grpcfg.engines |= BIT(eng->id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) engines++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) if (!engines)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) static int idxd_wqs_setup(struct idxd_device *idxd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) struct idxd_wq *wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) struct idxd_group *group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) int i, j, configured = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) struct device *dev = &idxd->pdev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) for (i = 0; i < idxd->max_groups; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) group = &idxd->groups[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) for (j = 0; j < 4; j++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) group->grpcfg.wqs[j] = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) for (i = 0; i < idxd->max_wqs; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) wq = &idxd->wqs[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) group = wq->group;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) if (!wq->group)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) if (!wq->size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) if (!wq_dedicated(wq)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) dev_warn(dev, "No shared workqueue support.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) group->grpcfg.wqs[wq->id / 64] |= BIT(wq->id % 64);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) configured++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) if (configured == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) int idxd_device_config(struct idxd_device *idxd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) lockdep_assert_held(&idxd->dev_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) rc = idxd_wqs_setup(idxd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) if (rc < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) rc = idxd_engines_setup(idxd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) if (rc < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) idxd_group_flags_setup(idxd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) rc = idxd_wqs_config_write(idxd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) if (rc < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) rc = idxd_groups_config_write(idxd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) if (rc < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags