^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) // Copyright 2017 IBM Corp.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) #include <linux/sched/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) #include <linux/mutex.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include <linux/mm_types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/mmu_context.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <asm/copro.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <asm/pnv-ocxl.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <asm/xive.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <misc/ocxl.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include "ocxl_internal.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include "trace.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #define SPA_PASID_BITS 15
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #define SPA_PASID_MAX ((1 << SPA_PASID_BITS) - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #define SPA_PE_MASK SPA_PASID_MAX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #define SPA_SPA_SIZE_LOG 22 /* Each SPA is 4 Mb */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #define SPA_CFG_SF (1ull << (63-0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #define SPA_CFG_TA (1ull << (63-1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #define SPA_CFG_HV (1ull << (63-3))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #define SPA_CFG_UV (1ull << (63-4))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #define SPA_CFG_XLAT_hpt (0ull << (63-6)) /* Hashed page table (HPT) mode */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #define SPA_CFG_XLAT_roh (2ull << (63-6)) /* Radix on HPT mode */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #define SPA_CFG_XLAT_ror (3ull << (63-6)) /* Radix on Radix mode */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #define SPA_CFG_PR (1ull << (63-49))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #define SPA_CFG_TC (1ull << (63-54))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #define SPA_CFG_DR (1ull << (63-59))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #define SPA_XSL_TF (1ull << (63-3)) /* Translation fault */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #define SPA_XSL_S (1ull << (63-38)) /* Store operation */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #define SPA_PE_VALID 0x80000000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) struct pe_data {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) struct mm_struct *mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) /* callback to trigger when a translation fault occurs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) void (*xsl_err_cb)(void *data, u64 addr, u64 dsisr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) /* opaque pointer to be passed to the above callback */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) void *xsl_err_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) struct rcu_head rcu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) struct spa {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) struct ocxl_process_element *spa_mem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) int spa_order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) struct mutex spa_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) struct radix_tree_root pe_tree; /* Maps PE handles to pe_data */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) char *irq_name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) int virq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) void __iomem *reg_dsisr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) void __iomem *reg_dar;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) void __iomem *reg_tfc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) void __iomem *reg_pe_handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) * The following field are used by the memory fault
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) * interrupt handler. We can only have one interrupt at a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) * time. The NPU won't raise another interrupt until the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) * previous one has been ack'd by writing to the TFC register
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) struct xsl_fault {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) struct work_struct fault_work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) u64 pe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) u64 dsisr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) u64 dar;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) struct pe_data pe_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) } xsl_fault;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) * A opencapi link can be used be by several PCI functions. We have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) * one link per device slot.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) * A linked list of opencapi links should suffice, as there's a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) * limited number of opencapi slots on a system and lookup is only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) * done when the device is probed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) struct ocxl_link {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) struct list_head list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) struct kref ref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) int domain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) int bus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) int dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) atomic_t irq_available;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) struct spa *spa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) void *platform_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) static struct list_head links_list = LIST_HEAD_INIT(links_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) static DEFINE_MUTEX(links_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) enum xsl_response {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) CONTINUE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) ADDRESS_ERROR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) RESTART,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) static void read_irq(struct spa *spa, u64 *dsisr, u64 *dar, u64 *pe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) u64 reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) *dsisr = in_be64(spa->reg_dsisr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) *dar = in_be64(spa->reg_dar);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) reg = in_be64(spa->reg_pe_handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) *pe = reg & SPA_PE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) static void ack_irq(struct spa *spa, enum xsl_response r)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) u64 reg = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) /* continue is not supported */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) if (r == RESTART)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) reg = PPC_BIT(31);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) else if (r == ADDRESS_ERROR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) reg = PPC_BIT(30);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) WARN(1, "Invalid irq response %d\n", r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) if (reg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) trace_ocxl_fault_ack(spa->spa_mem, spa->xsl_fault.pe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) spa->xsl_fault.dsisr, spa->xsl_fault.dar, reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) out_be64(spa->reg_tfc, reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) static void xsl_fault_handler_bh(struct work_struct *fault_work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) vm_fault_t flt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) unsigned long access, flags, inv_flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) enum xsl_response r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) struct xsl_fault *fault = container_of(fault_work, struct xsl_fault,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) fault_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) struct spa *spa = container_of(fault, struct spa, xsl_fault);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) * We must release a reference on mm_users whenever exiting this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) * function (taken in the memory fault interrupt handler)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) rc = copro_handle_mm_fault(fault->pe_data.mm, fault->dar, fault->dsisr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) &flt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) pr_debug("copro_handle_mm_fault failed: %d\n", rc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) if (fault->pe_data.xsl_err_cb) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) fault->pe_data.xsl_err_cb(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) fault->pe_data.xsl_err_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) fault->dar, fault->dsisr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) r = ADDRESS_ERROR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) goto ack;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) if (!radix_enabled()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) * update_mmu_cache() will not have loaded the hash
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) * since current->trap is not a 0x400 or 0x300, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) * just call hash_page_mm() here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) access = _PAGE_PRESENT | _PAGE_READ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) if (fault->dsisr & SPA_XSL_S)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) access |= _PAGE_WRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) if (get_region_id(fault->dar) != USER_REGION_ID)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) access |= _PAGE_PRIVILEGED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) hash_page_mm(fault->pe_data.mm, fault->dar, access, 0x300,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) inv_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) r = RESTART;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) ack:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) mmput(fault->pe_data.mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) ack_irq(spa, r);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) static irqreturn_t xsl_fault_handler(int irq, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) struct ocxl_link *link = (struct ocxl_link *) data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) struct spa *spa = link->spa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) u64 dsisr, dar, pe_handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) struct pe_data *pe_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) struct ocxl_process_element *pe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) int pid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) bool schedule = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) read_irq(spa, &dsisr, &dar, &pe_handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) trace_ocxl_fault(spa->spa_mem, pe_handle, dsisr, dar, -1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) WARN_ON(pe_handle > SPA_PE_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) pe = spa->spa_mem + pe_handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) pid = be32_to_cpu(pe->pid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) /* We could be reading all null values here if the PE is being
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) * removed while an interrupt kicks in. It's not supposed to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) * happen if the driver notified the AFU to terminate the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) * PASID, and the AFU waited for pending operations before
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) * acknowledging. But even if it happens, we won't find a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) * memory context below and fail silently, so it should be ok.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) if (!(dsisr & SPA_XSL_TF)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) WARN(1, "Invalid xsl interrupt fault register %#llx\n", dsisr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) ack_irq(spa, ADDRESS_ERROR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) return IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) pe_data = radix_tree_lookup(&spa->pe_tree, pe_handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) if (!pe_data) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) * Could only happen if the driver didn't notify the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) * AFU about PASID termination before removing the PE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) * or the AFU didn't wait for all memory access to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) * have completed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) * Either way, we fail early, but we shouldn't log an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) * error message, as it is a valid (if unexpected)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) * scenario
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) pr_debug("Unknown mm context for xsl interrupt\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) ack_irq(spa, ADDRESS_ERROR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) return IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) if (!pe_data->mm) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) * translation fault from a kernel context - an OpenCAPI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) * device tried to access a bad kernel address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) pr_warn("Unresolved OpenCAPI xsl fault in kernel context\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) ack_irq(spa, ADDRESS_ERROR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) return IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) WARN_ON(pe_data->mm->context.id != pid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) if (mmget_not_zero(pe_data->mm)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) spa->xsl_fault.pe = pe_handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) spa->xsl_fault.dar = dar;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) spa->xsl_fault.dsisr = dsisr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) spa->xsl_fault.pe_data = *pe_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) schedule = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) /* mm_users count released by bottom half */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) if (schedule)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) schedule_work(&spa->xsl_fault.fault_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) ack_irq(spa, ADDRESS_ERROR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) return IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) static void unmap_irq_registers(struct spa *spa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) pnv_ocxl_unmap_xsl_regs(spa->reg_dsisr, spa->reg_dar, spa->reg_tfc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) spa->reg_pe_handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) static int map_irq_registers(struct pci_dev *dev, struct spa *spa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) return pnv_ocxl_map_xsl_regs(dev, &spa->reg_dsisr, &spa->reg_dar,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) &spa->reg_tfc, &spa->reg_pe_handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) static int setup_xsl_irq(struct pci_dev *dev, struct ocxl_link *link)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) struct spa *spa = link->spa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) int hwirq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) rc = pnv_ocxl_get_xsl_irq(dev, &hwirq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) rc = map_irq_registers(dev, spa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) spa->irq_name = kasprintf(GFP_KERNEL, "ocxl-xsl-%x-%x-%x",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) link->domain, link->bus, link->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) if (!spa->irq_name) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) dev_err(&dev->dev, "Can't allocate name for xsl interrupt\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) rc = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) goto err_xsl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) * At some point, we'll need to look into allowing a higher
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) * number of interrupts. Could we have an IRQ domain per link?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) spa->virq = irq_create_mapping(NULL, hwirq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) if (!spa->virq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) dev_err(&dev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) "irq_create_mapping failed for translation interrupt\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) rc = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) goto err_name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) dev_dbg(&dev->dev, "hwirq %d mapped to virq %d\n", hwirq, spa->virq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) rc = request_irq(spa->virq, xsl_fault_handler, 0, spa->irq_name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) dev_err(&dev->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) "request_irq failed for translation interrupt: %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) rc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) rc = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) goto err_mapping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) err_mapping:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) irq_dispose_mapping(spa->virq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) err_name:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) kfree(spa->irq_name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) err_xsl:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) unmap_irq_registers(spa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) static void release_xsl_irq(struct ocxl_link *link)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) struct spa *spa = link->spa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) if (spa->virq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) free_irq(spa->virq, link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) irq_dispose_mapping(spa->virq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) kfree(spa->irq_name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) unmap_irq_registers(spa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) static int alloc_spa(struct pci_dev *dev, struct ocxl_link *link)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) struct spa *spa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) spa = kzalloc(sizeof(struct spa), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) if (!spa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) mutex_init(&spa->spa_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) INIT_RADIX_TREE(&spa->pe_tree, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) INIT_WORK(&spa->xsl_fault.fault_work, xsl_fault_handler_bh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) spa->spa_order = SPA_SPA_SIZE_LOG - PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) spa->spa_mem = (struct ocxl_process_element *)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) __get_free_pages(GFP_KERNEL | __GFP_ZERO, spa->spa_order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) if (!spa->spa_mem) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) dev_err(&dev->dev, "Can't allocate Shared Process Area\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) kfree(spa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) pr_debug("Allocated SPA for %x:%x:%x at %p\n", link->domain, link->bus,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) link->dev, spa->spa_mem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) link->spa = spa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) static void free_spa(struct ocxl_link *link)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) struct spa *spa = link->spa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) pr_debug("Freeing SPA for %x:%x:%x\n", link->domain, link->bus,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) link->dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) if (spa && spa->spa_mem) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) free_pages((unsigned long) spa->spa_mem, spa->spa_order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) kfree(spa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) link->spa = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) static int alloc_link(struct pci_dev *dev, int PE_mask, struct ocxl_link **out_link)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) struct ocxl_link *link;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) link = kzalloc(sizeof(struct ocxl_link), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) if (!link)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) kref_init(&link->ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) link->domain = pci_domain_nr(dev->bus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) link->bus = dev->bus->number;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) link->dev = PCI_SLOT(dev->devfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) atomic_set(&link->irq_available, MAX_IRQ_PER_LINK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) rc = alloc_spa(dev, link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) goto err_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) rc = setup_xsl_irq(dev, link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) goto err_spa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) /* platform specific hook */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) rc = pnv_ocxl_spa_setup(dev, link->spa->spa_mem, PE_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) &link->platform_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) goto err_xsl_irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) *out_link = link;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) err_xsl_irq:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) release_xsl_irq(link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) err_spa:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) free_spa(link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) err_free:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) kfree(link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) static void free_link(struct ocxl_link *link)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) release_xsl_irq(link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) free_spa(link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) kfree(link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) int ocxl_link_setup(struct pci_dev *dev, int PE_mask, void **link_handle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) int rc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) struct ocxl_link *link;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) mutex_lock(&links_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) list_for_each_entry(link, &links_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) /* The functions of a device all share the same link */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) if (link->domain == pci_domain_nr(dev->bus) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) link->bus == dev->bus->number &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) link->dev == PCI_SLOT(dev->devfn)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) kref_get(&link->ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) *link_handle = link;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) rc = alloc_link(dev, PE_mask, &link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) list_add(&link->list, &links_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) *link_handle = link;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) mutex_unlock(&links_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) EXPORT_SYMBOL_GPL(ocxl_link_setup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) static void release_xsl(struct kref *ref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) struct ocxl_link *link = container_of(ref, struct ocxl_link, ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) list_del(&link->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) /* call platform code before releasing data */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) pnv_ocxl_spa_release(link->platform_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) free_link(link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) void ocxl_link_release(struct pci_dev *dev, void *link_handle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) struct ocxl_link *link = (struct ocxl_link *) link_handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) mutex_lock(&links_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) kref_put(&link->ref, release_xsl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) mutex_unlock(&links_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) EXPORT_SYMBOL_GPL(ocxl_link_release);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) static u64 calculate_cfg_state(bool kernel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) u64 state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) state = SPA_CFG_DR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) if (mfspr(SPRN_LPCR) & LPCR_TC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) state |= SPA_CFG_TC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) if (radix_enabled())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) state |= SPA_CFG_XLAT_ror;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) state |= SPA_CFG_XLAT_hpt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) state |= SPA_CFG_HV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) if (kernel) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) if (mfmsr() & MSR_SF)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) state |= SPA_CFG_SF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) state |= SPA_CFG_PR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) if (!test_tsk_thread_flag(current, TIF_32BIT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) state |= SPA_CFG_SF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) return state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) int ocxl_link_add_pe(void *link_handle, int pasid, u32 pidr, u32 tidr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) u64 amr, struct mm_struct *mm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) void (*xsl_err_cb)(void *data, u64 addr, u64 dsisr),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) void *xsl_err_data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) struct ocxl_link *link = (struct ocxl_link *) link_handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) struct spa *spa = link->spa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) struct ocxl_process_element *pe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) int pe_handle, rc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) struct pe_data *pe_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) BUILD_BUG_ON(sizeof(struct ocxl_process_element) != 128);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) if (pasid > SPA_PASID_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) mutex_lock(&spa->spa_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) pe_handle = pasid & SPA_PE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) pe = spa->spa_mem + pe_handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) if (pe->software_state) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) rc = -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) pe_data = kmalloc(sizeof(*pe_data), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) if (!pe_data) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) rc = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) pe_data->mm = mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) pe_data->xsl_err_cb = xsl_err_cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) pe_data->xsl_err_data = xsl_err_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) memset(pe, 0, sizeof(struct ocxl_process_element));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) pe->config_state = cpu_to_be64(calculate_cfg_state(pidr == 0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) pe->lpid = cpu_to_be32(mfspr(SPRN_LPID));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) pe->pid = cpu_to_be32(pidr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) pe->tid = cpu_to_be32(tidr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) pe->amr = cpu_to_be64(amr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) pe->software_state = cpu_to_be32(SPA_PE_VALID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) * For user contexts, register a copro so that TLBIs are seen
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) * by the nest MMU. If we have a kernel context, TLBIs are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) * already global.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) if (mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) mm_context_add_copro(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) * Barrier is to make sure PE is visible in the SPA before it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) * is used by the device. It also helps with the global TLBI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) * invalidation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) radix_tree_insert(&spa->pe_tree, pe_handle, pe_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) * The mm must stay valid for as long as the device uses it. We
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) * lower the count when the context is removed from the SPA.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) * We grab mm_count (and not mm_users), as we don't want to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) * end up in a circular dependency if a process mmaps its
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) * mmio, therefore incrementing the file ref count when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) * calling mmap(), and forgets to unmap before exiting. In
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) * that scenario, when the kernel handles the death of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) * process, the file is not cleaned because unmap was not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) * called, and the mm wouldn't be freed because we would still
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) * have a reference on mm_users. Incrementing mm_count solves
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) * the problem.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) if (mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) mmgrab(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) trace_ocxl_context_add(current->pid, spa->spa_mem, pasid, pidr, tidr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) mutex_unlock(&spa->spa_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) EXPORT_SYMBOL_GPL(ocxl_link_add_pe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) int ocxl_link_update_pe(void *link_handle, int pasid, __u16 tid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) struct ocxl_link *link = (struct ocxl_link *) link_handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) struct spa *spa = link->spa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) struct ocxl_process_element *pe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) int pe_handle, rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) if (pasid > SPA_PASID_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) pe_handle = pasid & SPA_PE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) pe = spa->spa_mem + pe_handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) mutex_lock(&spa->spa_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) pe->tid = cpu_to_be32(tid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) * The barrier makes sure the PE is updated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) * before we clear the NPU context cache below, so that the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) * old PE cannot be reloaded erroneously.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) * hook to platform code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) * On powerpc, the entry needs to be cleared from the context
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) * cache of the NPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) rc = pnv_ocxl_spa_remove_pe_from_cache(link->platform_data, pe_handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) WARN_ON(rc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) mutex_unlock(&spa->spa_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) int ocxl_link_remove_pe(void *link_handle, int pasid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) struct ocxl_link *link = (struct ocxl_link *) link_handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) struct spa *spa = link->spa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) struct ocxl_process_element *pe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) struct pe_data *pe_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) int pe_handle, rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) if (pasid > SPA_PASID_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) * About synchronization with our memory fault handler:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) * Before removing the PE, the driver is supposed to have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) * notified the AFU, which should have cleaned up and make
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) * sure the PASID is no longer in use, including pending
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) * interrupts. However, there's no way to be sure...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) * We clear the PE and remove the context from our radix
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) * tree. From that point on, any new interrupt for that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) * context will fail silently, which is ok. As mentioned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) * above, that's not expected, but it could happen if the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) * driver or AFU didn't do the right thing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) * There could still be a bottom half running, but we don't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) * need to wait/flush, as it is managing a reference count on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) * the mm it reads from the radix tree.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) pe_handle = pasid & SPA_PE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) pe = spa->spa_mem + pe_handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) mutex_lock(&spa->spa_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) if (!(be32_to_cpu(pe->software_state) & SPA_PE_VALID)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) rc = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) trace_ocxl_context_remove(current->pid, spa->spa_mem, pasid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) be32_to_cpu(pe->pid), be32_to_cpu(pe->tid));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) memset(pe, 0, sizeof(struct ocxl_process_element));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) * The barrier makes sure the PE is removed from the SPA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) * before we clear the NPU context cache below, so that the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) * old PE cannot be reloaded erroneously.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) * hook to platform code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) * On powerpc, the entry needs to be cleared from the context
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) * cache of the NPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) rc = pnv_ocxl_spa_remove_pe_from_cache(link->platform_data, pe_handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) WARN_ON(rc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) pe_data = radix_tree_delete(&spa->pe_tree, pe_handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) if (!pe_data) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) WARN(1, "Couldn't find pe data when removing PE\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) if (pe_data->mm) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) mm_context_remove_copro(pe_data->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) mmdrop(pe_data->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) kfree_rcu(pe_data, rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) mutex_unlock(&spa->spa_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) EXPORT_SYMBOL_GPL(ocxl_link_remove_pe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) int ocxl_link_irq_alloc(void *link_handle, int *hw_irq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) struct ocxl_link *link = (struct ocxl_link *) link_handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) int irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) if (atomic_dec_if_positive(&link->irq_available) < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) irq = xive_native_alloc_irq();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) if (!irq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) atomic_inc(&link->irq_available);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) return -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) *hw_irq = irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) EXPORT_SYMBOL_GPL(ocxl_link_irq_alloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) void ocxl_link_free_irq(void *link_handle, int hw_irq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) struct ocxl_link *link = (struct ocxl_link *) link_handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) xive_native_free_irq(hw_irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) atomic_inc(&link->irq_available);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) EXPORT_SYMBOL_GPL(ocxl_link_free_irq);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags