^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-or-later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * SN Platform GRU Driver
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * FILE OPERATIONS & DRIVER INITIALIZATION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * This file supports the user system call for file open, close, mmap, etc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * This also incudes the driver initialization code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * (C) Copyright 2020 Hewlett Packard Enterprise Development LP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * Copyright (c) 2008-2014 Silicon Graphics, Inc. All Rights Reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/io.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/device.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/miscdevice.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/proc_fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <asm/uv/uv_irq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <asm/uv/uv.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include "gru.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include "grulib.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include "grutables.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <asm/uv/uv_hub.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <asm/uv/uv_mmrs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) struct gru_blade_state *gru_base[GRU_MAX_BLADES] __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) unsigned long gru_start_paddr __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) void *gru_start_vaddr __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) unsigned long gru_end_paddr __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) unsigned int gru_max_gids __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) struct gru_stats_s gru_stats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) /* Guaranteed user available resources on each node */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) static int max_user_cbrs, max_user_dsr_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) static struct miscdevice gru_miscdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) static int gru_supported(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) return is_uv_system() &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) (uv_hub_info->hub_revision < UV3_HUB_REVISION_BASE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) * gru_vma_close
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) * Called when unmapping a device mapping. Frees all gru resources
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) * and tables belonging to the vma.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) static void gru_vma_close(struct vm_area_struct *vma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) struct gru_vma_data *vdata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) struct gru_thread_state *gts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) struct list_head *entry, *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) if (!vma->vm_private_data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) vdata = vma->vm_private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) vma->vm_private_data = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) gru_dbg(grudev, "vma %p, file %p, vdata %p\n", vma, vma->vm_file,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) vdata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) list_for_each_safe(entry, next, &vdata->vd_head) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) gts =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) list_entry(entry, struct gru_thread_state, ts_next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) list_del(>s->ts_next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) mutex_lock(>s->ts_ctxlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) if (gts->ts_gru)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) gru_unload_context(gts, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) mutex_unlock(>s->ts_ctxlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) gts_drop(gts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) kfree(vdata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) STAT(vdata_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) * gru_file_mmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) * Called when mmapping the device. Initializes the vma with a fault handler
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) * and private data structure necessary to allocate, track, and free the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) * underlying pages.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) static int gru_file_mmap(struct file *file, struct vm_area_struct *vma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) if ((vma->vm_flags & (VM_SHARED | VM_WRITE)) != (VM_SHARED | VM_WRITE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) return -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) if (vma->vm_start & (GRU_GSEG_PAGESIZE - 1) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) vma->vm_end & (GRU_GSEG_PAGESIZE - 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) vma->vm_flags |= VM_IO | VM_PFNMAP | VM_LOCKED |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) vma->vm_page_prot = PAGE_SHARED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) vma->vm_ops = &gru_vm_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) vma->vm_private_data = gru_alloc_vma_data(vma, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) if (!vma->vm_private_data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) gru_dbg(grudev, "file %p, vaddr 0x%lx, vma %p, vdata %p\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) file, vma->vm_start, vma, vma->vm_private_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) * Create a new GRU context
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) static int gru_create_new_context(unsigned long arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) struct gru_create_context_req req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) struct vm_area_struct *vma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) struct gru_vma_data *vdata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) int ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) if (copy_from_user(&req, (void __user *)arg, sizeof(req)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) if (req.data_segment_bytes > max_user_dsr_bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) if (req.control_blocks > max_user_cbrs || !req.maximum_thread_count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) if (!(req.options & GRU_OPT_MISS_MASK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) req.options |= GRU_OPT_MISS_FMM_INTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) mmap_write_lock(current->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) vma = gru_find_vma(req.gseg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) if (vma) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) vdata = vma->vm_private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) vdata->vd_user_options = req.options;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) vdata->vd_dsr_au_count =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) GRU_DS_BYTES_TO_AU(req.data_segment_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) vdata->vd_cbr_au_count = GRU_CB_COUNT_TO_AU(req.control_blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) vdata->vd_tlb_preload_count = req.tlb_preload_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) mmap_write_unlock(current->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) * Get GRU configuration info (temp - for emulator testing)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) static long gru_get_config_info(unsigned long arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) struct gru_config_info info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) int nodesperblade;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) if (num_online_nodes() > 1 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) (uv_node_to_blade_id(1) == uv_node_to_blade_id(0)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) nodesperblade = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) nodesperblade = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) memset(&info, 0, sizeof(info));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) info.cpus = num_online_cpus();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) info.nodes = num_online_nodes();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) info.blades = info.nodes / nodesperblade;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) info.chiplets = GRU_CHIPLETS_PER_BLADE * info.blades;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) if (copy_to_user((void __user *)arg, &info, sizeof(info)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) * gru_file_unlocked_ioctl
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) * Called to update file attributes via IOCTL calls.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) static long gru_file_unlocked_ioctl(struct file *file, unsigned int req,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) unsigned long arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) int err = -EBADRQC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) gru_dbg(grudev, "file %p, req 0x%x, 0x%lx\n", file, req, arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) switch (req) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) case GRU_CREATE_CONTEXT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) err = gru_create_new_context(arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) case GRU_SET_CONTEXT_OPTION:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) err = gru_set_context_option(arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) case GRU_USER_GET_EXCEPTION_DETAIL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) err = gru_get_exception_detail(arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) case GRU_USER_UNLOAD_CONTEXT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) err = gru_user_unload_context(arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) case GRU_USER_FLUSH_TLB:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) err = gru_user_flush_tlb(arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) case GRU_USER_CALL_OS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) err = gru_handle_user_call_os(arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) case GRU_GET_GSEG_STATISTICS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) err = gru_get_gseg_statistics(arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) case GRU_KTEST:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) err = gru_ktest(arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) case GRU_GET_CONFIG_INFO:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) err = gru_get_config_info(arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) case GRU_DUMP_CHIPLET_STATE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) err = gru_dump_chiplet_request(arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) * Called at init time to build tables for all GRUs that are present in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) * system.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) static void gru_init_chiplet(struct gru_state *gru, unsigned long paddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) void *vaddr, int blade_id, int chiplet_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) spin_lock_init(&gru->gs_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) spin_lock_init(&gru->gs_asid_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) gru->gs_gru_base_paddr = paddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) gru->gs_gru_base_vaddr = vaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) gru->gs_gid = blade_id * GRU_CHIPLETS_PER_BLADE + chiplet_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) gru->gs_blade = gru_base[blade_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) gru->gs_blade_id = blade_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) gru->gs_chiplet_id = chiplet_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) gru->gs_cbr_map = (GRU_CBR_AU == 64) ? ~0 : (1UL << GRU_CBR_AU) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) gru->gs_dsr_map = (1UL << GRU_DSR_AU) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) gru->gs_asid_limit = MAX_ASID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) gru_tgh_flush_init(gru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) if (gru->gs_gid >= gru_max_gids)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) gru_max_gids = gru->gs_gid + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) gru_dbg(grudev, "bid %d, gid %d, vaddr %p (0x%lx)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) blade_id, gru->gs_gid, gru->gs_gru_base_vaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) gru->gs_gru_base_paddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) static int gru_init_tables(unsigned long gru_base_paddr, void *gru_base_vaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) int pnode, nid, bid, chip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) int cbrs, dsrbytes, n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) int order = get_order(sizeof(struct gru_blade_state));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) struct gru_state *gru;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) unsigned long paddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) void *vaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) max_user_cbrs = GRU_NUM_CB;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) max_user_dsr_bytes = GRU_NUM_DSR_BYTES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) for_each_possible_blade(bid) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) pnode = uv_blade_to_pnode(bid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) nid = uv_blade_to_memory_nid(bid);/* -1 if no memory on blade */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) page = alloc_pages_node(nid, GFP_KERNEL, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) if (!page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) gru_base[bid] = page_address(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) memset(gru_base[bid], 0, sizeof(struct gru_blade_state));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) gru_base[bid]->bs_lru_gru = &gru_base[bid]->bs_grus[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) spin_lock_init(&gru_base[bid]->bs_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) init_rwsem(&gru_base[bid]->bs_kgts_sema);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) dsrbytes = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) cbrs = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) for (gru = gru_base[bid]->bs_grus, chip = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) chip < GRU_CHIPLETS_PER_BLADE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) chip++, gru++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) paddr = gru_chiplet_paddr(gru_base_paddr, pnode, chip);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) vaddr = gru_chiplet_vaddr(gru_base_vaddr, pnode, chip);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) gru_init_chiplet(gru, paddr, vaddr, bid, chip);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) n = hweight64(gru->gs_cbr_map) * GRU_CBR_AU_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) cbrs = max(cbrs, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) n = hweight64(gru->gs_dsr_map) * GRU_DSR_AU_BYTES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) dsrbytes = max(dsrbytes, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) max_user_cbrs = min(max_user_cbrs, cbrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) max_user_dsr_bytes = min(max_user_dsr_bytes, dsrbytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) fail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) for (bid--; bid >= 0; bid--)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) free_pages((unsigned long)gru_base[bid], order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) static void gru_free_tables(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) int bid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) int order = get_order(sizeof(struct gru_state) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) GRU_CHIPLETS_PER_BLADE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) for (bid = 0; bid < GRU_MAX_BLADES; bid++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) free_pages((unsigned long)gru_base[bid], order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) static unsigned long gru_chiplet_cpu_to_mmr(int chiplet, int cpu, int *corep)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) unsigned long mmr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) int core;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) * We target the cores of a blade and not the hyperthreads themselves.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) * There is a max of 8 cores per socket and 2 sockets per blade,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) * making for a max total of 16 cores (i.e., 16 CPUs without
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) * hyperthreading and 32 CPUs with hyperthreading).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) core = uv_cpu_core_number(cpu) + UV_MAX_INT_CORES * uv_cpu_socket_number(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) if (core >= GRU_NUM_TFM || uv_cpu_ht_number(cpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) if (chiplet == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) mmr = UVH_GR0_TLB_INT0_CONFIG +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) core * (UVH_GR0_TLB_INT1_CONFIG - UVH_GR0_TLB_INT0_CONFIG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) } else if (chiplet == 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) mmr = UVH_GR1_TLB_INT0_CONFIG +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) core * (UVH_GR1_TLB_INT1_CONFIG - UVH_GR1_TLB_INT0_CONFIG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) *corep = core;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) return mmr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) #ifdef CONFIG_IA64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) static int gru_irq_count[GRU_CHIPLETS_PER_BLADE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) static void gru_noop(struct irq_data *d)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) static struct irq_chip gru_chip[GRU_CHIPLETS_PER_BLADE] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) [0 ... GRU_CHIPLETS_PER_BLADE - 1] {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) .irq_mask = gru_noop,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) .irq_unmask = gru_noop,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) .irq_ack = gru_noop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) static int gru_chiplet_setup_tlb_irq(int chiplet, char *irq_name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) irq_handler_t irq_handler, int cpu, int blade)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) unsigned long mmr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) int irq = IRQ_GRU + chiplet;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) int ret, core;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) mmr = gru_chiplet_cpu_to_mmr(chiplet, cpu, &core);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) if (mmr == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) if (gru_irq_count[chiplet] == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) gru_chip[chiplet].name = irq_name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) ret = irq_set_chip(irq, &gru_chip[chiplet]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) printk(KERN_ERR "%s: set_irq_chip failed, errno=%d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) GRU_DRIVER_ID_STR, -ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) ret = request_irq(irq, irq_handler, 0, irq_name, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) printk(KERN_ERR "%s: request_irq failed, errno=%d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) GRU_DRIVER_ID_STR, -ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) gru_irq_count[chiplet]++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) static void gru_chiplet_teardown_tlb_irq(int chiplet, int cpu, int blade)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) unsigned long mmr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) int core, irq = IRQ_GRU + chiplet;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) if (gru_irq_count[chiplet] == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) mmr = gru_chiplet_cpu_to_mmr(chiplet, cpu, &core);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) if (mmr == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) if (--gru_irq_count[chiplet] == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) free_irq(irq, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) #elif defined CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) static int gru_chiplet_setup_tlb_irq(int chiplet, char *irq_name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) irq_handler_t irq_handler, int cpu, int blade)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) unsigned long mmr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) int irq, core;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) mmr = gru_chiplet_cpu_to_mmr(chiplet, cpu, &core);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) if (mmr == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) irq = uv_setup_irq(irq_name, cpu, blade, mmr, UV_AFFINITY_CPU);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) if (irq < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) printk(KERN_ERR "%s: uv_setup_irq failed, errno=%d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) GRU_DRIVER_ID_STR, -irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) return irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) ret = request_irq(irq, irq_handler, 0, irq_name, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) uv_teardown_irq(irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) printk(KERN_ERR "%s: request_irq failed, errno=%d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) GRU_DRIVER_ID_STR, -ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) gru_base[blade]->bs_grus[chiplet].gs_irq[core] = irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) static void gru_chiplet_teardown_tlb_irq(int chiplet, int cpu, int blade)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) int irq, core;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) unsigned long mmr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) mmr = gru_chiplet_cpu_to_mmr(chiplet, cpu, &core);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) if (mmr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) irq = gru_base[blade]->bs_grus[chiplet].gs_irq[core];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) if (irq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) free_irq(irq, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) uv_teardown_irq(irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) static void gru_teardown_tlb_irqs(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) int blade;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) for_each_online_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) blade = uv_cpu_to_blade_id(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) gru_chiplet_teardown_tlb_irq(0, cpu, blade);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) gru_chiplet_teardown_tlb_irq(1, cpu, blade);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) for_each_possible_blade(blade) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) if (uv_blade_nr_possible_cpus(blade))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) gru_chiplet_teardown_tlb_irq(0, 0, blade);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) gru_chiplet_teardown_tlb_irq(1, 0, blade);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) static int gru_setup_tlb_irqs(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) int blade;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) for_each_online_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) blade = uv_cpu_to_blade_id(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) ret = gru_chiplet_setup_tlb_irq(0, "GRU0_TLB", gru0_intr, cpu, blade);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) if (ret != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) goto exit1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) ret = gru_chiplet_setup_tlb_irq(1, "GRU1_TLB", gru1_intr, cpu, blade);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) if (ret != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) goto exit1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) for_each_possible_blade(blade) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) if (uv_blade_nr_possible_cpus(blade))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) ret = gru_chiplet_setup_tlb_irq(0, "GRU0_TLB", gru_intr_mblade, 0, blade);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) if (ret != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) goto exit1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) ret = gru_chiplet_setup_tlb_irq(1, "GRU1_TLB", gru_intr_mblade, 0, blade);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) if (ret != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) goto exit1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) exit1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) gru_teardown_tlb_irqs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) * gru_init
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) * Called at boot or module load time to initialize the GRUs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) static int __init gru_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) if (!gru_supported())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) #if defined CONFIG_IA64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) gru_start_paddr = 0xd000000000UL; /* ZZZZZZZZZZZZZZZZZZZ fixme */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) gru_start_paddr = uv_read_local_mmr(UVH_RH_GAM_GRU_OVERLAY_CONFIG) &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) 0x7fffffffffffUL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) gru_start_vaddr = __va(gru_start_paddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) gru_end_paddr = gru_start_paddr + GRU_MAX_BLADES * GRU_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) printk(KERN_INFO "GRU space: 0x%lx - 0x%lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) gru_start_paddr, gru_end_paddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) ret = misc_register(&gru_miscdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) printk(KERN_ERR "%s: misc_register failed\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) GRU_DRIVER_ID_STR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) goto exit0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) ret = gru_proc_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) printk(KERN_ERR "%s: proc init failed\n", GRU_DRIVER_ID_STR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) goto exit1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) ret = gru_init_tables(gru_start_paddr, gru_start_vaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) printk(KERN_ERR "%s: init tables failed\n", GRU_DRIVER_ID_STR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) goto exit2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) ret = gru_setup_tlb_irqs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) if (ret != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) goto exit3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) gru_kservices_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) printk(KERN_INFO "%s: v%s\n", GRU_DRIVER_ID_STR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) GRU_DRIVER_VERSION_STR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) exit3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) gru_free_tables();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) exit2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) gru_proc_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) exit1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) misc_deregister(&gru_miscdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) exit0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) static void __exit gru_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) if (!gru_supported())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) gru_teardown_tlb_irqs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) gru_kservices_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) gru_free_tables();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) misc_deregister(&gru_miscdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) gru_proc_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) mmu_notifier_synchronize();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) static const struct file_operations gru_fops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) .owner = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) .unlocked_ioctl = gru_file_unlocked_ioctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) .mmap = gru_file_mmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) .llseek = noop_llseek,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) static struct miscdevice gru_miscdev = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) .minor = MISC_DYNAMIC_MINOR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) .name = "gru",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) .fops = &gru_fops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) const struct vm_operations_struct gru_vm_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) .close = gru_vma_close,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) .fault = gru_fault,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) #ifndef MODULE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) fs_initcall(gru_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) module_init(gru_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) module_exit(gru_exit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) module_param(gru_options, ulong, 0644);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) MODULE_PARM_DESC(gru_options, "Various debug options");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) MODULE_AUTHOR("Silicon Graphics, Inc.");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) MODULE_LICENSE("GPL");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) MODULE_DESCRIPTION(GRU_DRIVER_ID_STR GRU_DRIVER_VERSION_STR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) MODULE_VERSION(GRU_DRIVER_VERSION_STR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612)
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags