^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-or-later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * SN Platform GRU Driver
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * DRIVER TABLE MANAGER + GRU CONTEXT LOAD/UNLOAD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/device.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/list.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/err.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/prefetch.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <asm/uv/uv_hub.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include "gru.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include "grutables.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include "gruhandles.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) unsigned long gru_options __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) static struct device_driver gru_driver = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) .name = "gru"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) static struct device gru_device = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) .init_name = "",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) .driver = &gru_driver,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) struct device *grudev = &gru_device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * Select a gru fault map to be used by the current cpu. Note that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) * multiple cpus may be using the same map.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) * ZZZ should be inline but did not work on emulator
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) int gru_cpu_fault_map_id(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #ifdef CONFIG_IA64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) return uv_blade_processor_id() % GRU_NUM_TFM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) int cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) int id, core;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) core = uv_cpu_core_number(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) id = core + UV_MAX_INT_CORES * uv_cpu_socket_number(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) return id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) /*--------- ASID Management -------------------------------------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) * Initially, assign asids sequentially from MIN_ASID .. MAX_ASID.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) * Once MAX is reached, flush the TLB & start over. However,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) * some asids may still be in use. There won't be many (percentage wise) still
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) * in use. Search active contexts & determine the value of the first
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) * asid in use ("x"s below). Set "limit" to this value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) * This defines a block of assignable asids.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) * When "limit" is reached, search forward from limit+1 and determine the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) * next block of assignable asids.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) * Repeat until MAX_ASID is reached, then start over again.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) * Each time MAX_ASID is reached, increment the asid generation. Since
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) * the search for in-use asids only checks contexts with GRUs currently
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) * assigned, asids in some contexts will be missed. Prior to loading
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) * a context, the asid generation of the GTS asid is rechecked. If it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) * doesn't match the current generation, a new asid will be assigned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) * 0---------------x------------x---------------------x----|
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) * ^-next ^-limit ^-MAX_ASID
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) * All asid manipulation & context loading/unloading is protected by the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) * gs_lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) /* Hit the asid limit. Start over */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) static int gru_wrap_asid(struct gru_state *gru)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) gru_dbg(grudev, "gid %d\n", gru->gs_gid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) STAT(asid_wrap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) gru->gs_asid_gen++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) return MIN_ASID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) /* Find the next chunk of unused asids */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) static int gru_reset_asid_limit(struct gru_state *gru, int asid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) int i, gid, inuse_asid, limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) gru_dbg(grudev, "gid %d, asid 0x%x\n", gru->gs_gid, asid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) STAT(asid_next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) limit = MAX_ASID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) if (asid >= limit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) asid = gru_wrap_asid(gru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) gru_flush_all_tlb(gru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) gid = gru->gs_gid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) for (i = 0; i < GRU_NUM_CCH; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) if (!gru->gs_gts[i] || is_kernel_context(gru->gs_gts[i]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) inuse_asid = gru->gs_gts[i]->ts_gms->ms_asids[gid].mt_asid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) gru_dbg(grudev, "gid %d, gts %p, gms %p, inuse 0x%x, cxt %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) gru->gs_gid, gru->gs_gts[i], gru->gs_gts[i]->ts_gms,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) inuse_asid, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) if (inuse_asid == asid) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) asid += ASID_INC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) if (asid >= limit) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) * empty range: reset the range limit and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) * start over
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) limit = MAX_ASID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) if (asid >= MAX_ASID)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) asid = gru_wrap_asid(gru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) if ((inuse_asid > asid) && (inuse_asid < limit))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) limit = inuse_asid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) gru->gs_asid_limit = limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) gru->gs_asid = asid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) gru_dbg(grudev, "gid %d, new asid 0x%x, new_limit 0x%x\n", gru->gs_gid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) asid, limit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) return asid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) /* Assign a new ASID to a thread context. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) static int gru_assign_asid(struct gru_state *gru)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) int asid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) gru->gs_asid += ASID_INC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) asid = gru->gs_asid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) if (asid >= gru->gs_asid_limit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) asid = gru_reset_asid_limit(gru, asid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) gru_dbg(grudev, "gid %d, asid 0x%x\n", gru->gs_gid, asid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) return asid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) * Clear n bits in a word. Return a word indicating the bits that were cleared.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) * Optionally, build an array of chars that contain the bit numbers allocated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) static unsigned long reserve_resources(unsigned long *p, int n, int mmax,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) char *idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) unsigned long bits = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) while (n--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) i = find_first_bit(p, mmax);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) if (i == mmax)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) __clear_bit(i, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) __set_bit(i, &bits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) if (idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) *idx++ = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) return bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) unsigned long gru_reserve_cb_resources(struct gru_state *gru, int cbr_au_count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) char *cbmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) return reserve_resources(&gru->gs_cbr_map, cbr_au_count, GRU_CBR_AU,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) cbmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) unsigned long gru_reserve_ds_resources(struct gru_state *gru, int dsr_au_count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) char *dsmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) return reserve_resources(&gru->gs_dsr_map, dsr_au_count, GRU_DSR_AU,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) dsmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) static void reserve_gru_resources(struct gru_state *gru,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) struct gru_thread_state *gts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) gru->gs_active_contexts++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) gts->ts_cbr_map =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) gru_reserve_cb_resources(gru, gts->ts_cbr_au_count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) gts->ts_cbr_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) gts->ts_dsr_map =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) gru_reserve_ds_resources(gru, gts->ts_dsr_au_count, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) static void free_gru_resources(struct gru_state *gru,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) struct gru_thread_state *gts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) gru->gs_active_contexts--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) gru->gs_cbr_map |= gts->ts_cbr_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) gru->gs_dsr_map |= gts->ts_dsr_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) * Check if a GRU has sufficient free resources to satisfy an allocation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) * request. Note: GRU locks may or may not be held when this is called. If
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) * not held, recheck after acquiring the appropriate locks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) * Returns 1 if sufficient resources, 0 if not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) static int check_gru_resources(struct gru_state *gru, int cbr_au_count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) int dsr_au_count, int max_active_contexts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) return hweight64(gru->gs_cbr_map) >= cbr_au_count
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) && hweight64(gru->gs_dsr_map) >= dsr_au_count
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) && gru->gs_active_contexts < max_active_contexts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) * TLB manangment requires tracking all GRU chiplets that have loaded a GSEG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) * context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) static int gru_load_mm_tracker(struct gru_state *gru,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) struct gru_thread_state *gts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) struct gru_mm_struct *gms = gts->ts_gms;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) struct gru_mm_tracker *asids = &gms->ms_asids[gru->gs_gid];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) unsigned short ctxbitmap = (1 << gts->ts_ctxnum);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) int asid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) spin_lock(&gms->ms_asid_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) asid = asids->mt_asid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) spin_lock(&gru->gs_asid_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) if (asid == 0 || (asids->mt_ctxbitmap == 0 && asids->mt_asid_gen !=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) gru->gs_asid_gen)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) asid = gru_assign_asid(gru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) asids->mt_asid = asid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) asids->mt_asid_gen = gru->gs_asid_gen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) STAT(asid_new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) STAT(asid_reuse);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) spin_unlock(&gru->gs_asid_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) BUG_ON(asids->mt_ctxbitmap & ctxbitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) asids->mt_ctxbitmap |= ctxbitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) if (!test_bit(gru->gs_gid, gms->ms_asidmap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) __set_bit(gru->gs_gid, gms->ms_asidmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) spin_unlock(&gms->ms_asid_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) gru_dbg(grudev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) "gid %d, gts %p, gms %p, ctxnum %d, asid 0x%x, asidmap 0x%lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) gru->gs_gid, gts, gms, gts->ts_ctxnum, asid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) gms->ms_asidmap[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) return asid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) static void gru_unload_mm_tracker(struct gru_state *gru,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) struct gru_thread_state *gts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) struct gru_mm_struct *gms = gts->ts_gms;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) struct gru_mm_tracker *asids;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) unsigned short ctxbitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) asids = &gms->ms_asids[gru->gs_gid];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) ctxbitmap = (1 << gts->ts_ctxnum);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) spin_lock(&gms->ms_asid_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) spin_lock(&gru->gs_asid_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) BUG_ON((asids->mt_ctxbitmap & ctxbitmap) != ctxbitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) asids->mt_ctxbitmap ^= ctxbitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) gru_dbg(grudev, "gid %d, gts %p, gms %p, ctxnum %d, asidmap 0x%lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) gru->gs_gid, gts, gms, gts->ts_ctxnum, gms->ms_asidmap[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) spin_unlock(&gru->gs_asid_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) spin_unlock(&gms->ms_asid_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) * Decrement the reference count on a GTS structure. Free the structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) * if the reference count goes to zero.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) void gts_drop(struct gru_thread_state *gts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) if (gts && atomic_dec_return(>s->ts_refcnt) == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) if (gts->ts_gms)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) gru_drop_mmu_notifier(gts->ts_gms);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) kfree(gts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) STAT(gts_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) * Locate the GTS structure for the current thread.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) static struct gru_thread_state *gru_find_current_gts_nolock(struct gru_vma_data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) *vdata, int tsid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) struct gru_thread_state *gts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) list_for_each_entry(gts, &vdata->vd_head, ts_next)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) if (gts->ts_tsid == tsid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) return gts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) * Allocate a thread state structure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) struct gru_thread_state *gru_alloc_gts(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) int cbr_au_count, int dsr_au_count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) unsigned char tlb_preload_count, int options, int tsid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) struct gru_thread_state *gts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) struct gru_mm_struct *gms;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) int bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) bytes = DSR_BYTES(dsr_au_count) + CBR_BYTES(cbr_au_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) bytes += sizeof(struct gru_thread_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) gts = kmalloc(bytes, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) if (!gts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) STAT(gts_alloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) memset(gts, 0, sizeof(struct gru_thread_state)); /* zero out header */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) atomic_set(>s->ts_refcnt, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) mutex_init(>s->ts_ctxlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) gts->ts_cbr_au_count = cbr_au_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) gts->ts_dsr_au_count = dsr_au_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) gts->ts_tlb_preload_count = tlb_preload_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) gts->ts_user_options = options;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) gts->ts_user_blade_id = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) gts->ts_user_chiplet_id = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) gts->ts_tsid = tsid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) gts->ts_ctxnum = NULLCTX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) gts->ts_tlb_int_select = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) gts->ts_cch_req_slice = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) gts->ts_sizeavail = GRU_SIZEAVAIL(PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) if (vma) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) gts->ts_mm = current->mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) gts->ts_vma = vma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) gms = gru_register_mmu_notifier();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) if (IS_ERR(gms))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) gts->ts_gms = gms;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) gru_dbg(grudev, "alloc gts %p\n", gts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) return gts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) gts_drop(gts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) return ERR_CAST(gms);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) * Allocate a vma private data structure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) struct gru_vma_data *gru_alloc_vma_data(struct vm_area_struct *vma, int tsid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) struct gru_vma_data *vdata = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) vdata = kmalloc(sizeof(*vdata), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) if (!vdata)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) STAT(vdata_alloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) INIT_LIST_HEAD(&vdata->vd_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) spin_lock_init(&vdata->vd_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) gru_dbg(grudev, "alloc vdata %p\n", vdata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) return vdata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) * Find the thread state structure for the current thread.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) struct gru_thread_state *gru_find_thread_state(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) int tsid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) struct gru_vma_data *vdata = vma->vm_private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) struct gru_thread_state *gts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) spin_lock(&vdata->vd_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) gts = gru_find_current_gts_nolock(vdata, tsid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) spin_unlock(&vdata->vd_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) gru_dbg(grudev, "vma %p, gts %p\n", vma, gts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) return gts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) * Allocate a new thread state for a GSEG. Note that races may allow
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) * another thread to race to create a gts.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) struct gru_thread_state *gru_alloc_thread_state(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) int tsid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) struct gru_vma_data *vdata = vma->vm_private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) struct gru_thread_state *gts, *ngts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) gts = gru_alloc_gts(vma, vdata->vd_cbr_au_count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) vdata->vd_dsr_au_count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) vdata->vd_tlb_preload_count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) vdata->vd_user_options, tsid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) if (IS_ERR(gts))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) return gts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) spin_lock(&vdata->vd_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) ngts = gru_find_current_gts_nolock(vdata, tsid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) if (ngts) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) gts_drop(gts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) gts = ngts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) STAT(gts_double_allocate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) list_add(>s->ts_next, &vdata->vd_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) spin_unlock(&vdata->vd_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) gru_dbg(grudev, "vma %p, gts %p\n", vma, gts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) return gts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) * Free the GRU context assigned to the thread state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) static void gru_free_gru_context(struct gru_thread_state *gts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) struct gru_state *gru;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) gru = gts->ts_gru;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) gru_dbg(grudev, "gts %p, gid %d\n", gts, gru->gs_gid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) spin_lock(&gru->gs_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) gru->gs_gts[gts->ts_ctxnum] = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) free_gru_resources(gru, gts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) BUG_ON(test_bit(gts->ts_ctxnum, &gru->gs_context_map) == 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) __clear_bit(gts->ts_ctxnum, &gru->gs_context_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) gts->ts_ctxnum = NULLCTX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) gts->ts_gru = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) gts->ts_blade = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) spin_unlock(&gru->gs_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) gts_drop(gts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) STAT(free_context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) * Prefetching cachelines help hardware performance.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) * (Strictly a performance enhancement. Not functionally required).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) static void prefetch_data(void *p, int num, int stride)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) while (num-- > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) prefetchw(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) p += stride;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) static inline long gru_copy_handle(void *d, void *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) memcpy(d, s, GRU_HANDLE_BYTES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) return GRU_HANDLE_BYTES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) static void gru_prefetch_context(void *gseg, void *cb, void *cbe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) unsigned long cbrmap, unsigned long length)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) int i, scr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) prefetch_data(gseg + GRU_DS_BASE, length / GRU_CACHE_LINE_BYTES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) GRU_CACHE_LINE_BYTES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) for_each_cbr_in_allocation_map(i, &cbrmap, scr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) prefetch_data(cb, 1, GRU_CACHE_LINE_BYTES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) prefetch_data(cbe + i * GRU_HANDLE_STRIDE, 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) GRU_CACHE_LINE_BYTES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) cb += GRU_HANDLE_STRIDE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) static void gru_load_context_data(void *save, void *grubase, int ctxnum,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) unsigned long cbrmap, unsigned long dsrmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) int data_valid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) void *gseg, *cb, *cbe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) unsigned long length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) int i, scr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) gseg = grubase + ctxnum * GRU_GSEG_STRIDE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) cb = gseg + GRU_CB_BASE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) cbe = grubase + GRU_CBE_BASE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) length = hweight64(dsrmap) * GRU_DSR_AU_BYTES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) gru_prefetch_context(gseg, cb, cbe, cbrmap, length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) for_each_cbr_in_allocation_map(i, &cbrmap, scr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) if (data_valid) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) save += gru_copy_handle(cb, save);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) save += gru_copy_handle(cbe + i * GRU_HANDLE_STRIDE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) save);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) memset(cb, 0, GRU_CACHE_LINE_BYTES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) memset(cbe + i * GRU_HANDLE_STRIDE, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) GRU_CACHE_LINE_BYTES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) /* Flush CBE to hide race in context restart */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) gru_flush_cache(cbe + i * GRU_HANDLE_STRIDE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) cb += GRU_HANDLE_STRIDE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) if (data_valid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) memcpy(gseg + GRU_DS_BASE, save, length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) memset(gseg + GRU_DS_BASE, 0, length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) static void gru_unload_context_data(void *save, void *grubase, int ctxnum,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) unsigned long cbrmap, unsigned long dsrmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) void *gseg, *cb, *cbe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) unsigned long length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) int i, scr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) gseg = grubase + ctxnum * GRU_GSEG_STRIDE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) cb = gseg + GRU_CB_BASE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) cbe = grubase + GRU_CBE_BASE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) length = hweight64(dsrmap) * GRU_DSR_AU_BYTES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) /* CBEs may not be coherent. Flush them from cache */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) for_each_cbr_in_allocation_map(i, &cbrmap, scr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) gru_flush_cache(cbe + i * GRU_HANDLE_STRIDE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) mb(); /* Let the CL flush complete */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) gru_prefetch_context(gseg, cb, cbe, cbrmap, length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) for_each_cbr_in_allocation_map(i, &cbrmap, scr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) save += gru_copy_handle(save, cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) save += gru_copy_handle(save, cbe + i * GRU_HANDLE_STRIDE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) cb += GRU_HANDLE_STRIDE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) memcpy(save, gseg + GRU_DS_BASE, length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) void gru_unload_context(struct gru_thread_state *gts, int savestate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) struct gru_state *gru = gts->ts_gru;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) struct gru_context_configuration_handle *cch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) int ctxnum = gts->ts_ctxnum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) if (!is_kernel_context(gts))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) zap_vma_ptes(gts->ts_vma, UGRUADDR(gts), GRU_GSEG_PAGESIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) cch = get_cch(gru->gs_gru_base_vaddr, ctxnum);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) gru_dbg(grudev, "gts %p, cbrmap 0x%lx, dsrmap 0x%lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) gts, gts->ts_cbr_map, gts->ts_dsr_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) lock_cch_handle(cch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) if (cch_interrupt_sync(cch))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) if (!is_kernel_context(gts))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) gru_unload_mm_tracker(gru, gts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) if (savestate) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) gru_unload_context_data(gts->ts_gdata, gru->gs_gru_base_vaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) ctxnum, gts->ts_cbr_map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) gts->ts_dsr_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) gts->ts_data_valid = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) if (cch_deallocate(cch))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) unlock_cch_handle(cch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) gru_free_gru_context(gts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) * Load a GRU context by copying it from the thread data structure in memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) * to the GRU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) void gru_load_context(struct gru_thread_state *gts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) struct gru_state *gru = gts->ts_gru;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) struct gru_context_configuration_handle *cch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) int i, err, asid, ctxnum = gts->ts_ctxnum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) cch = get_cch(gru->gs_gru_base_vaddr, ctxnum);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) lock_cch_handle(cch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) cch->tfm_fault_bit_enable =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) (gts->ts_user_options == GRU_OPT_MISS_FMM_POLL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) || gts->ts_user_options == GRU_OPT_MISS_FMM_INTR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) cch->tlb_int_enable = (gts->ts_user_options == GRU_OPT_MISS_FMM_INTR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) if (cch->tlb_int_enable) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) gts->ts_tlb_int_select = gru_cpu_fault_map_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) cch->tlb_int_select = gts->ts_tlb_int_select;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) if (gts->ts_cch_req_slice >= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) cch->req_slice_set_enable = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) cch->req_slice = gts->ts_cch_req_slice;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) cch->req_slice_set_enable =0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) cch->tfm_done_bit_enable = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) cch->dsr_allocation_map = gts->ts_dsr_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) cch->cbr_allocation_map = gts->ts_cbr_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) if (is_kernel_context(gts)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) cch->unmap_enable = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) cch->tfm_done_bit_enable = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) cch->cb_int_enable = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) cch->tlb_int_select = 0; /* For now, ints go to cpu 0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) cch->unmap_enable = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) cch->tfm_done_bit_enable = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) cch->cb_int_enable = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) asid = gru_load_mm_tracker(gru, gts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) for (i = 0; i < 8; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) cch->asid[i] = asid + i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) cch->sizeavail[i] = gts->ts_sizeavail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) err = cch_allocate(cch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) gru_dbg(grudev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) "err %d: cch %p, gts %p, cbr 0x%lx, dsr 0x%lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) err, cch, gts, gts->ts_cbr_map, gts->ts_dsr_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) gru_load_context_data(gts->ts_gdata, gru->gs_gru_base_vaddr, ctxnum,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) gts->ts_cbr_map, gts->ts_dsr_map, gts->ts_data_valid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) if (cch_start(cch))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) unlock_cch_handle(cch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) gru_dbg(grudev, "gid %d, gts %p, cbrmap 0x%lx, dsrmap 0x%lx, tie %d, tis %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) gts->ts_gru->gs_gid, gts, gts->ts_cbr_map, gts->ts_dsr_map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) (gts->ts_user_options == GRU_OPT_MISS_FMM_INTR), gts->ts_tlb_int_select);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) * Update fields in an active CCH:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) * - retarget interrupts on local blade
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) * - update sizeavail mask
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) int gru_update_cch(struct gru_thread_state *gts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) struct gru_context_configuration_handle *cch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) struct gru_state *gru = gts->ts_gru;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) int i, ctxnum = gts->ts_ctxnum, ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) cch = get_cch(gru->gs_gru_base_vaddr, ctxnum);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) lock_cch_handle(cch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) if (cch->state == CCHSTATE_ACTIVE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) if (gru->gs_gts[gts->ts_ctxnum] != gts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) goto exit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) if (cch_interrupt(cch))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) for (i = 0; i < 8; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) cch->sizeavail[i] = gts->ts_sizeavail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) gts->ts_tlb_int_select = gru_cpu_fault_map_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) cch->tlb_int_select = gru_cpu_fault_map_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) cch->tfm_fault_bit_enable =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) (gts->ts_user_options == GRU_OPT_MISS_FMM_POLL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) || gts->ts_user_options == GRU_OPT_MISS_FMM_INTR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) if (cch_start(cch))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) ret = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) exit:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) unlock_cch_handle(cch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) * Update CCH tlb interrupt select. Required when all the following is true:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) * - task's GRU context is loaded into a GRU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) * - task is using interrupt notification for TLB faults
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) * - task has migrated to a different cpu on the same blade where
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) * it was previously running.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) static int gru_retarget_intr(struct gru_thread_state *gts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) if (gts->ts_tlb_int_select < 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) || gts->ts_tlb_int_select == gru_cpu_fault_map_id())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) gru_dbg(grudev, "retarget from %d to %d\n", gts->ts_tlb_int_select,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) gru_cpu_fault_map_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) return gru_update_cch(gts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) * Check if a GRU context is allowed to use a specific chiplet. By default
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) * a context is assigned to any blade-local chiplet. However, users can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) * override this.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) * Returns 1 if assignment allowed, 0 otherwise
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) static int gru_check_chiplet_assignment(struct gru_state *gru,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) struct gru_thread_state *gts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) int blade_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) int chiplet_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) blade_id = gts->ts_user_blade_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) if (blade_id < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) blade_id = uv_numa_blade_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) chiplet_id = gts->ts_user_chiplet_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) return gru->gs_blade_id == blade_id &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) (chiplet_id < 0 || chiplet_id == gru->gs_chiplet_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) * Unload the gru context if it is not assigned to the correct blade or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) * chiplet. Misassignment can occur if the process migrates to a different
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) * blade or if the user changes the selected blade/chiplet.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) void gru_check_context_placement(struct gru_thread_state *gts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) struct gru_state *gru;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) * If the current task is the context owner, verify that the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) * context is correctly placed. This test is skipped for non-owner
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) * references. Pthread apps use non-owner references to the CBRs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) gru = gts->ts_gru;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) if (!gru || gts->ts_tgid_owner != current->tgid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) if (!gru_check_chiplet_assignment(gru, gts)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) STAT(check_context_unload);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) gru_unload_context(gts, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) } else if (gru_retarget_intr(gts)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) STAT(check_context_retarget_intr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) * Insufficient GRU resources available on the local blade. Steal a context from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) * a process. This is a hack until a _real_ resource scheduler is written....
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) #define next_ctxnum(n) ((n) < GRU_NUM_CCH - 2 ? (n) + 1 : 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) #define next_gru(b, g) (((g) < &(b)->bs_grus[GRU_CHIPLETS_PER_BLADE - 1]) ? \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) ((g)+1) : &(b)->bs_grus[0])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) static int is_gts_stealable(struct gru_thread_state *gts,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) struct gru_blade_state *bs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) if (is_kernel_context(gts))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) return down_write_trylock(&bs->bs_kgts_sema);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) return mutex_trylock(>s->ts_ctxlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) static void gts_stolen(struct gru_thread_state *gts,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) struct gru_blade_state *bs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) if (is_kernel_context(gts)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) up_write(&bs->bs_kgts_sema);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) STAT(steal_kernel_context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) mutex_unlock(>s->ts_ctxlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) STAT(steal_user_context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) void gru_steal_context(struct gru_thread_state *gts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) struct gru_blade_state *blade;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) struct gru_state *gru, *gru0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) struct gru_thread_state *ngts = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) int ctxnum, ctxnum0, flag = 0, cbr, dsr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) int blade_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) blade_id = gts->ts_user_blade_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) if (blade_id < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) blade_id = uv_numa_blade_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) cbr = gts->ts_cbr_au_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) dsr = gts->ts_dsr_au_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) blade = gru_base[blade_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) spin_lock(&blade->bs_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) ctxnum = next_ctxnum(blade->bs_lru_ctxnum);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) gru = blade->bs_lru_gru;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) if (ctxnum == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) gru = next_gru(blade, gru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) blade->bs_lru_gru = gru;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) blade->bs_lru_ctxnum = ctxnum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) ctxnum0 = ctxnum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) gru0 = gru;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) if (gru_check_chiplet_assignment(gru, gts)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) if (check_gru_resources(gru, cbr, dsr, GRU_NUM_CCH))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) spin_lock(&gru->gs_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) for (; ctxnum < GRU_NUM_CCH; ctxnum++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) if (flag && gru == gru0 && ctxnum == ctxnum0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) ngts = gru->gs_gts[ctxnum];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) * We are grabbing locks out of order, so trylock is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) * needed. GTSs are usually not locked, so the odds of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) * success are high. If trylock fails, try to steal a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) * different GSEG.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) if (ngts && is_gts_stealable(ngts, blade))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) ngts = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) spin_unlock(&gru->gs_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) if (ngts || (flag && gru == gru0 && ctxnum == ctxnum0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) if (flag && gru == gru0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) flag = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) ctxnum = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) gru = next_gru(blade, gru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) spin_unlock(&blade->bs_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) if (ngts) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) gts->ustats.context_stolen++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) ngts->ts_steal_jiffies = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) gru_unload_context(ngts, is_kernel_context(ngts) ? 0 : 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) gts_stolen(ngts, blade);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) STAT(steal_context_failed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) gru_dbg(grudev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) "stole gid %d, ctxnum %d from gts %p. Need cb %d, ds %d;"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) " avail cb %ld, ds %ld\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) gru->gs_gid, ctxnum, ngts, cbr, dsr, hweight64(gru->gs_cbr_map),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) hweight64(gru->gs_dsr_map));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) * Assign a gru context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) static int gru_assign_context_number(struct gru_state *gru)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) int ctxnum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) ctxnum = find_first_zero_bit(&gru->gs_context_map, GRU_NUM_CCH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) __set_bit(ctxnum, &gru->gs_context_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) return ctxnum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) * Scan the GRUs on the local blade & assign a GRU context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) struct gru_state *gru_assign_gru_context(struct gru_thread_state *gts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) struct gru_state *gru, *grux;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) int i, max_active_contexts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) int blade_id = gts->ts_user_blade_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) if (blade_id < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) blade_id = uv_numa_blade_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) gru = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) max_active_contexts = GRU_NUM_CCH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) for_each_gru_on_blade(grux, blade_id, i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) if (!gru_check_chiplet_assignment(grux, gts))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) if (check_gru_resources(grux, gts->ts_cbr_au_count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) gts->ts_dsr_au_count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) max_active_contexts)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) gru = grux;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) max_active_contexts = grux->gs_active_contexts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) if (max_active_contexts == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) if (gru) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) spin_lock(&gru->gs_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) if (!check_gru_resources(gru, gts->ts_cbr_au_count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) gts->ts_dsr_au_count, GRU_NUM_CCH)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) spin_unlock(&gru->gs_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) reserve_gru_resources(gru, gts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) gts->ts_gru = gru;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) gts->ts_blade = gru->gs_blade_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) gts->ts_ctxnum = gru_assign_context_number(gru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) atomic_inc(>s->ts_refcnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) gru->gs_gts[gts->ts_ctxnum] = gts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) spin_unlock(&gru->gs_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) STAT(assign_context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) gru_dbg(grudev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) "gseg %p, gts %p, gid %d, ctx %d, cbr %d, dsr %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) gseg_virtual_address(gts->ts_gru, gts->ts_ctxnum), gts,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) gts->ts_gru->gs_gid, gts->ts_ctxnum,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) gts->ts_cbr_au_count, gts->ts_dsr_au_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) gru_dbg(grudev, "failed to allocate a GTS %s\n", "");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) STAT(assign_context_failed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) return gru;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) * gru_nopage
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) * Map the user's GRU segment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) * Note: gru segments alway mmaped on GRU_GSEG_PAGESIZE boundaries.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) vm_fault_t gru_fault(struct vm_fault *vmf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) struct vm_area_struct *vma = vmf->vma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) struct gru_thread_state *gts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) unsigned long paddr, vaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) unsigned long expires;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) vaddr = vmf->address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) gru_dbg(grudev, "vma %p, vaddr 0x%lx (0x%lx)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) vma, vaddr, GSEG_BASE(vaddr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) STAT(nopfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) /* The following check ensures vaddr is a valid address in the VMA */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) gts = gru_find_thread_state(vma, TSID(vaddr, vma));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) if (!gts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) return VM_FAULT_SIGBUS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) mutex_lock(>s->ts_ctxlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) gru_check_context_placement(gts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) if (!gts->ts_gru) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) STAT(load_user_context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) if (!gru_assign_gru_context(gts)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) mutex_unlock(>s->ts_ctxlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) set_current_state(TASK_INTERRUPTIBLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) schedule_timeout(GRU_ASSIGN_DELAY); /* true hack ZZZ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) expires = gts->ts_steal_jiffies + GRU_STEAL_DELAY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) if (time_before(expires, jiffies))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) gru_steal_context(gts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) gru_load_context(gts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) paddr = gseg_physical_address(gts->ts_gru, gts->ts_ctxnum);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) remap_pfn_range(vma, vaddr & ~(GRU_GSEG_PAGESIZE - 1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) paddr >> PAGE_SHIFT, GRU_GSEG_PAGESIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) vma->vm_page_prot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) mutex_unlock(>s->ts_ctxlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) return VM_FAULT_NOPAGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963)
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags