^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-or-later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * GRU KERNEL MCS INSTRUCTIONS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include "gru.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include "grulib.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include "grutables.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) /* 10 sec */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #ifdef CONFIG_IA64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <asm/processor.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #define GRU_OPERATION_TIMEOUT (((cycles_t) local_cpu_data->itc_freq)*10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #define CLKS2NSEC(c) ((c) *1000000000 / local_cpu_data->itc_freq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/sync_core.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <asm/tsc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #define GRU_OPERATION_TIMEOUT ((cycles_t) tsc_khz*10*1000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #define CLKS2NSEC(c) ((c) * 1000000 / tsc_khz)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) /* Extract the status field from a kernel handle */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #define GET_MSEG_HANDLE_STATUS(h) (((*(unsigned long *)(h)) >> 16) & 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) struct mcs_op_statistic mcs_op_statistics[mcsop_last];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) static void update_mcs_stats(enum mcs_op op, unsigned long clks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) unsigned long nsec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) nsec = CLKS2NSEC(clks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) atomic_long_inc(&mcs_op_statistics[op].count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) atomic_long_add(nsec, &mcs_op_statistics[op].total);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) if (mcs_op_statistics[op].max < nsec)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) mcs_op_statistics[op].max = nsec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) static void start_instruction(void *h)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) unsigned long *w0 = h;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) wmb(); /* setting CMD/STATUS bits must be last */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) *w0 = *w0 | 0x20001;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) gru_flush_cache(h);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) static void report_instruction_timeout(void *h)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) unsigned long goff = GSEGPOFF((unsigned long)h);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) char *id = "???";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) if (TYPE_IS(CCH, goff))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) id = "CCH";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) else if (TYPE_IS(TGH, goff))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) id = "TGH";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) else if (TYPE_IS(TFH, goff))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) id = "TFH";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) panic(KERN_ALERT "GRU %p (%s) is malfunctioning\n", h, id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) static int wait_instruction_complete(void *h, enum mcs_op opc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) int status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) unsigned long start_time = get_cycles();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) cpu_relax();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) status = GET_MSEG_HANDLE_STATUS(h);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) if (status != CCHSTATUS_ACTIVE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) if (GRU_OPERATION_TIMEOUT < (get_cycles() - start_time)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) report_instruction_timeout(h);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) start_time = get_cycles();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) if (gru_options & OPT_STATS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) update_mcs_stats(opc, get_cycles() - start_time);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) return status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) int cch_allocate(struct gru_context_configuration_handle *cch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) cch->opc = CCHOP_ALLOCATE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) start_instruction(cch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) ret = wait_instruction_complete(cch, cchop_allocate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) * Stop speculation into the GSEG being mapped by the previous ALLOCATE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) * The GSEG memory does not exist until the ALLOCATE completes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) sync_core();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) int cch_start(struct gru_context_configuration_handle *cch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) cch->opc = CCHOP_START;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) start_instruction(cch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) return wait_instruction_complete(cch, cchop_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) int cch_interrupt(struct gru_context_configuration_handle *cch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) cch->opc = CCHOP_INTERRUPT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) start_instruction(cch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) return wait_instruction_complete(cch, cchop_interrupt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) int cch_deallocate(struct gru_context_configuration_handle *cch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) cch->opc = CCHOP_DEALLOCATE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) start_instruction(cch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) ret = wait_instruction_complete(cch, cchop_deallocate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) * Stop speculation into the GSEG being unmapped by the previous
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) * DEALLOCATE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) sync_core();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) int cch_interrupt_sync(struct gru_context_configuration_handle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) *cch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) cch->opc = CCHOP_INTERRUPT_SYNC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) start_instruction(cch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) return wait_instruction_complete(cch, cchop_interrupt_sync);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) int tgh_invalidate(struct gru_tlb_global_handle *tgh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) unsigned long vaddr, unsigned long vaddrmask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) int asid, int pagesize, int global, int n,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) unsigned short ctxbitmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) tgh->vaddr = vaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) tgh->asid = asid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) tgh->pagesize = pagesize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) tgh->n = n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) tgh->global = global;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) tgh->vaddrmask = vaddrmask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) tgh->ctxbitmap = ctxbitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) tgh->opc = TGHOP_TLBINV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) start_instruction(tgh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) return wait_instruction_complete(tgh, tghop_invalidate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) int tfh_write_only(struct gru_tlb_fault_handle *tfh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) unsigned long paddr, int gaa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) unsigned long vaddr, int asid, int dirty,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) int pagesize)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) tfh->fillasid = asid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) tfh->fillvaddr = vaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) tfh->pfn = paddr >> GRU_PADDR_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) tfh->gaa = gaa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) tfh->dirty = dirty;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) tfh->pagesize = pagesize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) tfh->opc = TFHOP_WRITE_ONLY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) start_instruction(tfh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) return wait_instruction_complete(tfh, tfhop_write_only);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) void tfh_write_restart(struct gru_tlb_fault_handle *tfh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) unsigned long paddr, int gaa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) unsigned long vaddr, int asid, int dirty,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) int pagesize)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) tfh->fillasid = asid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) tfh->fillvaddr = vaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) tfh->pfn = paddr >> GRU_PADDR_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) tfh->gaa = gaa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) tfh->dirty = dirty;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) tfh->pagesize = pagesize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) tfh->opc = TFHOP_WRITE_RESTART;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) start_instruction(tfh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) void tfh_user_polling_mode(struct gru_tlb_fault_handle *tfh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) tfh->opc = TFHOP_USER_POLLING_MODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) start_instruction(tfh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) void tfh_exception(struct gru_tlb_fault_handle *tfh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) tfh->opc = TFHOP_EXCEPTION;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) start_instruction(tfh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags