^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * In-kernel vector facility support functions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright IBM Corp. 2015
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <asm/fpu/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <asm/fpu/api.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) asm(".include \"asm/vx-insn.h\"\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) void __kernel_fpu_begin(struct kernel_fpu *state, u32 flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * Limit the save to the FPU/vector registers already
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * in use by the previous context
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) flags &= state->mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) if (flags & KERNEL_FPC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) /* Save floating point control */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) asm volatile("stfpc %0" : "=Q" (state->fpc));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) if (!MACHINE_HAS_VX) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) if (flags & KERNEL_VXR_V0V7) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) /* Save floating-point registers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) asm volatile("std 0,%0" : "=Q" (state->fprs[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) asm volatile("std 1,%0" : "=Q" (state->fprs[1]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) asm volatile("std 2,%0" : "=Q" (state->fprs[2]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) asm volatile("std 3,%0" : "=Q" (state->fprs[3]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) asm volatile("std 4,%0" : "=Q" (state->fprs[4]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) asm volatile("std 5,%0" : "=Q" (state->fprs[5]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) asm volatile("std 6,%0" : "=Q" (state->fprs[6]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) asm volatile("std 7,%0" : "=Q" (state->fprs[7]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) asm volatile("std 8,%0" : "=Q" (state->fprs[8]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) asm volatile("std 9,%0" : "=Q" (state->fprs[9]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) asm volatile("std 10,%0" : "=Q" (state->fprs[10]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) asm volatile("std 11,%0" : "=Q" (state->fprs[11]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) asm volatile("std 12,%0" : "=Q" (state->fprs[12]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) asm volatile("std 13,%0" : "=Q" (state->fprs[13]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) asm volatile("std 14,%0" : "=Q" (state->fprs[14]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) asm volatile("std 15,%0" : "=Q" (state->fprs[15]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) /* Test and save vector registers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) asm volatile (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) * Test if any vector register must be saved and, if so,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) * test if all register can be saved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) " la 1,%[vxrs]\n" /* load save area */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) " tmll %[m],30\n" /* KERNEL_VXR */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) " jz 7f\n" /* no work -> done */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) " jo 5f\n" /* -> save V0..V31 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) * Test for special case KERNEL_FPU_MID only. In this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) * case a vstm V8..V23 is the best instruction
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) " chi %[m],12\n" /* KERNEL_VXR_MID */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) " jne 0f\n" /* -> save V8..V23 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) " VSTM 8,23,128,1\n" /* vstm %v8,%v23,128(%r1) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) " j 7f\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) /* Test and save the first half of 16 vector registers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) "0: tmll %[m],6\n" /* KERNEL_VXR_LOW */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) " jz 3f\n" /* -> KERNEL_VXR_HIGH */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) " jo 2f\n" /* 11 -> save V0..V15 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) " brc 2,1f\n" /* 10 -> save V8..V15 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) " VSTM 0,7,0,1\n" /* vstm %v0,%v7,0(%r1) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) " j 3f\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) "1: VSTM 8,15,128,1\n" /* vstm %v8,%v15,128(%r1) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) " j 3f\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) "2: VSTM 0,15,0,1\n" /* vstm %v0,%v15,0(%r1) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) /* Test and save the second half of 16 vector registers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) "3: tmll %[m],24\n" /* KERNEL_VXR_HIGH */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) " jz 7f\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) " jo 6f\n" /* 11 -> save V16..V31 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) " brc 2,4f\n" /* 10 -> save V24..V31 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) " VSTM 16,23,256,1\n" /* vstm %v16,%v23,256(%r1) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) " j 7f\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) "4: VSTM 24,31,384,1\n" /* vstm %v24,%v31,384(%r1) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) " j 7f\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) "5: VSTM 0,15,0,1\n" /* vstm %v0,%v15,0(%r1) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) "6: VSTM 16,31,256,1\n" /* vstm %v16,%v31,256(%r1) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) "7:"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) : [vxrs] "=Q" (*(struct vx_array *) &state->vxrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) : [m] "d" (flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) : "1", "cc");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) EXPORT_SYMBOL(__kernel_fpu_begin);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) void __kernel_fpu_end(struct kernel_fpu *state, u32 flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) * Limit the restore to the FPU/vector registers of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) * previous context that have been overwritte by the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) * current context
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) flags &= state->mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) if (flags & KERNEL_FPC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) /* Restore floating-point controls */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) asm volatile("lfpc %0" : : "Q" (state->fpc));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) if (!MACHINE_HAS_VX) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) if (flags & KERNEL_VXR_V0V7) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) /* Restore floating-point registers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) asm volatile("ld 0,%0" : : "Q" (state->fprs[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) asm volatile("ld 1,%0" : : "Q" (state->fprs[1]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) asm volatile("ld 2,%0" : : "Q" (state->fprs[2]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) asm volatile("ld 3,%0" : : "Q" (state->fprs[3]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) asm volatile("ld 4,%0" : : "Q" (state->fprs[4]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) asm volatile("ld 5,%0" : : "Q" (state->fprs[5]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) asm volatile("ld 6,%0" : : "Q" (state->fprs[6]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) asm volatile("ld 7,%0" : : "Q" (state->fprs[7]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) asm volatile("ld 8,%0" : : "Q" (state->fprs[8]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) asm volatile("ld 9,%0" : : "Q" (state->fprs[9]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) asm volatile("ld 10,%0" : : "Q" (state->fprs[10]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) asm volatile("ld 11,%0" : : "Q" (state->fprs[11]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) asm volatile("ld 12,%0" : : "Q" (state->fprs[12]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) asm volatile("ld 13,%0" : : "Q" (state->fprs[13]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) asm volatile("ld 14,%0" : : "Q" (state->fprs[14]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) asm volatile("ld 15,%0" : : "Q" (state->fprs[15]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) /* Test and restore (load) vector registers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) asm volatile (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) * Test if any vector register must be loaded and, if so,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) * test if all registers can be loaded at once.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) " la 1,%[vxrs]\n" /* load restore area */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) " tmll %[m],30\n" /* KERNEL_VXR */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) " jz 7f\n" /* no work -> done */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) " jo 5f\n" /* -> restore V0..V31 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) * Test for special case KERNEL_FPU_MID only. In this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) * case a vlm V8..V23 is the best instruction
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) " chi %[m],12\n" /* KERNEL_VXR_MID */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) " jne 0f\n" /* -> restore V8..V23 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) " VLM 8,23,128,1\n" /* vlm %v8,%v23,128(%r1) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) " j 7f\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) /* Test and restore the first half of 16 vector registers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) "0: tmll %[m],6\n" /* KERNEL_VXR_LOW */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) " jz 3f\n" /* -> KERNEL_VXR_HIGH */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) " jo 2f\n" /* 11 -> restore V0..V15 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) " brc 2,1f\n" /* 10 -> restore V8..V15 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) " VLM 0,7,0,1\n" /* vlm %v0,%v7,0(%r1) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) " j 3f\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) "1: VLM 8,15,128,1\n" /* vlm %v8,%v15,128(%r1) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) " j 3f\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) "2: VLM 0,15,0,1\n" /* vlm %v0,%v15,0(%r1) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) /* Test and restore the second half of 16 vector registers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) "3: tmll %[m],24\n" /* KERNEL_VXR_HIGH */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) " jz 7f\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) " jo 6f\n" /* 11 -> restore V16..V31 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) " brc 2,4f\n" /* 10 -> restore V24..V31 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) " VLM 16,23,256,1\n" /* vlm %v16,%v23,256(%r1) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) " j 7f\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) "4: VLM 24,31,384,1\n" /* vlm %v24,%v31,384(%r1) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) " j 7f\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) "5: VLM 0,15,0,1\n" /* vlm %v0,%v15,0(%r1) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) "6: VLM 16,31,256,1\n" /* vlm %v16,%v31,256(%r1) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) "7:"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) : [vxrs] "=Q" (*(struct vx_array *) &state->vxrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) : [m] "d" (flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) : "1", "cc");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) EXPORT_SYMBOL(__kernel_fpu_end);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags