^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-or-later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /* -*- linux-c -*- ------------------------------------------------------- *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Copyright 2002 H. Peter Anvin - All Rights Reserved
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * ----------------------------------------------------------------------- */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * raid6/mmx.c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * MMX implementation of RAID-6 syndrome functions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #ifdef CONFIG_X86_32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/raid/pq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include "x86.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) /* Shared with raid6/sse1.c */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) const struct raid6_mmx_constants {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) u64 x1d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) } raid6_mmx_constants = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) 0x1d1d1d1d1d1d1d1dULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) static int raid6_have_mmx(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) /* Not really "boot_cpu" but "all_cpus" */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) return boot_cpu_has(X86_FEATURE_MMX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * Plain MMX implementation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) static void raid6_mmx1_gen_syndrome(int disks, size_t bytes, void **ptrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) u8 **dptr = (u8 **)ptrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) u8 *p, *q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) int d, z, z0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) z0 = disks - 3; /* Highest data disk */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) p = dptr[z0+1]; /* XOR parity */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) q = dptr[z0+2]; /* RS syndrome */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) kernel_fpu_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) asm volatile("pxor %mm5,%mm5"); /* Zero temp */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) for ( d = 0 ; d < bytes ; d += 8 ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) asm volatile("movq %0,%%mm2" : : "m" (dptr[z0][d])); /* P[0] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) asm volatile("movq %mm2,%mm4"); /* Q[0] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) for ( z = z0-1 ; z >= 0 ; z-- ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) asm volatile("movq %0,%%mm6" : : "m" (dptr[z][d]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) asm volatile("pcmpgtb %mm4,%mm5");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) asm volatile("paddb %mm4,%mm4");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) asm volatile("pand %mm0,%mm5");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) asm volatile("pxor %mm5,%mm4");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) asm volatile("pxor %mm5,%mm5");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) asm volatile("pxor %mm6,%mm2");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) asm volatile("pxor %mm6,%mm4");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) asm volatile("movq %%mm2,%0" : "=m" (p[d]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) asm volatile("pxor %mm2,%mm2");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) asm volatile("movq %%mm4,%0" : "=m" (q[d]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) asm volatile("pxor %mm4,%mm4");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) kernel_fpu_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) const struct raid6_calls raid6_mmxx1 = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) raid6_mmx1_gen_syndrome,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) NULL, /* XOR not yet implemented */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) raid6_have_mmx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) "mmxx1",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) * Unrolled-by-2 MMX implementation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) static void raid6_mmx2_gen_syndrome(int disks, size_t bytes, void **ptrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) u8 **dptr = (u8 **)ptrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) u8 *p, *q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) int d, z, z0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) z0 = disks - 3; /* Highest data disk */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) p = dptr[z0+1]; /* XOR parity */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) q = dptr[z0+2]; /* RS syndrome */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) kernel_fpu_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) asm volatile("pxor %mm5,%mm5"); /* Zero temp */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) asm volatile("pxor %mm7,%mm7"); /* Zero temp */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) for ( d = 0 ; d < bytes ; d += 16 ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) asm volatile("movq %0,%%mm2" : : "m" (dptr[z0][d])); /* P[0] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) asm volatile("movq %0,%%mm3" : : "m" (dptr[z0][d+8]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) asm volatile("movq %mm2,%mm4"); /* Q[0] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) asm volatile("movq %mm3,%mm6"); /* Q[1] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) for ( z = z0-1 ; z >= 0 ; z-- ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) asm volatile("pcmpgtb %mm4,%mm5");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) asm volatile("pcmpgtb %mm6,%mm7");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) asm volatile("paddb %mm4,%mm4");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) asm volatile("paddb %mm6,%mm6");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) asm volatile("pand %mm0,%mm5");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) asm volatile("pand %mm0,%mm7");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) asm volatile("pxor %mm5,%mm4");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) asm volatile("pxor %mm7,%mm6");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) asm volatile("movq %0,%%mm5" : : "m" (dptr[z][d]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) asm volatile("movq %0,%%mm7" : : "m" (dptr[z][d+8]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) asm volatile("pxor %mm5,%mm2");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) asm volatile("pxor %mm7,%mm3");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) asm volatile("pxor %mm5,%mm4");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) asm volatile("pxor %mm7,%mm6");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) asm volatile("pxor %mm5,%mm5");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) asm volatile("pxor %mm7,%mm7");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) asm volatile("movq %%mm2,%0" : "=m" (p[d]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) asm volatile("movq %%mm3,%0" : "=m" (p[d+8]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) asm volatile("movq %%mm4,%0" : "=m" (q[d]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) asm volatile("movq %%mm6,%0" : "=m" (q[d+8]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) kernel_fpu_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) const struct raid6_calls raid6_mmxx2 = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) raid6_mmx2_gen_syndrome,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) NULL, /* XOR not yet implemented */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) raid6_have_mmx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) "mmxx2",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) #endif
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags