^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (C) 2012 Intel Corporation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/raid/pq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include "x86.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) static int raid6_has_ssse3(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) return boot_cpu_has(X86_FEATURE_XMM) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) boot_cpu_has(X86_FEATURE_XMM2) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) boot_cpu_has(X86_FEATURE_SSSE3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) static void raid6_2data_recov_ssse3(int disks, size_t bytes, int faila,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) int failb, void **ptrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) u8 *p, *q, *dp, *dq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) const u8 *pbmul; /* P multiplier table for B data */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) const u8 *qmul; /* Q multiplier table (for both) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) static const u8 __aligned(16) x0f[16] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) p = (u8 *)ptrs[disks-2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) q = (u8 *)ptrs[disks-1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) /* Compute syndrome with zero for the missing data pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) Use the dead data pages as temporary storage for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) delta p and delta q */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) dp = (u8 *)ptrs[faila];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) ptrs[faila] = (void *)raid6_empty_zero_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) ptrs[disks-2] = dp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) dq = (u8 *)ptrs[failb];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) ptrs[failb] = (void *)raid6_empty_zero_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) ptrs[disks-1] = dq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) raid6_call.gen_syndrome(disks, bytes, ptrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) /* Restore pointer table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) ptrs[faila] = dp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) ptrs[failb] = dq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) ptrs[disks-2] = p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) ptrs[disks-1] = q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) /* Now, pick the proper data tables */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) raid6_gfexp[failb]]];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) kernel_fpu_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) asm volatile("movdqa %0,%%xmm7" : : "m" (x0f[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) asm volatile("movdqa %0,%%xmm6" : : "m" (qmul[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) asm volatile("movdqa %0,%%xmm14" : : "m" (pbmul[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) asm volatile("movdqa %0,%%xmm15" : : "m" (pbmul[16]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) /* Now do it... */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) while (bytes) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) /* xmm6, xmm14, xmm15 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) asm volatile("movdqa %0,%%xmm1" : : "m" (q[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) asm volatile("movdqa %0,%%xmm9" : : "m" (q[16]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) asm volatile("movdqa %0,%%xmm0" : : "m" (p[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) asm volatile("movdqa %0,%%xmm8" : : "m" (p[16]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) asm volatile("pxor %0,%%xmm1" : : "m" (dq[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) asm volatile("pxor %0,%%xmm9" : : "m" (dq[16]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) asm volatile("pxor %0,%%xmm0" : : "m" (dp[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) asm volatile("pxor %0,%%xmm8" : : "m" (dp[16]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) /* xmm0/8 = px */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) asm volatile("movdqa %xmm6,%xmm4");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) asm volatile("movdqa %0,%%xmm5" : : "m" (qmul[16]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) asm volatile("movdqa %xmm6,%xmm12");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) asm volatile("movdqa %xmm5,%xmm13");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) asm volatile("movdqa %xmm1,%xmm3");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) asm volatile("movdqa %xmm9,%xmm11");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) asm volatile("movdqa %xmm0,%xmm2"); /* xmm2/10 = px */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) asm volatile("movdqa %xmm8,%xmm10");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) asm volatile("psraw $4,%xmm1");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) asm volatile("psraw $4,%xmm9");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) asm volatile("pand %xmm7,%xmm3");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) asm volatile("pand %xmm7,%xmm11");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) asm volatile("pand %xmm7,%xmm1");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) asm volatile("pand %xmm7,%xmm9");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) asm volatile("pshufb %xmm3,%xmm4");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) asm volatile("pshufb %xmm11,%xmm12");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) asm volatile("pshufb %xmm1,%xmm5");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) asm volatile("pshufb %xmm9,%xmm13");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) asm volatile("pxor %xmm4,%xmm5");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) asm volatile("pxor %xmm12,%xmm13");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) /* xmm5/13 = qx */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) asm volatile("movdqa %xmm14,%xmm4");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) asm volatile("movdqa %xmm15,%xmm1");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) asm volatile("movdqa %xmm14,%xmm12");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) asm volatile("movdqa %xmm15,%xmm9");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) asm volatile("movdqa %xmm2,%xmm3");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) asm volatile("movdqa %xmm10,%xmm11");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) asm volatile("psraw $4,%xmm2");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) asm volatile("psraw $4,%xmm10");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) asm volatile("pand %xmm7,%xmm3");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) asm volatile("pand %xmm7,%xmm11");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) asm volatile("pand %xmm7,%xmm2");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) asm volatile("pand %xmm7,%xmm10");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) asm volatile("pshufb %xmm3,%xmm4");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) asm volatile("pshufb %xmm11,%xmm12");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) asm volatile("pshufb %xmm2,%xmm1");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) asm volatile("pshufb %xmm10,%xmm9");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) asm volatile("pxor %xmm4,%xmm1");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) asm volatile("pxor %xmm12,%xmm9");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) /* xmm1/9 = pbmul[px] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) asm volatile("pxor %xmm5,%xmm1");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) asm volatile("pxor %xmm13,%xmm9");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) /* xmm1/9 = db = DQ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) asm volatile("movdqa %%xmm1,%0" : "=m" (dq[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) asm volatile("movdqa %%xmm9,%0" : "=m" (dq[16]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) asm volatile("pxor %xmm1,%xmm0");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) asm volatile("pxor %xmm9,%xmm8");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) asm volatile("movdqa %%xmm0,%0" : "=m" (dp[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) asm volatile("movdqa %%xmm8,%0" : "=m" (dp[16]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) bytes -= 32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) p += 32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) q += 32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) dp += 32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) dq += 32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) asm volatile("movdqa %0,%%xmm1" : : "m" (*q));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) asm volatile("movdqa %0,%%xmm0" : : "m" (*p));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) asm volatile("pxor %0,%%xmm1" : : "m" (*dq));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) asm volatile("pxor %0,%%xmm0" : : "m" (*dp));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) /* 1 = dq ^ q
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) * 0 = dp ^ p
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) asm volatile("movdqa %0,%%xmm4" : : "m" (qmul[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) asm volatile("movdqa %0,%%xmm5" : : "m" (qmul[16]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) asm volatile("movdqa %xmm1,%xmm3");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) asm volatile("psraw $4,%xmm1");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) asm volatile("pand %xmm7,%xmm3");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) asm volatile("pand %xmm7,%xmm1");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) asm volatile("pshufb %xmm3,%xmm4");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) asm volatile("pshufb %xmm1,%xmm5");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) asm volatile("pxor %xmm4,%xmm5");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) asm volatile("movdqa %xmm0,%xmm2"); /* xmm2 = px */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) /* xmm5 = qx */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) asm volatile("movdqa %0,%%xmm4" : : "m" (pbmul[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) asm volatile("movdqa %0,%%xmm1" : : "m" (pbmul[16]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) asm volatile("movdqa %xmm2,%xmm3");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) asm volatile("psraw $4,%xmm2");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) asm volatile("pand %xmm7,%xmm3");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) asm volatile("pand %xmm7,%xmm2");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) asm volatile("pshufb %xmm3,%xmm4");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) asm volatile("pshufb %xmm2,%xmm1");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) asm volatile("pxor %xmm4,%xmm1");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) /* xmm1 = pbmul[px] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) asm volatile("pxor %xmm5,%xmm1");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) /* xmm1 = db = DQ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) asm volatile("movdqa %%xmm1,%0" : "=m" (*dq));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) asm volatile("pxor %xmm1,%xmm0");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) asm volatile("movdqa %%xmm0,%0" : "=m" (*dp));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) bytes -= 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) p += 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) q += 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) dp += 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) dq += 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) kernel_fpu_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) static void raid6_datap_recov_ssse3(int disks, size_t bytes, int faila,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) void **ptrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) u8 *p, *q, *dq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) const u8 *qmul; /* Q multiplier table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) static const u8 __aligned(16) x0f[16] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) p = (u8 *)ptrs[disks-2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) q = (u8 *)ptrs[disks-1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) /* Compute syndrome with zero for the missing data page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) Use the dead data page as temporary storage for delta q */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) dq = (u8 *)ptrs[faila];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) ptrs[faila] = (void *)raid6_empty_zero_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) ptrs[disks-1] = dq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) raid6_call.gen_syndrome(disks, bytes, ptrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) /* Restore pointer table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) ptrs[faila] = dq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) ptrs[disks-1] = q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) /* Now, pick the proper data tables */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) kernel_fpu_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) asm volatile("movdqa %0, %%xmm7" : : "m" (x0f[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) while (bytes) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) asm volatile("movdqa %0, %%xmm3" : : "m" (dq[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) asm volatile("movdqa %0, %%xmm4" : : "m" (dq[16]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) asm volatile("pxor %0, %%xmm3" : : "m" (q[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) asm volatile("movdqa %0, %%xmm0" : : "m" (qmul[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) /* xmm3 = q[0] ^ dq[0] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) asm volatile("pxor %0, %%xmm4" : : "m" (q[16]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) asm volatile("movdqa %0, %%xmm1" : : "m" (qmul[16]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) /* xmm4 = q[16] ^ dq[16] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) asm volatile("movdqa %xmm3, %xmm6");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) asm volatile("movdqa %xmm4, %xmm8");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) /* xmm4 = xmm8 = q[16] ^ dq[16] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) asm volatile("psraw $4, %xmm3");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) asm volatile("pand %xmm7, %xmm6");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) asm volatile("pand %xmm7, %xmm3");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) asm volatile("pshufb %xmm6, %xmm0");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) asm volatile("pshufb %xmm3, %xmm1");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) asm volatile("movdqa %0, %%xmm10" : : "m" (qmul[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) asm volatile("pxor %xmm0, %xmm1");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) asm volatile("movdqa %0, %%xmm11" : : "m" (qmul[16]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) /* xmm1 = qmul[q[0] ^ dq[0]] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) asm volatile("psraw $4, %xmm4");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) asm volatile("pand %xmm7, %xmm8");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) asm volatile("pand %xmm7, %xmm4");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) asm volatile("pshufb %xmm8, %xmm10");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) asm volatile("pshufb %xmm4, %xmm11");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) asm volatile("movdqa %0, %%xmm2" : : "m" (p[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) asm volatile("pxor %xmm10, %xmm11");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) asm volatile("movdqa %0, %%xmm12" : : "m" (p[16]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) /* xmm11 = qmul[q[16] ^ dq[16]] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) asm volatile("pxor %xmm1, %xmm2");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) /* xmm2 = p[0] ^ qmul[q[0] ^ dq[0]] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) asm volatile("pxor %xmm11, %xmm12");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) /* xmm12 = p[16] ^ qmul[q[16] ^ dq[16]] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) asm volatile("movdqa %%xmm1, %0" : "=m" (dq[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) asm volatile("movdqa %%xmm11, %0" : "=m" (dq[16]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) asm volatile("movdqa %%xmm2, %0" : "=m" (p[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) asm volatile("movdqa %%xmm12, %0" : "=m" (p[16]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) bytes -= 32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) p += 32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) q += 32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) dq += 32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) asm volatile("movdqa %0, %%xmm3" : : "m" (dq[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) asm volatile("movdqa %0, %%xmm0" : : "m" (qmul[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) asm volatile("pxor %0, %%xmm3" : : "m" (q[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) asm volatile("movdqa %0, %%xmm1" : : "m" (qmul[16]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) /* xmm3 = *q ^ *dq */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) asm volatile("movdqa %xmm3, %xmm6");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) asm volatile("movdqa %0, %%xmm2" : : "m" (p[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) asm volatile("psraw $4, %xmm3");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) asm volatile("pand %xmm7, %xmm6");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) asm volatile("pand %xmm7, %xmm3");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) asm volatile("pshufb %xmm6, %xmm0");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) asm volatile("pshufb %xmm3, %xmm1");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) asm volatile("pxor %xmm0, %xmm1");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) /* xmm1 = qmul[*q ^ *dq */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) asm volatile("pxor %xmm1, %xmm2");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) /* xmm2 = *p ^ qmul[*q ^ *dq] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) asm volatile("movdqa %%xmm1, %0" : "=m" (dq[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) asm volatile("movdqa %%xmm2, %0" : "=m" (p[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) bytes -= 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) p += 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) q += 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) dq += 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) kernel_fpu_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) const struct raid6_recov_calls raid6_recov_ssse3 = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) .data2 = raid6_2data_recov_ssse3,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) .datap = raid6_datap_recov_ssse3,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) .valid = raid6_has_ssse3,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) .name = "ssse3x2",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) .name = "ssse3x1",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) .priority = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) };
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags