1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright 2024 Institute of Software, CAS.
4 * Author: Chunyan Zhang <zhangchunyan@iscas.ac.cn>
5 */
6
7#include <linux/raid/pq.h>
8#include "rvv.h"
9
10static void __raid6_2data_recov_rvv(int bytes, u8 *p, u8 *q, u8 *dp,
11 u8 *dq, const u8 *pbmul,
12 const u8 *qmul)
13{
14 asm volatile (".option push\n"
15 ".option arch,+v\n"
16 "vsetvli x0, %[avl], e8, m1, ta, ma\n"
17 ".option pop\n"
18 : :
19 [avl]"r"(16)
20 );
21
22 /*
23 * while ( bytes-- ) {
24 * uint8_t px, qx, db;
25 *
26 * px = *p ^ *dp;
27 * qx = qmul[*q ^ *dq];
28 * *dq++ = db = pbmul[px] ^ qx;
29 * *dp++ = db ^ px;
30 * p++; q++;
31 * }
32 */
33 while (bytes) {
34 /*
35 * v0:px, v1:dp,
36 * v2:qx, v3:dq,
37 * v4:vx, v5:vy,
38 * v6:qm0, v7:qm1,
39 * v8:pm0, v9:pm1,
40 * v14:p/qm[vx], v15:p/qm[vy]
41 */
42 asm volatile (".option push\n"
43 ".option arch,+v\n"
44 "vle8.v v0, (%[px])\n"
45 "vle8.v v1, (%[dp])\n"
46 "vxor.vv v0, v0, v1\n"
47 "vle8.v v2, (%[qx])\n"
48 "vle8.v v3, (%[dq])\n"
49 "vxor.vv v4, v2, v3\n"
50 "vsrl.vi v5, v4, 4\n"
51 "vand.vi v4, v4, 0xf\n"
52 "vle8.v v6, (%[qm0])\n"
53 "vle8.v v7, (%[qm1])\n"
54 "vrgather.vv v14, v6, v4\n" /* v14 = qm[vx] */
55 "vrgather.vv v15, v7, v5\n" /* v15 = qm[vy] */
56 "vxor.vv v2, v14, v15\n" /* v2 = qmul[*q ^ *dq] */
57
58 "vsrl.vi v5, v0, 4\n"
59 "vand.vi v4, v0, 0xf\n"
60 "vle8.v v8, (%[pm0])\n"
61 "vle8.v v9, (%[pm1])\n"
62 "vrgather.vv v14, v8, v4\n" /* v14 = pm[vx] */
63 "vrgather.vv v15, v9, v5\n" /* v15 = pm[vy] */
64 "vxor.vv v4, v14, v15\n" /* v4 = pbmul[px] */
65 "vxor.vv v3, v4, v2\n" /* v3 = db = pbmul[px] ^ qx */
66 "vxor.vv v1, v3, v0\n" /* v1 = db ^ px; */
67 "vse8.v v3, (%[dq])\n"
68 "vse8.v v1, (%[dp])\n"
69 ".option pop\n"
70 : :
71 [px]"r"(p),
72 [dp]"r"(dp),
73 [qx]"r"(q),
74 [dq]"r"(dq),
75 [qm0]"r"(qmul),
76 [qm1]"r"(qmul + 16),
77 [pm0]"r"(pbmul),
78 [pm1]"r"(pbmul + 16)
79 :);
80
81 bytes -= 16;
82 p += 16;
83 q += 16;
84 dp += 16;
85 dq += 16;
86 }
87}
88
89static void __raid6_datap_recov_rvv(int bytes, u8 *p, u8 *q,
90 u8 *dq, const u8 *qmul)
91{
92 asm volatile (".option push\n"
93 ".option arch,+v\n"
94 "vsetvli x0, %[avl], e8, m1, ta, ma\n"
95 ".option pop\n"
96 : :
97 [avl]"r"(16)
98 );
99
100 /*
101 * while (bytes--) {
102 * *p++ ^= *dq = qmul[*q ^ *dq];
103 * q++; dq++;
104 * }
105 */
106 while (bytes) {
107 /*
108 * v0:vx, v1:vy,
109 * v2:dq, v3:p,
110 * v4:qm0, v5:qm1,
111 * v10:m[vx], v11:m[vy]
112 */
113 asm volatile (".option push\n"
114 ".option arch,+v\n"
115 "vle8.v v0, (%[vx])\n"
116 "vle8.v v2, (%[dq])\n"
117 "vxor.vv v0, v0, v2\n"
118 "vsrl.vi v1, v0, 4\n"
119 "vand.vi v0, v0, 0xf\n"
120 "vle8.v v4, (%[qm0])\n"
121 "vle8.v v5, (%[qm1])\n"
122 "vrgather.vv v10, v4, v0\n"
123 "vrgather.vv v11, v5, v1\n"
124 "vxor.vv v0, v10, v11\n"
125 "vle8.v v1, (%[vy])\n"
126 "vxor.vv v1, v0, v1\n"
127 "vse8.v v0, (%[dq])\n"
128 "vse8.v v1, (%[vy])\n"
129 ".option pop\n"
130 : :
131 [vx]"r"(q),
132 [vy]"r"(p),
133 [dq]"r"(dq),
134 [qm0]"r"(qmul),
135 [qm1]"r"(qmul + 16)
136 :);
137
138 bytes -= 16;
139 p += 16;
140 q += 16;
141 dq += 16;
142 }
143}
144
145static void raid6_2data_recov_rvv(int disks, size_t bytes, int faila,
146 int failb, void **ptrs)
147{
148 u8 *p, *q, *dp, *dq;
149 const u8 *pbmul; /* P multiplier table for B data */
150 const u8 *qmul; /* Q multiplier table (for both) */
151
152 p = (u8 *)ptrs[disks - 2];
153 q = (u8 *)ptrs[disks - 1];
154
155 /*
156 * Compute syndrome with zero for the missing data pages
157 * Use the dead data pages as temporary storage for
158 * delta p and delta q
159 */
160 dp = (u8 *)ptrs[faila];
161 ptrs[faila] = raid6_get_zero_page();
162 ptrs[disks - 2] = dp;
163 dq = (u8 *)ptrs[failb];
164 ptrs[failb] = raid6_get_zero_page();
165 ptrs[disks - 1] = dq;
166
167 raid6_call.gen_syndrome(disks, bytes, ptrs);
168
169 /* Restore pointer table */
170 ptrs[faila] = dp;
171 ptrs[failb] = dq;
172 ptrs[disks - 2] = p;
173 ptrs[disks - 1] = q;
174
175 /* Now, pick the proper data tables */
176 pbmul = raid6_vgfmul[raid6_gfexi[failb - faila]];
177 qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
178 raid6_gfexp[failb]]];
179
180 kernel_vector_begin();
181 __raid6_2data_recov_rvv(bytes, p, q, dp, dq, pbmul, qmul);
182 kernel_vector_end();
183}
184
185static void raid6_datap_recov_rvv(int disks, size_t bytes, int faila,
186 void **ptrs)
187{
188 u8 *p, *q, *dq;
189 const u8 *qmul; /* Q multiplier table */
190
191 p = (u8 *)ptrs[disks - 2];
192 q = (u8 *)ptrs[disks - 1];
193
194 /*
195 * Compute syndrome with zero for the missing data page
196 * Use the dead data page as temporary storage for delta q
197 */
198 dq = (u8 *)ptrs[faila];
199 ptrs[faila] = raid6_get_zero_page();
200 ptrs[disks - 1] = dq;
201
202 raid6_call.gen_syndrome(disks, bytes, ptrs);
203
204 /* Restore pointer table */
205 ptrs[faila] = dq;
206 ptrs[disks - 1] = q;
207
208 /* Now, pick the proper data tables */
209 qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
210
211 kernel_vector_begin();
212 __raid6_datap_recov_rvv(bytes, p, q, dq, qmul);
213 kernel_vector_end();
214}
215
216const struct raid6_recov_calls raid6_recov_rvv = {
217 .data2 = raid6_2data_recov_rvv,
218 .datap = raid6_datap_recov_rvv,
219 .valid = rvv_has_vector,
220 .name = "rvv",
221 .priority = 1,
222};
223

source code of linux/lib/raid6/recov_rvv.c