1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* -*- linux-c -*- ------------------------------------------------------- * |
3 | * |
4 | * Copyright 2002 H. Peter Anvin - All Rights Reserved |
5 | * |
6 | * ----------------------------------------------------------------------- */ |
7 | |
8 | /* |
9 | * raid6/algos.c |
10 | * |
11 | * Algorithm list and algorithm selection for RAID-6 |
12 | */ |
13 | |
14 | #include <linux/raid/pq.h> |
15 | #ifndef __KERNEL__ |
16 | #include <sys/mman.h> |
17 | #include <stdio.h> |
18 | #else |
19 | #include <linux/module.h> |
20 | #include <linux/gfp.h> |
21 | /* In .bss so it's zeroed */ |
22 | const char raid6_empty_zero_page[PAGE_SIZE] __attribute__((aligned(256))); |
23 | EXPORT_SYMBOL(raid6_empty_zero_page); |
24 | #endif |
25 | |
26 | struct raid6_calls raid6_call; |
27 | EXPORT_SYMBOL_GPL(raid6_call); |
28 | |
29 | const struct raid6_calls * const raid6_algos[] = { |
30 | #if defined(__i386__) && !defined(__arch_um__) |
31 | #ifdef CONFIG_AS_AVX512 |
32 | &raid6_avx512x2, |
33 | &raid6_avx512x1, |
34 | #endif |
35 | &raid6_avx2x2, |
36 | &raid6_avx2x1, |
37 | &raid6_sse2x2, |
38 | &raid6_sse2x1, |
39 | &raid6_sse1x2, |
40 | &raid6_sse1x1, |
41 | &raid6_mmxx2, |
42 | &raid6_mmxx1, |
43 | #endif |
44 | #if defined(__x86_64__) && !defined(__arch_um__) |
45 | #ifdef CONFIG_AS_AVX512 |
46 | &raid6_avx512x4, |
47 | &raid6_avx512x2, |
48 | &raid6_avx512x1, |
49 | #endif |
50 | &raid6_avx2x4, |
51 | &raid6_avx2x2, |
52 | &raid6_avx2x1, |
53 | &raid6_sse2x4, |
54 | &raid6_sse2x2, |
55 | &raid6_sse2x1, |
56 | #endif |
57 | #ifdef CONFIG_ALTIVEC |
58 | &raid6_vpermxor8, |
59 | &raid6_vpermxor4, |
60 | &raid6_vpermxor2, |
61 | &raid6_vpermxor1, |
62 | &raid6_altivec8, |
63 | &raid6_altivec4, |
64 | &raid6_altivec2, |
65 | &raid6_altivec1, |
66 | #endif |
67 | #if defined(CONFIG_S390) |
68 | &raid6_s390vx8, |
69 | #endif |
70 | #ifdef CONFIG_KERNEL_MODE_NEON |
71 | &raid6_neonx8, |
72 | &raid6_neonx4, |
73 | &raid6_neonx2, |
74 | &raid6_neonx1, |
75 | #endif |
76 | #ifdef CONFIG_LOONGARCH |
77 | #ifdef CONFIG_CPU_HAS_LASX |
78 | &raid6_lasx, |
79 | #endif |
80 | #ifdef CONFIG_CPU_HAS_LSX |
81 | &raid6_lsx, |
82 | #endif |
83 | #endif |
84 | &raid6_intx8, |
85 | &raid6_intx4, |
86 | &raid6_intx2, |
87 | &raid6_intx1, |
88 | NULL |
89 | }; |
90 | |
91 | void (*raid6_2data_recov)(int, size_t, int, int, void **); |
92 | EXPORT_SYMBOL_GPL(raid6_2data_recov); |
93 | |
94 | void (*raid6_datap_recov)(int, size_t, int, void **); |
95 | EXPORT_SYMBOL_GPL(raid6_datap_recov); |
96 | |
97 | const struct raid6_recov_calls *const raid6_recov_algos[] = { |
98 | #ifdef CONFIG_X86 |
99 | #ifdef CONFIG_AS_AVX512 |
100 | &raid6_recov_avx512, |
101 | #endif |
102 | &raid6_recov_avx2, |
103 | &raid6_recov_ssse3, |
104 | #endif |
105 | #ifdef CONFIG_S390 |
106 | &raid6_recov_s390xc, |
107 | #endif |
108 | #if defined(CONFIG_KERNEL_MODE_NEON) |
109 | &raid6_recov_neon, |
110 | #endif |
111 | #ifdef CONFIG_LOONGARCH |
112 | #ifdef CONFIG_CPU_HAS_LASX |
113 | &raid6_recov_lasx, |
114 | #endif |
115 | #ifdef CONFIG_CPU_HAS_LSX |
116 | &raid6_recov_lsx, |
117 | #endif |
118 | #endif |
119 | &raid6_recov_intx1, |
120 | NULL |
121 | }; |
122 | |
123 | #ifdef __KERNEL__ |
124 | #define RAID6_TIME_JIFFIES_LG2 4 |
125 | #else |
126 | /* Need more time to be stable in userspace */ |
127 | #define RAID6_TIME_JIFFIES_LG2 9 |
128 | #define time_before(x, y) ((x) < (y)) |
129 | #endif |
130 | |
131 | #define RAID6_TEST_DISKS 8 |
132 | #define RAID6_TEST_DISKS_ORDER 3 |
133 | |
134 | static inline const struct raid6_recov_calls *raid6_choose_recov(void) |
135 | { |
136 | const struct raid6_recov_calls *const *algo; |
137 | const struct raid6_recov_calls *best; |
138 | |
139 | for (best = NULL, algo = raid6_recov_algos; *algo; algo++) |
140 | if (!best || (*algo)->priority > best->priority) |
141 | if (!(*algo)->valid || (*algo)->valid()) |
142 | best = *algo; |
143 | |
144 | if (best) { |
145 | raid6_2data_recov = best->data2; |
146 | raid6_datap_recov = best->datap; |
147 | |
148 | pr_info("raid6: using %s recovery algorithm\n" , best->name); |
149 | } else |
150 | pr_err("raid6: Yikes! No recovery algorithm found!\n" ); |
151 | |
152 | return best; |
153 | } |
154 | |
155 | static inline const struct raid6_calls *raid6_choose_gen( |
156 | void *(*const dptrs)[RAID6_TEST_DISKS], const int disks) |
157 | { |
158 | unsigned long perf, bestgenperf, j0, j1; |
159 | int start = (disks>>1)-1, stop = disks-3; /* work on the second half of the disks */ |
160 | const struct raid6_calls *const *algo; |
161 | const struct raid6_calls *best; |
162 | |
163 | for (bestgenperf = 0, best = NULL, algo = raid6_algos; *algo; algo++) { |
164 | if (!best || (*algo)->priority >= best->priority) { |
165 | if ((*algo)->valid && !(*algo)->valid()) |
166 | continue; |
167 | |
168 | if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) { |
169 | best = *algo; |
170 | break; |
171 | } |
172 | |
173 | perf = 0; |
174 | |
175 | preempt_disable(); |
176 | j0 = jiffies; |
177 | while ((j1 = jiffies) == j0) |
178 | cpu_relax(); |
179 | while (time_before(jiffies, |
180 | j1 + (1<<RAID6_TIME_JIFFIES_LG2))) { |
181 | (*algo)->gen_syndrome(disks, PAGE_SIZE, *dptrs); |
182 | perf++; |
183 | } |
184 | preempt_enable(); |
185 | |
186 | if (perf > bestgenperf) { |
187 | bestgenperf = perf; |
188 | best = *algo; |
189 | } |
190 | pr_info("raid6: %-8s gen() %5ld MB/s\n" , (*algo)->name, |
191 | (perf * HZ * (disks-2)) >> |
192 | (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2)); |
193 | } |
194 | } |
195 | |
196 | if (!best) { |
197 | pr_err("raid6: Yikes! No algorithm found!\n" ); |
198 | goto out; |
199 | } |
200 | |
201 | raid6_call = *best; |
202 | |
203 | if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) { |
204 | pr_info("raid6: skipped pq benchmark and selected %s\n" , |
205 | best->name); |
206 | goto out; |
207 | } |
208 | |
209 | pr_info("raid6: using algorithm %s gen() %ld MB/s\n" , |
210 | best->name, |
211 | (bestgenperf * HZ * (disks - 2)) >> |
212 | (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2)); |
213 | |
214 | if (best->xor_syndrome) { |
215 | perf = 0; |
216 | |
217 | preempt_disable(); |
218 | j0 = jiffies; |
219 | while ((j1 = jiffies) == j0) |
220 | cpu_relax(); |
221 | while (time_before(jiffies, |
222 | j1 + (1 << RAID6_TIME_JIFFIES_LG2))) { |
223 | best->xor_syndrome(disks, start, stop, |
224 | PAGE_SIZE, *dptrs); |
225 | perf++; |
226 | } |
227 | preempt_enable(); |
228 | |
229 | pr_info("raid6: .... xor() %ld MB/s, rmw enabled\n" , |
230 | (perf * HZ * (disks - 2)) >> |
231 | (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2 + 1)); |
232 | } |
233 | |
234 | out: |
235 | return best; |
236 | } |
237 | |
238 | |
239 | /* Try to pick the best algorithm */ |
240 | /* This code uses the gfmul table as convenient data set to abuse */ |
241 | |
242 | int __init raid6_select_algo(void) |
243 | { |
244 | const int disks = RAID6_TEST_DISKS; |
245 | |
246 | const struct raid6_calls *gen_best; |
247 | const struct raid6_recov_calls *rec_best; |
248 | char *disk_ptr, *p; |
249 | void *dptrs[RAID6_TEST_DISKS]; |
250 | int i, cycle; |
251 | |
252 | /* prepare the buffer and fill it circularly with gfmul table */ |
253 | disk_ptr = (char *)__get_free_pages(GFP_KERNEL, RAID6_TEST_DISKS_ORDER); |
254 | if (!disk_ptr) { |
255 | pr_err("raid6: Yikes! No memory available.\n" ); |
256 | return -ENOMEM; |
257 | } |
258 | |
259 | p = disk_ptr; |
260 | for (i = 0; i < disks; i++) |
261 | dptrs[i] = p + PAGE_SIZE * i; |
262 | |
263 | cycle = ((disks - 2) * PAGE_SIZE) / 65536; |
264 | for (i = 0; i < cycle; i++) { |
265 | memcpy(p, raid6_gfmul, 65536); |
266 | p += 65536; |
267 | } |
268 | |
269 | if ((disks - 2) * PAGE_SIZE % 65536) |
270 | memcpy(p, raid6_gfmul, (disks - 2) * PAGE_SIZE % 65536); |
271 | |
272 | /* select raid gen_syndrome function */ |
273 | gen_best = raid6_choose_gen(dptrs: &dptrs, disks); |
274 | |
275 | /* select raid recover functions */ |
276 | rec_best = raid6_choose_recov(); |
277 | |
278 | free_pages(addr: (unsigned long)disk_ptr, RAID6_TEST_DISKS_ORDER); |
279 | |
280 | return gen_best && rec_best ? 0 : -EINVAL; |
281 | } |
282 | |
283 | static void raid6_exit(void) |
284 | { |
285 | do { } while (0); |
286 | } |
287 | |
288 | subsys_initcall(raid6_select_algo); |
289 | module_exit(raid6_exit); |
290 | MODULE_LICENSE("GPL" ); |
291 | MODULE_DESCRIPTION("RAID6 Q-syndrome calculations" ); |
292 | |