1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * x86_64/AVX2/AES-NI assembler implementation of Camellia
4 *
5 * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
6 */
7
8#include <linux/linkage.h>
9#include <linux/cfi_types.h>
10#include <asm/frame.h>
11
12#define CAMELLIA_TABLE_BYTE_LEN 272
13
14/* struct camellia_ctx: */
15#define key_table 0
16#define key_length CAMELLIA_TABLE_BYTE_LEN
17
18/* register macros */
19#define CTX %rdi
20#define RIO %r8
21
22/**********************************************************************
23 helper macros
24 **********************************************************************/
25#define filter_8bit(x, lo_t, hi_t, mask4bit, tmp0) \
26 vpand x, mask4bit, tmp0; \
27 vpandn x, mask4bit, x; \
28 vpsrld $4, x, x; \
29 \
30 vpshufb tmp0, lo_t, tmp0; \
31 vpshufb x, hi_t, x; \
32 vpxor tmp0, x, x;
33
34#define ymm0_x xmm0
35#define ymm1_x xmm1
36#define ymm2_x xmm2
37#define ymm3_x xmm3
38#define ymm4_x xmm4
39#define ymm5_x xmm5
40#define ymm6_x xmm6
41#define ymm7_x xmm7
42#define ymm8_x xmm8
43#define ymm9_x xmm9
44#define ymm10_x xmm10
45#define ymm11_x xmm11
46#define ymm12_x xmm12
47#define ymm13_x xmm13
48#define ymm14_x xmm14
49#define ymm15_x xmm15
50
51/**********************************************************************
52 32-way camellia
53 **********************************************************************/
54
55/*
56 * IN:
57 * x0..x7: byte-sliced AB state
58 * mem_cd: register pointer storing CD state
59 * key: index for key material
60 * OUT:
61 * x0..x7: new byte-sliced CD state
62 */
63#define roundsm32(x0, x1, x2, x3, x4, x5, x6, x7, t0, t1, t2, t3, t4, t5, t6, \
64 t7, mem_cd, key) \
65 /* \
66 * S-function with AES subbytes \
67 */ \
68 vbroadcasti128 .Linv_shift_row(%rip), t4; \
69 vpbroadcastd .L0f0f0f0f(%rip), t7; \
70 vbroadcasti128 .Lpre_tf_lo_s1(%rip), t5; \
71 vbroadcasti128 .Lpre_tf_hi_s1(%rip), t6; \
72 vbroadcasti128 .Lpre_tf_lo_s4(%rip), t2; \
73 vbroadcasti128 .Lpre_tf_hi_s4(%rip), t3; \
74 \
75 /* AES inverse shift rows */ \
76 vpshufb t4, x0, x0; \
77 vpshufb t4, x7, x7; \
78 vpshufb t4, x3, x3; \
79 vpshufb t4, x6, x6; \
80 vpshufb t4, x2, x2; \
81 vpshufb t4, x5, x5; \
82 vpshufb t4, x1, x1; \
83 vpshufb t4, x4, x4; \
84 \
85 /* prefilter sboxes 1, 2 and 3 */ \
86 /* prefilter sbox 4 */ \
87 filter_8bit(x0, t5, t6, t7, t4); \
88 filter_8bit(x7, t5, t6, t7, t4); \
89 vextracti128 $1, x0, t0##_x; \
90 vextracti128 $1, x7, t1##_x; \
91 filter_8bit(x3, t2, t3, t7, t4); \
92 filter_8bit(x6, t2, t3, t7, t4); \
93 vextracti128 $1, x3, t3##_x; \
94 vextracti128 $1, x6, t2##_x; \
95 filter_8bit(x2, t5, t6, t7, t4); \
96 filter_8bit(x5, t5, t6, t7, t4); \
97 filter_8bit(x1, t5, t6, t7, t4); \
98 filter_8bit(x4, t5, t6, t7, t4); \
99 \
100 vpxor t4##_x, t4##_x, t4##_x; \
101 \
102 /* AES subbytes + AES shift rows */ \
103 vextracti128 $1, x2, t6##_x; \
104 vextracti128 $1, x5, t5##_x; \
105 vaesenclast t4##_x, x0##_x, x0##_x; \
106 vaesenclast t4##_x, t0##_x, t0##_x; \
107 vinserti128 $1, t0##_x, x0, x0; \
108 vaesenclast t4##_x, x7##_x, x7##_x; \
109 vaesenclast t4##_x, t1##_x, t1##_x; \
110 vinserti128 $1, t1##_x, x7, x7; \
111 vaesenclast t4##_x, x3##_x, x3##_x; \
112 vaesenclast t4##_x, t3##_x, t3##_x; \
113 vinserti128 $1, t3##_x, x3, x3; \
114 vaesenclast t4##_x, x6##_x, x6##_x; \
115 vaesenclast t4##_x, t2##_x, t2##_x; \
116 vinserti128 $1, t2##_x, x6, x6; \
117 vextracti128 $1, x1, t3##_x; \
118 vextracti128 $1, x4, t2##_x; \
119 vbroadcasti128 .Lpost_tf_lo_s1(%rip), t0; \
120 vbroadcasti128 .Lpost_tf_hi_s1(%rip), t1; \
121 vaesenclast t4##_x, x2##_x, x2##_x; \
122 vaesenclast t4##_x, t6##_x, t6##_x; \
123 vinserti128 $1, t6##_x, x2, x2; \
124 vaesenclast t4##_x, x5##_x, x5##_x; \
125 vaesenclast t4##_x, t5##_x, t5##_x; \
126 vinserti128 $1, t5##_x, x5, x5; \
127 vaesenclast t4##_x, x1##_x, x1##_x; \
128 vaesenclast t4##_x, t3##_x, t3##_x; \
129 vinserti128 $1, t3##_x, x1, x1; \
130 vaesenclast t4##_x, x4##_x, x4##_x; \
131 vaesenclast t4##_x, t2##_x, t2##_x; \
132 vinserti128 $1, t2##_x, x4, x4; \
133 \
134 /* postfilter sboxes 1 and 4 */ \
135 vbroadcasti128 .Lpost_tf_lo_s3(%rip), t2; \
136 vbroadcasti128 .Lpost_tf_hi_s3(%rip), t3; \
137 filter_8bit(x0, t0, t1, t7, t6); \
138 filter_8bit(x7, t0, t1, t7, t6); \
139 filter_8bit(x3, t0, t1, t7, t6); \
140 filter_8bit(x6, t0, t1, t7, t6); \
141 \
142 /* postfilter sbox 3 */ \
143 vbroadcasti128 .Lpost_tf_lo_s2(%rip), t4; \
144 vbroadcasti128 .Lpost_tf_hi_s2(%rip), t5; \
145 filter_8bit(x2, t2, t3, t7, t6); \
146 filter_8bit(x5, t2, t3, t7, t6); \
147 \
148 vpbroadcastq key, t0; /* higher 64-bit duplicate ignored */ \
149 \
150 /* postfilter sbox 2 */ \
151 filter_8bit(x1, t4, t5, t7, t2); \
152 filter_8bit(x4, t4, t5, t7, t2); \
153 vpxor t7, t7, t7; \
154 \
155 vpsrldq $1, t0, t1; \
156 vpsrldq $2, t0, t2; \
157 vpshufb t7, t1, t1; \
158 vpsrldq $3, t0, t3; \
159 \
160 /* P-function */ \
161 vpxor x5, x0, x0; \
162 vpxor x6, x1, x1; \
163 vpxor x7, x2, x2; \
164 vpxor x4, x3, x3; \
165 \
166 vpshufb t7, t2, t2; \
167 vpsrldq $4, t0, t4; \
168 vpshufb t7, t3, t3; \
169 vpsrldq $5, t0, t5; \
170 vpshufb t7, t4, t4; \
171 \
172 vpxor x2, x4, x4; \
173 vpxor x3, x5, x5; \
174 vpxor x0, x6, x6; \
175 vpxor x1, x7, x7; \
176 \
177 vpsrldq $6, t0, t6; \
178 vpshufb t7, t5, t5; \
179 vpshufb t7, t6, t6; \
180 \
181 vpxor x7, x0, x0; \
182 vpxor x4, x1, x1; \
183 vpxor x5, x2, x2; \
184 vpxor x6, x3, x3; \
185 \
186 vpxor x3, x4, x4; \
187 vpxor x0, x5, x5; \
188 vpxor x1, x6, x6; \
189 vpxor x2, x7, x7; /* note: high and low parts swapped */ \
190 \
191 /* Add key material and result to CD (x becomes new CD) */ \
192 \
193 vpxor t6, x1, x1; \
194 vpxor 5 * 32(mem_cd), x1, x1; \
195 \
196 vpsrldq $7, t0, t6; \
197 vpshufb t7, t0, t0; \
198 vpshufb t7, t6, t7; \
199 \
200 vpxor t7, x0, x0; \
201 vpxor 4 * 32(mem_cd), x0, x0; \
202 \
203 vpxor t5, x2, x2; \
204 vpxor 6 * 32(mem_cd), x2, x2; \
205 \
206 vpxor t4, x3, x3; \
207 vpxor 7 * 32(mem_cd), x3, x3; \
208 \
209 vpxor t3, x4, x4; \
210 vpxor 0 * 32(mem_cd), x4, x4; \
211 \
212 vpxor t2, x5, x5; \
213 vpxor 1 * 32(mem_cd), x5, x5; \
214 \
215 vpxor t1, x6, x6; \
216 vpxor 2 * 32(mem_cd), x6, x6; \
217 \
218 vpxor t0, x7, x7; \
219 vpxor 3 * 32(mem_cd), x7, x7;
220
221/*
222 * Size optimization... with inlined roundsm32 binary would be over 5 times
223 * larger and would only marginally faster.
224 */
225SYM_FUNC_START_LOCAL(roundsm32_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd)
226 roundsm32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
227 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, %ymm15,
228 %rcx, (%r9));
229 RET;
230SYM_FUNC_END(roundsm32_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd)
231
232SYM_FUNC_START_LOCAL(roundsm32_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab)
233 roundsm32(%ymm4, %ymm5, %ymm6, %ymm7, %ymm0, %ymm1, %ymm2, %ymm3,
234 %ymm12, %ymm13, %ymm14, %ymm15, %ymm8, %ymm9, %ymm10, %ymm11,
235 %rax, (%r9));
236 RET;
237SYM_FUNC_END(roundsm32_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab)
238
239/*
240 * IN/OUT:
241 * x0..x7: byte-sliced AB state preloaded
242 * mem_ab: byte-sliced AB state in memory
243 * mem_cb: byte-sliced CD state in memory
244 */
245#define two_roundsm32(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
246 y6, y7, mem_ab, mem_cd, i, dir, store_ab) \
247 leaq (key_table + (i) * 8)(CTX), %r9; \
248 call roundsm32_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd; \
249 \
250 vmovdqu x0, 4 * 32(mem_cd); \
251 vmovdqu x1, 5 * 32(mem_cd); \
252 vmovdqu x2, 6 * 32(mem_cd); \
253 vmovdqu x3, 7 * 32(mem_cd); \
254 vmovdqu x4, 0 * 32(mem_cd); \
255 vmovdqu x5, 1 * 32(mem_cd); \
256 vmovdqu x6, 2 * 32(mem_cd); \
257 vmovdqu x7, 3 * 32(mem_cd); \
258 \
259 leaq (key_table + ((i) + (dir)) * 8)(CTX), %r9; \
260 call roundsm32_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab; \
261 \
262 store_ab(x0, x1, x2, x3, x4, x5, x6, x7, mem_ab);
263
264#define dummy_store(x0, x1, x2, x3, x4, x5, x6, x7, mem_ab) /* do nothing */
265
266#define store_ab_state(x0, x1, x2, x3, x4, x5, x6, x7, mem_ab) \
267 /* Store new AB state */ \
268 vmovdqu x4, 4 * 32(mem_ab); \
269 vmovdqu x5, 5 * 32(mem_ab); \
270 vmovdqu x6, 6 * 32(mem_ab); \
271 vmovdqu x7, 7 * 32(mem_ab); \
272 vmovdqu x0, 0 * 32(mem_ab); \
273 vmovdqu x1, 1 * 32(mem_ab); \
274 vmovdqu x2, 2 * 32(mem_ab); \
275 vmovdqu x3, 3 * 32(mem_ab);
276
277#define enc_rounds32(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
278 y6, y7, mem_ab, mem_cd, i) \
279 two_roundsm32(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
280 y6, y7, mem_ab, mem_cd, (i) + 2, 1, store_ab_state); \
281 two_roundsm32(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
282 y6, y7, mem_ab, mem_cd, (i) + 4, 1, store_ab_state); \
283 two_roundsm32(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
284 y6, y7, mem_ab, mem_cd, (i) + 6, 1, dummy_store);
285
286#define dec_rounds32(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
287 y6, y7, mem_ab, mem_cd, i) \
288 two_roundsm32(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
289 y6, y7, mem_ab, mem_cd, (i) + 7, -1, store_ab_state); \
290 two_roundsm32(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
291 y6, y7, mem_ab, mem_cd, (i) + 5, -1, store_ab_state); \
292 two_roundsm32(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
293 y6, y7, mem_ab, mem_cd, (i) + 3, -1, dummy_store);
294
295/*
296 * IN:
297 * v0..3: byte-sliced 32-bit integers
298 * OUT:
299 * v0..3: (IN <<< 1)
300 */
301#define rol32_1_32(v0, v1, v2, v3, t0, t1, t2, zero) \
302 vpcmpgtb v0, zero, t0; \
303 vpaddb v0, v0, v0; \
304 vpabsb t0, t0; \
305 \
306 vpcmpgtb v1, zero, t1; \
307 vpaddb v1, v1, v1; \
308 vpabsb t1, t1; \
309 \
310 vpcmpgtb v2, zero, t2; \
311 vpaddb v2, v2, v2; \
312 vpabsb t2, t2; \
313 \
314 vpor t0, v1, v1; \
315 \
316 vpcmpgtb v3, zero, t0; \
317 vpaddb v3, v3, v3; \
318 vpabsb t0, t0; \
319 \
320 vpor t1, v2, v2; \
321 vpor t2, v3, v3; \
322 vpor t0, v0, v0;
323
324/*
325 * IN:
326 * r: byte-sliced AB state in memory
327 * l: byte-sliced CD state in memory
328 * OUT:
329 * x0..x7: new byte-sliced CD state
330 */
331#define fls32(l, l0, l1, l2, l3, l4, l5, l6, l7, r, t0, t1, t2, t3, tt0, \
332 tt1, tt2, tt3, kll, klr, krl, krr) \
333 /* \
334 * t0 = kll; \
335 * t0 &= ll; \
336 * lr ^= rol32(t0, 1); \
337 */ \
338 vpbroadcastd kll, t0; /* only lowest 32-bit used */ \
339 vpxor tt0, tt0, tt0; \
340 vpshufb tt0, t0, t3; \
341 vpsrldq $1, t0, t0; \
342 vpshufb tt0, t0, t2; \
343 vpsrldq $1, t0, t0; \
344 vpshufb tt0, t0, t1; \
345 vpsrldq $1, t0, t0; \
346 vpshufb tt0, t0, t0; \
347 \
348 vpand l0, t0, t0; \
349 vpand l1, t1, t1; \
350 vpand l2, t2, t2; \
351 vpand l3, t3, t3; \
352 \
353 rol32_1_32(t3, t2, t1, t0, tt1, tt2, tt3, tt0); \
354 \
355 vpxor l4, t0, l4; \
356 vpbroadcastd krr, t0; /* only lowest 32-bit used */ \
357 vmovdqu l4, 4 * 32(l); \
358 vpxor l5, t1, l5; \
359 vmovdqu l5, 5 * 32(l); \
360 vpxor l6, t2, l6; \
361 vmovdqu l6, 6 * 32(l); \
362 vpxor l7, t3, l7; \
363 vmovdqu l7, 7 * 32(l); \
364 \
365 /* \
366 * t2 = krr; \
367 * t2 |= rr; \
368 * rl ^= t2; \
369 */ \
370 \
371 vpshufb tt0, t0, t3; \
372 vpsrldq $1, t0, t0; \
373 vpshufb tt0, t0, t2; \
374 vpsrldq $1, t0, t0; \
375 vpshufb tt0, t0, t1; \
376 vpsrldq $1, t0, t0; \
377 vpshufb tt0, t0, t0; \
378 \
379 vpor 4 * 32(r), t0, t0; \
380 vpor 5 * 32(r), t1, t1; \
381 vpor 6 * 32(r), t2, t2; \
382 vpor 7 * 32(r), t3, t3; \
383 \
384 vpxor 0 * 32(r), t0, t0; \
385 vpxor 1 * 32(r), t1, t1; \
386 vpxor 2 * 32(r), t2, t2; \
387 vpxor 3 * 32(r), t3, t3; \
388 vmovdqu t0, 0 * 32(r); \
389 vpbroadcastd krl, t0; /* only lowest 32-bit used */ \
390 vmovdqu t1, 1 * 32(r); \
391 vmovdqu t2, 2 * 32(r); \
392 vmovdqu t3, 3 * 32(r); \
393 \
394 /* \
395 * t2 = krl; \
396 * t2 &= rl; \
397 * rr ^= rol32(t2, 1); \
398 */ \
399 vpshufb tt0, t0, t3; \
400 vpsrldq $1, t0, t0; \
401 vpshufb tt0, t0, t2; \
402 vpsrldq $1, t0, t0; \
403 vpshufb tt0, t0, t1; \
404 vpsrldq $1, t0, t0; \
405 vpshufb tt0, t0, t0; \
406 \
407 vpand 0 * 32(r), t0, t0; \
408 vpand 1 * 32(r), t1, t1; \
409 vpand 2 * 32(r), t2, t2; \
410 vpand 3 * 32(r), t3, t3; \
411 \
412 rol32_1_32(t3, t2, t1, t0, tt1, tt2, tt3, tt0); \
413 \
414 vpxor 4 * 32(r), t0, t0; \
415 vpxor 5 * 32(r), t1, t1; \
416 vpxor 6 * 32(r), t2, t2; \
417 vpxor 7 * 32(r), t3, t3; \
418 vmovdqu t0, 4 * 32(r); \
419 vpbroadcastd klr, t0; /* only lowest 32-bit used */ \
420 vmovdqu t1, 5 * 32(r); \
421 vmovdqu t2, 6 * 32(r); \
422 vmovdqu t3, 7 * 32(r); \
423 \
424 /* \
425 * t0 = klr; \
426 * t0 |= lr; \
427 * ll ^= t0; \
428 */ \
429 \
430 vpshufb tt0, t0, t3; \
431 vpsrldq $1, t0, t0; \
432 vpshufb tt0, t0, t2; \
433 vpsrldq $1, t0, t0; \
434 vpshufb tt0, t0, t1; \
435 vpsrldq $1, t0, t0; \
436 vpshufb tt0, t0, t0; \
437 \
438 vpor l4, t0, t0; \
439 vpor l5, t1, t1; \
440 vpor l6, t2, t2; \
441 vpor l7, t3, t3; \
442 \
443 vpxor l0, t0, l0; \
444 vmovdqu l0, 0 * 32(l); \
445 vpxor l1, t1, l1; \
446 vmovdqu l1, 1 * 32(l); \
447 vpxor l2, t2, l2; \
448 vmovdqu l2, 2 * 32(l); \
449 vpxor l3, t3, l3; \
450 vmovdqu l3, 3 * 32(l);
451
452#define transpose_4x4(x0, x1, x2, x3, t1, t2) \
453 vpunpckhdq x1, x0, t2; \
454 vpunpckldq x1, x0, x0; \
455 \
456 vpunpckldq x3, x2, t1; \
457 vpunpckhdq x3, x2, x2; \
458 \
459 vpunpckhqdq t1, x0, x1; \
460 vpunpcklqdq t1, x0, x0; \
461 \
462 vpunpckhqdq x2, t2, x3; \
463 vpunpcklqdq x2, t2, x2;
464
465#define byteslice_16x16b_fast(a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, \
466 a3, b3, c3, d3, st0, st1) \
467 vmovdqu d2, st0; \
468 vmovdqu d3, st1; \
469 transpose_4x4(a0, a1, a2, a3, d2, d3); \
470 transpose_4x4(b0, b1, b2, b3, d2, d3); \
471 vmovdqu st0, d2; \
472 vmovdqu st1, d3; \
473 \
474 vmovdqu a0, st0; \
475 vmovdqu a1, st1; \
476 transpose_4x4(c0, c1, c2, c3, a0, a1); \
477 transpose_4x4(d0, d1, d2, d3, a0, a1); \
478 \
479 vbroadcasti128 .Lshufb_16x16b(%rip), a0; \
480 vmovdqu st1, a1; \
481 vpshufb a0, a2, a2; \
482 vpshufb a0, a3, a3; \
483 vpshufb a0, b0, b0; \
484 vpshufb a0, b1, b1; \
485 vpshufb a0, b2, b2; \
486 vpshufb a0, b3, b3; \
487 vpshufb a0, a1, a1; \
488 vpshufb a0, c0, c0; \
489 vpshufb a0, c1, c1; \
490 vpshufb a0, c2, c2; \
491 vpshufb a0, c3, c3; \
492 vpshufb a0, d0, d0; \
493 vpshufb a0, d1, d1; \
494 vpshufb a0, d2, d2; \
495 vpshufb a0, d3, d3; \
496 vmovdqu d3, st1; \
497 vmovdqu st0, d3; \
498 vpshufb a0, d3, a0; \
499 vmovdqu d2, st0; \
500 \
501 transpose_4x4(a0, b0, c0, d0, d2, d3); \
502 transpose_4x4(a1, b1, c1, d1, d2, d3); \
503 vmovdqu st0, d2; \
504 vmovdqu st1, d3; \
505 \
506 vmovdqu b0, st0; \
507 vmovdqu b1, st1; \
508 transpose_4x4(a2, b2, c2, d2, b0, b1); \
509 transpose_4x4(a3, b3, c3, d3, b0, b1); \
510 vmovdqu st0, b0; \
511 vmovdqu st1, b1; \
512 /* does not adjust output bytes inside vectors */
513
514/* load blocks to registers and apply pre-whitening */
515#define inpack32_pre(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
516 y6, y7, rio, key) \
517 vpbroadcastq key, x0; \
518 vpshufb .Lpack_bswap(%rip), x0, x0; \
519 \
520 vpxor 0 * 32(rio), x0, y7; \
521 vpxor 1 * 32(rio), x0, y6; \
522 vpxor 2 * 32(rio), x0, y5; \
523 vpxor 3 * 32(rio), x0, y4; \
524 vpxor 4 * 32(rio), x0, y3; \
525 vpxor 5 * 32(rio), x0, y2; \
526 vpxor 6 * 32(rio), x0, y1; \
527 vpxor 7 * 32(rio), x0, y0; \
528 vpxor 8 * 32(rio), x0, x7; \
529 vpxor 9 * 32(rio), x0, x6; \
530 vpxor 10 * 32(rio), x0, x5; \
531 vpxor 11 * 32(rio), x0, x4; \
532 vpxor 12 * 32(rio), x0, x3; \
533 vpxor 13 * 32(rio), x0, x2; \
534 vpxor 14 * 32(rio), x0, x1; \
535 vpxor 15 * 32(rio), x0, x0;
536
537/* byteslice pre-whitened blocks and store to temporary memory */
538#define inpack32_post(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
539 y6, y7, mem_ab, mem_cd) \
540 byteslice_16x16b_fast(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, \
541 y4, y5, y6, y7, (mem_ab), (mem_cd)); \
542 \
543 vmovdqu x0, 0 * 32(mem_ab); \
544 vmovdqu x1, 1 * 32(mem_ab); \
545 vmovdqu x2, 2 * 32(mem_ab); \
546 vmovdqu x3, 3 * 32(mem_ab); \
547 vmovdqu x4, 4 * 32(mem_ab); \
548 vmovdqu x5, 5 * 32(mem_ab); \
549 vmovdqu x6, 6 * 32(mem_ab); \
550 vmovdqu x7, 7 * 32(mem_ab); \
551 vmovdqu y0, 0 * 32(mem_cd); \
552 vmovdqu y1, 1 * 32(mem_cd); \
553 vmovdqu y2, 2 * 32(mem_cd); \
554 vmovdqu y3, 3 * 32(mem_cd); \
555 vmovdqu y4, 4 * 32(mem_cd); \
556 vmovdqu y5, 5 * 32(mem_cd); \
557 vmovdqu y6, 6 * 32(mem_cd); \
558 vmovdqu y7, 7 * 32(mem_cd);
559
560/* de-byteslice, apply post-whitening and store blocks */
561#define outunpack32(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, \
562 y5, y6, y7, key, stack_tmp0, stack_tmp1) \
563 byteslice_16x16b_fast(y0, y4, x0, x4, y1, y5, x1, x5, y2, y6, x2, x6, \
564 y3, y7, x3, x7, stack_tmp0, stack_tmp1); \
565 \
566 vmovdqu x0, stack_tmp0; \
567 \
568 vpbroadcastq key, x0; \
569 vpshufb .Lpack_bswap(%rip), x0, x0; \
570 \
571 vpxor x0, y7, y7; \
572 vpxor x0, y6, y6; \
573 vpxor x0, y5, y5; \
574 vpxor x0, y4, y4; \
575 vpxor x0, y3, y3; \
576 vpxor x0, y2, y2; \
577 vpxor x0, y1, y1; \
578 vpxor x0, y0, y0; \
579 vpxor x0, x7, x7; \
580 vpxor x0, x6, x6; \
581 vpxor x0, x5, x5; \
582 vpxor x0, x4, x4; \
583 vpxor x0, x3, x3; \
584 vpxor x0, x2, x2; \
585 vpxor x0, x1, x1; \
586 vpxor stack_tmp0, x0, x0;
587
588#define write_output(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \
589 y6, y7, rio) \
590 vmovdqu x0, 0 * 32(rio); \
591 vmovdqu x1, 1 * 32(rio); \
592 vmovdqu x2, 2 * 32(rio); \
593 vmovdqu x3, 3 * 32(rio); \
594 vmovdqu x4, 4 * 32(rio); \
595 vmovdqu x5, 5 * 32(rio); \
596 vmovdqu x6, 6 * 32(rio); \
597 vmovdqu x7, 7 * 32(rio); \
598 vmovdqu y0, 8 * 32(rio); \
599 vmovdqu y1, 9 * 32(rio); \
600 vmovdqu y2, 10 * 32(rio); \
601 vmovdqu y3, 11 * 32(rio); \
602 vmovdqu y4, 12 * 32(rio); \
603 vmovdqu y5, 13 * 32(rio); \
604 vmovdqu y6, 14 * 32(rio); \
605 vmovdqu y7, 15 * 32(rio);
606
607
608.section .rodata.cst32.shufb_16x16b, "aM", @progbits, 32
609.align 32
610#define SHUFB_BYTES(idx) \
611 0 + (idx), 4 + (idx), 8 + (idx), 12 + (idx)
612.Lshufb_16x16b:
613 .byte SHUFB_BYTES(0), SHUFB_BYTES(1), SHUFB_BYTES(2), SHUFB_BYTES(3)
614 .byte SHUFB_BYTES(0), SHUFB_BYTES(1), SHUFB_BYTES(2), SHUFB_BYTES(3)
615
616.section .rodata.cst32.pack_bswap, "aM", @progbits, 32
617.align 32
618.Lpack_bswap:
619 .long 0x00010203, 0x04050607, 0x80808080, 0x80808080
620 .long 0x00010203, 0x04050607, 0x80808080, 0x80808080
621
622/* NB: section is mergeable, all elements must be aligned 16-byte blocks */
623.section .rodata.cst16, "aM", @progbits, 16
624.align 16
625
626/*
627 * pre-SubByte transform
628 *
629 * pre-lookup for sbox1, sbox2, sbox3:
630 * swap_bitendianness(
631 * isom_map_camellia_to_aes(
632 * camellia_f(
633 * swap_bitendianess(in)
634 * )
635 * )
636 * )
637 *
638 * (note: '⊕ 0xc5' inside camellia_f())
639 */
640.Lpre_tf_lo_s1:
641 .byte 0x45, 0xe8, 0x40, 0xed, 0x2e, 0x83, 0x2b, 0x86
642 .byte 0x4b, 0xe6, 0x4e, 0xe3, 0x20, 0x8d, 0x25, 0x88
643.Lpre_tf_hi_s1:
644 .byte 0x00, 0x51, 0xf1, 0xa0, 0x8a, 0xdb, 0x7b, 0x2a
645 .byte 0x09, 0x58, 0xf8, 0xa9, 0x83, 0xd2, 0x72, 0x23
646
647/*
648 * pre-SubByte transform
649 *
650 * pre-lookup for sbox4:
651 * swap_bitendianness(
652 * isom_map_camellia_to_aes(
653 * camellia_f(
654 * swap_bitendianess(in <<< 1)
655 * )
656 * )
657 * )
658 *
659 * (note: '⊕ 0xc5' inside camellia_f())
660 */
661.Lpre_tf_lo_s4:
662 .byte 0x45, 0x40, 0x2e, 0x2b, 0x4b, 0x4e, 0x20, 0x25
663 .byte 0x14, 0x11, 0x7f, 0x7a, 0x1a, 0x1f, 0x71, 0x74
664.Lpre_tf_hi_s4:
665 .byte 0x00, 0xf1, 0x8a, 0x7b, 0x09, 0xf8, 0x83, 0x72
666 .byte 0xad, 0x5c, 0x27, 0xd6, 0xa4, 0x55, 0x2e, 0xdf
667
668/*
669 * post-SubByte transform
670 *
671 * post-lookup for sbox1, sbox4:
672 * swap_bitendianness(
673 * camellia_h(
674 * isom_map_aes_to_camellia(
675 * swap_bitendianness(
676 * aes_inverse_affine_transform(in)
677 * )
678 * )
679 * )
680 * )
681 *
682 * (note: '⊕ 0x6e' inside camellia_h())
683 */
684.Lpost_tf_lo_s1:
685 .byte 0x3c, 0xcc, 0xcf, 0x3f, 0x32, 0xc2, 0xc1, 0x31
686 .byte 0xdc, 0x2c, 0x2f, 0xdf, 0xd2, 0x22, 0x21, 0xd1
687.Lpost_tf_hi_s1:
688 .byte 0x00, 0xf9, 0x86, 0x7f, 0xd7, 0x2e, 0x51, 0xa8
689 .byte 0xa4, 0x5d, 0x22, 0xdb, 0x73, 0x8a, 0xf5, 0x0c
690
691/*
692 * post-SubByte transform
693 *
694 * post-lookup for sbox2:
695 * swap_bitendianness(
696 * camellia_h(
697 * isom_map_aes_to_camellia(
698 * swap_bitendianness(
699 * aes_inverse_affine_transform(in)
700 * )
701 * )
702 * )
703 * ) <<< 1
704 *
705 * (note: '⊕ 0x6e' inside camellia_h())
706 */
707.Lpost_tf_lo_s2:
708 .byte 0x78, 0x99, 0x9f, 0x7e, 0x64, 0x85, 0x83, 0x62
709 .byte 0xb9, 0x58, 0x5e, 0xbf, 0xa5, 0x44, 0x42, 0xa3
710.Lpost_tf_hi_s2:
711 .byte 0x00, 0xf3, 0x0d, 0xfe, 0xaf, 0x5c, 0xa2, 0x51
712 .byte 0x49, 0xba, 0x44, 0xb7, 0xe6, 0x15, 0xeb, 0x18
713
714/*
715 * post-SubByte transform
716 *
717 * post-lookup for sbox3:
718 * swap_bitendianness(
719 * camellia_h(
720 * isom_map_aes_to_camellia(
721 * swap_bitendianness(
722 * aes_inverse_affine_transform(in)
723 * )
724 * )
725 * )
726 * ) >>> 1
727 *
728 * (note: '⊕ 0x6e' inside camellia_h())
729 */
730.Lpost_tf_lo_s3:
731 .byte 0x1e, 0x66, 0xe7, 0x9f, 0x19, 0x61, 0xe0, 0x98
732 .byte 0x6e, 0x16, 0x97, 0xef, 0x69, 0x11, 0x90, 0xe8
733.Lpost_tf_hi_s3:
734 .byte 0x00, 0xfc, 0x43, 0xbf, 0xeb, 0x17, 0xa8, 0x54
735 .byte 0x52, 0xae, 0x11, 0xed, 0xb9, 0x45, 0xfa, 0x06
736
737/* For isolating SubBytes from AESENCLAST, inverse shift row */
738.Linv_shift_row:
739 .byte 0x00, 0x0d, 0x0a, 0x07, 0x04, 0x01, 0x0e, 0x0b
740 .byte 0x08, 0x05, 0x02, 0x0f, 0x0c, 0x09, 0x06, 0x03
741
742.section .rodata.cst4.L0f0f0f0f, "aM", @progbits, 4
743.align 4
744/* 4-bit mask */
745.L0f0f0f0f:
746 .long 0x0f0f0f0f
747
748.text
749
750SYM_FUNC_START_LOCAL(__camellia_enc_blk32)
751 /* input:
752 * %rdi: ctx, CTX
753 * %rax: temporary storage, 512 bytes
754 * %ymm0..%ymm15: 32 plaintext blocks
755 * output:
756 * %ymm0..%ymm15: 32 encrypted blocks, order swapped:
757 * 7, 8, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8
758 */
759 FRAME_BEGIN
760
761 leaq 8 * 32(%rax), %rcx;
762
763 inpack32_post(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
764 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
765 %ymm15, %rax, %rcx);
766
767 enc_rounds32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
768 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
769 %ymm15, %rax, %rcx, 0);
770
771 fls32(%rax, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
772 %rcx, %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
773 %ymm15,
774 ((key_table + (8) * 8) + 0)(CTX),
775 ((key_table + (8) * 8) + 4)(CTX),
776 ((key_table + (8) * 8) + 8)(CTX),
777 ((key_table + (8) * 8) + 12)(CTX));
778
779 enc_rounds32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
780 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
781 %ymm15, %rax, %rcx, 8);
782
783 fls32(%rax, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
784 %rcx, %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
785 %ymm15,
786 ((key_table + (16) * 8) + 0)(CTX),
787 ((key_table + (16) * 8) + 4)(CTX),
788 ((key_table + (16) * 8) + 8)(CTX),
789 ((key_table + (16) * 8) + 12)(CTX));
790
791 enc_rounds32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
792 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
793 %ymm15, %rax, %rcx, 16);
794
795 movl $24, %r8d;
796 cmpl $16, key_length(CTX);
797 jne .Lenc_max32;
798
799.Lenc_done:
800 /* load CD for output */
801 vmovdqu 0 * 32(%rcx), %ymm8;
802 vmovdqu 1 * 32(%rcx), %ymm9;
803 vmovdqu 2 * 32(%rcx), %ymm10;
804 vmovdqu 3 * 32(%rcx), %ymm11;
805 vmovdqu 4 * 32(%rcx), %ymm12;
806 vmovdqu 5 * 32(%rcx), %ymm13;
807 vmovdqu 6 * 32(%rcx), %ymm14;
808 vmovdqu 7 * 32(%rcx), %ymm15;
809
810 outunpack32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
811 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
812 %ymm15, (key_table)(CTX, %r8, 8), (%rax), 1 * 32(%rax));
813
814 FRAME_END
815 RET;
816
817.align 8
818.Lenc_max32:
819 movl $32, %r8d;
820
821 fls32(%rax, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
822 %rcx, %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
823 %ymm15,
824 ((key_table + (24) * 8) + 0)(CTX),
825 ((key_table + (24) * 8) + 4)(CTX),
826 ((key_table + (24) * 8) + 8)(CTX),
827 ((key_table + (24) * 8) + 12)(CTX));
828
829 enc_rounds32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
830 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
831 %ymm15, %rax, %rcx, 24);
832
833 jmp .Lenc_done;
834SYM_FUNC_END(__camellia_enc_blk32)
835
836SYM_FUNC_START_LOCAL(__camellia_dec_blk32)
837 /* input:
838 * %rdi: ctx, CTX
839 * %rax: temporary storage, 512 bytes
840 * %r8d: 24 for 16 byte key, 32 for larger
841 * %ymm0..%ymm15: 16 encrypted blocks
842 * output:
843 * %ymm0..%ymm15: 16 plaintext blocks, order swapped:
844 * 7, 8, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8
845 */
846 FRAME_BEGIN
847
848 leaq 8 * 32(%rax), %rcx;
849
850 inpack32_post(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
851 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
852 %ymm15, %rax, %rcx);
853
854 cmpl $32, %r8d;
855 je .Ldec_max32;
856
857.Ldec_max24:
858 dec_rounds32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
859 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
860 %ymm15, %rax, %rcx, 16);
861
862 fls32(%rax, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
863 %rcx, %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
864 %ymm15,
865 ((key_table + (16) * 8) + 8)(CTX),
866 ((key_table + (16) * 8) + 12)(CTX),
867 ((key_table + (16) * 8) + 0)(CTX),
868 ((key_table + (16) * 8) + 4)(CTX));
869
870 dec_rounds32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
871 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
872 %ymm15, %rax, %rcx, 8);
873
874 fls32(%rax, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
875 %rcx, %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
876 %ymm15,
877 ((key_table + (8) * 8) + 8)(CTX),
878 ((key_table + (8) * 8) + 12)(CTX),
879 ((key_table + (8) * 8) + 0)(CTX),
880 ((key_table + (8) * 8) + 4)(CTX));
881
882 dec_rounds32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
883 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
884 %ymm15, %rax, %rcx, 0);
885
886 /* load CD for output */
887 vmovdqu 0 * 32(%rcx), %ymm8;
888 vmovdqu 1 * 32(%rcx), %ymm9;
889 vmovdqu 2 * 32(%rcx), %ymm10;
890 vmovdqu 3 * 32(%rcx), %ymm11;
891 vmovdqu 4 * 32(%rcx), %ymm12;
892 vmovdqu 5 * 32(%rcx), %ymm13;
893 vmovdqu 6 * 32(%rcx), %ymm14;
894 vmovdqu 7 * 32(%rcx), %ymm15;
895
896 outunpack32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
897 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
898 %ymm15, (key_table)(CTX), (%rax), 1 * 32(%rax));
899
900 FRAME_END
901 RET;
902
903.align 8
904.Ldec_max32:
905 dec_rounds32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
906 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
907 %ymm15, %rax, %rcx, 24);
908
909 fls32(%rax, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
910 %rcx, %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
911 %ymm15,
912 ((key_table + (24) * 8) + 8)(CTX),
913 ((key_table + (24) * 8) + 12)(CTX),
914 ((key_table + (24) * 8) + 0)(CTX),
915 ((key_table + (24) * 8) + 4)(CTX));
916
917 jmp .Ldec_max24;
918SYM_FUNC_END(__camellia_dec_blk32)
919
920SYM_FUNC_START(camellia_ecb_enc_32way)
921 /* input:
922 * %rdi: ctx, CTX
923 * %rsi: dst (32 blocks)
924 * %rdx: src (32 blocks)
925 */
926 FRAME_BEGIN
927
928 vzeroupper;
929
930 inpack32_pre(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
931 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
932 %ymm15, %rdx, (key_table)(CTX));
933
934 /* now dst can be used as temporary buffer (even in src == dst case) */
935 movq %rsi, %rax;
936
937 call __camellia_enc_blk32;
938
939 write_output(%ymm7, %ymm6, %ymm5, %ymm4, %ymm3, %ymm2, %ymm1, %ymm0,
940 %ymm15, %ymm14, %ymm13, %ymm12, %ymm11, %ymm10, %ymm9,
941 %ymm8, %rsi);
942
943 vzeroupper;
944
945 FRAME_END
946 RET;
947SYM_FUNC_END(camellia_ecb_enc_32way)
948
949SYM_FUNC_START(camellia_ecb_dec_32way)
950 /* input:
951 * %rdi: ctx, CTX
952 * %rsi: dst (32 blocks)
953 * %rdx: src (32 blocks)
954 */
955 FRAME_BEGIN
956
957 vzeroupper;
958
959 cmpl $16, key_length(CTX);
960 movl $32, %r8d;
961 movl $24, %eax;
962 cmovel %eax, %r8d; /* max */
963
964 inpack32_pre(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
965 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
966 %ymm15, %rdx, (key_table)(CTX, %r8, 8));
967
968 /* now dst can be used as temporary buffer (even in src == dst case) */
969 movq %rsi, %rax;
970
971 call __camellia_dec_blk32;
972
973 write_output(%ymm7, %ymm6, %ymm5, %ymm4, %ymm3, %ymm2, %ymm1, %ymm0,
974 %ymm15, %ymm14, %ymm13, %ymm12, %ymm11, %ymm10, %ymm9,
975 %ymm8, %rsi);
976
977 vzeroupper;
978
979 FRAME_END
980 RET;
981SYM_FUNC_END(camellia_ecb_dec_32way)
982
983SYM_FUNC_START(camellia_cbc_dec_32way)
984 /* input:
985 * %rdi: ctx, CTX
986 * %rsi: dst (32 blocks)
987 * %rdx: src (32 blocks)
988 */
989 FRAME_BEGIN
990 subq $(16 * 32), %rsp;
991
992 vzeroupper;
993
994 cmpl $16, key_length(CTX);
995 movl $32, %r8d;
996 movl $24, %eax;
997 cmovel %eax, %r8d; /* max */
998
999 inpack32_pre(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7,
1000 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
1001 %ymm15, %rdx, (key_table)(CTX, %r8, 8));
1002
1003 cmpq %rsi, %rdx;
1004 je .Lcbc_dec_use_stack;
1005
1006 /* dst can be used as temporary storage, src is not overwritten. */
1007 movq %rsi, %rax;
1008 jmp .Lcbc_dec_continue;
1009
1010.Lcbc_dec_use_stack:
1011 /*
1012 * dst still in-use (because dst == src), so use stack for temporary
1013 * storage.
1014 */
1015 movq %rsp, %rax;
1016
1017.Lcbc_dec_continue:
1018 call __camellia_dec_blk32;
1019
1020 vmovdqu %ymm7, (%rax);
1021 vpxor %ymm7, %ymm7, %ymm7;
1022 vinserti128 $1, (%rdx), %ymm7, %ymm7;
1023 vpxor (%rax), %ymm7, %ymm7;
1024 vpxor (0 * 32 + 16)(%rdx), %ymm6, %ymm6;
1025 vpxor (1 * 32 + 16)(%rdx), %ymm5, %ymm5;
1026 vpxor (2 * 32 + 16)(%rdx), %ymm4, %ymm4;
1027 vpxor (3 * 32 + 16)(%rdx), %ymm3, %ymm3;
1028 vpxor (4 * 32 + 16)(%rdx), %ymm2, %ymm2;
1029 vpxor (5 * 32 + 16)(%rdx), %ymm1, %ymm1;
1030 vpxor (6 * 32 + 16)(%rdx), %ymm0, %ymm0;
1031 vpxor (7 * 32 + 16)(%rdx), %ymm15, %ymm15;
1032 vpxor (8 * 32 + 16)(%rdx), %ymm14, %ymm14;
1033 vpxor (9 * 32 + 16)(%rdx), %ymm13, %ymm13;
1034 vpxor (10 * 32 + 16)(%rdx), %ymm12, %ymm12;
1035 vpxor (11 * 32 + 16)(%rdx), %ymm11, %ymm11;
1036 vpxor (12 * 32 + 16)(%rdx), %ymm10, %ymm10;
1037 vpxor (13 * 32 + 16)(%rdx), %ymm9, %ymm9;
1038 vpxor (14 * 32 + 16)(%rdx), %ymm8, %ymm8;
1039 write_output(%ymm7, %ymm6, %ymm5, %ymm4, %ymm3, %ymm2, %ymm1, %ymm0,
1040 %ymm15, %ymm14, %ymm13, %ymm12, %ymm11, %ymm10, %ymm9,
1041 %ymm8, %rsi);
1042
1043 vzeroupper;
1044
1045 addq $(16 * 32), %rsp;
1046 FRAME_END
1047 RET;
1048SYM_FUNC_END(camellia_cbc_dec_32way)
1049

source code of linux/arch/x86/crypto/camellia-aesni-avx2-asm_64.S