1	/* SPDX-License-Identifier: GPL-2.0-or-later */
2	/*
3	* ARIA Cipher 16-way parallel algorithm (AVX)
4	*
5	* Copyright (c) 2022 Taehee Yoo <ap420073@gmail.com>
6	*
7	*/
8
9	#include <linux/linkage.h>
10	#include <linux/cfi_types.h>
11	#include <asm/asm-offsets.h>
12	#include <asm/frame.h>
13
14	/* register macros */
15	#define CTX %rdi
16
17
18	#define BV8(a0, a1, a2, a3, a4, a5, a6, a7) \
19	( (((a0) & 1) << 0) | \
20	(((a1) & 1) << 1) | \
21	(((a2) & 1) << 2) | \
22	(((a3) & 1) << 3) | \
23	(((a4) & 1) << 4) | \
24	(((a5) & 1) << 5) | \
25	(((a6) & 1) << 6) | \
26	(((a7) & 1) << 7) )
27
28	#define BM8X8(l0, l1, l2, l3, l4, l5, l6, l7) \
29	( ((l7) << (0 * 8)) | \
30	((l6) << (1 * 8)) | \
31	((l5) << (2 * 8)) | \
32	((l4) << (3 * 8)) | \
33	((l3) << (4 * 8)) | \
34	((l2) << (5 * 8)) | \
35	((l1) << (6 * 8)) | \
36	((l0) << (7 * 8)) )
37
38	#define inc_le128(x, minus_one, tmp) \
39	vpcmpeqq minus_one, x, tmp; \
40	vpsubq minus_one, x, x; \
41	vpslldq $8, tmp, tmp; \
42	vpsubq tmp, x, x;
43
44	#define filter_8bit(x, lo_t, hi_t, mask4bit, tmp0) \
45	vpand x, mask4bit, tmp0; \
46	vpandn x, mask4bit, x; \
47	vpsrld $4, x, x; \
48	\
49	vpshufb tmp0, lo_t, tmp0; \
50	vpshufb x, hi_t, x; \
51	vpxor tmp0, x, x;
52
53	#define transpose_4x4(x0, x1, x2, x3, t1, t2) \
54	vpunpckhdq x1, x0, t2; \
55	vpunpckldq x1, x0, x0; \
56	\
57	vpunpckldq x3, x2, t1; \
58	vpunpckhdq x3, x2, x2; \
59	\
60	vpunpckhqdq t1, x0, x1; \
61	vpunpcklqdq t1, x0, x0; \
62	\
63	vpunpckhqdq x2, t2, x3; \
64	vpunpcklqdq x2, t2, x2;
65
66	#define byteslice_16x16b(a0, b0, c0, d0, \
67	a1, b1, c1, d1, \
68	a2, b2, c2, d2, \
69	a3, b3, c3, d3, \
70	st0, st1) \
71	vmovdqu d2, st0; \
72	vmovdqu d3, st1; \
73	transpose_4x4(a0, a1, a2, a3, d2, d3); \
74	transpose_4x4(b0, b1, b2, b3, d2, d3); \
75	vmovdqu st0, d2; \
76	vmovdqu st1, d3; \
77	\
78	vmovdqu a0, st0; \
79	vmovdqu a1, st1; \
80	transpose_4x4(c0, c1, c2, c3, a0, a1); \
81	transpose_4x4(d0, d1, d2, d3, a0, a1); \
82	\
83	vmovdqu .Lshufb_16x16b(%rip), a0; \
84	vmovdqu st1, a1; \
85	vpshufb a0, a2, a2; \
86	vpshufb a0, a3, a3; \
87	vpshufb a0, b0, b0; \
88	vpshufb a0, b1, b1; \
89	vpshufb a0, b2, b2; \
90	vpshufb a0, b3, b3; \
91	vpshufb a0, a1, a1; \
92	vpshufb a0, c0, c0; \
93	vpshufb a0, c1, c1; \
94	vpshufb a0, c2, c2; \
95	vpshufb a0, c3, c3; \
96	vpshufb a0, d0, d0; \
97	vpshufb a0, d1, d1; \
98	vpshufb a0, d2, d2; \
99	vpshufb a0, d3, d3; \
100	vmovdqu d3, st1; \
101	vmovdqu st0, d3; \
102	vpshufb a0, d3, a0; \
103	vmovdqu d2, st0; \
104	\
105	transpose_4x4(a0, b0, c0, d0, d2, d3); \
106	transpose_4x4(a1, b1, c1, d1, d2, d3); \
107	vmovdqu st0, d2; \
108	vmovdqu st1, d3; \
109	\
110	vmovdqu b0, st0; \
111	vmovdqu b1, st1; \
112	transpose_4x4(a2, b2, c2, d2, b0, b1); \
113	transpose_4x4(a3, b3, c3, d3, b0, b1); \
114	vmovdqu st0, b0; \
115	vmovdqu st1, b1; \
116	/* does not adjust output bytes inside vectors */
117
118	#define debyteslice_16x16b(a0, b0, c0, d0, \
119	a1, b1, c1, d1, \
120	a2, b2, c2, d2, \
121	a3, b3, c3, d3, \
122	st0, st1) \
123	vmovdqu d2, st0; \
124	vmovdqu d3, st1; \
125	transpose_4x4(a0, a1, a2, a3, d2, d3); \
126	transpose_4x4(b0, b1, b2, b3, d2, d3); \
127	vmovdqu st0, d2; \
128	vmovdqu st1, d3; \
129	\
130	vmovdqu a0, st0; \
131	vmovdqu a1, st1; \
132	transpose_4x4(c0, c1, c2, c3, a0, a1); \
133	transpose_4x4(d0, d1, d2, d3, a0, a1); \
134	\
135	vmovdqu .Lshufb_16x16b(%rip), a0; \
136	vmovdqu st1, a1; \
137	vpshufb a0, a2, a2; \
138	vpshufb a0, a3, a3; \
139	vpshufb a0, b0, b0; \
140	vpshufb a0, b1, b1; \
141	vpshufb a0, b2, b2; \
142	vpshufb a0, b3, b3; \
143	vpshufb a0, a1, a1; \
144	vpshufb a0, c0, c0; \
145	vpshufb a0, c1, c1; \
146	vpshufb a0, c2, c2; \
147	vpshufb a0, c3, c3; \
148	vpshufb a0, d0, d0; \
149	vpshufb a0, d1, d1; \
150	vpshufb a0, d2, d2; \
151	vpshufb a0, d3, d3; \
152	vmovdqu d3, st1; \
153	vmovdqu st0, d3; \
154	vpshufb a0, d3, a0; \
155	vmovdqu d2, st0; \
156	\
157	transpose_4x4(c0, d0, a0, b0, d2, d3); \
158	transpose_4x4(c1, d1, a1, b1, d2, d3); \
159	vmovdqu st0, d2; \
160	vmovdqu st1, d3; \
161	\
162	vmovdqu b0, st0; \
163	vmovdqu b1, st1; \
164	transpose_4x4(c2, d2, a2, b2, b0, b1); \
165	transpose_4x4(c3, d3, a3, b3, b0, b1); \
166	vmovdqu st0, b0; \
167	vmovdqu st1, b1; \
168	/* does not adjust output bytes inside vectors */
169
170	/* load blocks to registers and apply pre-whitening */
171	#define inpack16_pre(x0, x1, x2, x3, \
172	x4, x5, x6, x7, \
173	y0, y1, y2, y3, \
174	y4, y5, y6, y7, \
175	rio) \
176	vmovdqu (0 * 16)(rio), x0; \
177	vmovdqu (1 * 16)(rio), x1; \
178	vmovdqu (2 * 16)(rio), x2; \
179	vmovdqu (3 * 16)(rio), x3; \
180	vmovdqu (4 * 16)(rio), x4; \
181	vmovdqu (5 * 16)(rio), x5; \
182	vmovdqu (6 * 16)(rio), x6; \
183	vmovdqu (7 * 16)(rio), x7; \
184	vmovdqu (8 * 16)(rio), y0; \
185	vmovdqu (9 * 16)(rio), y1; \
186	vmovdqu (10 * 16)(rio), y2; \
187	vmovdqu (11 * 16)(rio), y3; \
188	vmovdqu (12 * 16)(rio), y4; \
189	vmovdqu (13 * 16)(rio), y5; \
190	vmovdqu (14 * 16)(rio), y6; \
191	vmovdqu (15 * 16)(rio), y7;
192
193	/* byteslice pre-whitened blocks and store to temporary memory */
194	#define inpack16_post(x0, x1, x2, x3, \
195	x4, x5, x6, x7, \
196	y0, y1, y2, y3, \
197	y4, y5, y6, y7, \
198	mem_ab, mem_cd) \
199	byteslice_16x16b(x0, x1, x2, x3, \
200	x4, x5, x6, x7, \
201	y0, y1, y2, y3, \
202	y4, y5, y6, y7, \
203	(mem_ab), (mem_cd)); \
204	\
205	vmovdqu x0, 0 * 16(mem_ab); \
206	vmovdqu x1, 1 * 16(mem_ab); \
207	vmovdqu x2, 2 * 16(mem_ab); \
208	vmovdqu x3, 3 * 16(mem_ab); \
209	vmovdqu x4, 4 * 16(mem_ab); \
210	vmovdqu x5, 5 * 16(mem_ab); \
211	vmovdqu x6, 6 * 16(mem_ab); \
212	vmovdqu x7, 7 * 16(mem_ab); \
213	vmovdqu y0, 0 * 16(mem_cd); \
214	vmovdqu y1, 1 * 16(mem_cd); \
215	vmovdqu y2, 2 * 16(mem_cd); \
216	vmovdqu y3, 3 * 16(mem_cd); \
217	vmovdqu y4, 4 * 16(mem_cd); \
218	vmovdqu y5, 5 * 16(mem_cd); \
219	vmovdqu y6, 6 * 16(mem_cd); \
220	vmovdqu y7, 7 * 16(mem_cd);
221
222	#define write_output(x0, x1, x2, x3, \
223	x4, x5, x6, x7, \
224	y0, y1, y2, y3, \
225	y4, y5, y6, y7, \
226	mem) \
227	vmovdqu x0, 0 * 16(mem); \
228	vmovdqu x1, 1 * 16(mem); \
229	vmovdqu x2, 2 * 16(mem); \
230	vmovdqu x3, 3 * 16(mem); \
231	vmovdqu x4, 4 * 16(mem); \
232	vmovdqu x5, 5 * 16(mem); \
233	vmovdqu x6, 6 * 16(mem); \
234	vmovdqu x7, 7 * 16(mem); \
235	vmovdqu y0, 8 * 16(mem); \
236	vmovdqu y1, 9 * 16(mem); \
237	vmovdqu y2, 10 * 16(mem); \
238	vmovdqu y3, 11 * 16(mem); \
239	vmovdqu y4, 12 * 16(mem); \
240	vmovdqu y5, 13 * 16(mem); \
241	vmovdqu y6, 14 * 16(mem); \
242	vmovdqu y7, 15 * 16(mem); \
243
244	#define aria_store_state_8way(x0, x1, x2, x3, \
245	x4, x5, x6, x7, \
246	mem_tmp, idx) \
247	vmovdqu x0, ((idx + 0) * 16)(mem_tmp); \
248	vmovdqu x1, ((idx + 1) * 16)(mem_tmp); \
249	vmovdqu x2, ((idx + 2) * 16)(mem_tmp); \
250	vmovdqu x3, ((idx + 3) * 16)(mem_tmp); \
251	vmovdqu x4, ((idx + 4) * 16)(mem_tmp); \
252	vmovdqu x5, ((idx + 5) * 16)(mem_tmp); \
253	vmovdqu x6, ((idx + 6) * 16)(mem_tmp); \
254	vmovdqu x7, ((idx + 7) * 16)(mem_tmp);
255
256	#define aria_load_state_8way(x0, x1, x2, x3, \
257	x4, x5, x6, x7, \
258	mem_tmp, idx) \
259	vmovdqu ((idx + 0) * 16)(mem_tmp), x0; \
260	vmovdqu ((idx + 1) * 16)(mem_tmp), x1; \
261	vmovdqu ((idx + 2) * 16)(mem_tmp), x2; \
262	vmovdqu ((idx + 3) * 16)(mem_tmp), x3; \
263	vmovdqu ((idx + 4) * 16)(mem_tmp), x4; \
264	vmovdqu ((idx + 5) * 16)(mem_tmp), x5; \
265	vmovdqu ((idx + 6) * 16)(mem_tmp), x6; \
266	vmovdqu ((idx + 7) * 16)(mem_tmp), x7;
267
268	#define aria_ark_8way(x0, x1, x2, x3, \
269	x4, x5, x6, x7, \
270	t0, t1, t2, rk, \
271	idx, round) \
272	/* AddRoundKey */ \
273	vbroadcastss ((round * 16) + idx + 0)(rk), t0; \
274	vpsrld $24, t0, t2; \
275	vpshufb t1, t2, t2; \
276	vpxor t2, x0, x0; \
277	vpsrld $16, t0, t2; \
278	vpshufb t1, t2, t2; \
279	vpxor t2, x1, x1; \
280	vpsrld $8, t0, t2; \
281	vpshufb t1, t2, t2; \
282	vpxor t2, x2, x2; \
283	vpshufb t1, t0, t2; \
284	vpxor t2, x3, x3; \
285	vbroadcastss ((round * 16) + idx + 4)(rk), t0; \
286	vpsrld $24, t0, t2; \
287	vpshufb t1, t2, t2; \
288	vpxor t2, x4, x4; \
289	vpsrld $16, t0, t2; \
290	vpshufb t1, t2, t2; \
291	vpxor t2, x5, x5; \
292	vpsrld $8, t0, t2; \
293	vpshufb t1, t2, t2; \
294	vpxor t2, x6, x6; \
295	vpshufb t1, t0, t2; \
296	vpxor t2, x7, x7;
297
298	#ifdef CONFIG_AS_GFNI
299	#define aria_sbox_8way_gfni(x0, x1, x2, x3, \
300	x4, x5, x6, x7, \
301	t0, t1, t2, t3, \
302	t4, t5, t6, t7) \
303	vmovdqa .Ltf_s2_bitmatrix(%rip), t0; \
304	vmovdqa .Ltf_inv_bitmatrix(%rip), t1; \
305	vmovdqa .Ltf_id_bitmatrix(%rip), t2; \
306	vmovdqa .Ltf_aff_bitmatrix(%rip), t3; \
307	vmovdqa .Ltf_x2_bitmatrix(%rip), t4; \
308	vgf2p8affineinvqb $(tf_s2_const), t0, x1, x1; \
309	vgf2p8affineinvqb $(tf_s2_const), t0, x5, x5; \
310	vgf2p8affineqb $(tf_inv_const), t1, x2, x2; \
311	vgf2p8affineqb $(tf_inv_const), t1, x6, x6; \
312	vgf2p8affineinvqb $0, t2, x2, x2; \
313	vgf2p8affineinvqb $0, t2, x6, x6; \
314	vgf2p8affineinvqb $(tf_aff_const), t3, x0, x0; \
315	vgf2p8affineinvqb $(tf_aff_const), t3, x4, x4; \
316	vgf2p8affineqb $(tf_x2_const), t4, x3, x3; \
317	vgf2p8affineqb $(tf_x2_const), t4, x7, x7; \
318	vgf2p8affineinvqb $0, t2, x3, x3; \
319	vgf2p8affineinvqb $0, t2, x7, x7
320
321	#endif /* CONFIG_AS_GFNI */
322
323	#define aria_sbox_8way(x0, x1, x2, x3, \
324	x4, x5, x6, x7, \
325	t0, t1, t2, t3, \
326	t4, t5, t6, t7) \
327	vmovdqa .Linv_shift_row(%rip), t0; \
328	vmovdqa .Lshift_row(%rip), t1; \
329	vbroadcastss .L0f0f0f0f(%rip), t6; \
330	vmovdqa .Ltf_lo__inv_aff__and__s2(%rip), t2; \
331	vmovdqa .Ltf_hi__inv_aff__and__s2(%rip), t3; \
332	vmovdqa .Ltf_lo__x2__and__fwd_aff(%rip), t4; \
333	vmovdqa .Ltf_hi__x2__and__fwd_aff(%rip), t5; \
334	\
335	vaesenclast t7, x0, x0; \
336	vaesenclast t7, x4, x4; \
337	vaesenclast t7, x1, x1; \
338	vaesenclast t7, x5, x5; \
339	vaesdeclast t7, x2, x2; \
340	vaesdeclast t7, x6, x6; \
341	\
342	/* AES inverse shift rows */ \
343	vpshufb t0, x0, x0; \
344	vpshufb t0, x4, x4; \
345	vpshufb t0, x1, x1; \
346	vpshufb t0, x5, x5; \
347	vpshufb t1, x3, x3; \
348	vpshufb t1, x7, x7; \
349	vpshufb t1, x2, x2; \
350	vpshufb t1, x6, x6; \
351	\
352	/* affine transformation for S2 */ \
353	filter_8bit(x1, t2, t3, t6, t0); \
354	/* affine transformation for S2 */ \
355	filter_8bit(x5, t2, t3, t6, t0); \
356	\
357	/* affine transformation for X2 */ \
358	filter_8bit(x3, t4, t5, t6, t0); \
359	/* affine transformation for X2 */ \
360	filter_8bit(x7, t4, t5, t6, t0); \
361	vaesdeclast t7, x3, x3; \
362	vaesdeclast t7, x7, x7;
363
364	#define aria_diff_m(x0, x1, x2, x3, \
365	t0, t1, t2, t3) \
366	/* T = rotr32(X, 8); */ \
367	/* X ^= T */ \
368	vpxor x0, x3, t0; \
369	vpxor x1, x0, t1; \
370	vpxor x2, x1, t2; \
371	vpxor x3, x2, t3; \
372	/* X = T ^ rotr(X, 16); */ \
373	vpxor t2, x0, x0; \
374	vpxor x1, t3, t3; \
375	vpxor t0, x2, x2; \
376	vpxor t1, x3, x1; \
377	vmovdqu t3, x3;
378
379	#define aria_diff_word(x0, x1, x2, x3, \
380	x4, x5, x6, x7, \
381	y0, y1, y2, y3, \
382	y4, y5, y6, y7) \
383	/* t1 ^= t2; */ \
384	vpxor y0, x4, x4; \
385	vpxor y1, x5, x5; \
386	vpxor y2, x6, x6; \
387	vpxor y3, x7, x7; \
388	\
389	/* t2 ^= t3; */ \
390	vpxor y4, y0, y0; \
391	vpxor y5, y1, y1; \
392	vpxor y6, y2, y2; \
393	vpxor y7, y3, y3; \
394	\
395	/* t0 ^= t1; */ \
396	vpxor x4, x0, x0; \
397	vpxor x5, x1, x1; \
398	vpxor x6, x2, x2; \
399	vpxor x7, x3, x3; \
400	\
401	/* t3 ^= t1; */ \
402	vpxor x4, y4, y4; \
403	vpxor x5, y5, y5; \
404	vpxor x6, y6, y6; \
405	vpxor x7, y7, y7; \
406	\
407	/* t2 ^= t0; */ \
408	vpxor x0, y0, y0; \
409	vpxor x1, y1, y1; \
410	vpxor x2, y2, y2; \
411	vpxor x3, y3, y3; \
412	\
413	/* t1 ^= t2; */ \
414	vpxor y0, x4, x4; \
415	vpxor y1, x5, x5; \
416	vpxor y2, x6, x6; \
417	vpxor y3, x7, x7;
418
419	#define aria_fe(x0, x1, x2, x3, \
420	x4, x5, x6, x7, \
421	y0, y1, y2, y3, \
422	y4, y5, y6, y7, \
423	mem_tmp, rk, round) \
424	vpxor y7, y7, y7; \
425	aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
426	y0, y7, y2, rk, 8, round); \
427	\
428	aria_sbox_8way(x2, x3, x0, x1, x6, x7, x4, x5, \
429	y0, y1, y2, y3, y4, y5, y6, y7); \
430	\
431	aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \
432	aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \
433	aria_store_state_8way(x0, x1, x2, x3, \
434	x4, x5, x6, x7, \
435	mem_tmp, 8); \
436	\
437	aria_load_state_8way(x0, x1, x2, x3, \
438	x4, x5, x6, x7, \
439	mem_tmp, 0); \
440	aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
441	y0, y7, y2, rk, 0, round); \
442	\
443	aria_sbox_8way(x2, x3, x0, x1, x6, x7, x4, x5, \
444	y0, y1, y2, y3, y4, y5, y6, y7); \
445	\
446	aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \
447	aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \
448	aria_store_state_8way(x0, x1, x2, x3, \
449	x4, x5, x6, x7, \
450	mem_tmp, 0); \
451	aria_load_state_8way(y0, y1, y2, y3, \
452	y4, y5, y6, y7, \
453	mem_tmp, 8); \
454	aria_diff_word(x0, x1, x2, x3, \
455	x4, x5, x6, x7, \
456	y0, y1, y2, y3, \
457	y4, y5, y6, y7); \
458	/* aria_diff_byte() \
459	* T3 = ABCD -> BADC \
460	* T3 = y4, y5, y6, y7 -> y5, y4, y7, y6 \
461	* T0 = ABCD -> CDAB \
462	* T0 = x0, x1, x2, x3 -> x2, x3, x0, x1 \
463	* T1 = ABCD -> DCBA \
464	* T1 = x4, x5, x6, x7 -> x7, x6, x5, x4 \
465	*/ \
466	aria_diff_word(x2, x3, x0, x1, \
467	x7, x6, x5, x4, \
468	y0, y1, y2, y3, \
469	y5, y4, y7, y6); \
470	aria_store_state_8way(x3, x2, x1, x0, \
471	x6, x7, x4, x5, \
472	mem_tmp, 0);
473
474	#define aria_fo(x0, x1, x2, x3, \
475	x4, x5, x6, x7, \
476	y0, y1, y2, y3, \
477	y4, y5, y6, y7, \
478	mem_tmp, rk, round) \
479	vpxor y7, y7, y7; \
480	aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
481	y0, y7, y2, rk, 8, round); \
482	\
483	aria_sbox_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
484	y0, y1, y2, y3, y4, y5, y6, y7); \
485	\
486	aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \
487	aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \
488	aria_store_state_8way(x0, x1, x2, x3, \
489	x4, x5, x6, x7, \
490	mem_tmp, 8); \
491	\
492	aria_load_state_8way(x0, x1, x2, x3, \
493	x4, x5, x6, x7, \
494	mem_tmp, 0); \
495	aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
496	y0, y7, y2, rk, 0, round); \
497	\
498	aria_sbox_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
499	y0, y1, y2, y3, y4, y5, y6, y7); \
500	\
501	aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \
502	aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \
503	aria_store_state_8way(x0, x1, x2, x3, \
504	x4, x5, x6, x7, \
505	mem_tmp, 0); \
506	aria_load_state_8way(y0, y1, y2, y3, \
507	y4, y5, y6, y7, \
508	mem_tmp, 8); \
509	aria_diff_word(x0, x1, x2, x3, \
510	x4, x5, x6, x7, \
511	y0, y1, y2, y3, \
512	y4, y5, y6, y7); \
513	/* aria_diff_byte() \
514	* T1 = ABCD -> BADC \
515	* T1 = x4, x5, x6, x7 -> x5, x4, x7, x6 \
516	* T2 = ABCD -> CDAB \
517	* T2 = y0, y1, y2, y3, -> y2, y3, y0, y1 \
518	* T3 = ABCD -> DCBA \
519	* T3 = y4, y5, y6, y7 -> y7, y6, y5, y4 \
520	*/ \
521	aria_diff_word(x0, x1, x2, x3, \
522	x5, x4, x7, x6, \
523	y2, y3, y0, y1, \
524	y7, y6, y5, y4); \
525	aria_store_state_8way(x3, x2, x1, x0, \
526	x6, x7, x4, x5, \
527	mem_tmp, 0);
528
529	#define aria_ff(x0, x1, x2, x3, \
530	x4, x5, x6, x7, \
531	y0, y1, y2, y3, \
532	y4, y5, y6, y7, \
533	mem_tmp, rk, round, last_round) \
534	vpxor y7, y7, y7; \
535	aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
536	y0, y7, y2, rk, 8, round); \
537	\
538	aria_sbox_8way(x2, x3, x0, x1, x6, x7, x4, x5, \
539	y0, y1, y2, y3, y4, y5, y6, y7); \
540	\
541	aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
542	y0, y7, y2, rk, 8, last_round); \
543	\
544	aria_store_state_8way(x0, x1, x2, x3, \
545	x4, x5, x6, x7, \
546	mem_tmp, 8); \
547	\
548	aria_load_state_8way(x0, x1, x2, x3, \
549	x4, x5, x6, x7, \
550	mem_tmp, 0); \
551	aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
552	y0, y7, y2, rk, 0, round); \
553	\
554	aria_sbox_8way(x2, x3, x0, x1, x6, x7, x4, x5, \
555	y0, y1, y2, y3, y4, y5, y6, y7); \
556	\
557	aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
558	y0, y7, y2, rk, 0, last_round); \
559	\
560	aria_load_state_8way(y0, y1, y2, y3, \
561	y4, y5, y6, y7, \
562	mem_tmp, 8);
563
564	#ifdef CONFIG_AS_GFNI
565	#define aria_fe_gfni(x0, x1, x2, x3, \
566	x4, x5, x6, x7, \
567	y0, y1, y2, y3, \
568	y4, y5, y6, y7, \
569	mem_tmp, rk, round) \
570	vpxor y7, y7, y7; \
571	aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
572	y0, y7, y2, rk, 8, round); \
573	\
574	aria_sbox_8way_gfni(x2, x3, x0, x1, \
575	x6, x7, x4, x5, \
576	y0, y1, y2, y3, \
577	y4, y5, y6, y7); \
578	\
579	aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \
580	aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \
581	aria_store_state_8way(x0, x1, x2, x3, \
582	x4, x5, x6, x7, \
583	mem_tmp, 8); \
584	\
585	aria_load_state_8way(x0, x1, x2, x3, \
586	x4, x5, x6, x7, \
587	mem_tmp, 0); \
588	aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
589	y0, y7, y2, rk, 0, round); \
590	\
591	aria_sbox_8way_gfni(x2, x3, x0, x1, \
592	x6, x7, x4, x5, \
593	y0, y1, y2, y3, \
594	y4, y5, y6, y7); \
595	\
596	aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \
597	aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \
598	aria_store_state_8way(x0, x1, x2, x3, \
599	x4, x5, x6, x7, \
600	mem_tmp, 0); \
601	aria_load_state_8way(y0, y1, y2, y3, \
602	y4, y5, y6, y7, \
603	mem_tmp, 8); \
604	aria_diff_word(x0, x1, x2, x3, \
605	x4, x5, x6, x7, \
606	y0, y1, y2, y3, \
607	y4, y5, y6, y7); \
608	/* aria_diff_byte() \
609	* T3 = ABCD -> BADC \
610	* T3 = y4, y5, y6, y7 -> y5, y4, y7, y6 \
611	* T0 = ABCD -> CDAB \
612	* T0 = x0, x1, x2, x3 -> x2, x3, x0, x1 \
613	* T1 = ABCD -> DCBA \
614	* T1 = x4, x5, x6, x7 -> x7, x6, x5, x4 \
615	*/ \
616	aria_diff_word(x2, x3, x0, x1, \
617	x7, x6, x5, x4, \
618	y0, y1, y2, y3, \
619	y5, y4, y7, y6); \
620	aria_store_state_8way(x3, x2, x1, x0, \
621	x6, x7, x4, x5, \
622	mem_tmp, 0);
623
624	#define aria_fo_gfni(x0, x1, x2, x3, \
625	x4, x5, x6, x7, \
626	y0, y1, y2, y3, \
627	y4, y5, y6, y7, \
628	mem_tmp, rk, round) \
629	vpxor y7, y7, y7; \
630	aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
631	y0, y7, y2, rk, 8, round); \
632	\
633	aria_sbox_8way_gfni(x0, x1, x2, x3, \
634	x4, x5, x6, x7, \
635	y0, y1, y2, y3, \
636	y4, y5, y6, y7); \
637	\
638	aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \
639	aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \
640	aria_store_state_8way(x0, x1, x2, x3, \
641	x4, x5, x6, x7, \
642	mem_tmp, 8); \
643	\
644	aria_load_state_8way(x0, x1, x2, x3, \
645	x4, x5, x6, x7, \
646	mem_tmp, 0); \
647	aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
648	y0, y7, y2, rk, 0, round); \
649	\
650	aria_sbox_8way_gfni(x0, x1, x2, x3, \
651	x4, x5, x6, x7, \
652	y0, y1, y2, y3, \
653	y4, y5, y6, y7); \
654	\
655	aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \
656	aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \
657	aria_store_state_8way(x0, x1, x2, x3, \
658	x4, x5, x6, x7, \
659	mem_tmp, 0); \
660	aria_load_state_8way(y0, y1, y2, y3, \
661	y4, y5, y6, y7, \
662	mem_tmp, 8); \
663	aria_diff_word(x0, x1, x2, x3, \
664	x4, x5, x6, x7, \
665	y0, y1, y2, y3, \
666	y4, y5, y6, y7); \
667	/* aria_diff_byte() \
668	* T1 = ABCD -> BADC \
669	* T1 = x4, x5, x6, x7 -> x5, x4, x7, x6 \
670	* T2 = ABCD -> CDAB \
671	* T2 = y0, y1, y2, y3, -> y2, y3, y0, y1 \
672	* T3 = ABCD -> DCBA \
673	* T3 = y4, y5, y6, y7 -> y7, y6, y5, y4 \
674	*/ \
675	aria_diff_word(x0, x1, x2, x3, \
676	x5, x4, x7, x6, \
677	y2, y3, y0, y1, \
678	y7, y6, y5, y4); \
679	aria_store_state_8way(x3, x2, x1, x0, \
680	x6, x7, x4, x5, \
681	mem_tmp, 0);
682
683	#define aria_ff_gfni(x0, x1, x2, x3, \
684	x4, x5, x6, x7, \
685	y0, y1, y2, y3, \
686	y4, y5, y6, y7, \
687	mem_tmp, rk, round, last_round) \
688	vpxor y7, y7, y7; \
689	aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
690	y0, y7, y2, rk, 8, round); \
691	\
692	aria_sbox_8way_gfni(x2, x3, x0, x1, \
693	x6, x7, x4, x5, \
694	y0, y1, y2, y3, \
695	y4, y5, y6, y7); \
696	\
697	aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
698	y0, y7, y2, rk, 8, last_round); \
699	\
700	aria_store_state_8way(x0, x1, x2, x3, \
701	x4, x5, x6, x7, \
702	mem_tmp, 8); \
703	\
704	aria_load_state_8way(x0, x1, x2, x3, \
705	x4, x5, x6, x7, \
706	mem_tmp, 0); \
707	aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
708	y0, y7, y2, rk, 0, round); \
709	\
710	aria_sbox_8way_gfni(x2, x3, x0, x1, \
711	x6, x7, x4, x5, \
712	y0, y1, y2, y3, \
713	y4, y5, y6, y7); \
714	\
715	aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
716	y0, y7, y2, rk, 0, last_round); \
717	\
718	aria_load_state_8way(y0, y1, y2, y3, \
719	y4, y5, y6, y7, \
720	mem_tmp, 8);
721
722	#endif /* CONFIG_AS_GFNI */
723
724	/* NB: section is mergeable, all elements must be aligned 16-byte blocks */
725	.section .rodata.cst16, "aM", @progbits, 16
726	.align 16
727
728	#define SHUFB_BYTES(idx) \
729	0 + (idx), 4 + (idx), 8 + (idx), 12 + (idx)
730
731	.Lshufb_16x16b:
732	.byte SHUFB_BYTES(0), SHUFB_BYTES(1), SHUFB_BYTES(2), SHUFB_BYTES(3);
733	/* For isolating SubBytes from AESENCLAST, inverse shift row */
734	.Linv_shift_row:
735	.byte 0x00, 0x0d, 0x0a, 0x07, 0x04, 0x01, 0x0e, 0x0b
736	.byte 0x08, 0x05, 0x02, 0x0f, 0x0c, 0x09, 0x06, 0x03
737	.Lshift_row:
738	.byte 0x00, 0x05, 0x0a, 0x0f, 0x04, 0x09, 0x0e, 0x03
739	.byte 0x08, 0x0d, 0x02, 0x07, 0x0c, 0x01, 0x06, 0x0b
740	/* For CTR-mode IV byteswap */
741	.Lbswap128_mask:
742	.byte 0x0f, 0x0e, 0x0d, 0x0c, 0x0b, 0x0a, 0x09, 0x08
743	.byte 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00
744
745	/* AES inverse affine and S2 combined:
746	* 1 1 0 0 0 0 0 1 x0 0
747	* 0 1 0 0 1 0 0 0 x1 0
748	* 1 1 0 0 1 1 1 1 x2 0
749	* 0 1 1 0 1 0 0 1 x3 1
750	* 0 1 0 0 1 1 0 0 * x4 + 0
751	* 0 1 0 1 1 0 0 0 x5 0
752	* 0 0 0 0 0 1 0 1 x6 0
753	* 1 1 1 0 0 1 1 1 x7 1
754	*/
755	.Ltf_lo__inv_aff__and__s2:
756	.octa 0x92172DA81A9FA520B2370D883ABF8500
757	.Ltf_hi__inv_aff__and__s2:
758	.octa 0x2B15FFC1AF917B45E6D8320C625CB688
759
760	/* X2 and AES forward affine combined:
761	* 1 0 1 1 0 0 0 1 x0 0
762	* 0 1 1 1 1 0 1 1 x1 0
763	* 0 0 0 1 1 0 1 0 x2 1
764	* 0 1 0 0 0 1 0 0 x3 0
765	* 0 0 1 1 1 0 1 1 * x4 + 0
766	* 0 1 0 0 1 0 0 0 x5 0
767	* 1 1 0 1 0 0 1 1 x6 0
768	* 0 1 0 0 1 0 1 0 x7 0
769	*/
770	.Ltf_lo__x2__and__fwd_aff:
771	.octa 0xEFAE0544FCBD1657B8F95213ABEA4100
772	.Ltf_hi__x2__and__fwd_aff:
773	.octa 0x3F893781E95FE1576CDA64D2BA0CB204
774
775	#ifdef CONFIG_AS_GFNI
776	/* AES affine: */
777	#define tf_aff_const BV8(1, 1, 0, 0, 0, 1, 1, 0)
778	.Ltf_aff_bitmatrix:
779	.quad BM8X8(BV8(1, 0, 0, 0, 1, 1, 1, 1),
780	BV8(1, 1, 0, 0, 0, 1, 1, 1),
781	BV8(1, 1, 1, 0, 0, 0, 1, 1),
782	BV8(1, 1, 1, 1, 0, 0, 0, 1),
783	BV8(1, 1, 1, 1, 1, 0, 0, 0),
784	BV8(0, 1, 1, 1, 1, 1, 0, 0),
785	BV8(0, 0, 1, 1, 1, 1, 1, 0),
786	BV8(0, 0, 0, 1, 1, 1, 1, 1))
787	.quad BM8X8(BV8(1, 0, 0, 0, 1, 1, 1, 1),
788	BV8(1, 1, 0, 0, 0, 1, 1, 1),
789	BV8(1, 1, 1, 0, 0, 0, 1, 1),
790	BV8(1, 1, 1, 1, 0, 0, 0, 1),
791	BV8(1, 1, 1, 1, 1, 0, 0, 0),
792	BV8(0, 1, 1, 1, 1, 1, 0, 0),
793	BV8(0, 0, 1, 1, 1, 1, 1, 0),
794	BV8(0, 0, 0, 1, 1, 1, 1, 1))
795
796	/* AES inverse affine: */
797	#define tf_inv_const BV8(1, 0, 1, 0, 0, 0, 0, 0)
798	.Ltf_inv_bitmatrix:
799	.quad BM8X8(BV8(0, 0, 1, 0, 0, 1, 0, 1),
800	BV8(1, 0, 0, 1, 0, 0, 1, 0),
801	BV8(0, 1, 0, 0, 1, 0, 0, 1),
802	BV8(1, 0, 1, 0, 0, 1, 0, 0),
803	BV8(0, 1, 0, 1, 0, 0, 1, 0),
804	BV8(0, 0, 1, 0, 1, 0, 0, 1),
805	BV8(1, 0, 0, 1, 0, 1, 0, 0),
806	BV8(0, 1, 0, 0, 1, 0, 1, 0))
807	.quad BM8X8(BV8(0, 0, 1, 0, 0, 1, 0, 1),
808	BV8(1, 0, 0, 1, 0, 0, 1, 0),
809	BV8(0, 1, 0, 0, 1, 0, 0, 1),
810	BV8(1, 0, 1, 0, 0, 1, 0, 0),
811	BV8(0, 1, 0, 1, 0, 0, 1, 0),
812	BV8(0, 0, 1, 0, 1, 0, 0, 1),
813	BV8(1, 0, 0, 1, 0, 1, 0, 0),
814	BV8(0, 1, 0, 0, 1, 0, 1, 0))
815
816	/* S2: */
817	#define tf_s2_const BV8(0, 1, 0, 0, 0, 1, 1, 1)
818	.Ltf_s2_bitmatrix:
819	.quad BM8X8(BV8(0, 1, 0, 1, 0, 1, 1, 1),
820	BV8(0, 0, 1, 1, 1, 1, 1, 1),
821	BV8(1, 1, 1, 0, 1, 1, 0, 1),
822	BV8(1, 1, 0, 0, 0, 0, 1, 1),
823	BV8(0, 1, 0, 0, 0, 0, 1, 1),
824	BV8(1, 1, 0, 0, 1, 1, 1, 0),
825	BV8(0, 1, 1, 0, 0, 0, 1, 1),
826	BV8(1, 1, 1, 1, 0, 1, 1, 0))
827	.quad BM8X8(BV8(0, 1, 0, 1, 0, 1, 1, 1),
828	BV8(0, 0, 1, 1, 1, 1, 1, 1),
829	BV8(1, 1, 1, 0, 1, 1, 0, 1),
830	BV8(1, 1, 0, 0, 0, 0, 1, 1),
831	BV8(0, 1, 0, 0, 0, 0, 1, 1),
832	BV8(1, 1, 0, 0, 1, 1, 1, 0),
833	BV8(0, 1, 1, 0, 0, 0, 1, 1),
834	BV8(1, 1, 1, 1, 0, 1, 1, 0))
835
836	/* X2: */
837	#define tf_x2_const BV8(0, 0, 1, 1, 0, 1, 0, 0)
838	.Ltf_x2_bitmatrix:
839	.quad BM8X8(BV8(0, 0, 0, 1, 1, 0, 0, 0),
840	BV8(0, 0, 1, 0, 0, 1, 1, 0),
841	BV8(0, 0, 0, 0, 1, 0, 1, 0),
842	BV8(1, 1, 1, 0, 0, 0, 1, 1),
843	BV8(1, 1, 1, 0, 1, 1, 0, 0),
844	BV8(0, 1, 1, 0, 1, 0, 1, 1),
845	BV8(1, 0, 1, 1, 1, 1, 0, 1),
846	BV8(1, 0, 0, 1, 0, 0, 1, 1))
847	.quad BM8X8(BV8(0, 0, 0, 1, 1, 0, 0, 0),
848	BV8(0, 0, 1, 0, 0, 1, 1, 0),
849	BV8(0, 0, 0, 0, 1, 0, 1, 0),
850	BV8(1, 1, 1, 0, 0, 0, 1, 1),
851	BV8(1, 1, 1, 0, 1, 1, 0, 0),
852	BV8(0, 1, 1, 0, 1, 0, 1, 1),
853	BV8(1, 0, 1, 1, 1, 1, 0, 1),
854	BV8(1, 0, 0, 1, 0, 0, 1, 1))
855
856	/* Identity matrix: */
857	.Ltf_id_bitmatrix:
858	.quad BM8X8(BV8(1, 0, 0, 0, 0, 0, 0, 0),
859	BV8(0, 1, 0, 0, 0, 0, 0, 0),
860	BV8(0, 0, 1, 0, 0, 0, 0, 0),
861	BV8(0, 0, 0, 1, 0, 0, 0, 0),
862	BV8(0, 0, 0, 0, 1, 0, 0, 0),
863	BV8(0, 0, 0, 0, 0, 1, 0, 0),
864	BV8(0, 0, 0, 0, 0, 0, 1, 0),
865	BV8(0, 0, 0, 0, 0, 0, 0, 1))
866	.quad BM8X8(BV8(1, 0, 0, 0, 0, 0, 0, 0),
867	BV8(0, 1, 0, 0, 0, 0, 0, 0),
868	BV8(0, 0, 1, 0, 0, 0, 0, 0),
869	BV8(0, 0, 0, 1, 0, 0, 0, 0),
870	BV8(0, 0, 0, 0, 1, 0, 0, 0),
871	BV8(0, 0, 0, 0, 0, 1, 0, 0),
872	BV8(0, 0, 0, 0, 0, 0, 1, 0),
873	BV8(0, 0, 0, 0, 0, 0, 0, 1))
874	#endif /* CONFIG_AS_GFNI */
875
876	/* 4-bit mask */
877	.section .rodata.cst4.L0f0f0f0f, "aM", @progbits, 4
878	.align 4
879	.L0f0f0f0f:
880	.long 0x0f0f0f0f
881
882	.text
883
884	SYM_FUNC_START_LOCAL(__aria_aesni_avx_crypt_16way)
885	/* input:
886	* %r9: rk
887	* %rsi: dst
888	* %rdx: src
889	* %xmm0..%xmm15: 16 byte-sliced blocks
890	*/
891
892	FRAME_BEGIN
893
894	movq %rsi, %rax;
895	leaq 8 * 16(%rax), %r8;
896
897	inpack16_post(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
898	%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
899	%xmm15, %rax, %r8);
900	aria_fo(%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm15,
901	%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
902	%rax, %r9, 0);
903	aria_fe(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
904	%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
905	%xmm15, %rax, %r9, 1);
906	aria_fo(%xmm9, %xmm8, %xmm11, %xmm10, %xmm12, %xmm13, %xmm14, %xmm15,
907	%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
908	%rax, %r9, 2);
909	aria_fe(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
910	%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
911	%xmm15, %rax, %r9, 3);
912	aria_fo(%xmm9, %xmm8, %xmm11, %xmm10, %xmm12, %xmm13, %xmm14, %xmm15,
913	%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
914	%rax, %r9, 4);
915	aria_fe(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
916	%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
917	%xmm15, %rax, %r9, 5);
918	aria_fo(%xmm9, %xmm8, %xmm11, %xmm10, %xmm12, %xmm13, %xmm14, %xmm15,
919	%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
920	%rax, %r9, 6);
921	aria_fe(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
922	%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
923	%xmm15, %rax, %r9, 7);
924	aria_fo(%xmm9, %xmm8, %xmm11, %xmm10, %xmm12, %xmm13, %xmm14, %xmm15,
925	%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
926	%rax, %r9, 8);
927	aria_fe(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
928	%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
929	%xmm15, %rax, %r9, 9);
930	aria_fo(%xmm9, %xmm8, %xmm11, %xmm10, %xmm12, %xmm13, %xmm14, %xmm15,
931	%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
932	%rax, %r9, 10);
933	cmpl $12, ARIA_CTX_rounds(CTX);
934	jne .Laria_192;
935	aria_ff(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
936	%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
937	%xmm15, %rax, %r9, 11, 12);
938	jmp .Laria_end;
939	.Laria_192:
940	aria_fe(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
941	%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
942	%xmm15, %rax, %r9, 11);
943	aria_fo(%xmm9, %xmm8, %xmm11, %xmm10, %xmm12, %xmm13, %xmm14, %xmm15,
944	%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
945	%rax, %r9, 12);
946	cmpl $14, ARIA_CTX_rounds(CTX);
947	jne .Laria_256;
948	aria_ff(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
949	%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
950	%xmm15, %rax, %r9, 13, 14);
951	jmp .Laria_end;
952	.Laria_256:
953	aria_fe(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
954	%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
955	%xmm15, %rax, %r9, 13);
956	aria_fo(%xmm9, %xmm8, %xmm11, %xmm10, %xmm12, %xmm13, %xmm14, %xmm15,
957	%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
958	%rax, %r9, 14);
959	aria_ff(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
960	%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
961	%xmm15, %rax, %r9, 15, 16);
962	.Laria_end:
963	debyteslice_16x16b(%xmm8, %xmm12, %xmm1, %xmm4,
964	%xmm9, %xmm13, %xmm0, %xmm5,
965	%xmm10, %xmm14, %xmm3, %xmm6,
966	%xmm11, %xmm15, %xmm2, %xmm7,
967	(%rax), (%r8));
968
969	FRAME_END
970	RET;
971	SYM_FUNC_END(__aria_aesni_avx_crypt_16way)
972
973	SYM_TYPED_FUNC_START(aria_aesni_avx_encrypt_16way)
974	/* input:
975	* %rdi: ctx, CTX
976	* %rsi: dst
977	* %rdx: src
978	*/
979
980	FRAME_BEGIN
981
982	leaq ARIA_CTX_enc_key(CTX), %r9;
983
984	inpack16_pre(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
985	%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
986	%xmm15, %rdx);
987
988	call __aria_aesni_avx_crypt_16way;
989
990	write_output(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
991	%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
992	%xmm15, %rax);
993
994	FRAME_END
995	RET;
996	SYM_FUNC_END(aria_aesni_avx_encrypt_16way)
997
998	SYM_TYPED_FUNC_START(aria_aesni_avx_decrypt_16way)
999	/* input:
1000	* %rdi: ctx, CTX
1001	* %rsi: dst
1002	* %rdx: src
1003	*/
1004
1005	FRAME_BEGIN
1006
1007	leaq ARIA_CTX_dec_key(CTX), %r9;
1008
1009	inpack16_pre(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
1010	%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
1011	%xmm15, %rdx);
1012
1013	call __aria_aesni_avx_crypt_16way;
1014
1015	write_output(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
1016	%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
1017	%xmm15, %rax);
1018
1019	FRAME_END
1020	RET;
1021	SYM_FUNC_END(aria_aesni_avx_decrypt_16way)
1022
1023	SYM_FUNC_START_LOCAL(__aria_aesni_avx_ctr_gen_keystream_16way)
1024	/* input:
1025	* %rdi: ctx
1026	* %rsi: dst
1027	* %rdx: src
1028	* %rcx: keystream
1029	* %r8: iv (big endian, 128bit)
1030	*/
1031
1032	FRAME_BEGIN
1033	/* load IV and byteswap */
1034	vmovdqu (%r8), %xmm8;
1035
1036	vmovdqa .Lbswap128_mask (%rip), %xmm1;
1037	vpshufb %xmm1, %xmm8, %xmm3; /* be => le */
1038
1039	vpcmpeqd %xmm0, %xmm0, %xmm0;
1040	vpsrldq $8, %xmm0, %xmm0; /* low: -1, high: 0 */
1041
1042	/* construct IVs */
1043	inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
1044	vpshufb %xmm1, %xmm3, %xmm9;
1045	inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
1046	vpshufb %xmm1, %xmm3, %xmm10;
1047	inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
1048	vpshufb %xmm1, %xmm3, %xmm11;
1049	inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
1050	vpshufb %xmm1, %xmm3, %xmm12;
1051	inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
1052	vpshufb %xmm1, %xmm3, %xmm13;
1053	inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
1054	vpshufb %xmm1, %xmm3, %xmm14;
1055	inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
1056	vpshufb %xmm1, %xmm3, %xmm15;
1057	vmovdqu %xmm8, (0 * 16)(%rcx);
1058	vmovdqu %xmm9, (1 * 16)(%rcx);
1059	vmovdqu %xmm10, (2 * 16)(%rcx);
1060	vmovdqu %xmm11, (3 * 16)(%rcx);
1061	vmovdqu %xmm12, (4 * 16)(%rcx);
1062	vmovdqu %xmm13, (5 * 16)(%rcx);
1063	vmovdqu %xmm14, (6 * 16)(%rcx);
1064	vmovdqu %xmm15, (7 * 16)(%rcx);
1065
1066	inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
1067	vpshufb %xmm1, %xmm3, %xmm8;
1068	inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
1069	vpshufb %xmm1, %xmm3, %xmm9;
1070	inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
1071	vpshufb %xmm1, %xmm3, %xmm10;
1072	inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
1073	vpshufb %xmm1, %xmm3, %xmm11;
1074	inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
1075	vpshufb %xmm1, %xmm3, %xmm12;
1076	inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
1077	vpshufb %xmm1, %xmm3, %xmm13;
1078	inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
1079	vpshufb %xmm1, %xmm3, %xmm14;
1080	inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
1081	vpshufb %xmm1, %xmm3, %xmm15;
1082	inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
1083	vpshufb %xmm1, %xmm3, %xmm4;
1084	vmovdqu %xmm4, (%r8);
1085
1086	vmovdqu (0 * 16)(%rcx), %xmm0;
1087	vmovdqu (1 * 16)(%rcx), %xmm1;
1088	vmovdqu (2 * 16)(%rcx), %xmm2;
1089	vmovdqu (3 * 16)(%rcx), %xmm3;
1090	vmovdqu (4 * 16)(%rcx), %xmm4;
1091	vmovdqu (5 * 16)(%rcx), %xmm5;
1092	vmovdqu (6 * 16)(%rcx), %xmm6;
1093	vmovdqu (7 * 16)(%rcx), %xmm7;
1094
1095	FRAME_END
1096	RET;
1097	SYM_FUNC_END(__aria_aesni_avx_ctr_gen_keystream_16way)
1098
1099	SYM_TYPED_FUNC_START(aria_aesni_avx_ctr_crypt_16way)
1100	/* input:
1101	* %rdi: ctx
1102	* %rsi: dst
1103	* %rdx: src
1104	* %rcx: keystream
1105	* %r8: iv (big endian, 128bit)
1106	*/
1107	FRAME_BEGIN
1108
1109	call __aria_aesni_avx_ctr_gen_keystream_16way;
1110
1111	leaq (%rsi), %r10;
1112	leaq (%rdx), %r11;
1113	leaq (%rcx), %rsi;
1114	leaq (%rcx), %rdx;
1115	leaq ARIA_CTX_enc_key(CTX), %r9;
1116
1117	call __aria_aesni_avx_crypt_16way;
1118
1119	vpxor (0 * 16)(%r11), %xmm1, %xmm1;
1120	vpxor (1 * 16)(%r11), %xmm0, %xmm0;
1121	vpxor (2 * 16)(%r11), %xmm3, %xmm3;
1122	vpxor (3 * 16)(%r11), %xmm2, %xmm2;
1123	vpxor (4 * 16)(%r11), %xmm4, %xmm4;
1124	vpxor (5 * 16)(%r11), %xmm5, %xmm5;
1125	vpxor (6 * 16)(%r11), %xmm6, %xmm6;
1126	vpxor (7 * 16)(%r11), %xmm7, %xmm7;
1127	vpxor (8 * 16)(%r11), %xmm8, %xmm8;
1128	vpxor (9 * 16)(%r11), %xmm9, %xmm9;
1129	vpxor (10 * 16)(%r11), %xmm10, %xmm10;
1130	vpxor (11 * 16)(%r11), %xmm11, %xmm11;
1131	vpxor (12 * 16)(%r11), %xmm12, %xmm12;
1132	vpxor (13 * 16)(%r11), %xmm13, %xmm13;
1133	vpxor (14 * 16)(%r11), %xmm14, %xmm14;
1134	vpxor (15 * 16)(%r11), %xmm15, %xmm15;
1135	write_output(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
1136	%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
1137	%xmm15, %r10);
1138
1139	FRAME_END
1140	RET;
1141	SYM_FUNC_END(aria_aesni_avx_ctr_crypt_16way)
1142
1143	#ifdef CONFIG_AS_GFNI
1144	SYM_FUNC_START_LOCAL(__aria_aesni_avx_gfni_crypt_16way)
1145	/* input:
1146	* %r9: rk
1147	* %rsi: dst
1148	* %rdx: src
1149	* %xmm0..%xmm15: 16 byte-sliced blocks
1150	*/
1151
1152	FRAME_BEGIN
1153
1154	movq %rsi, %rax;
1155	leaq 8 * 16(%rax), %r8;
1156
1157	inpack16_post(%xmm0, %xmm1, %xmm2, %xmm3,
1158	%xmm4, %xmm5, %xmm6, %xmm7,
1159	%xmm8, %xmm9, %xmm10, %xmm11,
1160	%xmm12, %xmm13, %xmm14,
1161	%xmm15, %rax, %r8);
1162	aria_fo_gfni(%xmm8, %xmm9, %xmm10, %xmm11,
1163	%xmm12, %xmm13, %xmm14, %xmm15,
1164	%xmm0, %xmm1, %xmm2, %xmm3,
1165	%xmm4, %xmm5, %xmm6, %xmm7,
1166	%rax, %r9, 0);
1167	aria_fe_gfni(%xmm1, %xmm0, %xmm3, %xmm2,
1168	%xmm4, %xmm5, %xmm6, %xmm7,
1169	%xmm8, %xmm9, %xmm10, %xmm11,
1170	%xmm12, %xmm13, %xmm14,
1171	%xmm15, %rax, %r9, 1);
1172	aria_fo_gfni(%xmm9, %xmm8, %xmm11, %xmm10,
1173	%xmm12, %xmm13, %xmm14, %xmm15,
1174	%xmm0, %xmm1, %xmm2, %xmm3,
1175	%xmm4, %xmm5, %xmm6, %xmm7,
1176	%rax, %r9, 2);
1177	aria_fe_gfni(%xmm1, %xmm0, %xmm3, %xmm2,
1178	%xmm4, %xmm5, %xmm6, %xmm7,
1179	%xmm8, %xmm9, %xmm10, %xmm11,
1180	%xmm12, %xmm13, %xmm14,
1181	%xmm15, %rax, %r9, 3);
1182	aria_fo_gfni(%xmm9, %xmm8, %xmm11, %xmm10,
1183	%xmm12, %xmm13, %xmm14, %xmm15,
1184	%xmm0, %xmm1, %xmm2, %xmm3,
1185	%xmm4, %xmm5, %xmm6, %xmm7,
1186	%rax, %r9, 4);
1187	aria_fe_gfni(%xmm1, %xmm0, %xmm3, %xmm2,
1188	%xmm4, %xmm5, %xmm6, %xmm7,
1189	%xmm8, %xmm9, %xmm10, %xmm11,
1190	%xmm12, %xmm13, %xmm14,
1191	%xmm15, %rax, %r9, 5);
1192	aria_fo_gfni(%xmm9, %xmm8, %xmm11, %xmm10,
1193	%xmm12, %xmm13, %xmm14, %xmm15,
1194	%xmm0, %xmm1, %xmm2, %xmm3,
1195	%xmm4, %xmm5, %xmm6, %xmm7,
1196	%rax, %r9, 6);
1197	aria_fe_gfni(%xmm1, %xmm0, %xmm3, %xmm2,
1198	%xmm4, %xmm5, %xmm6, %xmm7,
1199	%xmm8, %xmm9, %xmm10, %xmm11,
1200	%xmm12, %xmm13, %xmm14,
1201	%xmm15, %rax, %r9, 7);
1202	aria_fo_gfni(%xmm9, %xmm8, %xmm11, %xmm10,
1203	%xmm12, %xmm13, %xmm14, %xmm15,
1204	%xmm0, %xmm1, %xmm2, %xmm3,
1205	%xmm4, %xmm5, %xmm6, %xmm7,
1206	%rax, %r9, 8);
1207	aria_fe_gfni(%xmm1, %xmm0, %xmm3, %xmm2,
1208	%xmm4, %xmm5, %xmm6, %xmm7,
1209	%xmm8, %xmm9, %xmm10, %xmm11,
1210	%xmm12, %xmm13, %xmm14,
1211	%xmm15, %rax, %r9, 9);
1212	aria_fo_gfni(%xmm9, %xmm8, %xmm11, %xmm10,
1213	%xmm12, %xmm13, %xmm14, %xmm15,
1214	%xmm0, %xmm1, %xmm2, %xmm3,
1215	%xmm4, %xmm5, %xmm6, %xmm7,
1216	%rax, %r9, 10);
1217	cmpl $12, ARIA_CTX_rounds(CTX);
1218	jne .Laria_gfni_192;
1219	aria_ff_gfni(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
1220	%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
1221	%xmm15, %rax, %r9, 11, 12);
1222	jmp .Laria_gfni_end;
1223	.Laria_gfni_192:
1224	aria_fe_gfni(%xmm1, %xmm0, %xmm3, %xmm2,
1225	%xmm4, %xmm5, %xmm6, %xmm7,
1226	%xmm8, %xmm9, %xmm10, %xmm11,
1227	%xmm12, %xmm13, %xmm14,
1228	%xmm15, %rax, %r9, 11);
1229	aria_fo_gfni(%xmm9, %xmm8, %xmm11, %xmm10,
1230	%xmm12, %xmm13, %xmm14, %xmm15,
1231	%xmm0, %xmm1, %xmm2, %xmm3,
1232	%xmm4, %xmm5, %xmm6, %xmm7,
1233	%rax, %r9, 12);
1234	cmpl $14, ARIA_CTX_rounds(CTX);
1235	jne .Laria_gfni_256;
1236	aria_ff_gfni(%xmm1, %xmm0, %xmm3, %xmm2,
1237	%xmm4, %xmm5, %xmm6, %xmm7,
1238	%xmm8, %xmm9, %xmm10, %xmm11,
1239	%xmm12, %xmm13, %xmm14,
1240	%xmm15, %rax, %r9, 13, 14);
1241	jmp .Laria_gfni_end;
1242	.Laria_gfni_256:
1243	aria_fe_gfni(%xmm1, %xmm0, %xmm3, %xmm2,
1244	%xmm4, %xmm5, %xmm6, %xmm7,
1245	%xmm8, %xmm9, %xmm10, %xmm11,
1246	%xmm12, %xmm13, %xmm14,
1247	%xmm15, %rax, %r9, 13);
1248	aria_fo_gfni(%xmm9, %xmm8, %xmm11, %xmm10,
1249	%xmm12, %xmm13, %xmm14, %xmm15,
1250	%xmm0, %xmm1, %xmm2, %xmm3,
1251	%xmm4, %xmm5, %xmm6, %xmm7,
1252	%rax, %r9, 14);
1253	aria_ff_gfni(%xmm1, %xmm0, %xmm3, %xmm2,
1254	%xmm4, %xmm5, %xmm6, %xmm7,
1255	%xmm8, %xmm9, %xmm10, %xmm11,
1256	%xmm12, %xmm13, %xmm14,
1257	%xmm15, %rax, %r9, 15, 16);
1258	.Laria_gfni_end:
1259	debyteslice_16x16b(%xmm8, %xmm12, %xmm1, %xmm4,
1260	%xmm9, %xmm13, %xmm0, %xmm5,
1261	%xmm10, %xmm14, %xmm3, %xmm6,
1262	%xmm11, %xmm15, %xmm2, %xmm7,
1263	(%rax), (%r8));
1264
1265	FRAME_END
1266	RET;
1267	SYM_FUNC_END(__aria_aesni_avx_gfni_crypt_16way)
1268
1269	SYM_TYPED_FUNC_START(aria_aesni_avx_gfni_encrypt_16way)
1270	/* input:
1271	* %rdi: ctx, CTX
1272	* %rsi: dst
1273	* %rdx: src
1274	*/
1275
1276	FRAME_BEGIN
1277
1278	leaq ARIA_CTX_enc_key(CTX), %r9;
1279
1280	inpack16_pre(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
1281	%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
1282	%xmm15, %rdx);
1283
1284	call __aria_aesni_avx_gfni_crypt_16way;
1285
1286	write_output(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
1287	%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
1288	%xmm15, %rax);
1289
1290	FRAME_END
1291	RET;
1292	SYM_FUNC_END(aria_aesni_avx_gfni_encrypt_16way)
1293
1294	SYM_TYPED_FUNC_START(aria_aesni_avx_gfni_decrypt_16way)
1295	/* input:
1296	* %rdi: ctx, CTX
1297	* %rsi: dst
1298	* %rdx: src
1299	*/
1300
1301	FRAME_BEGIN
1302
1303	leaq ARIA_CTX_dec_key(CTX), %r9;
1304
1305	inpack16_pre(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
1306	%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
1307	%xmm15, %rdx);
1308
1309	call __aria_aesni_avx_gfni_crypt_16way;
1310
1311	write_output(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
1312	%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
1313	%xmm15, %rax);
1314
1315	FRAME_END
1316	RET;
1317	SYM_FUNC_END(aria_aesni_avx_gfni_decrypt_16way)
1318
1319	SYM_TYPED_FUNC_START(aria_aesni_avx_gfni_ctr_crypt_16way)
1320	/* input:
1321	* %rdi: ctx
1322	* %rsi: dst
1323	* %rdx: src
1324	* %rcx: keystream
1325	* %r8: iv (big endian, 128bit)
1326	*/
1327	FRAME_BEGIN
1328
1329	call __aria_aesni_avx_ctr_gen_keystream_16way
1330
1331	leaq (%rsi), %r10;
1332	leaq (%rdx), %r11;
1333	leaq (%rcx), %rsi;
1334	leaq (%rcx), %rdx;
1335	leaq ARIA_CTX_enc_key(CTX), %r9;
1336
1337	call __aria_aesni_avx_gfni_crypt_16way;
1338
1339	vpxor (0 * 16)(%r11), %xmm1, %xmm1;
1340	vpxor (1 * 16)(%r11), %xmm0, %xmm0;
1341	vpxor (2 * 16)(%r11), %xmm3, %xmm3;
1342	vpxor (3 * 16)(%r11), %xmm2, %xmm2;
1343	vpxor (4 * 16)(%r11), %xmm4, %xmm4;
1344	vpxor (5 * 16)(%r11), %xmm5, %xmm5;
1345	vpxor (6 * 16)(%r11), %xmm6, %xmm6;
1346	vpxor (7 * 16)(%r11), %xmm7, %xmm7;
1347	vpxor (8 * 16)(%r11), %xmm8, %xmm8;
1348	vpxor (9 * 16)(%r11), %xmm9, %xmm9;
1349	vpxor (10 * 16)(%r11), %xmm10, %xmm10;
1350	vpxor (11 * 16)(%r11), %xmm11, %xmm11;
1351	vpxor (12 * 16)(%r11), %xmm12, %xmm12;
1352	vpxor (13 * 16)(%r11), %xmm13, %xmm13;
1353	vpxor (14 * 16)(%r11), %xmm14, %xmm14;
1354	vpxor (15 * 16)(%r11), %xmm15, %xmm15;
1355	write_output(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
1356	%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
1357	%xmm15, %r10);
1358
1359	FRAME_END
1360	RET;
1361	SYM_FUNC_END(aria_aesni_avx_gfni_ctr_crypt_16way)
1362	#endif /* CONFIG_AS_GFNI */
1363