1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * ChaCha 256-bit cipher algorithm, x64 SSSE3 functions
4 *
5 * Copyright (C) 2015 Martin Willi
6 */
7
8#include <linux/linkage.h>
9#include <asm/frame.h>
10
11.section .rodata.cst16.ROT8, "aM", @progbits, 16
12.align 16
13ROT8: .octa 0x0e0d0c0f0a09080b0605040702010003
14.section .rodata.cst16.ROT16, "aM", @progbits, 16
15.align 16
16ROT16: .octa 0x0d0c0f0e09080b0a0504070601000302
17.section .rodata.cst16.CTRINC, "aM", @progbits, 16
18.align 16
19CTRINC: .octa 0x00000003000000020000000100000000
20
21.text
22
23/*
24 * chacha_permute - permute one block
25 *
26 * Permute one 64-byte block where the state matrix is in %xmm0-%xmm3. This
27 * function performs matrix operations on four words in parallel, but requires
28 * shuffling to rearrange the words after each round. 8/16-bit word rotation is
29 * done with the slightly better performing SSSE3 byte shuffling, 7/12-bit word
30 * rotation uses traditional shift+OR.
31 *
32 * The round count is given in %r8d.
33 *
34 * Clobbers: %r8d, %xmm4-%xmm7
35 */
36SYM_FUNC_START_LOCAL(chacha_permute)
37
38 movdqa ROT8(%rip),%xmm4
39 movdqa ROT16(%rip),%xmm5
40
41.Ldoubleround:
42 # x0 += x1, x3 = rotl32(x3 ^ x0, 16)
43 paddd %xmm1,%xmm0
44 pxor %xmm0,%xmm3
45 pshufb %xmm5,%xmm3
46
47 # x2 += x3, x1 = rotl32(x1 ^ x2, 12)
48 paddd %xmm3,%xmm2
49 pxor %xmm2,%xmm1
50 movdqa %xmm1,%xmm6
51 pslld $12,%xmm6
52 psrld $20,%xmm1
53 por %xmm6,%xmm1
54
55 # x0 += x1, x3 = rotl32(x3 ^ x0, 8)
56 paddd %xmm1,%xmm0
57 pxor %xmm0,%xmm3
58 pshufb %xmm4,%xmm3
59
60 # x2 += x3, x1 = rotl32(x1 ^ x2, 7)
61 paddd %xmm3,%xmm2
62 pxor %xmm2,%xmm1
63 movdqa %xmm1,%xmm7
64 pslld $7,%xmm7
65 psrld $25,%xmm1
66 por %xmm7,%xmm1
67
68 # x1 = shuffle32(x1, MASK(0, 3, 2, 1))
69 pshufd $0x39,%xmm1,%xmm1
70 # x2 = shuffle32(x2, MASK(1, 0, 3, 2))
71 pshufd $0x4e,%xmm2,%xmm2
72 # x3 = shuffle32(x3, MASK(2, 1, 0, 3))
73 pshufd $0x93,%xmm3,%xmm3
74
75 # x0 += x1, x3 = rotl32(x3 ^ x0, 16)
76 paddd %xmm1,%xmm0
77 pxor %xmm0,%xmm3
78 pshufb %xmm5,%xmm3
79
80 # x2 += x3, x1 = rotl32(x1 ^ x2, 12)
81 paddd %xmm3,%xmm2
82 pxor %xmm2,%xmm1
83 movdqa %xmm1,%xmm6
84 pslld $12,%xmm6
85 psrld $20,%xmm1
86 por %xmm6,%xmm1
87
88 # x0 += x1, x3 = rotl32(x3 ^ x0, 8)
89 paddd %xmm1,%xmm0
90 pxor %xmm0,%xmm3
91 pshufb %xmm4,%xmm3
92
93 # x2 += x3, x1 = rotl32(x1 ^ x2, 7)
94 paddd %xmm3,%xmm2
95 pxor %xmm2,%xmm1
96 movdqa %xmm1,%xmm7
97 pslld $7,%xmm7
98 psrld $25,%xmm1
99 por %xmm7,%xmm1
100
101 # x1 = shuffle32(x1, MASK(2, 1, 0, 3))
102 pshufd $0x93,%xmm1,%xmm1
103 # x2 = shuffle32(x2, MASK(1, 0, 3, 2))
104 pshufd $0x4e,%xmm2,%xmm2
105 # x3 = shuffle32(x3, MASK(0, 3, 2, 1))
106 pshufd $0x39,%xmm3,%xmm3
107
108 sub $2,%r8d
109 jnz .Ldoubleround
110
111 RET
112SYM_FUNC_END(chacha_permute)
113
114SYM_FUNC_START(chacha_block_xor_ssse3)
115 # %rdi: Input state matrix, s
116 # %rsi: up to 1 data block output, o
117 # %rdx: up to 1 data block input, i
118 # %rcx: input/output length in bytes
119 # %r8d: nrounds
120 FRAME_BEGIN
121
122 # x0..3 = s0..3
123 movdqu 0x00(%rdi),%xmm0
124 movdqu 0x10(%rdi),%xmm1
125 movdqu 0x20(%rdi),%xmm2
126 movdqu 0x30(%rdi),%xmm3
127 movdqa %xmm0,%xmm8
128 movdqa %xmm1,%xmm9
129 movdqa %xmm2,%xmm10
130 movdqa %xmm3,%xmm11
131
132 mov %rcx,%rax
133 call chacha_permute
134
135 # o0 = i0 ^ (x0 + s0)
136 paddd %xmm8,%xmm0
137 cmp $0x10,%rax
138 jl .Lxorpart
139 movdqu 0x00(%rdx),%xmm4
140 pxor %xmm4,%xmm0
141 movdqu %xmm0,0x00(%rsi)
142 # o1 = i1 ^ (x1 + s1)
143 paddd %xmm9,%xmm1
144 movdqa %xmm1,%xmm0
145 cmp $0x20,%rax
146 jl .Lxorpart
147 movdqu 0x10(%rdx),%xmm0
148 pxor %xmm1,%xmm0
149 movdqu %xmm0,0x10(%rsi)
150 # o2 = i2 ^ (x2 + s2)
151 paddd %xmm10,%xmm2
152 movdqa %xmm2,%xmm0
153 cmp $0x30,%rax
154 jl .Lxorpart
155 movdqu 0x20(%rdx),%xmm0
156 pxor %xmm2,%xmm0
157 movdqu %xmm0,0x20(%rsi)
158 # o3 = i3 ^ (x3 + s3)
159 paddd %xmm11,%xmm3
160 movdqa %xmm3,%xmm0
161 cmp $0x40,%rax
162 jl .Lxorpart
163 movdqu 0x30(%rdx),%xmm0
164 pxor %xmm3,%xmm0
165 movdqu %xmm0,0x30(%rsi)
166
167.Ldone:
168 FRAME_END
169 RET
170
171.Lxorpart:
172 # xor remaining bytes from partial register into output
173 mov %rax,%r9
174 and $0x0f,%r9
175 jz .Ldone
176 and $~0x0f,%rax
177
178 mov %rsi,%r11
179
180 lea 8(%rsp),%r10
181 sub $0x10,%rsp
182 and $~31,%rsp
183
184 lea (%rdx,%rax),%rsi
185 mov %rsp,%rdi
186 mov %r9,%rcx
187 rep movsb
188
189 pxor 0x00(%rsp),%xmm0
190 movdqa %xmm0,0x00(%rsp)
191
192 mov %rsp,%rsi
193 lea (%r11,%rax),%rdi
194 mov %r9,%rcx
195 rep movsb
196
197 lea -8(%r10),%rsp
198 jmp .Ldone
199
200SYM_FUNC_END(chacha_block_xor_ssse3)
201
202SYM_FUNC_START(hchacha_block_ssse3)
203 # %rdi: Input state matrix, s
204 # %rsi: output (8 32-bit words)
205 # %edx: nrounds
206 FRAME_BEGIN
207
208 movdqu 0x00(%rdi),%xmm0
209 movdqu 0x10(%rdi),%xmm1
210 movdqu 0x20(%rdi),%xmm2
211 movdqu 0x30(%rdi),%xmm3
212
213 mov %edx,%r8d
214 call chacha_permute
215
216 movdqu %xmm0,0x00(%rsi)
217 movdqu %xmm3,0x10(%rsi)
218
219 FRAME_END
220 RET
221SYM_FUNC_END(hchacha_block_ssse3)
222
223SYM_FUNC_START(chacha_4block_xor_ssse3)
224 # %rdi: Input state matrix, s
225 # %rsi: up to 4 data blocks output, o
226 # %rdx: up to 4 data blocks input, i
227 # %rcx: input/output length in bytes
228 # %r8d: nrounds
229
230 # This function encrypts four consecutive ChaCha blocks by loading the
231 # the state matrix in SSE registers four times. As we need some scratch
232 # registers, we save the first four registers on the stack. The
233 # algorithm performs each operation on the corresponding word of each
234 # state matrix, hence requires no word shuffling. For final XORing step
235 # we transpose the matrix by interleaving 32- and then 64-bit words,
236 # which allows us to do XOR in SSE registers. 8/16-bit word rotation is
237 # done with the slightly better performing SSSE3 byte shuffling,
238 # 7/12-bit word rotation uses traditional shift+OR.
239
240 lea 8(%rsp),%r10
241 sub $0x80,%rsp
242 and $~63,%rsp
243 mov %rcx,%rax
244
245 # x0..15[0-3] = s0..3[0..3]
246 movq 0x00(%rdi),%xmm1
247 pshufd $0x00,%xmm1,%xmm0
248 pshufd $0x55,%xmm1,%xmm1
249 movq 0x08(%rdi),%xmm3
250 pshufd $0x00,%xmm3,%xmm2
251 pshufd $0x55,%xmm3,%xmm3
252 movq 0x10(%rdi),%xmm5
253 pshufd $0x00,%xmm5,%xmm4
254 pshufd $0x55,%xmm5,%xmm5
255 movq 0x18(%rdi),%xmm7
256 pshufd $0x00,%xmm7,%xmm6
257 pshufd $0x55,%xmm7,%xmm7
258 movq 0x20(%rdi),%xmm9
259 pshufd $0x00,%xmm9,%xmm8
260 pshufd $0x55,%xmm9,%xmm9
261 movq 0x28(%rdi),%xmm11
262 pshufd $0x00,%xmm11,%xmm10
263 pshufd $0x55,%xmm11,%xmm11
264 movq 0x30(%rdi),%xmm13
265 pshufd $0x00,%xmm13,%xmm12
266 pshufd $0x55,%xmm13,%xmm13
267 movq 0x38(%rdi),%xmm15
268 pshufd $0x00,%xmm15,%xmm14
269 pshufd $0x55,%xmm15,%xmm15
270 # x0..3 on stack
271 movdqa %xmm0,0x00(%rsp)
272 movdqa %xmm1,0x10(%rsp)
273 movdqa %xmm2,0x20(%rsp)
274 movdqa %xmm3,0x30(%rsp)
275
276 movdqa CTRINC(%rip),%xmm1
277 movdqa ROT8(%rip),%xmm2
278 movdqa ROT16(%rip),%xmm3
279
280 # x12 += counter values 0-3
281 paddd %xmm1,%xmm12
282
283.Ldoubleround4:
284 # x0 += x4, x12 = rotl32(x12 ^ x0, 16)
285 movdqa 0x00(%rsp),%xmm0
286 paddd %xmm4,%xmm0
287 movdqa %xmm0,0x00(%rsp)
288 pxor %xmm0,%xmm12
289 pshufb %xmm3,%xmm12
290 # x1 += x5, x13 = rotl32(x13 ^ x1, 16)
291 movdqa 0x10(%rsp),%xmm0
292 paddd %xmm5,%xmm0
293 movdqa %xmm0,0x10(%rsp)
294 pxor %xmm0,%xmm13
295 pshufb %xmm3,%xmm13
296 # x2 += x6, x14 = rotl32(x14 ^ x2, 16)
297 movdqa 0x20(%rsp),%xmm0
298 paddd %xmm6,%xmm0
299 movdqa %xmm0,0x20(%rsp)
300 pxor %xmm0,%xmm14
301 pshufb %xmm3,%xmm14
302 # x3 += x7, x15 = rotl32(x15 ^ x3, 16)
303 movdqa 0x30(%rsp),%xmm0
304 paddd %xmm7,%xmm0
305 movdqa %xmm0,0x30(%rsp)
306 pxor %xmm0,%xmm15
307 pshufb %xmm3,%xmm15
308
309 # x8 += x12, x4 = rotl32(x4 ^ x8, 12)
310 paddd %xmm12,%xmm8
311 pxor %xmm8,%xmm4
312 movdqa %xmm4,%xmm0
313 pslld $12,%xmm0
314 psrld $20,%xmm4
315 por %xmm0,%xmm4
316 # x9 += x13, x5 = rotl32(x5 ^ x9, 12)
317 paddd %xmm13,%xmm9
318 pxor %xmm9,%xmm5
319 movdqa %xmm5,%xmm0
320 pslld $12,%xmm0
321 psrld $20,%xmm5
322 por %xmm0,%xmm5
323 # x10 += x14, x6 = rotl32(x6 ^ x10, 12)
324 paddd %xmm14,%xmm10
325 pxor %xmm10,%xmm6
326 movdqa %xmm6,%xmm0
327 pslld $12,%xmm0
328 psrld $20,%xmm6
329 por %xmm0,%xmm6
330 # x11 += x15, x7 = rotl32(x7 ^ x11, 12)
331 paddd %xmm15,%xmm11
332 pxor %xmm11,%xmm7
333 movdqa %xmm7,%xmm0
334 pslld $12,%xmm0
335 psrld $20,%xmm7
336 por %xmm0,%xmm7
337
338 # x0 += x4, x12 = rotl32(x12 ^ x0, 8)
339 movdqa 0x00(%rsp),%xmm0
340 paddd %xmm4,%xmm0
341 movdqa %xmm0,0x00(%rsp)
342 pxor %xmm0,%xmm12
343 pshufb %xmm2,%xmm12
344 # x1 += x5, x13 = rotl32(x13 ^ x1, 8)
345 movdqa 0x10(%rsp),%xmm0
346 paddd %xmm5,%xmm0
347 movdqa %xmm0,0x10(%rsp)
348 pxor %xmm0,%xmm13
349 pshufb %xmm2,%xmm13
350 # x2 += x6, x14 = rotl32(x14 ^ x2, 8)
351 movdqa 0x20(%rsp),%xmm0
352 paddd %xmm6,%xmm0
353 movdqa %xmm0,0x20(%rsp)
354 pxor %xmm0,%xmm14
355 pshufb %xmm2,%xmm14
356 # x3 += x7, x15 = rotl32(x15 ^ x3, 8)
357 movdqa 0x30(%rsp),%xmm0
358 paddd %xmm7,%xmm0
359 movdqa %xmm0,0x30(%rsp)
360 pxor %xmm0,%xmm15
361 pshufb %xmm2,%xmm15
362
363 # x8 += x12, x4 = rotl32(x4 ^ x8, 7)
364 paddd %xmm12,%xmm8
365 pxor %xmm8,%xmm4
366 movdqa %xmm4,%xmm0
367 pslld $7,%xmm0
368 psrld $25,%xmm4
369 por %xmm0,%xmm4
370 # x9 += x13, x5 = rotl32(x5 ^ x9, 7)
371 paddd %xmm13,%xmm9
372 pxor %xmm9,%xmm5
373 movdqa %xmm5,%xmm0
374 pslld $7,%xmm0
375 psrld $25,%xmm5
376 por %xmm0,%xmm5
377 # x10 += x14, x6 = rotl32(x6 ^ x10, 7)
378 paddd %xmm14,%xmm10
379 pxor %xmm10,%xmm6
380 movdqa %xmm6,%xmm0
381 pslld $7,%xmm0
382 psrld $25,%xmm6
383 por %xmm0,%xmm6
384 # x11 += x15, x7 = rotl32(x7 ^ x11, 7)
385 paddd %xmm15,%xmm11
386 pxor %xmm11,%xmm7
387 movdqa %xmm7,%xmm0
388 pslld $7,%xmm0
389 psrld $25,%xmm7
390 por %xmm0,%xmm7
391
392 # x0 += x5, x15 = rotl32(x15 ^ x0, 16)
393 movdqa 0x00(%rsp),%xmm0
394 paddd %xmm5,%xmm0
395 movdqa %xmm0,0x00(%rsp)
396 pxor %xmm0,%xmm15
397 pshufb %xmm3,%xmm15
398 # x1 += x6, x12 = rotl32(x12 ^ x1, 16)
399 movdqa 0x10(%rsp),%xmm0
400 paddd %xmm6,%xmm0
401 movdqa %xmm0,0x10(%rsp)
402 pxor %xmm0,%xmm12
403 pshufb %xmm3,%xmm12
404 # x2 += x7, x13 = rotl32(x13 ^ x2, 16)
405 movdqa 0x20(%rsp),%xmm0
406 paddd %xmm7,%xmm0
407 movdqa %xmm0,0x20(%rsp)
408 pxor %xmm0,%xmm13
409 pshufb %xmm3,%xmm13
410 # x3 += x4, x14 = rotl32(x14 ^ x3, 16)
411 movdqa 0x30(%rsp),%xmm0
412 paddd %xmm4,%xmm0
413 movdqa %xmm0,0x30(%rsp)
414 pxor %xmm0,%xmm14
415 pshufb %xmm3,%xmm14
416
417 # x10 += x15, x5 = rotl32(x5 ^ x10, 12)
418 paddd %xmm15,%xmm10
419 pxor %xmm10,%xmm5
420 movdqa %xmm5,%xmm0
421 pslld $12,%xmm0
422 psrld $20,%xmm5
423 por %xmm0,%xmm5
424 # x11 += x12, x6 = rotl32(x6 ^ x11, 12)
425 paddd %xmm12,%xmm11
426 pxor %xmm11,%xmm6
427 movdqa %xmm6,%xmm0
428 pslld $12,%xmm0
429 psrld $20,%xmm6
430 por %xmm0,%xmm6
431 # x8 += x13, x7 = rotl32(x7 ^ x8, 12)
432 paddd %xmm13,%xmm8
433 pxor %xmm8,%xmm7
434 movdqa %xmm7,%xmm0
435 pslld $12,%xmm0
436 psrld $20,%xmm7
437 por %xmm0,%xmm7
438 # x9 += x14, x4 = rotl32(x4 ^ x9, 12)
439 paddd %xmm14,%xmm9
440 pxor %xmm9,%xmm4
441 movdqa %xmm4,%xmm0
442 pslld $12,%xmm0
443 psrld $20,%xmm4
444 por %xmm0,%xmm4
445
446 # x0 += x5, x15 = rotl32(x15 ^ x0, 8)
447 movdqa 0x00(%rsp),%xmm0
448 paddd %xmm5,%xmm0
449 movdqa %xmm0,0x00(%rsp)
450 pxor %xmm0,%xmm15
451 pshufb %xmm2,%xmm15
452 # x1 += x6, x12 = rotl32(x12 ^ x1, 8)
453 movdqa 0x10(%rsp),%xmm0
454 paddd %xmm6,%xmm0
455 movdqa %xmm0,0x10(%rsp)
456 pxor %xmm0,%xmm12
457 pshufb %xmm2,%xmm12
458 # x2 += x7, x13 = rotl32(x13 ^ x2, 8)
459 movdqa 0x20(%rsp),%xmm0
460 paddd %xmm7,%xmm0
461 movdqa %xmm0,0x20(%rsp)
462 pxor %xmm0,%xmm13
463 pshufb %xmm2,%xmm13
464 # x3 += x4, x14 = rotl32(x14 ^ x3, 8)
465 movdqa 0x30(%rsp),%xmm0
466 paddd %xmm4,%xmm0
467 movdqa %xmm0,0x30(%rsp)
468 pxor %xmm0,%xmm14
469 pshufb %xmm2,%xmm14
470
471 # x10 += x15, x5 = rotl32(x5 ^ x10, 7)
472 paddd %xmm15,%xmm10
473 pxor %xmm10,%xmm5
474 movdqa %xmm5,%xmm0
475 pslld $7,%xmm0
476 psrld $25,%xmm5
477 por %xmm0,%xmm5
478 # x11 += x12, x6 = rotl32(x6 ^ x11, 7)
479 paddd %xmm12,%xmm11
480 pxor %xmm11,%xmm6
481 movdqa %xmm6,%xmm0
482 pslld $7,%xmm0
483 psrld $25,%xmm6
484 por %xmm0,%xmm6
485 # x8 += x13, x7 = rotl32(x7 ^ x8, 7)
486 paddd %xmm13,%xmm8
487 pxor %xmm8,%xmm7
488 movdqa %xmm7,%xmm0
489 pslld $7,%xmm0
490 psrld $25,%xmm7
491 por %xmm0,%xmm7
492 # x9 += x14, x4 = rotl32(x4 ^ x9, 7)
493 paddd %xmm14,%xmm9
494 pxor %xmm9,%xmm4
495 movdqa %xmm4,%xmm0
496 pslld $7,%xmm0
497 psrld $25,%xmm4
498 por %xmm0,%xmm4
499
500 sub $2,%r8d
501 jnz .Ldoubleround4
502
503 # x0[0-3] += s0[0]
504 # x1[0-3] += s0[1]
505 movq 0x00(%rdi),%xmm3
506 pshufd $0x00,%xmm3,%xmm2
507 pshufd $0x55,%xmm3,%xmm3
508 paddd 0x00(%rsp),%xmm2
509 movdqa %xmm2,0x00(%rsp)
510 paddd 0x10(%rsp),%xmm3
511 movdqa %xmm3,0x10(%rsp)
512 # x2[0-3] += s0[2]
513 # x3[0-3] += s0[3]
514 movq 0x08(%rdi),%xmm3
515 pshufd $0x00,%xmm3,%xmm2
516 pshufd $0x55,%xmm3,%xmm3
517 paddd 0x20(%rsp),%xmm2
518 movdqa %xmm2,0x20(%rsp)
519 paddd 0x30(%rsp),%xmm3
520 movdqa %xmm3,0x30(%rsp)
521
522 # x4[0-3] += s1[0]
523 # x5[0-3] += s1[1]
524 movq 0x10(%rdi),%xmm3
525 pshufd $0x00,%xmm3,%xmm2
526 pshufd $0x55,%xmm3,%xmm3
527 paddd %xmm2,%xmm4
528 paddd %xmm3,%xmm5
529 # x6[0-3] += s1[2]
530 # x7[0-3] += s1[3]
531 movq 0x18(%rdi),%xmm3
532 pshufd $0x00,%xmm3,%xmm2
533 pshufd $0x55,%xmm3,%xmm3
534 paddd %xmm2,%xmm6
535 paddd %xmm3,%xmm7
536
537 # x8[0-3] += s2[0]
538 # x9[0-3] += s2[1]
539 movq 0x20(%rdi),%xmm3
540 pshufd $0x00,%xmm3,%xmm2
541 pshufd $0x55,%xmm3,%xmm3
542 paddd %xmm2,%xmm8
543 paddd %xmm3,%xmm9
544 # x10[0-3] += s2[2]
545 # x11[0-3] += s2[3]
546 movq 0x28(%rdi),%xmm3
547 pshufd $0x00,%xmm3,%xmm2
548 pshufd $0x55,%xmm3,%xmm3
549 paddd %xmm2,%xmm10
550 paddd %xmm3,%xmm11
551
552 # x12[0-3] += s3[0]
553 # x13[0-3] += s3[1]
554 movq 0x30(%rdi),%xmm3
555 pshufd $0x00,%xmm3,%xmm2
556 pshufd $0x55,%xmm3,%xmm3
557 paddd %xmm2,%xmm12
558 paddd %xmm3,%xmm13
559 # x14[0-3] += s3[2]
560 # x15[0-3] += s3[3]
561 movq 0x38(%rdi),%xmm3
562 pshufd $0x00,%xmm3,%xmm2
563 pshufd $0x55,%xmm3,%xmm3
564 paddd %xmm2,%xmm14
565 paddd %xmm3,%xmm15
566
567 # x12 += counter values 0-3
568 paddd %xmm1,%xmm12
569
570 # interleave 32-bit words in state n, n+1
571 movdqa 0x00(%rsp),%xmm0
572 movdqa 0x10(%rsp),%xmm1
573 movdqa %xmm0,%xmm2
574 punpckldq %xmm1,%xmm2
575 punpckhdq %xmm1,%xmm0
576 movdqa %xmm2,0x00(%rsp)
577 movdqa %xmm0,0x10(%rsp)
578 movdqa 0x20(%rsp),%xmm0
579 movdqa 0x30(%rsp),%xmm1
580 movdqa %xmm0,%xmm2
581 punpckldq %xmm1,%xmm2
582 punpckhdq %xmm1,%xmm0
583 movdqa %xmm2,0x20(%rsp)
584 movdqa %xmm0,0x30(%rsp)
585 movdqa %xmm4,%xmm0
586 punpckldq %xmm5,%xmm4
587 punpckhdq %xmm5,%xmm0
588 movdqa %xmm0,%xmm5
589 movdqa %xmm6,%xmm0
590 punpckldq %xmm7,%xmm6
591 punpckhdq %xmm7,%xmm0
592 movdqa %xmm0,%xmm7
593 movdqa %xmm8,%xmm0
594 punpckldq %xmm9,%xmm8
595 punpckhdq %xmm9,%xmm0
596 movdqa %xmm0,%xmm9
597 movdqa %xmm10,%xmm0
598 punpckldq %xmm11,%xmm10
599 punpckhdq %xmm11,%xmm0
600 movdqa %xmm0,%xmm11
601 movdqa %xmm12,%xmm0
602 punpckldq %xmm13,%xmm12
603 punpckhdq %xmm13,%xmm0
604 movdqa %xmm0,%xmm13
605 movdqa %xmm14,%xmm0
606 punpckldq %xmm15,%xmm14
607 punpckhdq %xmm15,%xmm0
608 movdqa %xmm0,%xmm15
609
610 # interleave 64-bit words in state n, n+2
611 movdqa 0x00(%rsp),%xmm0
612 movdqa 0x20(%rsp),%xmm1
613 movdqa %xmm0,%xmm2
614 punpcklqdq %xmm1,%xmm2
615 punpckhqdq %xmm1,%xmm0
616 movdqa %xmm2,0x00(%rsp)
617 movdqa %xmm0,0x20(%rsp)
618 movdqa 0x10(%rsp),%xmm0
619 movdqa 0x30(%rsp),%xmm1
620 movdqa %xmm0,%xmm2
621 punpcklqdq %xmm1,%xmm2
622 punpckhqdq %xmm1,%xmm0
623 movdqa %xmm2,0x10(%rsp)
624 movdqa %xmm0,0x30(%rsp)
625 movdqa %xmm4,%xmm0
626 punpcklqdq %xmm6,%xmm4
627 punpckhqdq %xmm6,%xmm0
628 movdqa %xmm0,%xmm6
629 movdqa %xmm5,%xmm0
630 punpcklqdq %xmm7,%xmm5
631 punpckhqdq %xmm7,%xmm0
632 movdqa %xmm0,%xmm7
633 movdqa %xmm8,%xmm0
634 punpcklqdq %xmm10,%xmm8
635 punpckhqdq %xmm10,%xmm0
636 movdqa %xmm0,%xmm10
637 movdqa %xmm9,%xmm0
638 punpcklqdq %xmm11,%xmm9
639 punpckhqdq %xmm11,%xmm0
640 movdqa %xmm0,%xmm11
641 movdqa %xmm12,%xmm0
642 punpcklqdq %xmm14,%xmm12
643 punpckhqdq %xmm14,%xmm0
644 movdqa %xmm0,%xmm14
645 movdqa %xmm13,%xmm0
646 punpcklqdq %xmm15,%xmm13
647 punpckhqdq %xmm15,%xmm0
648 movdqa %xmm0,%xmm15
649
650 # xor with corresponding input, write to output
651 movdqa 0x00(%rsp),%xmm0
652 cmp $0x10,%rax
653 jl .Lxorpart4
654 movdqu 0x00(%rdx),%xmm1
655 pxor %xmm1,%xmm0
656 movdqu %xmm0,0x00(%rsi)
657
658 movdqu %xmm4,%xmm0
659 cmp $0x20,%rax
660 jl .Lxorpart4
661 movdqu 0x10(%rdx),%xmm1
662 pxor %xmm1,%xmm0
663 movdqu %xmm0,0x10(%rsi)
664
665 movdqu %xmm8,%xmm0
666 cmp $0x30,%rax
667 jl .Lxorpart4
668 movdqu 0x20(%rdx),%xmm1
669 pxor %xmm1,%xmm0
670 movdqu %xmm0,0x20(%rsi)
671
672 movdqu %xmm12,%xmm0
673 cmp $0x40,%rax
674 jl .Lxorpart4
675 movdqu 0x30(%rdx),%xmm1
676 pxor %xmm1,%xmm0
677 movdqu %xmm0,0x30(%rsi)
678
679 movdqa 0x20(%rsp),%xmm0
680 cmp $0x50,%rax
681 jl .Lxorpart4
682 movdqu 0x40(%rdx),%xmm1
683 pxor %xmm1,%xmm0
684 movdqu %xmm0,0x40(%rsi)
685
686 movdqu %xmm6,%xmm0
687 cmp $0x60,%rax
688 jl .Lxorpart4
689 movdqu 0x50(%rdx),%xmm1
690 pxor %xmm1,%xmm0
691 movdqu %xmm0,0x50(%rsi)
692
693 movdqu %xmm10,%xmm0
694 cmp $0x70,%rax
695 jl .Lxorpart4
696 movdqu 0x60(%rdx),%xmm1
697 pxor %xmm1,%xmm0
698 movdqu %xmm0,0x60(%rsi)
699
700 movdqu %xmm14,%xmm0
701 cmp $0x80,%rax
702 jl .Lxorpart4
703 movdqu 0x70(%rdx),%xmm1
704 pxor %xmm1,%xmm0
705 movdqu %xmm0,0x70(%rsi)
706
707 movdqa 0x10(%rsp),%xmm0
708 cmp $0x90,%rax
709 jl .Lxorpart4
710 movdqu 0x80(%rdx),%xmm1
711 pxor %xmm1,%xmm0
712 movdqu %xmm0,0x80(%rsi)
713
714 movdqu %xmm5,%xmm0
715 cmp $0xa0,%rax
716 jl .Lxorpart4
717 movdqu 0x90(%rdx),%xmm1
718 pxor %xmm1,%xmm0
719 movdqu %xmm0,0x90(%rsi)
720
721 movdqu %xmm9,%xmm0
722 cmp $0xb0,%rax
723 jl .Lxorpart4
724 movdqu 0xa0(%rdx),%xmm1
725 pxor %xmm1,%xmm0
726 movdqu %xmm0,0xa0(%rsi)
727
728 movdqu %xmm13,%xmm0
729 cmp $0xc0,%rax
730 jl .Lxorpart4
731 movdqu 0xb0(%rdx),%xmm1
732 pxor %xmm1,%xmm0
733 movdqu %xmm0,0xb0(%rsi)
734
735 movdqa 0x30(%rsp),%xmm0
736 cmp $0xd0,%rax
737 jl .Lxorpart4
738 movdqu 0xc0(%rdx),%xmm1
739 pxor %xmm1,%xmm0
740 movdqu %xmm0,0xc0(%rsi)
741
742 movdqu %xmm7,%xmm0
743 cmp $0xe0,%rax
744 jl .Lxorpart4
745 movdqu 0xd0(%rdx),%xmm1
746 pxor %xmm1,%xmm0
747 movdqu %xmm0,0xd0(%rsi)
748
749 movdqu %xmm11,%xmm0
750 cmp $0xf0,%rax
751 jl .Lxorpart4
752 movdqu 0xe0(%rdx),%xmm1
753 pxor %xmm1,%xmm0
754 movdqu %xmm0,0xe0(%rsi)
755
756 movdqu %xmm15,%xmm0
757 cmp $0x100,%rax
758 jl .Lxorpart4
759 movdqu 0xf0(%rdx),%xmm1
760 pxor %xmm1,%xmm0
761 movdqu %xmm0,0xf0(%rsi)
762
763.Ldone4:
764 lea -8(%r10),%rsp
765 RET
766
767.Lxorpart4:
768 # xor remaining bytes from partial register into output
769 mov %rax,%r9
770 and $0x0f,%r9
771 jz .Ldone4
772 and $~0x0f,%rax
773
774 mov %rsi,%r11
775
776 lea (%rdx,%rax),%rsi
777 mov %rsp,%rdi
778 mov %r9,%rcx
779 rep movsb
780
781 pxor 0x00(%rsp),%xmm0
782 movdqa %xmm0,0x00(%rsp)
783
784 mov %rsp,%rsi
785 lea (%r11,%rax),%rdi
786 mov %r9,%rcx
787 rep movsb
788
789 jmp .Ldone4
790
791SYM_FUNC_END(chacha_4block_xor_ssse3)
792

source code of linux/lib/crypto/x86/chacha-ssse3-x86_64.S