1 | ######################################################################## |
2 | # Implement fast SHA-256 with AVX2 instructions. (x86_64) |
3 | # |
4 | # Copyright (C) 2013 Intel Corporation. |
5 | # |
6 | # Authors: |
7 | # James Guilford <james.guilford@intel.com> |
8 | # Kirk Yap <kirk.s.yap@intel.com> |
9 | # Tim Chen <tim.c.chen@linux.intel.com> |
10 | # |
11 | # This software is available to you under a choice of one of two |
12 | # licenses. You may choose to be licensed under the terms of the GNU |
13 | # General Public License (GPL) Version 2, available from the file |
14 | # COPYING in the main directory of this source tree, or the |
15 | # OpenIB.org BSD license below: |
16 | # |
17 | # Redistribution and use in source and binary forms, with or |
18 | # without modification, are permitted provided that the following |
19 | # conditions are met: |
20 | # |
21 | # - Redistributions of source code must retain the above |
22 | # copyright notice, this list of conditions and the following |
23 | # disclaimer. |
24 | # |
25 | # - Redistributions in binary form must reproduce the above |
26 | # copyright notice, this list of conditions and the following |
27 | # disclaimer in the documentation and/or other materials |
28 | # provided with the distribution. |
29 | # |
30 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
31 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
32 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
33 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
34 | # BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
35 | # ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
36 | # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
37 | # SOFTWARE. |
38 | # |
39 | ######################################################################## |
40 | # |
41 | # This code is described in an Intel White-Paper: |
42 | # "Fast SHA-256 Implementations on Intel Architecture Processors" |
43 | # |
44 | # To find it, surf to http://www.intel.com/p/en_US/embedded |
45 | # and search for that title. |
46 | # |
47 | ######################################################################## |
48 | # This code schedules 2 blocks at a time, with 4 lanes per block |
49 | ######################################################################## |
50 | |
51 | #include <linux/linkage.h> |
52 | #include <linux/cfi_types.h> |
53 | |
54 | ## assume buffers not aligned |
55 | #define VMOVDQ vmovdqu |
56 | |
57 | ################################ Define Macros |
58 | |
59 | # addm [mem], reg |
60 | # Add reg to mem using reg-mem add and store |
61 | .macro addm p1 p2 |
62 | add \p1, \p2 |
63 | mov \p2, \p1 |
64 | .endm |
65 | |
66 | ################################ |
67 | |
68 | X0 = %ymm4 |
69 | X1 = %ymm5 |
70 | X2 = %ymm6 |
71 | X3 = %ymm7 |
72 | |
73 | # XMM versions of above |
74 | XWORD0 = %xmm4 |
75 | XWORD1 = %xmm5 |
76 | XWORD2 = %xmm6 |
77 | XWORD3 = %xmm7 |
78 | |
79 | XTMP0 = %ymm0 |
80 | XTMP1 = %ymm1 |
81 | XTMP2 = %ymm2 |
82 | XTMP3 = %ymm3 |
83 | XTMP4 = %ymm8 |
84 | XFER = %ymm9 |
85 | XTMP5 = %ymm11 |
86 | |
87 | SHUF_00BA = %ymm10 # shuffle xBxA -> 00BA |
88 | SHUF_DC00 = %ymm12 # shuffle xDxC -> DC00 |
89 | BYTE_FLIP_MASK = %ymm13 |
90 | |
91 | X_BYTE_FLIP_MASK = %xmm13 # XMM version of BYTE_FLIP_MASK |
92 | |
93 | NUM_BLKS = %rdx # 3rd arg |
94 | INP = %rsi # 2nd arg |
95 | CTX = %rdi # 1st arg |
96 | c = %ecx |
97 | d = %r8d |
98 | e = %edx # clobbers NUM_BLKS |
99 | y3 = %esi # clobbers INP |
100 | |
101 | SRND = CTX # SRND is same register as CTX |
102 | |
103 | a = %eax |
104 | b = %ebx |
105 | f = %r9d |
106 | g = %r10d |
107 | h = %r11d |
108 | old_h = %r11d |
109 | |
110 | T1 = %r12d |
111 | y0 = %r13d |
112 | y1 = %r14d |
113 | y2 = %r15d |
114 | |
115 | |
116 | _XFER_SIZE = 2*64*4 # 2 blocks, 64 rounds, 4 bytes/round |
117 | _XMM_SAVE_SIZE = 0 |
118 | _INP_END_SIZE = 8 |
119 | _INP_SIZE = 8 |
120 | _CTX_SIZE = 8 |
121 | |
122 | _XFER = 0 |
123 | _XMM_SAVE = _XFER + _XFER_SIZE |
124 | _INP_END = _XMM_SAVE + _XMM_SAVE_SIZE |
125 | _INP = _INP_END + _INP_END_SIZE |
126 | _CTX = _INP + _INP_SIZE |
127 | STACK_SIZE = _CTX + _CTX_SIZE |
128 | |
129 | # rotate_Xs |
130 | # Rotate values of symbols X0...X3 |
131 | .macro rotate_Xs |
132 | X_ = X0 |
133 | X0 = X1 |
134 | X1 = X2 |
135 | X2 = X3 |
136 | X3 = X_ |
137 | .endm |
138 | |
139 | # ROTATE_ARGS |
140 | # Rotate values of symbols a...h |
141 | .macro ROTATE_ARGS |
142 | old_h = h |
143 | TMP_ = h |
144 | h = g |
145 | g = f |
146 | f = e |
147 | e = d |
148 | d = c |
149 | c = b |
150 | b = a |
151 | a = TMP_ |
152 | .endm |
153 | |
154 | .macro FOUR_ROUNDS_AND_SCHED disp |
155 | ################################### RND N + 0 ############################ |
156 | |
157 | mov a, y3 # y3 = a # MAJA |
158 | rorx $25, e, y0 # y0 = e >> 25 # S1A |
159 | rorx $11, e, y1 # y1 = e >> 11 # S1B |
160 | |
161 | addl \disp(%rsp, SRND), h # h = k + w + h # -- |
162 | or c, y3 # y3 = a|c # MAJA |
163 | vpalignr $4, X2, X3, XTMP0 # XTMP0 = W[-7] |
164 | mov f, y2 # y2 = f # CH |
165 | rorx $13, a, T1 # T1 = a >> 13 # S0B |
166 | |
167 | xor y1, y0 # y0 = (e>>25) ^ (e>>11) # S1 |
168 | xor g, y2 # y2 = f^g # CH |
169 | vpaddd X0, XTMP0, XTMP0 # XTMP0 = W[-7] + W[-16]# y1 = (e >> 6)# S1 |
170 | rorx $6, e, y1 # y1 = (e >> 6) # S1 |
171 | |
172 | and e, y2 # y2 = (f^g)&e # CH |
173 | xor y1, y0 # y0 = (e>>25) ^ (e>>11) ^ (e>>6) # S1 |
174 | rorx $22, a, y1 # y1 = a >> 22 # S0A |
175 | add h, d # d = k + w + h + d # -- |
176 | |
177 | and b, y3 # y3 = (a|c)&b # MAJA |
178 | vpalignr $4, X0, X1, XTMP1 # XTMP1 = W[-15] |
179 | xor T1, y1 # y1 = (a>>22) ^ (a>>13) # S0 |
180 | rorx $2, a, T1 # T1 = (a >> 2) # S0 |
181 | |
182 | xor g, y2 # y2 = CH = ((f^g)&e)^g # CH |
183 | vpsrld $7, XTMP1, XTMP2 |
184 | xor T1, y1 # y1 = (a>>22) ^ (a>>13) ^ (a>>2) # S0 |
185 | mov a, T1 # T1 = a # MAJB |
186 | and c, T1 # T1 = a&c # MAJB |
187 | |
188 | add y0, y2 # y2 = S1 + CH # -- |
189 | vpslld $(32-7), XTMP1, XTMP3 |
190 | or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ |
191 | add y1, h # h = k + w + h + S0 # -- |
192 | |
193 | add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- |
194 | vpor XTMP2, XTMP3, XTMP3 # XTMP3 = W[-15] ror 7 |
195 | |
196 | vpsrld $18, XTMP1, XTMP2 |
197 | add y2, h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- |
198 | add y3, h # h = t1 + S0 + MAJ # -- |
199 | |
200 | |
201 | ROTATE_ARGS |
202 | |
203 | ################################### RND N + 1 ############################ |
204 | |
205 | mov a, y3 # y3 = a # MAJA |
206 | rorx $25, e, y0 # y0 = e >> 25 # S1A |
207 | rorx $11, e, y1 # y1 = e >> 11 # S1B |
208 | offset = \disp + 1*4 |
209 | addl offset(%rsp, SRND), h # h = k + w + h # -- |
210 | or c, y3 # y3 = a|c # MAJA |
211 | |
212 | |
213 | vpsrld $3, XTMP1, XTMP4 # XTMP4 = W[-15] >> 3 |
214 | mov f, y2 # y2 = f # CH |
215 | rorx $13, a, T1 # T1 = a >> 13 # S0B |
216 | xor y1, y0 # y0 = (e>>25) ^ (e>>11) # S1 |
217 | xor g, y2 # y2 = f^g # CH |
218 | |
219 | |
220 | rorx $6, e, y1 # y1 = (e >> 6) # S1 |
221 | xor y1, y0 # y0 = (e>>25) ^ (e>>11) ^ (e>>6) # S1 |
222 | rorx $22, a, y1 # y1 = a >> 22 # S0A |
223 | and e, y2 # y2 = (f^g)&e # CH |
224 | add h, d # d = k + w + h + d # -- |
225 | |
226 | vpslld $(32-18), XTMP1, XTMP1 |
227 | and b, y3 # y3 = (a|c)&b # MAJA |
228 | xor T1, y1 # y1 = (a>>22) ^ (a>>13) # S0 |
229 | |
230 | vpxor XTMP1, XTMP3, XTMP3 |
231 | rorx $2, a, T1 # T1 = (a >> 2) # S0 |
232 | xor g, y2 # y2 = CH = ((f^g)&e)^g # CH |
233 | |
234 | vpxor XTMP2, XTMP3, XTMP3 # XTMP3 = W[-15] ror 7 ^ W[-15] ror 18 |
235 | xor T1, y1 # y1 = (a>>22) ^ (a>>13) ^ (a>>2) # S0 |
236 | mov a, T1 # T1 = a # MAJB |
237 | and c, T1 # T1 = a&c # MAJB |
238 | add y0, y2 # y2 = S1 + CH # -- |
239 | |
240 | vpxor XTMP4, XTMP3, XTMP1 # XTMP1 = s0 |
241 | vpshufd $0b11111010, X3, XTMP2 # XTMP2 = W[-2] {BBAA} |
242 | or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ |
243 | add y1, h # h = k + w + h + S0 # -- |
244 | |
245 | vpaddd XTMP1, XTMP0, XTMP0 # XTMP0 = W[-16] + W[-7] + s0 |
246 | add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- |
247 | add y2, h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- |
248 | add y3, h # h = t1 + S0 + MAJ # -- |
249 | |
250 | vpsrld $10, XTMP2, XTMP4 # XTMP4 = W[-2] >> 10 {BBAA} |
251 | |
252 | |
253 | ROTATE_ARGS |
254 | |
255 | ################################### RND N + 2 ############################ |
256 | |
257 | mov a, y3 # y3 = a # MAJA |
258 | rorx $25, e, y0 # y0 = e >> 25 # S1A |
259 | offset = \disp + 2*4 |
260 | addl offset(%rsp, SRND), h # h = k + w + h # -- |
261 | |
262 | vpsrlq $19, XTMP2, XTMP3 # XTMP3 = W[-2] ror 19 {xBxA} |
263 | rorx $11, e, y1 # y1 = e >> 11 # S1B |
264 | or c, y3 # y3 = a|c # MAJA |
265 | mov f, y2 # y2 = f # CH |
266 | xor g, y2 # y2 = f^g # CH |
267 | |
268 | rorx $13, a, T1 # T1 = a >> 13 # S0B |
269 | xor y1, y0 # y0 = (e>>25) ^ (e>>11) # S1 |
270 | vpsrlq $17, XTMP2, XTMP2 # XTMP2 = W[-2] ror 17 {xBxA} |
271 | and e, y2 # y2 = (f^g)&e # CH |
272 | |
273 | rorx $6, e, y1 # y1 = (e >> 6) # S1 |
274 | vpxor XTMP3, XTMP2, XTMP2 |
275 | add h, d # d = k + w + h + d # -- |
276 | and b, y3 # y3 = (a|c)&b # MAJA |
277 | |
278 | xor y1, y0 # y0 = (e>>25) ^ (e>>11) ^ (e>>6) # S1 |
279 | rorx $22, a, y1 # y1 = a >> 22 # S0A |
280 | vpxor XTMP2, XTMP4, XTMP4 # XTMP4 = s1 {xBxA} |
281 | xor g, y2 # y2 = CH = ((f^g)&e)^g # CH |
282 | |
283 | vpshufb SHUF_00BA, XTMP4, XTMP4 # XTMP4 = s1 {00BA} |
284 | xor T1, y1 # y1 = (a>>22) ^ (a>>13) # S0 |
285 | rorx $2, a ,T1 # T1 = (a >> 2) # S0 |
286 | vpaddd XTMP4, XTMP0, XTMP0 # XTMP0 = {..., ..., W[1], W[0]} |
287 | |
288 | xor T1, y1 # y1 = (a>>22) ^ (a>>13) ^ (a>>2) # S0 |
289 | mov a, T1 # T1 = a # MAJB |
290 | and c, T1 # T1 = a&c # MAJB |
291 | add y0, y2 # y2 = S1 + CH # -- |
292 | vpshufd $0b01010000, XTMP0, XTMP2 # XTMP2 = W[-2] {DDCC} |
293 | |
294 | or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ |
295 | add y1,h # h = k + w + h + S0 # -- |
296 | add y2,d # d = k + w + h + d + S1 + CH = d + t1 # -- |
297 | add y2,h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- |
298 | |
299 | add y3,h # h = t1 + S0 + MAJ # -- |
300 | |
301 | |
302 | ROTATE_ARGS |
303 | |
304 | ################################### RND N + 3 ############################ |
305 | |
306 | mov a, y3 # y3 = a # MAJA |
307 | rorx $25, e, y0 # y0 = e >> 25 # S1A |
308 | rorx $11, e, y1 # y1 = e >> 11 # S1B |
309 | offset = \disp + 3*4 |
310 | addl offset(%rsp, SRND), h # h = k + w + h # -- |
311 | or c, y3 # y3 = a|c # MAJA |
312 | |
313 | |
314 | vpsrld $10, XTMP2, XTMP5 # XTMP5 = W[-2] >> 10 {DDCC} |
315 | mov f, y2 # y2 = f # CH |
316 | rorx $13, a, T1 # T1 = a >> 13 # S0B |
317 | xor y1, y0 # y0 = (e>>25) ^ (e>>11) # S1 |
318 | xor g, y2 # y2 = f^g # CH |
319 | |
320 | |
321 | vpsrlq $19, XTMP2, XTMP3 # XTMP3 = W[-2] ror 19 {xDxC} |
322 | rorx $6, e, y1 # y1 = (e >> 6) # S1 |
323 | and e, y2 # y2 = (f^g)&e # CH |
324 | add h, d # d = k + w + h + d # -- |
325 | and b, y3 # y3 = (a|c)&b # MAJA |
326 | |
327 | vpsrlq $17, XTMP2, XTMP2 # XTMP2 = W[-2] ror 17 {xDxC} |
328 | xor y1, y0 # y0 = (e>>25) ^ (e>>11) ^ (e>>6) # S1 |
329 | xor g, y2 # y2 = CH = ((f^g)&e)^g # CH |
330 | |
331 | vpxor XTMP3, XTMP2, XTMP2 |
332 | rorx $22, a, y1 # y1 = a >> 22 # S0A |
333 | add y0, y2 # y2 = S1 + CH # -- |
334 | |
335 | vpxor XTMP2, XTMP5, XTMP5 # XTMP5 = s1 {xDxC} |
336 | xor T1, y1 # y1 = (a>>22) ^ (a>>13) # S0 |
337 | add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- |
338 | |
339 | rorx $2, a, T1 # T1 = (a >> 2) # S0 |
340 | vpshufb SHUF_DC00, XTMP5, XTMP5 # XTMP5 = s1 {DC00} |
341 | |
342 | vpaddd XTMP0, XTMP5, X0 # X0 = {W[3], W[2], W[1], W[0]} |
343 | xor T1, y1 # y1 = (a>>22) ^ (a>>13) ^ (a>>2) # S0 |
344 | mov a, T1 # T1 = a # MAJB |
345 | and c, T1 # T1 = a&c # MAJB |
346 | or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ |
347 | |
348 | add y1, h # h = k + w + h + S0 # -- |
349 | add y2, h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- |
350 | add y3, h # h = t1 + S0 + MAJ # -- |
351 | |
352 | ROTATE_ARGS |
353 | rotate_Xs |
354 | .endm |
355 | |
356 | .macro DO_4ROUNDS disp |
357 | ################################### RND N + 0 ########################### |
358 | |
359 | mov f, y2 # y2 = f # CH |
360 | rorx $25, e, y0 # y0 = e >> 25 # S1A |
361 | rorx $11, e, y1 # y1 = e >> 11 # S1B |
362 | xor g, y2 # y2 = f^g # CH |
363 | |
364 | xor y1, y0 # y0 = (e>>25) ^ (e>>11) # S1 |
365 | rorx $6, e, y1 # y1 = (e >> 6) # S1 |
366 | and e, y2 # y2 = (f^g)&e # CH |
367 | |
368 | xor y1, y0 # y0 = (e>>25) ^ (e>>11) ^ (e>>6) # S1 |
369 | rorx $13, a, T1 # T1 = a >> 13 # S0B |
370 | xor g, y2 # y2 = CH = ((f^g)&e)^g # CH |
371 | rorx $22, a, y1 # y1 = a >> 22 # S0A |
372 | mov a, y3 # y3 = a # MAJA |
373 | |
374 | xor T1, y1 # y1 = (a>>22) ^ (a>>13) # S0 |
375 | rorx $2, a, T1 # T1 = (a >> 2) # S0 |
376 | addl \disp(%rsp, SRND), h # h = k + w + h # -- |
377 | or c, y3 # y3 = a|c # MAJA |
378 | |
379 | xor T1, y1 # y1 = (a>>22) ^ (a>>13) ^ (a>>2) # S0 |
380 | mov a, T1 # T1 = a # MAJB |
381 | and b, y3 # y3 = (a|c)&b # MAJA |
382 | and c, T1 # T1 = a&c # MAJB |
383 | add y0, y2 # y2 = S1 + CH # -- |
384 | |
385 | |
386 | add h, d # d = k + w + h + d # -- |
387 | or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ |
388 | add y1, h # h = k + w + h + S0 # -- |
389 | add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- |
390 | |
391 | ROTATE_ARGS |
392 | |
393 | ################################### RND N + 1 ########################### |
394 | |
395 | add y2, old_h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- |
396 | mov f, y2 # y2 = f # CH |
397 | rorx $25, e, y0 # y0 = e >> 25 # S1A |
398 | rorx $11, e, y1 # y1 = e >> 11 # S1B |
399 | xor g, y2 # y2 = f^g # CH |
400 | |
401 | xor y1, y0 # y0 = (e>>25) ^ (e>>11) # S1 |
402 | rorx $6, e, y1 # y1 = (e >> 6) # S1 |
403 | and e, y2 # y2 = (f^g)&e # CH |
404 | add y3, old_h # h = t1 + S0 + MAJ # -- |
405 | |
406 | xor y1, y0 # y0 = (e>>25) ^ (e>>11) ^ (e>>6) # S1 |
407 | rorx $13, a, T1 # T1 = a >> 13 # S0B |
408 | xor g, y2 # y2 = CH = ((f^g)&e)^g # CH |
409 | rorx $22, a, y1 # y1 = a >> 22 # S0A |
410 | mov a, y3 # y3 = a # MAJA |
411 | |
412 | xor T1, y1 # y1 = (a>>22) ^ (a>>13) # S0 |
413 | rorx $2, a, T1 # T1 = (a >> 2) # S0 |
414 | offset = 4*1 + \disp |
415 | addl offset(%rsp, SRND), h # h = k + w + h # -- |
416 | or c, y3 # y3 = a|c # MAJA |
417 | |
418 | xor T1, y1 # y1 = (a>>22) ^ (a>>13) ^ (a>>2) # S0 |
419 | mov a, T1 # T1 = a # MAJB |
420 | and b, y3 # y3 = (a|c)&b # MAJA |
421 | and c, T1 # T1 = a&c # MAJB |
422 | add y0, y2 # y2 = S1 + CH # -- |
423 | |
424 | |
425 | add h, d # d = k + w + h + d # -- |
426 | or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ |
427 | add y1, h # h = k + w + h + S0 # -- |
428 | |
429 | add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- |
430 | |
431 | ROTATE_ARGS |
432 | |
433 | ################################### RND N + 2 ############################## |
434 | |
435 | add y2, old_h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- |
436 | mov f, y2 # y2 = f # CH |
437 | rorx $25, e, y0 # y0 = e >> 25 # S1A |
438 | rorx $11, e, y1 # y1 = e >> 11 # S1B |
439 | xor g, y2 # y2 = f^g # CH |
440 | |
441 | xor y1, y0 # y0 = (e>>25) ^ (e>>11) # S1 |
442 | rorx $6, e, y1 # y1 = (e >> 6) # S1 |
443 | and e, y2 # y2 = (f^g)&e # CH |
444 | add y3, old_h # h = t1 + S0 + MAJ # -- |
445 | |
446 | xor y1, y0 # y0 = (e>>25) ^ (e>>11) ^ (e>>6) # S1 |
447 | rorx $13, a, T1 # T1 = a >> 13 # S0B |
448 | xor g, y2 # y2 = CH = ((f^g)&e)^g # CH |
449 | rorx $22, a, y1 # y1 = a >> 22 # S0A |
450 | mov a, y3 # y3 = a # MAJA |
451 | |
452 | xor T1, y1 # y1 = (a>>22) ^ (a>>13) # S0 |
453 | rorx $2, a, T1 # T1 = (a >> 2) # S0 |
454 | offset = 4*2 + \disp |
455 | addl offset(%rsp, SRND), h # h = k + w + h # -- |
456 | or c, y3 # y3 = a|c # MAJA |
457 | |
458 | xor T1, y1 # y1 = (a>>22) ^ (a>>13) ^ (a>>2) # S0 |
459 | mov a, T1 # T1 = a # MAJB |
460 | and b, y3 # y3 = (a|c)&b # MAJA |
461 | and c, T1 # T1 = a&c # MAJB |
462 | add y0, y2 # y2 = S1 + CH # -- |
463 | |
464 | |
465 | add h, d # d = k + w + h + d # -- |
466 | or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ |
467 | add y1, h # h = k + w + h + S0 # -- |
468 | |
469 | add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- |
470 | |
471 | ROTATE_ARGS |
472 | |
473 | ################################### RND N + 3 ########################### |
474 | |
475 | add y2, old_h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- |
476 | mov f, y2 # y2 = f # CH |
477 | rorx $25, e, y0 # y0 = e >> 25 # S1A |
478 | rorx $11, e, y1 # y1 = e >> 11 # S1B |
479 | xor g, y2 # y2 = f^g # CH |
480 | |
481 | xor y1, y0 # y0 = (e>>25) ^ (e>>11) # S1 |
482 | rorx $6, e, y1 # y1 = (e >> 6) # S1 |
483 | and e, y2 # y2 = (f^g)&e # CH |
484 | add y3, old_h # h = t1 + S0 + MAJ # -- |
485 | |
486 | xor y1, y0 # y0 = (e>>25) ^ (e>>11) ^ (e>>6) # S1 |
487 | rorx $13, a, T1 # T1 = a >> 13 # S0B |
488 | xor g, y2 # y2 = CH = ((f^g)&e)^g # CH |
489 | rorx $22, a, y1 # y1 = a >> 22 # S0A |
490 | mov a, y3 # y3 = a # MAJA |
491 | |
492 | xor T1, y1 # y1 = (a>>22) ^ (a>>13) # S0 |
493 | rorx $2, a, T1 # T1 = (a >> 2) # S0 |
494 | offset = 4*3 + \disp |
495 | addl offset(%rsp, SRND), h # h = k + w + h # -- |
496 | or c, y3 # y3 = a|c # MAJA |
497 | |
498 | xor T1, y1 # y1 = (a>>22) ^ (a>>13) ^ (a>>2) # S0 |
499 | mov a, T1 # T1 = a # MAJB |
500 | and b, y3 # y3 = (a|c)&b # MAJA |
501 | and c, T1 # T1 = a&c # MAJB |
502 | add y0, y2 # y2 = S1 + CH # -- |
503 | |
504 | |
505 | add h, d # d = k + w + h + d # -- |
506 | or T1, y3 # y3 = MAJ = (a|c)&b)|(a&c) # MAJ |
507 | add y1, h # h = k + w + h + S0 # -- |
508 | |
509 | add y2, d # d = k + w + h + d + S1 + CH = d + t1 # -- |
510 | |
511 | |
512 | add y2, h # h = k + w + h + S0 + S1 + CH = t1 + S0# -- |
513 | |
514 | add y3, h # h = t1 + S0 + MAJ # -- |
515 | |
516 | ROTATE_ARGS |
517 | |
518 | .endm |
519 | |
520 | ######################################################################## |
521 | ## void sha256_transform_rorx(struct sha256_state *state, const u8 *data, int blocks) |
522 | ## arg 1 : pointer to state |
523 | ## arg 2 : pointer to input data |
524 | ## arg 3 : Num blocks |
525 | ######################################################################## |
526 | .text |
527 | SYM_TYPED_FUNC_START(sha256_transform_rorx) |
528 | pushq %rbx |
529 | pushq %r12 |
530 | pushq %r13 |
531 | pushq %r14 |
532 | pushq %r15 |
533 | |
534 | push %rbp |
535 | mov %rsp, %rbp |
536 | |
537 | subq $STACK_SIZE, %rsp |
538 | and $-32, %rsp # align rsp to 32 byte boundary |
539 | |
540 | shl $6, NUM_BLKS # convert to bytes |
541 | jz .Ldone_hash |
542 | lea -64(INP, NUM_BLKS), NUM_BLKS # pointer to last block |
543 | mov NUM_BLKS, _INP_END(%rsp) |
544 | |
545 | cmp NUM_BLKS, INP |
546 | je .Lonly_one_block |
547 | |
548 | ## load initial digest |
549 | mov (CTX), a |
550 | mov 4*1(CTX), b |
551 | mov 4*2(CTX), c |
552 | mov 4*3(CTX), d |
553 | mov 4*4(CTX), e |
554 | mov 4*5(CTX), f |
555 | mov 4*6(CTX), g |
556 | mov 4*7(CTX), h |
557 | |
558 | vmovdqa PSHUFFLE_BYTE_FLIP_MASK(%rip), BYTE_FLIP_MASK |
559 | vmovdqa _SHUF_00BA(%rip), SHUF_00BA |
560 | vmovdqa _SHUF_DC00(%rip), SHUF_DC00 |
561 | |
562 | mov CTX, _CTX(%rsp) |
563 | |
564 | .Lloop0: |
565 | ## Load first 16 dwords from two blocks |
566 | VMOVDQ 0*32(INP),XTMP0 |
567 | VMOVDQ 1*32(INP),XTMP1 |
568 | VMOVDQ 2*32(INP),XTMP2 |
569 | VMOVDQ 3*32(INP),XTMP3 |
570 | |
571 | ## byte swap data |
572 | vpshufb BYTE_FLIP_MASK, XTMP0, XTMP0 |
573 | vpshufb BYTE_FLIP_MASK, XTMP1, XTMP1 |
574 | vpshufb BYTE_FLIP_MASK, XTMP2, XTMP2 |
575 | vpshufb BYTE_FLIP_MASK, XTMP3, XTMP3 |
576 | |
577 | ## transpose data into high/low halves |
578 | vperm2i128 $0x20, XTMP2, XTMP0, X0 |
579 | vperm2i128 $0x31, XTMP2, XTMP0, X1 |
580 | vperm2i128 $0x20, XTMP3, XTMP1, X2 |
581 | vperm2i128 $0x31, XTMP3, XTMP1, X3 |
582 | |
583 | .Llast_block_enter: |
584 | add $64, INP |
585 | mov INP, _INP(%rsp) |
586 | |
587 | ## schedule 48 input dwords, by doing 3 rounds of 12 each |
588 | xor SRND, SRND |
589 | |
590 | .align 16 |
591 | .Lloop1: |
592 | leaq K256+0*32(%rip), INP ## reuse INP as scratch reg |
593 | vpaddd (INP, SRND), X0, XFER |
594 | vmovdqa XFER, 0*32+_XFER(%rsp, SRND) |
595 | FOUR_ROUNDS_AND_SCHED _XFER + 0*32 |
596 | |
597 | leaq K256+1*32(%rip), INP |
598 | vpaddd (INP, SRND), X0, XFER |
599 | vmovdqa XFER, 1*32+_XFER(%rsp, SRND) |
600 | FOUR_ROUNDS_AND_SCHED _XFER + 1*32 |
601 | |
602 | leaq K256+2*32(%rip), INP |
603 | vpaddd (INP, SRND), X0, XFER |
604 | vmovdqa XFER, 2*32+_XFER(%rsp, SRND) |
605 | FOUR_ROUNDS_AND_SCHED _XFER + 2*32 |
606 | |
607 | leaq K256+3*32(%rip), INP |
608 | vpaddd (INP, SRND), X0, XFER |
609 | vmovdqa XFER, 3*32+_XFER(%rsp, SRND) |
610 | FOUR_ROUNDS_AND_SCHED _XFER + 3*32 |
611 | |
612 | add $4*32, SRND |
613 | cmp $3*4*32, SRND |
614 | jb .Lloop1 |
615 | |
616 | .Lloop2: |
617 | ## Do last 16 rounds with no scheduling |
618 | leaq K256+0*32(%rip), INP |
619 | vpaddd (INP, SRND), X0, XFER |
620 | vmovdqa XFER, 0*32+_XFER(%rsp, SRND) |
621 | DO_4ROUNDS _XFER + 0*32 |
622 | |
623 | leaq K256+1*32(%rip), INP |
624 | vpaddd (INP, SRND), X1, XFER |
625 | vmovdqa XFER, 1*32+_XFER(%rsp, SRND) |
626 | DO_4ROUNDS _XFER + 1*32 |
627 | add $2*32, SRND |
628 | |
629 | vmovdqa X2, X0 |
630 | vmovdqa X3, X1 |
631 | |
632 | cmp $4*4*32, SRND |
633 | jb .Lloop2 |
634 | |
635 | mov _CTX(%rsp), CTX |
636 | mov _INP(%rsp), INP |
637 | |
638 | addm (4*0)(CTX),a |
639 | addm (4*1)(CTX),b |
640 | addm (4*2)(CTX),c |
641 | addm (4*3)(CTX),d |
642 | addm (4*4)(CTX),e |
643 | addm (4*5)(CTX),f |
644 | addm (4*6)(CTX),g |
645 | addm (4*7)(CTX),h |
646 | |
647 | cmp _INP_END(%rsp), INP |
648 | ja .Ldone_hash |
649 | |
650 | #### Do second block using previously scheduled results |
651 | xor SRND, SRND |
652 | .align 16 |
653 | .Lloop3: |
654 | DO_4ROUNDS _XFER + 0*32 + 16 |
655 | DO_4ROUNDS _XFER + 1*32 + 16 |
656 | add $2*32, SRND |
657 | cmp $4*4*32, SRND |
658 | jb .Lloop3 |
659 | |
660 | mov _CTX(%rsp), CTX |
661 | mov _INP(%rsp), INP |
662 | add $64, INP |
663 | |
664 | addm (4*0)(CTX),a |
665 | addm (4*1)(CTX),b |
666 | addm (4*2)(CTX),c |
667 | addm (4*3)(CTX),d |
668 | addm (4*4)(CTX),e |
669 | addm (4*5)(CTX),f |
670 | addm (4*6)(CTX),g |
671 | addm (4*7)(CTX),h |
672 | |
673 | cmp _INP_END(%rsp), INP |
674 | jb .Lloop0 |
675 | ja .Ldone_hash |
676 | |
677 | .Ldo_last_block: |
678 | VMOVDQ 0*16(INP),XWORD0 |
679 | VMOVDQ 1*16(INP),XWORD1 |
680 | VMOVDQ 2*16(INP),XWORD2 |
681 | VMOVDQ 3*16(INP),XWORD3 |
682 | |
683 | vpshufb X_BYTE_FLIP_MASK, XWORD0, XWORD0 |
684 | vpshufb X_BYTE_FLIP_MASK, XWORD1, XWORD1 |
685 | vpshufb X_BYTE_FLIP_MASK, XWORD2, XWORD2 |
686 | vpshufb X_BYTE_FLIP_MASK, XWORD3, XWORD3 |
687 | |
688 | jmp .Llast_block_enter |
689 | |
690 | .Lonly_one_block: |
691 | |
692 | ## load initial digest |
693 | mov (4*0)(CTX),a |
694 | mov (4*1)(CTX),b |
695 | mov (4*2)(CTX),c |
696 | mov (4*3)(CTX),d |
697 | mov (4*4)(CTX),e |
698 | mov (4*5)(CTX),f |
699 | mov (4*6)(CTX),g |
700 | mov (4*7)(CTX),h |
701 | |
702 | vmovdqa PSHUFFLE_BYTE_FLIP_MASK(%rip), BYTE_FLIP_MASK |
703 | vmovdqa _SHUF_00BA(%rip), SHUF_00BA |
704 | vmovdqa _SHUF_DC00(%rip), SHUF_DC00 |
705 | |
706 | mov CTX, _CTX(%rsp) |
707 | jmp .Ldo_last_block |
708 | |
709 | .Ldone_hash: |
710 | |
711 | mov %rbp, %rsp |
712 | pop %rbp |
713 | |
714 | popq %r15 |
715 | popq %r14 |
716 | popq %r13 |
717 | popq %r12 |
718 | popq %rbx |
719 | RET |
720 | SYM_FUNC_END(sha256_transform_rorx) |
721 | |
722 | .section .rodata.cst512.K256, "aM" , @progbits, 512 |
723 | .align 64 |
724 | K256: |
725 | .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 |
726 | .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 |
727 | .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 |
728 | .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 |
729 | .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 |
730 | .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 |
731 | .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 |
732 | .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 |
733 | .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc |
734 | .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc |
735 | .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da |
736 | .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da |
737 | .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 |
738 | .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 |
739 | .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 |
740 | .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 |
741 | .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 |
742 | .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 |
743 | .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 |
744 | .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 |
745 | .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 |
746 | .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 |
747 | .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 |
748 | .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 |
749 | .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 |
750 | .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 |
751 | .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 |
752 | .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 |
753 | .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 |
754 | .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 |
755 | .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 |
756 | .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 |
757 | |
758 | .section .rodata.cst32.PSHUFFLE_BYTE_FLIP_MASK, "aM" , @progbits, 32 |
759 | .align 32 |
760 | PSHUFFLE_BYTE_FLIP_MASK: |
761 | .octa 0x0c0d0e0f08090a0b0405060700010203,0x0c0d0e0f08090a0b0405060700010203 |
762 | |
763 | # shuffle xBxA -> 00BA |
764 | .section .rodata.cst32._SHUF_00BA, "aM" , @progbits, 32 |
765 | .align 32 |
766 | _SHUF_00BA: |
767 | .octa 0xFFFFFFFFFFFFFFFF0b0a090803020100,0xFFFFFFFFFFFFFFFF0b0a090803020100 |
768 | |
769 | # shuffle xDxC -> DC00 |
770 | .section .rodata.cst32._SHUF_DC00, "aM" , @progbits, 32 |
771 | .align 32 |
772 | _SHUF_DC00: |
773 | .octa 0x0b0a090803020100FFFFFFFFFFFFFFFF,0x0b0a090803020100FFFFFFFFFFFFFFFF |
774 | |