1/* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */
2//
3// AES-GCM implementation for x86_64 CPUs that support the following CPU
4// features: VAES && VPCLMULQDQ && AVX2
5//
6// Copyright 2025 Google LLC
7//
8// Author: Eric Biggers <ebiggers@google.com>
9//
10//------------------------------------------------------------------------------
11//
12// This file is dual-licensed, meaning that you can use it under your choice of
13// either of the following two licenses:
14//
15// Licensed under the Apache License 2.0 (the "License"). You may obtain a copy
16// of the License at
17//
18// http://www.apache.org/licenses/LICENSE-2.0
19//
20// Unless required by applicable law or agreed to in writing, software
21// distributed under the License is distributed on an "AS IS" BASIS,
22// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
23// See the License for the specific language governing permissions and
24// limitations under the License.
25//
26// or
27//
28// Redistribution and use in source and binary forms, with or without
29// modification, are permitted provided that the following conditions are met:
30//
31// 1. Redistributions of source code must retain the above copyright notice,
32// this list of conditions and the following disclaimer.
33//
34// 2. Redistributions in binary form must reproduce the above copyright
35// notice, this list of conditions and the following disclaimer in the
36// documentation and/or other materials provided with the distribution.
37//
38// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
39// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
40// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
41// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
42// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
43// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
44// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
45// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
46// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
47// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
48// POSSIBILITY OF SUCH DAMAGE.
49//
50// -----------------------------------------------------------------------------
51//
52// This is similar to aes-gcm-vaes-avx512.S, but it uses AVX2 instead of AVX512.
53// This means it can only use 16 vector registers instead of 32, the maximum
54// vector length is 32 bytes, and some instructions such as vpternlogd and
55// masked loads/stores are unavailable. However, it is able to run on CPUs that
56// have VAES without AVX512, namely AMD Zen 3 (including "Milan" server CPUs),
57// various Intel client CPUs such as Alder Lake, and Intel Sierra Forest.
58//
59// This implementation also uses Karatsuba multiplication instead of schoolbook
60// multiplication for GHASH in its main loop. This does not help much on Intel,
61// but it improves performance by ~5% on AMD Zen 3. Other factors weighing
62// slightly in favor of Karatsuba multiplication in this implementation are the
63// lower maximum vector length (which means there are fewer key powers, so we
64// can cache the halves of each key power XOR'd together and still use less
65// memory than the AVX512 implementation), and the unavailability of the
66// vpternlogd instruction (which helped schoolbook a bit more than Karatsuba).
67
68#include <linux/linkage.h>
69
70.section .rodata
71.p2align 4
72
73 // The below three 16-byte values must be in the order that they are, as
74 // they are really two 32-byte tables and a 16-byte value that overlap:
75 //
76 // - The first 32-byte table begins at .Lselect_high_bytes_table.
77 // For 0 <= len <= 16, the 16-byte value at
78 // '.Lselect_high_bytes_table + len' selects the high 'len' bytes of
79 // another 16-byte value when AND'ed with it.
80 //
81 // - The second 32-byte table begins at .Lrshift_and_bswap_table.
82 // For 0 <= len <= 16, the 16-byte value at
83 // '.Lrshift_and_bswap_table + len' is a vpshufb mask that does the
84 // following operation: right-shift by '16 - len' bytes (shifting in
85 // zeroes), then reflect all 16 bytes.
86 //
87 // - The 16-byte value at .Lbswap_mask is a vpshufb mask that reflects
88 // all 16 bytes.
89.Lselect_high_bytes_table:
90 .octa 0
91.Lrshift_and_bswap_table:
92 .octa 0xffffffffffffffffffffffffffffffff
93.Lbswap_mask:
94 .octa 0x000102030405060708090a0b0c0d0e0f
95
96 // Sixteen 0x0f bytes. By XOR'ing an entry of .Lrshift_and_bswap_table
97 // with this, we get a mask that left-shifts by '16 - len' bytes.
98.Lfifteens:
99 .octa 0x0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f
100
101 // This is the GHASH reducing polynomial without its constant term, i.e.
102 // x^128 + x^7 + x^2 + x, represented using the backwards mapping
103 // between bits and polynomial coefficients.
104 //
105 // Alternatively, it can be interpreted as the naturally-ordered
106 // representation of the polynomial x^127 + x^126 + x^121 + 1, i.e. the
107 // "reversed" GHASH reducing polynomial without its x^128 term.
108.Lgfpoly:
109 .octa 0xc2000000000000000000000000000001
110
111 // Same as above, but with the (1 << 64) bit set.
112.Lgfpoly_and_internal_carrybit:
113 .octa 0xc2000000000000010000000000000001
114
115 // Values needed to prepare the initial vector of counter blocks.
116.Lctr_pattern:
117 .octa 0
118 .octa 1
119
120 // The number of AES blocks per vector, as a 128-bit value.
121.Linc_2blocks:
122 .octa 2
123
124// Offsets in struct aes_gcm_key_vaes_avx2
125#define OFFSETOF_AESKEYLEN 480
126#define OFFSETOF_H_POWERS 512
127#define NUM_H_POWERS 8
128#define OFFSETOFEND_H_POWERS (OFFSETOF_H_POWERS + (NUM_H_POWERS * 16))
129#define OFFSETOF_H_POWERS_XORED OFFSETOFEND_H_POWERS
130
131.text
132
133// Do one step of GHASH-multiplying the 128-bit lanes of \a by the 128-bit lanes
134// of \b and storing the reduced products in \dst. Uses schoolbook
135// multiplication.
136.macro _ghash_mul_step i, a, b, dst, gfpoly, t0, t1, t2
137.if \i == 0
138 vpclmulqdq $0x00, \a, \b, \t0 // LO = a_L * b_L
139 vpclmulqdq $0x01, \a, \b, \t1 // MI_0 = a_L * b_H
140.elseif \i == 1
141 vpclmulqdq $0x10, \a, \b, \t2 // MI_1 = a_H * b_L
142.elseif \i == 2
143 vpxor \t2, \t1, \t1 // MI = MI_0 + MI_1
144.elseif \i == 3
145 vpclmulqdq $0x01, \t0, \gfpoly, \t2 // LO_L*(x^63 + x^62 + x^57)
146.elseif \i == 4
147 vpshufd $0x4e, \t0, \t0 // Swap halves of LO
148.elseif \i == 5
149 vpxor \t0, \t1, \t1 // Fold LO into MI (part 1)
150 vpxor \t2, \t1, \t1 // Fold LO into MI (part 2)
151.elseif \i == 6
152 vpclmulqdq $0x11, \a, \b, \dst // HI = a_H * b_H
153.elseif \i == 7
154 vpclmulqdq $0x01, \t1, \gfpoly, \t0 // MI_L*(x^63 + x^62 + x^57)
155.elseif \i == 8
156 vpshufd $0x4e, \t1, \t1 // Swap halves of MI
157.elseif \i == 9
158 vpxor \t1, \dst, \dst // Fold MI into HI (part 1)
159 vpxor \t0, \dst, \dst // Fold MI into HI (part 2)
160.endif
161.endm
162
163// GHASH-multiply the 128-bit lanes of \a by the 128-bit lanes of \b and store
164// the reduced products in \dst. See _ghash_mul_step for full explanation.
165.macro _ghash_mul a, b, dst, gfpoly, t0, t1, t2
166.irp i, 0,1,2,3,4,5,6,7,8,9
167 _ghash_mul_step \i, \a, \b, \dst, \gfpoly, \t0, \t1, \t2
168.endr
169.endm
170
171// GHASH-multiply the 128-bit lanes of \a by the 128-bit lanes of \b and add the
172// *unreduced* products to \lo, \mi, and \hi.
173.macro _ghash_mul_noreduce a, b, lo, mi, hi, t0
174 vpclmulqdq $0x00, \a, \b, \t0 // a_L * b_L
175 vpxor \t0, \lo, \lo
176 vpclmulqdq $0x01, \a, \b, \t0 // a_L * b_H
177 vpxor \t0, \mi, \mi
178 vpclmulqdq $0x10, \a, \b, \t0 // a_H * b_L
179 vpxor \t0, \mi, \mi
180 vpclmulqdq $0x11, \a, \b, \t0 // a_H * b_H
181 vpxor \t0, \hi, \hi
182.endm
183
184// Reduce the unreduced products from \lo, \mi, and \hi and store the 128-bit
185// reduced products in \hi. See _ghash_mul_step for explanation of reduction.
186.macro _ghash_reduce lo, mi, hi, gfpoly, t0
187 vpclmulqdq $0x01, \lo, \gfpoly, \t0
188 vpshufd $0x4e, \lo, \lo
189 vpxor \lo, \mi, \mi
190 vpxor \t0, \mi, \mi
191 vpclmulqdq $0x01, \mi, \gfpoly, \t0
192 vpshufd $0x4e, \mi, \mi
193 vpxor \mi, \hi, \hi
194 vpxor \t0, \hi, \hi
195.endm
196
197// This is a specialized version of _ghash_mul that computes \a * \a, i.e. it
198// squares \a. It skips computing MI = (a_L * a_H) + (a_H * a_L) = 0.
199.macro _ghash_square a, dst, gfpoly, t0, t1
200 vpclmulqdq $0x00, \a, \a, \t0 // LO = a_L * a_L
201 vpclmulqdq $0x11, \a, \a, \dst // HI = a_H * a_H
202 vpclmulqdq $0x01, \t0, \gfpoly, \t1 // LO_L*(x^63 + x^62 + x^57)
203 vpshufd $0x4e, \t0, \t0 // Swap halves of LO
204 vpxor \t0, \t1, \t1 // Fold LO into MI
205 vpclmulqdq $0x01, \t1, \gfpoly, \t0 // MI_L*(x^63 + x^62 + x^57)
206 vpshufd $0x4e, \t1, \t1 // Swap halves of MI
207 vpxor \t1, \dst, \dst // Fold MI into HI (part 1)
208 vpxor \t0, \dst, \dst // Fold MI into HI (part 2)
209.endm
210
211// void aes_gcm_precompute_vaes_avx2(struct aes_gcm_key_vaes_avx2 *key);
212//
213// Given the expanded AES key |key->base.aes_key|, derive the GHASH subkey and
214// initialize |key->h_powers| and |key->h_powers_xored|.
215//
216// We use h_powers[0..7] to store H^8 through H^1, and h_powers_xored[0..7] to
217// store the 64-bit halves of the key powers XOR'd together (for Karatsuba
218// multiplication) in the order 8,6,7,5,4,2,3,1.
219SYM_FUNC_START(aes_gcm_precompute_vaes_avx2)
220
221 // Function arguments
222 .set KEY, %rdi
223
224 // Additional local variables
225 .set POWERS_PTR, %rsi
226 .set RNDKEYLAST_PTR, %rdx
227 .set TMP0, %ymm0
228 .set TMP0_XMM, %xmm0
229 .set TMP1, %ymm1
230 .set TMP1_XMM, %xmm1
231 .set TMP2, %ymm2
232 .set TMP2_XMM, %xmm2
233 .set H_CUR, %ymm3
234 .set H_CUR_XMM, %xmm3
235 .set H_CUR2, %ymm4
236 .set H_INC, %ymm5
237 .set H_INC_XMM, %xmm5
238 .set GFPOLY, %ymm6
239 .set GFPOLY_XMM, %xmm6
240
241 // Encrypt an all-zeroes block to get the raw hash subkey.
242 movl OFFSETOF_AESKEYLEN(KEY), %eax
243 lea 6*16(KEY,%rax,4), RNDKEYLAST_PTR
244 vmovdqu (KEY), H_CUR_XMM // Zero-th round key XOR all-zeroes block
245 lea 16(KEY), %rax
2461:
247 vaesenc (%rax), H_CUR_XMM, H_CUR_XMM
248 add $16, %rax
249 cmp %rax, RNDKEYLAST_PTR
250 jne 1b
251 vaesenclast (RNDKEYLAST_PTR), H_CUR_XMM, H_CUR_XMM
252
253 // Reflect the bytes of the raw hash subkey.
254 vpshufb .Lbswap_mask(%rip), H_CUR_XMM, H_CUR_XMM
255
256 // Finish preprocessing the byte-reflected hash subkey by multiplying it
257 // by x^-1 ("standard" interpretation of polynomial coefficients) or
258 // equivalently x^1 (natural interpretation). This gets the key into a
259 // format that avoids having to bit-reflect the data blocks later.
260 vpshufd $0xd3, H_CUR_XMM, TMP0_XMM
261 vpsrad $31, TMP0_XMM, TMP0_XMM
262 vpaddq H_CUR_XMM, H_CUR_XMM, H_CUR_XMM
263 vpand .Lgfpoly_and_internal_carrybit(%rip), TMP0_XMM, TMP0_XMM
264 vpxor TMP0_XMM, H_CUR_XMM, H_CUR_XMM
265
266 // Load the gfpoly constant.
267 vbroadcasti128 .Lgfpoly(%rip), GFPOLY
268
269 // Square H^1 to get H^2.
270 _ghash_square H_CUR_XMM, H_INC_XMM, GFPOLY_XMM, TMP0_XMM, TMP1_XMM
271
272 // Create H_CUR = [H^2, H^1] and H_INC = [H^2, H^2].
273 vinserti128 $1, H_CUR_XMM, H_INC, H_CUR
274 vinserti128 $1, H_INC_XMM, H_INC, H_INC
275
276 // Compute H_CUR2 = [H^4, H^3].
277 _ghash_mul H_INC, H_CUR, H_CUR2, GFPOLY, TMP0, TMP1, TMP2
278
279 // Store [H^2, H^1] and [H^4, H^3].
280 vmovdqu H_CUR, OFFSETOF_H_POWERS+3*32(KEY)
281 vmovdqu H_CUR2, OFFSETOF_H_POWERS+2*32(KEY)
282
283 // For Karatsuba multiplication: compute and store the two 64-bit halves
284 // of each key power XOR'd together. Order is 4,2,3,1.
285 vpunpcklqdq H_CUR, H_CUR2, TMP0
286 vpunpckhqdq H_CUR, H_CUR2, TMP1
287 vpxor TMP1, TMP0, TMP0
288 vmovdqu TMP0, OFFSETOF_H_POWERS_XORED+32(KEY)
289
290 // Compute and store H_CUR = [H^6, H^5] and H_CUR2 = [H^8, H^7].
291 _ghash_mul H_INC, H_CUR2, H_CUR, GFPOLY, TMP0, TMP1, TMP2
292 _ghash_mul H_INC, H_CUR, H_CUR2, GFPOLY, TMP0, TMP1, TMP2
293 vmovdqu H_CUR, OFFSETOF_H_POWERS+1*32(KEY)
294 vmovdqu H_CUR2, OFFSETOF_H_POWERS+0*32(KEY)
295
296 // Again, compute and store the two 64-bit halves of each key power
297 // XOR'd together. Order is 8,6,7,5.
298 vpunpcklqdq H_CUR, H_CUR2, TMP0
299 vpunpckhqdq H_CUR, H_CUR2, TMP1
300 vpxor TMP1, TMP0, TMP0
301 vmovdqu TMP0, OFFSETOF_H_POWERS_XORED(KEY)
302
303 vzeroupper
304 RET
305SYM_FUNC_END(aes_gcm_precompute_vaes_avx2)
306
307// Do one step of the GHASH update of four vectors of data blocks.
308// \i: the step to do, 0 through 9
309// \ghashdata_ptr: pointer to the data blocks (ciphertext or AAD)
310// KEY: pointer to struct aes_gcm_key_vaes_avx2
311// BSWAP_MASK: mask for reflecting the bytes of blocks
312// H_POW[2-1]_XORED: cached values from KEY->h_powers_xored
313// TMP[0-2]: temporary registers. TMP[1-2] must be preserved across steps.
314// LO, MI: working state for this macro that must be preserved across steps
315// GHASH_ACC: the GHASH accumulator (input/output)
316.macro _ghash_step_4x i, ghashdata_ptr
317 .set HI, GHASH_ACC # alias
318 .set HI_XMM, GHASH_ACC_XMM
319.if \i == 0
320 // First vector
321 vmovdqu 0*32(\ghashdata_ptr), TMP1
322 vpshufb BSWAP_MASK, TMP1, TMP1
323 vmovdqu OFFSETOF_H_POWERS+0*32(KEY), TMP2
324 vpxor GHASH_ACC, TMP1, TMP1
325 vpclmulqdq $0x00, TMP2, TMP1, LO
326 vpclmulqdq $0x11, TMP2, TMP1, HI
327 vpunpckhqdq TMP1, TMP1, TMP0
328 vpxor TMP1, TMP0, TMP0
329 vpclmulqdq $0x00, H_POW2_XORED, TMP0, MI
330.elseif \i == 1
331.elseif \i == 2
332 // Second vector
333 vmovdqu 1*32(\ghashdata_ptr), TMP1
334 vpshufb BSWAP_MASK, TMP1, TMP1
335 vmovdqu OFFSETOF_H_POWERS+1*32(KEY), TMP2
336 vpclmulqdq $0x00, TMP2, TMP1, TMP0
337 vpxor TMP0, LO, LO
338 vpclmulqdq $0x11, TMP2, TMP1, TMP0
339 vpxor TMP0, HI, HI
340 vpunpckhqdq TMP1, TMP1, TMP0
341 vpxor TMP1, TMP0, TMP0
342 vpclmulqdq $0x10, H_POW2_XORED, TMP0, TMP0
343 vpxor TMP0, MI, MI
344.elseif \i == 3
345 // Third vector
346 vmovdqu 2*32(\ghashdata_ptr), TMP1
347 vpshufb BSWAP_MASK, TMP1, TMP1
348 vmovdqu OFFSETOF_H_POWERS+2*32(KEY), TMP2
349.elseif \i == 4
350 vpclmulqdq $0x00, TMP2, TMP1, TMP0
351 vpxor TMP0, LO, LO
352 vpclmulqdq $0x11, TMP2, TMP1, TMP0
353 vpxor TMP0, HI, HI
354.elseif \i == 5
355 vpunpckhqdq TMP1, TMP1, TMP0
356 vpxor TMP1, TMP0, TMP0
357 vpclmulqdq $0x00, H_POW1_XORED, TMP0, TMP0
358 vpxor TMP0, MI, MI
359
360 // Fourth vector
361 vmovdqu 3*32(\ghashdata_ptr), TMP1
362 vpshufb BSWAP_MASK, TMP1, TMP1
363.elseif \i == 6
364 vmovdqu OFFSETOF_H_POWERS+3*32(KEY), TMP2
365 vpclmulqdq $0x00, TMP2, TMP1, TMP0
366 vpxor TMP0, LO, LO
367 vpclmulqdq $0x11, TMP2, TMP1, TMP0
368 vpxor TMP0, HI, HI
369 vpunpckhqdq TMP1, TMP1, TMP0
370 vpxor TMP1, TMP0, TMP0
371 vpclmulqdq $0x10, H_POW1_XORED, TMP0, TMP0
372 vpxor TMP0, MI, MI
373.elseif \i == 7
374 // Finalize 'mi' following Karatsuba multiplication.
375 vpxor LO, MI, MI
376 vpxor HI, MI, MI
377
378 // Fold lo into mi.
379 vbroadcasti128 .Lgfpoly(%rip), TMP2
380 vpclmulqdq $0x01, LO, TMP2, TMP0
381 vpshufd $0x4e, LO, LO
382 vpxor LO, MI, MI
383 vpxor TMP0, MI, MI
384.elseif \i == 8
385 // Fold mi into hi.
386 vpclmulqdq $0x01, MI, TMP2, TMP0
387 vpshufd $0x4e, MI, MI
388 vpxor MI, HI, HI
389 vpxor TMP0, HI, HI
390.elseif \i == 9
391 vextracti128 $1, HI, TMP0_XMM
392 vpxor TMP0_XMM, HI_XMM, GHASH_ACC_XMM
393.endif
394.endm
395
396// Update GHASH with four vectors of data blocks. See _ghash_step_4x for full
397// explanation.
398.macro _ghash_4x ghashdata_ptr
399.irp i, 0,1,2,3,4,5,6,7,8,9
400 _ghash_step_4x \i, \ghashdata_ptr
401.endr
402.endm
403
404// Load 1 <= %ecx <= 16 bytes from the pointer \src into the xmm register \dst
405// and zeroize any remaining bytes. Clobbers %rax, %rcx, and \tmp{64,32}.
406.macro _load_partial_block src, dst, tmp64, tmp32
407 sub $8, %ecx // LEN - 8
408 jle .Lle8\@
409
410 // Load 9 <= LEN <= 16 bytes.
411 vmovq (\src), \dst // Load first 8 bytes
412 mov (\src, %rcx), %rax // Load last 8 bytes
413 neg %ecx
414 shl $3, %ecx
415 shr %cl, %rax // Discard overlapping bytes
416 vpinsrq $1, %rax, \dst, \dst
417 jmp .Ldone\@
418
419.Lle8\@:
420 add $4, %ecx // LEN - 4
421 jl .Llt4\@
422
423 // Load 4 <= LEN <= 8 bytes.
424 mov (\src), %eax // Load first 4 bytes
425 mov (\src, %rcx), \tmp32 // Load last 4 bytes
426 jmp .Lcombine\@
427
428.Llt4\@:
429 // Load 1 <= LEN <= 3 bytes.
430 add $2, %ecx // LEN - 2
431 movzbl (\src), %eax // Load first byte
432 jl .Lmovq\@
433 movzwl (\src, %rcx), \tmp32 // Load last 2 bytes
434.Lcombine\@:
435 shl $3, %ecx
436 shl %cl, \tmp64
437 or \tmp64, %rax // Combine the two parts
438.Lmovq\@:
439 vmovq %rax, \dst
440.Ldone\@:
441.endm
442
443// Store 1 <= %ecx <= 16 bytes from the xmm register \src to the pointer \dst.
444// Clobbers %rax, %rcx, and \tmp{64,32}.
445.macro _store_partial_block src, dst, tmp64, tmp32
446 sub $8, %ecx // LEN - 8
447 jl .Llt8\@
448
449 // Store 8 <= LEN <= 16 bytes.
450 vpextrq $1, \src, %rax
451 mov %ecx, \tmp32
452 shl $3, %ecx
453 ror %cl, %rax
454 mov %rax, (\dst, \tmp64) // Store last LEN - 8 bytes
455 vmovq \src, (\dst) // Store first 8 bytes
456 jmp .Ldone\@
457
458.Llt8\@:
459 add $4, %ecx // LEN - 4
460 jl .Llt4\@
461
462 // Store 4 <= LEN <= 7 bytes.
463 vpextrd $1, \src, %eax
464 mov %ecx, \tmp32
465 shl $3, %ecx
466 ror %cl, %eax
467 mov %eax, (\dst, \tmp64) // Store last LEN - 4 bytes
468 vmovd \src, (\dst) // Store first 4 bytes
469 jmp .Ldone\@
470
471.Llt4\@:
472 // Store 1 <= LEN <= 3 bytes.
473 vpextrb $0, \src, 0(\dst)
474 cmp $-2, %ecx // LEN - 4 == -2, i.e. LEN == 2?
475 jl .Ldone\@
476 vpextrb $1, \src, 1(\dst)
477 je .Ldone\@
478 vpextrb $2, \src, 2(\dst)
479.Ldone\@:
480.endm
481
482// void aes_gcm_aad_update_vaes_avx2(const struct aes_gcm_key_vaes_avx2 *key,
483// u8 ghash_acc[16],
484// const u8 *aad, int aadlen);
485//
486// This function processes the AAD (Additional Authenticated Data) in GCM.
487// Using the key |key|, it updates the GHASH accumulator |ghash_acc| with the
488// data given by |aad| and |aadlen|. On the first call, |ghash_acc| must be all
489// zeroes. |aadlen| must be a multiple of 16, except on the last call where it
490// can be any length. The caller must do any buffering needed to ensure this.
491//
492// This handles large amounts of AAD efficiently, while also keeping overhead
493// low for small amounts which is the common case. TLS and IPsec use less than
494// one block of AAD, but (uncommonly) other use cases may use much more.
495SYM_FUNC_START(aes_gcm_aad_update_vaes_avx2)
496
497 // Function arguments
498 .set KEY, %rdi
499 .set GHASH_ACC_PTR, %rsi
500 .set AAD, %rdx
501 .set AADLEN, %ecx // Must be %ecx for _load_partial_block
502 .set AADLEN64, %rcx // Zero-extend AADLEN before using!
503
504 // Additional local variables.
505 // %rax and %r8 are used as temporary registers.
506 .set TMP0, %ymm0
507 .set TMP0_XMM, %xmm0
508 .set TMP1, %ymm1
509 .set TMP1_XMM, %xmm1
510 .set TMP2, %ymm2
511 .set TMP2_XMM, %xmm2
512 .set LO, %ymm3
513 .set LO_XMM, %xmm3
514 .set MI, %ymm4
515 .set MI_XMM, %xmm4
516 .set GHASH_ACC, %ymm5
517 .set GHASH_ACC_XMM, %xmm5
518 .set BSWAP_MASK, %ymm6
519 .set BSWAP_MASK_XMM, %xmm6
520 .set GFPOLY, %ymm7
521 .set GFPOLY_XMM, %xmm7
522 .set H_POW2_XORED, %ymm8
523 .set H_POW1_XORED, %ymm9
524
525 // Load the bswap_mask and gfpoly constants. Since AADLEN is usually
526 // small, usually only 128-bit vectors will be used. So as an
527 // optimization, don't broadcast these constants to both 128-bit lanes
528 // quite yet.
529 vmovdqu .Lbswap_mask(%rip), BSWAP_MASK_XMM
530 vmovdqu .Lgfpoly(%rip), GFPOLY_XMM
531
532 // Load the GHASH accumulator.
533 vmovdqu (GHASH_ACC_PTR), GHASH_ACC_XMM
534
535 // Check for the common case of AADLEN <= 16, as well as AADLEN == 0.
536 test AADLEN, AADLEN
537 jz .Laad_done
538 cmp $16, AADLEN
539 jle .Laad_lastblock
540
541 // AADLEN > 16, so we'll operate on full vectors. Broadcast bswap_mask
542 // and gfpoly to both 128-bit lanes.
543 vinserti128 $1, BSWAP_MASK_XMM, BSWAP_MASK, BSWAP_MASK
544 vinserti128 $1, GFPOLY_XMM, GFPOLY, GFPOLY
545
546 // If AADLEN >= 128, update GHASH with 128 bytes of AAD at a time.
547 add $-128, AADLEN // 128 is 4 bytes, -128 is 1 byte
548 jl .Laad_loop_4x_done
549 vmovdqu OFFSETOF_H_POWERS_XORED(KEY), H_POW2_XORED
550 vmovdqu OFFSETOF_H_POWERS_XORED+32(KEY), H_POW1_XORED
551.Laad_loop_4x:
552 _ghash_4x AAD
553 sub $-128, AAD
554 add $-128, AADLEN
555 jge .Laad_loop_4x
556.Laad_loop_4x_done:
557
558 // If AADLEN >= 32, update GHASH with 32 bytes of AAD at a time.
559 add $96, AADLEN
560 jl .Laad_loop_1x_done
561.Laad_loop_1x:
562 vmovdqu (AAD), TMP0
563 vpshufb BSWAP_MASK, TMP0, TMP0
564 vpxor TMP0, GHASH_ACC, GHASH_ACC
565 vmovdqu OFFSETOFEND_H_POWERS-32(KEY), TMP0
566 _ghash_mul TMP0, GHASH_ACC, GHASH_ACC, GFPOLY, TMP1, TMP2, LO
567 vextracti128 $1, GHASH_ACC, TMP0_XMM
568 vpxor TMP0_XMM, GHASH_ACC_XMM, GHASH_ACC_XMM
569 add $32, AAD
570 sub $32, AADLEN
571 jge .Laad_loop_1x
572.Laad_loop_1x_done:
573 add $32, AADLEN
574 // Now 0 <= AADLEN < 32.
575
576 jz .Laad_done
577 cmp $16, AADLEN
578 jle .Laad_lastblock
579
580 // Update GHASH with the remaining 17 <= AADLEN <= 31 bytes of AAD.
581 mov AADLEN, AADLEN // Zero-extend AADLEN to AADLEN64.
582 vmovdqu (AAD), TMP0_XMM
583 vmovdqu -16(AAD, AADLEN64), TMP1_XMM
584 vpshufb BSWAP_MASK_XMM, TMP0_XMM, TMP0_XMM
585 vpxor TMP0_XMM, GHASH_ACC_XMM, GHASH_ACC_XMM
586 lea .Lrshift_and_bswap_table(%rip), %rax
587 vpshufb -16(%rax, AADLEN64), TMP1_XMM, TMP1_XMM
588 vinserti128 $1, TMP1_XMM, GHASH_ACC, GHASH_ACC
589 vmovdqu OFFSETOFEND_H_POWERS-32(KEY), TMP0
590 _ghash_mul TMP0, GHASH_ACC, GHASH_ACC, GFPOLY, TMP1, TMP2, LO
591 vextracti128 $1, GHASH_ACC, TMP0_XMM
592 vpxor TMP0_XMM, GHASH_ACC_XMM, GHASH_ACC_XMM
593 jmp .Laad_done
594
595.Laad_lastblock:
596 // Update GHASH with the remaining 1 <= AADLEN <= 16 bytes of AAD.
597 _load_partial_block AAD, TMP0_XMM, %r8, %r8d
598 vpshufb BSWAP_MASK_XMM, TMP0_XMM, TMP0_XMM
599 vpxor TMP0_XMM, GHASH_ACC_XMM, GHASH_ACC_XMM
600 vmovdqu OFFSETOFEND_H_POWERS-16(KEY), TMP0_XMM
601 _ghash_mul TMP0_XMM, GHASH_ACC_XMM, GHASH_ACC_XMM, GFPOLY_XMM, \
602 TMP1_XMM, TMP2_XMM, LO_XMM
603
604.Laad_done:
605 // Store the updated GHASH accumulator back to memory.
606 vmovdqu GHASH_ACC_XMM, (GHASH_ACC_PTR)
607
608 vzeroupper
609 RET
610SYM_FUNC_END(aes_gcm_aad_update_vaes_avx2)
611
612// Do one non-last round of AES encryption on the blocks in the given AESDATA
613// vectors using the round key that has been broadcast to all 128-bit lanes of
614// \round_key.
615.macro _vaesenc round_key, vecs:vararg
616.irp i, \vecs
617 vaesenc \round_key, AESDATA\i, AESDATA\i
618.endr
619.endm
620
621// Generate counter blocks in the given AESDATA vectors, then do the zero-th AES
622// round on them. Clobbers TMP0.
623.macro _ctr_begin vecs:vararg
624 vbroadcasti128 .Linc_2blocks(%rip), TMP0
625.irp i, \vecs
626 vpshufb BSWAP_MASK, LE_CTR, AESDATA\i
627 vpaddd TMP0, LE_CTR, LE_CTR
628.endr
629.irp i, \vecs
630 vpxor RNDKEY0, AESDATA\i, AESDATA\i
631.endr
632.endm
633
634// Generate and encrypt counter blocks in the given AESDATA vectors, excluding
635// the last AES round. Clobbers %rax and TMP0.
636.macro _aesenc_loop vecs:vararg
637 _ctr_begin \vecs
638 lea 16(KEY), %rax
639.Laesenc_loop\@:
640 vbroadcasti128 (%rax), TMP0
641 _vaesenc TMP0, \vecs
642 add $16, %rax
643 cmp %rax, RNDKEYLAST_PTR
644 jne .Laesenc_loop\@
645.endm
646
647// Finalize the keystream blocks in the given AESDATA vectors by doing the last
648// AES round, then XOR those keystream blocks with the corresponding data.
649// Reduce latency by doing the XOR before the vaesenclast, utilizing the
650// property vaesenclast(key, a) ^ b == vaesenclast(key ^ b, a). Clobbers TMP0.
651.macro _aesenclast_and_xor vecs:vararg
652.irp i, \vecs
653 vpxor \i*32(SRC), RNDKEYLAST, TMP0
654 vaesenclast TMP0, AESDATA\i, AESDATA\i
655.endr
656.irp i, \vecs
657 vmovdqu AESDATA\i, \i*32(DST)
658.endr
659.endm
660
661// void aes_gcm_{enc,dec}_update_vaes_avx2(const struct aes_gcm_key_vaes_avx2 *key,
662// const u32 le_ctr[4], u8 ghash_acc[16],
663// const u8 *src, u8 *dst, int datalen);
664//
665// This macro generates a GCM encryption or decryption update function with the
666// above prototype (with \enc selecting which one). The function computes the
667// next portion of the CTR keystream, XOR's it with |datalen| bytes from |src|,
668// and writes the resulting encrypted or decrypted data to |dst|. It also
669// updates the GHASH accumulator |ghash_acc| using the next |datalen| ciphertext
670// bytes.
671//
672// |datalen| must be a multiple of 16, except on the last call where it can be
673// any length. The caller must do any buffering needed to ensure this. Both
674// in-place and out-of-place en/decryption are supported.
675//
676// |le_ctr| must give the current counter in little-endian format. This
677// function loads the counter from |le_ctr| and increments the loaded counter as
678// needed, but it does *not* store the updated counter back to |le_ctr|. The
679// caller must update |le_ctr| if any more data segments follow. Internally,
680// only the low 32-bit word of the counter is incremented, following the GCM
681// standard.
682.macro _aes_gcm_update enc
683
684 // Function arguments
685 .set KEY, %rdi
686 .set LE_CTR_PTR, %rsi
687 .set LE_CTR_PTR32, %esi
688 .set GHASH_ACC_PTR, %rdx
689 .set SRC, %rcx // Assumed to be %rcx.
690 // See .Ltail_xor_and_ghash_1to16bytes
691 .set DST, %r8
692 .set DATALEN, %r9d
693 .set DATALEN64, %r9 // Zero-extend DATALEN before using!
694
695 // Additional local variables
696
697 // %rax is used as a temporary register. LE_CTR_PTR is also available
698 // as a temporary register after the counter is loaded.
699
700 // AES key length in bytes
701 .set AESKEYLEN, %r10d
702 .set AESKEYLEN64, %r10
703
704 // Pointer to the last AES round key for the chosen AES variant
705 .set RNDKEYLAST_PTR, %r11
706
707 // BSWAP_MASK is the shuffle mask for byte-reflecting 128-bit values
708 // using vpshufb, copied to all 128-bit lanes.
709 .set BSWAP_MASK, %ymm0
710 .set BSWAP_MASK_XMM, %xmm0
711
712 // GHASH_ACC is the accumulator variable for GHASH. When fully reduced,
713 // only the lowest 128-bit lane can be nonzero. When not fully reduced,
714 // more than one lane may be used, and they need to be XOR'd together.
715 .set GHASH_ACC, %ymm1
716 .set GHASH_ACC_XMM, %xmm1
717
718 // TMP[0-2] are temporary registers.
719 .set TMP0, %ymm2
720 .set TMP0_XMM, %xmm2
721 .set TMP1, %ymm3
722 .set TMP1_XMM, %xmm3
723 .set TMP2, %ymm4
724 .set TMP2_XMM, %xmm4
725
726 // LO and MI are used to accumulate unreduced GHASH products.
727 .set LO, %ymm5
728 .set LO_XMM, %xmm5
729 .set MI, %ymm6
730 .set MI_XMM, %xmm6
731
732 // H_POW[2-1]_XORED contain cached values from KEY->h_powers_xored. The
733 // descending numbering reflects the order of the key powers.
734 .set H_POW2_XORED, %ymm7
735 .set H_POW2_XORED_XMM, %xmm7
736 .set H_POW1_XORED, %ymm8
737
738 // RNDKEY0 caches the zero-th round key, and RNDKEYLAST the last one.
739 .set RNDKEY0, %ymm9
740 .set RNDKEYLAST, %ymm10
741
742 // LE_CTR contains the next set of little-endian counter blocks.
743 .set LE_CTR, %ymm11
744
745 // AESDATA[0-3] hold the counter blocks that are being encrypted by AES.
746 .set AESDATA0, %ymm12
747 .set AESDATA0_XMM, %xmm12
748 .set AESDATA1, %ymm13
749 .set AESDATA1_XMM, %xmm13
750 .set AESDATA2, %ymm14
751 .set AESDATA3, %ymm15
752
753.if \enc
754 .set GHASHDATA_PTR, DST
755.else
756 .set GHASHDATA_PTR, SRC
757.endif
758
759 vbroadcasti128 .Lbswap_mask(%rip), BSWAP_MASK
760
761 // Load the GHASH accumulator and the starting counter.
762 vmovdqu (GHASH_ACC_PTR), GHASH_ACC_XMM
763 vbroadcasti128 (LE_CTR_PTR), LE_CTR
764
765 // Load the AES key length in bytes.
766 movl OFFSETOF_AESKEYLEN(KEY), AESKEYLEN
767
768 // Make RNDKEYLAST_PTR point to the last AES round key. This is the
769 // round key with index 10, 12, or 14 for AES-128, AES-192, or AES-256
770 // respectively. Then load the zero-th and last round keys.
771 lea 6*16(KEY,AESKEYLEN64,4), RNDKEYLAST_PTR
772 vbroadcasti128 (KEY), RNDKEY0
773 vbroadcasti128 (RNDKEYLAST_PTR), RNDKEYLAST
774
775 // Finish initializing LE_CTR by adding 1 to the second block.
776 vpaddd .Lctr_pattern(%rip), LE_CTR, LE_CTR
777
778 // If there are at least 128 bytes of data, then continue into the loop
779 // that processes 128 bytes of data at a time. Otherwise skip it.
780 add $-128, DATALEN // 128 is 4 bytes, -128 is 1 byte
781 jl .Lcrypt_loop_4x_done\@
782
783 vmovdqu OFFSETOF_H_POWERS_XORED(KEY), H_POW2_XORED
784 vmovdqu OFFSETOF_H_POWERS_XORED+32(KEY), H_POW1_XORED
785
786 // Main loop: en/decrypt and hash 4 vectors (128 bytes) at a time.
787
788.if \enc
789 // Encrypt the first 4 vectors of plaintext blocks.
790 _aesenc_loop 0,1,2,3
791 _aesenclast_and_xor 0,1,2,3
792 sub $-128, SRC // 128 is 4 bytes, -128 is 1 byte
793 add $-128, DATALEN
794 jl .Lghash_last_ciphertext_4x\@
795.endif
796
797.align 16
798.Lcrypt_loop_4x\@:
799
800 // Start the AES encryption of the counter blocks.
801 _ctr_begin 0,1,2,3
802 cmp $24, AESKEYLEN
803 jl 128f // AES-128?
804 je 192f // AES-192?
805 // AES-256
806 vbroadcasti128 -13*16(RNDKEYLAST_PTR), TMP0
807 _vaesenc TMP0, 0,1,2,3
808 vbroadcasti128 -12*16(RNDKEYLAST_PTR), TMP0
809 _vaesenc TMP0, 0,1,2,3
810192:
811 vbroadcasti128 -11*16(RNDKEYLAST_PTR), TMP0
812 _vaesenc TMP0, 0,1,2,3
813 vbroadcasti128 -10*16(RNDKEYLAST_PTR), TMP0
814 _vaesenc TMP0, 0,1,2,3
815128:
816
817 // Finish the AES encryption of the counter blocks in AESDATA[0-3],
818 // interleaved with the GHASH update of the ciphertext blocks.
819.irp i, 9,8,7,6,5,4,3,2,1
820 _ghash_step_4x (9 - \i), GHASHDATA_PTR
821 vbroadcasti128 -\i*16(RNDKEYLAST_PTR), TMP0
822 _vaesenc TMP0, 0,1,2,3
823.endr
824 _ghash_step_4x 9, GHASHDATA_PTR
825.if \enc
826 sub $-128, DST // 128 is 4 bytes, -128 is 1 byte
827.endif
828 _aesenclast_and_xor 0,1,2,3
829 sub $-128, SRC
830.if !\enc
831 sub $-128, DST
832.endif
833 add $-128, DATALEN
834 jge .Lcrypt_loop_4x\@
835
836.if \enc
837.Lghash_last_ciphertext_4x\@:
838 // Update GHASH with the last set of ciphertext blocks.
839 _ghash_4x DST
840 sub $-128, DST
841.endif
842
843.Lcrypt_loop_4x_done\@:
844
845 // Undo the extra subtraction by 128 and check whether data remains.
846 sub $-128, DATALEN // 128 is 4 bytes, -128 is 1 byte
847 jz .Ldone\@
848
849 // The data length isn't a multiple of 128 bytes. Process the remaining
850 // data of length 1 <= DATALEN < 128.
851 //
852 // Since there are enough key powers available for all remaining data,
853 // there is no need to do a GHASH reduction after each iteration.
854 // Instead, multiply each remaining block by its own key power, and only
855 // do a GHASH reduction at the very end.
856
857 // Make POWERS_PTR point to the key powers [H^N, H^(N-1), ...] where N
858 // is the number of blocks that remain.
859 .set POWERS_PTR, LE_CTR_PTR // LE_CTR_PTR is free to be reused.
860 .set POWERS_PTR32, LE_CTR_PTR32
861 mov DATALEN, %eax
862 neg %rax
863 and $~15, %rax // -round_up(DATALEN, 16)
864 lea OFFSETOFEND_H_POWERS(KEY,%rax), POWERS_PTR
865
866 // Start collecting the unreduced GHASH intermediate value LO, MI, HI.
867 .set HI, H_POW2_XORED // H_POW2_XORED is free to be reused.
868 .set HI_XMM, H_POW2_XORED_XMM
869 vpxor LO_XMM, LO_XMM, LO_XMM
870 vpxor MI_XMM, MI_XMM, MI_XMM
871 vpxor HI_XMM, HI_XMM, HI_XMM
872
873 // 1 <= DATALEN < 128. Generate 2 or 4 more vectors of keystream blocks
874 // excluding the last AES round, depending on the remaining DATALEN.
875 cmp $64, DATALEN
876 jg .Ltail_gen_4_keystream_vecs\@
877 _aesenc_loop 0,1
878 cmp $32, DATALEN
879 jge .Ltail_xor_and_ghash_full_vec_loop\@
880 jmp .Ltail_xor_and_ghash_partial_vec\@
881.Ltail_gen_4_keystream_vecs\@:
882 _aesenc_loop 0,1,2,3
883
884 // XOR the remaining data and accumulate the unreduced GHASH products
885 // for DATALEN >= 32, starting with one full 32-byte vector at a time.
886.Ltail_xor_and_ghash_full_vec_loop\@:
887.if \enc
888 _aesenclast_and_xor 0
889 vpshufb BSWAP_MASK, AESDATA0, AESDATA0
890.else
891 vmovdqu (SRC), TMP1
892 vpxor TMP1, RNDKEYLAST, TMP0
893 vaesenclast TMP0, AESDATA0, AESDATA0
894 vmovdqu AESDATA0, (DST)
895 vpshufb BSWAP_MASK, TMP1, AESDATA0
896.endif
897 // The ciphertext blocks (i.e. GHASH input data) are now in AESDATA0.
898 vpxor GHASH_ACC, AESDATA0, AESDATA0
899 vmovdqu (POWERS_PTR), TMP2
900 _ghash_mul_noreduce TMP2, AESDATA0, LO, MI, HI, TMP0
901 vmovdqa AESDATA1, AESDATA0
902 vmovdqa AESDATA2, AESDATA1
903 vmovdqa AESDATA3, AESDATA2
904 vpxor GHASH_ACC_XMM, GHASH_ACC_XMM, GHASH_ACC_XMM
905 add $32, SRC
906 add $32, DST
907 add $32, POWERS_PTR
908 sub $32, DATALEN
909 cmp $32, DATALEN
910 jge .Ltail_xor_and_ghash_full_vec_loop\@
911 test DATALEN, DATALEN
912 jz .Ltail_ghash_reduce\@
913
914.Ltail_xor_and_ghash_partial_vec\@:
915 // XOR the remaining data and accumulate the unreduced GHASH products,
916 // for 1 <= DATALEN < 32.
917 vaesenclast RNDKEYLAST, AESDATA0, AESDATA0
918 cmp $16, DATALEN
919 jle .Ltail_xor_and_ghash_1to16bytes\@
920
921 // Handle 17 <= DATALEN < 32.
922
923 // Load a vpshufb mask that will right-shift by '32 - DATALEN' bytes
924 // (shifting in zeroes), then reflect all 16 bytes.
925 lea .Lrshift_and_bswap_table(%rip), %rax
926 vmovdqu -16(%rax, DATALEN64), TMP2_XMM
927
928 // Move the second keystream block to its own register and left-align it
929 vextracti128 $1, AESDATA0, AESDATA1_XMM
930 vpxor .Lfifteens(%rip), TMP2_XMM, TMP0_XMM
931 vpshufb TMP0_XMM, AESDATA1_XMM, AESDATA1_XMM
932
933 // Using overlapping loads and stores, XOR the source data with the
934 // keystream and write the destination data. Then prepare the GHASH
935 // input data: the full ciphertext block and the zero-padded partial
936 // ciphertext block, both byte-reflected, in AESDATA0.
937.if \enc
938 vpxor -16(SRC, DATALEN64), AESDATA1_XMM, AESDATA1_XMM
939 vpxor (SRC), AESDATA0_XMM, AESDATA0_XMM
940 vmovdqu AESDATA1_XMM, -16(DST, DATALEN64)
941 vmovdqu AESDATA0_XMM, (DST)
942 vpshufb TMP2_XMM, AESDATA1_XMM, AESDATA1_XMM
943 vpshufb BSWAP_MASK_XMM, AESDATA0_XMM, AESDATA0_XMM
944.else
945 vmovdqu -16(SRC, DATALEN64), TMP1_XMM
946 vmovdqu (SRC), TMP0_XMM
947 vpxor TMP1_XMM, AESDATA1_XMM, AESDATA1_XMM
948 vpxor TMP0_XMM, AESDATA0_XMM, AESDATA0_XMM
949 vmovdqu AESDATA1_XMM, -16(DST, DATALEN64)
950 vmovdqu AESDATA0_XMM, (DST)
951 vpshufb TMP2_XMM, TMP1_XMM, AESDATA1_XMM
952 vpshufb BSWAP_MASK_XMM, TMP0_XMM, AESDATA0_XMM
953.endif
954 vpxor GHASH_ACC_XMM, AESDATA0_XMM, AESDATA0_XMM
955 vinserti128 $1, AESDATA1_XMM, AESDATA0, AESDATA0
956 vmovdqu (POWERS_PTR), TMP2
957 jmp .Ltail_ghash_last_vec\@
958
959.Ltail_xor_and_ghash_1to16bytes\@:
960 // Handle 1 <= DATALEN <= 16. Carefully load and store the
961 // possibly-partial block, which we mustn't access out of bounds.
962 vmovdqu (POWERS_PTR), TMP2_XMM
963 mov SRC, KEY // Free up %rcx, assuming SRC == %rcx
964 mov DATALEN, %ecx
965 _load_partial_block KEY, TMP0_XMM, POWERS_PTR, POWERS_PTR32
966 vpxor TMP0_XMM, AESDATA0_XMM, AESDATA0_XMM
967 mov DATALEN, %ecx
968 _store_partial_block AESDATA0_XMM, DST, POWERS_PTR, POWERS_PTR32
969.if \enc
970 lea .Lselect_high_bytes_table(%rip), %rax
971 vpshufb BSWAP_MASK_XMM, AESDATA0_XMM, AESDATA0_XMM
972 vpand (%rax, DATALEN64), AESDATA0_XMM, AESDATA0_XMM
973.else
974 vpshufb BSWAP_MASK_XMM, TMP0_XMM, AESDATA0_XMM
975.endif
976 vpxor GHASH_ACC_XMM, AESDATA0_XMM, AESDATA0_XMM
977
978.Ltail_ghash_last_vec\@:
979 // Accumulate the unreduced GHASH products for the last 1-2 blocks. The
980 // GHASH input data is in AESDATA0. If only one block remains, then the
981 // second block in AESDATA0 is zero and does not affect the result.
982 _ghash_mul_noreduce TMP2, AESDATA0, LO, MI, HI, TMP0
983
984.Ltail_ghash_reduce\@:
985 // Finally, do the GHASH reduction.
986 vbroadcasti128 .Lgfpoly(%rip), TMP0
987 _ghash_reduce LO, MI, HI, TMP0, TMP1
988 vextracti128 $1, HI, GHASH_ACC_XMM
989 vpxor HI_XMM, GHASH_ACC_XMM, GHASH_ACC_XMM
990
991.Ldone\@:
992 // Store the updated GHASH accumulator back to memory.
993 vmovdqu GHASH_ACC_XMM, (GHASH_ACC_PTR)
994
995 vzeroupper
996 RET
997.endm
998
999// void aes_gcm_enc_final_vaes_avx2(const struct aes_gcm_key_vaes_avx2 *key,
1000// const u32 le_ctr[4], u8 ghash_acc[16],
1001// u64 total_aadlen, u64 total_datalen);
1002// bool aes_gcm_dec_final_vaes_avx2(const struct aes_gcm_key_vaes_avx2 *key,
1003// const u32 le_ctr[4], const u8 ghash_acc[16],
1004// u64 total_aadlen, u64 total_datalen,
1005// const u8 tag[16], int taglen);
1006//
1007// This macro generates one of the above two functions (with \enc selecting
1008// which one). Both functions finish computing the GCM authentication tag by
1009// updating GHASH with the lengths block and encrypting the GHASH accumulator.
1010// |total_aadlen| and |total_datalen| must be the total length of the additional
1011// authenticated data and the en/decrypted data in bytes, respectively.
1012//
1013// The encryption function then stores the full-length (16-byte) computed
1014// authentication tag to |ghash_acc|. The decryption function instead loads the
1015// expected authentication tag (the one that was transmitted) from the 16-byte
1016// buffer |tag|, compares the first 4 <= |taglen| <= 16 bytes of it to the
1017// computed tag in constant time, and returns true if and only if they match.
1018.macro _aes_gcm_final enc
1019
1020 // Function arguments
1021 .set KEY, %rdi
1022 .set LE_CTR_PTR, %rsi
1023 .set GHASH_ACC_PTR, %rdx
1024 .set TOTAL_AADLEN, %rcx
1025 .set TOTAL_DATALEN, %r8
1026 .set TAG, %r9
1027 .set TAGLEN, %r10d // Originally at 8(%rsp)
1028 .set TAGLEN64, %r10
1029
1030 // Additional local variables.
1031 // %rax and %xmm0-%xmm3 are used as temporary registers.
1032 .set AESKEYLEN, %r11d
1033 .set AESKEYLEN64, %r11
1034 .set GFPOLY, %xmm4
1035 .set BSWAP_MASK, %xmm5
1036 .set LE_CTR, %xmm6
1037 .set GHASH_ACC, %xmm7
1038 .set H_POW1, %xmm8
1039
1040 // Load some constants.
1041 vmovdqa .Lgfpoly(%rip), GFPOLY
1042 vmovdqa .Lbswap_mask(%rip), BSWAP_MASK
1043
1044 // Load the AES key length in bytes.
1045 movl OFFSETOF_AESKEYLEN(KEY), AESKEYLEN
1046
1047 // Set up a counter block with 1 in the low 32-bit word. This is the
1048 // counter that produces the ciphertext needed to encrypt the auth tag.
1049 // GFPOLY has 1 in the low word, so grab the 1 from there using a blend.
1050 vpblendd $0xe, (LE_CTR_PTR), GFPOLY, LE_CTR
1051
1052 // Build the lengths block and XOR it with the GHASH accumulator.
1053 // Although the lengths block is defined as the AAD length followed by
1054 // the en/decrypted data length, both in big-endian byte order, a byte
1055 // reflection of the full block is needed because of the way we compute
1056 // GHASH (see _ghash_mul_step). By using little-endian values in the
1057 // opposite order, we avoid having to reflect any bytes here.
1058 vmovq TOTAL_DATALEN, %xmm0
1059 vpinsrq $1, TOTAL_AADLEN, %xmm0, %xmm0
1060 vpsllq $3, %xmm0, %xmm0 // Bytes to bits
1061 vpxor (GHASH_ACC_PTR), %xmm0, GHASH_ACC
1062
1063 // Load the first hash key power (H^1), which is stored last.
1064 vmovdqu OFFSETOFEND_H_POWERS-16(KEY), H_POW1
1065
1066 // Load TAGLEN if decrypting.
1067.if !\enc
1068 movl 8(%rsp), TAGLEN
1069.endif
1070
1071 // Make %rax point to the last AES round key for the chosen AES variant.
1072 lea 6*16(KEY,AESKEYLEN64,4), %rax
1073
1074 // Start the AES encryption of the counter block by swapping the counter
1075 // block to big-endian and XOR-ing it with the zero-th AES round key.
1076 vpshufb BSWAP_MASK, LE_CTR, %xmm0
1077 vpxor (KEY), %xmm0, %xmm0
1078
1079 // Complete the AES encryption and multiply GHASH_ACC by H^1.
1080 // Interleave the AES and GHASH instructions to improve performance.
1081 cmp $24, AESKEYLEN
1082 jl 128f // AES-128?
1083 je 192f // AES-192?
1084 // AES-256
1085 vaesenc -13*16(%rax), %xmm0, %xmm0
1086 vaesenc -12*16(%rax), %xmm0, %xmm0
1087192:
1088 vaesenc -11*16(%rax), %xmm0, %xmm0
1089 vaesenc -10*16(%rax), %xmm0, %xmm0
1090128:
1091.irp i, 0,1,2,3,4,5,6,7,8
1092 _ghash_mul_step \i, H_POW1, GHASH_ACC, GHASH_ACC, GFPOLY, \
1093 %xmm1, %xmm2, %xmm3
1094 vaesenc (\i-9)*16(%rax), %xmm0, %xmm0
1095.endr
1096 _ghash_mul_step 9, H_POW1, GHASH_ACC, GHASH_ACC, GFPOLY, \
1097 %xmm1, %xmm2, %xmm3
1098
1099 // Undo the byte reflection of the GHASH accumulator.
1100 vpshufb BSWAP_MASK, GHASH_ACC, GHASH_ACC
1101
1102 // Do the last AES round and XOR the resulting keystream block with the
1103 // GHASH accumulator to produce the full computed authentication tag.
1104 //
1105 // Reduce latency by taking advantage of the property vaesenclast(key,
1106 // a) ^ b == vaesenclast(key ^ b, a). I.e., XOR GHASH_ACC into the last
1107 // round key, instead of XOR'ing the final AES output with GHASH_ACC.
1108 //
1109 // enc_final then returns the computed auth tag, while dec_final
1110 // compares it with the transmitted one and returns a bool. To compare
1111 // the tags, dec_final XORs them together and uses vptest to check
1112 // whether the result is all-zeroes. This should be constant-time.
1113 // dec_final applies the vaesenclast optimization to this additional
1114 // value XOR'd too.
1115.if \enc
1116 vpxor (%rax), GHASH_ACC, %xmm1
1117 vaesenclast %xmm1, %xmm0, GHASH_ACC
1118 vmovdqu GHASH_ACC, (GHASH_ACC_PTR)
1119.else
1120 vpxor (TAG), GHASH_ACC, GHASH_ACC
1121 vpxor (%rax), GHASH_ACC, GHASH_ACC
1122 vaesenclast GHASH_ACC, %xmm0, %xmm0
1123 lea .Lselect_high_bytes_table(%rip), %rax
1124 vmovdqu (%rax, TAGLEN64), %xmm1
1125 vpshufb BSWAP_MASK, %xmm1, %xmm1 // select low bytes, not high
1126 xor %eax, %eax
1127 vptest %xmm1, %xmm0
1128 sete %al
1129.endif
1130 // No need for vzeroupper here, since only used xmm registers were used.
1131 RET
1132.endm
1133
1134SYM_FUNC_START(aes_gcm_enc_update_vaes_avx2)
1135 _aes_gcm_update 1
1136SYM_FUNC_END(aes_gcm_enc_update_vaes_avx2)
1137SYM_FUNC_START(aes_gcm_dec_update_vaes_avx2)
1138 _aes_gcm_update 0
1139SYM_FUNC_END(aes_gcm_dec_update_vaes_avx2)
1140
1141SYM_FUNC_START(aes_gcm_enc_final_vaes_avx2)
1142 _aes_gcm_final 1
1143SYM_FUNC_END(aes_gcm_enc_final_vaes_avx2)
1144SYM_FUNC_START(aes_gcm_dec_final_vaes_avx2)
1145 _aes_gcm_final 0
1146SYM_FUNC_END(aes_gcm_dec_final_vaes_avx2)
1147

source code of linux/arch/x86/crypto/aes-gcm-vaes-avx2.S