1// Copyright 2015-2016 Brian Smith.
2//
3// Permission to use, copy, modify, and/or distribute this software for any
4// purpose with or without fee is hereby granted, provided that the above
5// copyright notice and this permission notice appear in all copies.
6//
7// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHORS DISCLAIM ALL WARRANTIES
8// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
9// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY
10// SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
11// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
12// OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
13// CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
14
15use super::{
16 aes::{self, Counter},
17 block::{Block, BLOCK_LEN},
18 gcm, shift, Aad, Nonce, Tag,
19};
20use crate::{aead, cpu, error, polyfill::usize_from_u64_saturated};
21use core::ops::RangeFrom;
22
23/// AES-128 in GCM mode with 128-bit tags and 96 bit nonces.
24pub static AES_128_GCM: aead::Algorithm = aead::Algorithm {
25 key_len: 16,
26 init: init_128,
27 seal: aes_gcm_seal,
28 open: aes_gcm_open,
29 id: aead::AlgorithmID::AES_128_GCM,
30};
31
32/// AES-256 in GCM mode with 128-bit tags and 96 bit nonces.
33pub static AES_256_GCM: aead::Algorithm = aead::Algorithm {
34 key_len: 32,
35 init: init_256,
36 seal: aes_gcm_seal,
37 open: aes_gcm_open,
38 id: aead::AlgorithmID::AES_256_GCM,
39};
40
41#[derive(Clone)]
42pub struct Key {
43 gcm_key: gcm::Key, // First because it has a large alignment requirement.
44 aes_key: aes::Key,
45}
46
47fn init_128(key: &[u8], cpu_features: cpu::Features) -> Result<aead::KeyInner, error::Unspecified> {
48 init(key, aes::Variant::AES_128, cpu_features)
49}
50
51fn init_256(key: &[u8], cpu_features: cpu::Features) -> Result<aead::KeyInner, error::Unspecified> {
52 init(key, aes::Variant::AES_256, cpu_features)
53}
54
55fn init(
56 key: &[u8],
57 variant: aes::Variant,
58 cpu_features: cpu::Features,
59) -> Result<aead::KeyInner, error::Unspecified> {
60 let aes_key: Key = aes::Key::new(bytes:key, variant, cpu_features)?;
61 let gcm_key: Key = gcm::Key::new(
62 h_be:aes_key.encrypt_block(a:Block::zero(), cpu_features),
63 cpu_features,
64 );
65 Ok(aead::KeyInner::AesGcm(Key { gcm_key, aes_key }))
66}
67
68const CHUNK_BLOCKS: usize = 3 * 1024 / 16;
69
70fn aes_gcm_seal(
71 key: &aead::KeyInner,
72 nonce: Nonce,
73 aad: Aad<&[u8]>,
74 in_out: &mut [u8],
75 cpu_features: cpu::Features,
76) -> Result<Tag, error::Unspecified> {
77 let Key { gcm_key, aes_key } = match key {
78 aead::KeyInner::AesGcm(key) => key,
79 _ => unreachable!(),
80 };
81
82 let mut auth = gcm::Context::new(gcm_key, aad, in_out.len(), cpu_features)?;
83
84 let mut ctr = Counter::one(nonce);
85 let tag_iv = ctr.increment();
86
87 #[cfg(target_arch = "x86_64")]
88 let in_out = {
89 if !aes_key.is_aes_hw(cpu_features) || !auth.is_avx() {
90 in_out
91 } else {
92 use crate::c;
93 let (htable, xi) = auth.inner();
94 prefixed_extern! {
95 // `HTable` and `Xi` should be 128-bit aligned. TODO: Can we shrink `HTable`? The
96 // assembly says it needs just nine values in that array.
97 fn aesni_gcm_encrypt(
98 input: *const u8,
99 output: *mut u8,
100 len: c::size_t,
101 key: &aes::AES_KEY,
102 ivec: &mut Counter,
103 Htable: &gcm::HTable,
104 Xi: &mut gcm::Xi) -> c::size_t;
105 }
106 let processed = unsafe {
107 aesni_gcm_encrypt(
108 in_out.as_ptr(),
109 in_out.as_mut_ptr(),
110 in_out.len(),
111 aes_key.inner_less_safe(),
112 &mut ctr,
113 htable,
114 xi,
115 )
116 };
117
118 &mut in_out[processed..]
119 }
120 };
121
122 #[cfg(target_arch = "aarch64")]
123 let in_out = {
124 if !aes_key.is_aes_hw(cpu_features) || !auth.is_clmul() {
125 in_out
126 } else {
127 let whole_block_bits = auth.in_out_whole_block_bits();
128 if whole_block_bits.as_bits() > 0 {
129 use crate::{bits::BitLength, c};
130 let (htable, xi) = auth.inner();
131 prefixed_extern! {
132 fn aes_gcm_enc_kernel(
133 input: *const u8,
134 in_bits: BitLength<c::size_t>,
135 output: *mut u8,
136 Xi: &mut gcm::Xi,
137 ivec: &mut Counter,
138 key: &aes::AES_KEY,
139 Htable: &gcm::HTable);
140 }
141 unsafe {
142 aes_gcm_enc_kernel(
143 in_out.as_ptr(),
144 whole_block_bits,
145 in_out.as_mut_ptr(),
146 xi,
147 &mut ctr,
148 aes_key.inner_less_safe(),
149 htable,
150 )
151 }
152 }
153
154 &mut in_out[whole_block_bits.as_usize_bytes_rounded_up()..]
155 }
156 };
157
158 let (whole, remainder) = {
159 let in_out_len = in_out.len();
160 let whole_len = in_out_len - (in_out_len % BLOCK_LEN);
161 in_out.split_at_mut(whole_len)
162 };
163
164 for chunk in whole.chunks_mut(CHUNK_BLOCKS * BLOCK_LEN) {
165 aes_key.ctr32_encrypt_within(chunk, 0.., &mut ctr, cpu_features);
166 auth.update_blocks(chunk);
167 }
168
169 if !remainder.is_empty() {
170 let mut input = Block::zero();
171 input.overwrite_part_at(0, remainder);
172 let mut output = aes_key.encrypt_iv_xor_block(ctr.into(), input, cpu_features);
173 output.zero_from(remainder.len());
174 auth.update_block(output);
175 remainder.copy_from_slice(&output.as_ref()[..remainder.len()]);
176 }
177
178 Ok(finish(aes_key, auth, tag_iv))
179}
180
181fn aes_gcm_open(
182 key: &aead::KeyInner,
183 nonce: Nonce,
184 aad: Aad<&[u8]>,
185 in_out: &mut [u8],
186 src: RangeFrom<usize>,
187 cpu_features: cpu::Features,
188) -> Result<Tag, error::Unspecified> {
189 let Key { gcm_key, aes_key } = match key {
190 aead::KeyInner::AesGcm(key) => key,
191 _ => unreachable!(),
192 };
193
194 let mut auth = {
195 let unprefixed_len = in_out
196 .len()
197 .checked_sub(src.start)
198 .ok_or(error::Unspecified)?;
199 gcm::Context::new(gcm_key, aad, unprefixed_len, cpu_features)
200 }?;
201
202 let mut ctr = Counter::one(nonce);
203 let tag_iv = ctr.increment();
204
205 let in_prefix_len = src.start;
206
207 #[cfg(target_arch = "x86_64")]
208 let in_out = {
209 if !aes_key.is_aes_hw(cpu_features) || !auth.is_avx() {
210 in_out
211 } else {
212 use crate::c;
213 let (htable, xi) = auth.inner();
214 prefixed_extern! {
215 // `HTable` and `Xi` should be 128-bit aligned. TODO: Can we shrink `HTable`? The
216 // assembly says it needs just nine values in that array.
217 fn aesni_gcm_decrypt(
218 input: *const u8,
219 output: *mut u8,
220 len: c::size_t,
221 key: &aes::AES_KEY,
222 ivec: &mut Counter,
223 Htable: &gcm::HTable,
224 Xi: &mut gcm::Xi) -> c::size_t;
225 }
226
227 let processed = unsafe {
228 aesni_gcm_decrypt(
229 in_out[src.clone()].as_ptr(),
230 in_out.as_mut_ptr(),
231 in_out.len() - src.start,
232 aes_key.inner_less_safe(),
233 &mut ctr,
234 htable,
235 xi,
236 )
237 };
238 &mut in_out[processed..]
239 }
240 };
241
242 #[cfg(target_arch = "aarch64")]
243 let in_out = {
244 if !aes_key.is_aes_hw(cpu_features) || !auth.is_clmul() {
245 in_out
246 } else {
247 let whole_block_bits = auth.in_out_whole_block_bits();
248 if whole_block_bits.as_bits() > 0 {
249 use crate::{bits::BitLength, c};
250 let (htable, xi) = auth.inner();
251 prefixed_extern! {
252 fn aes_gcm_dec_kernel(
253 input: *const u8,
254 in_bits: BitLength<c::size_t>,
255 output: *mut u8,
256 Xi: &mut gcm::Xi,
257 ivec: &mut Counter,
258 key: &aes::AES_KEY,
259 Htable: &gcm::HTable);
260 }
261
262 unsafe {
263 aes_gcm_dec_kernel(
264 in_out[src.clone()].as_ptr(),
265 whole_block_bits,
266 in_out.as_mut_ptr(),
267 xi,
268 &mut ctr,
269 aes_key.inner_less_safe(),
270 htable,
271 )
272 }
273 }
274
275 &mut in_out[whole_block_bits.as_usize_bytes_rounded_up()..]
276 }
277 };
278
279 let whole_len = {
280 let in_out_len = in_out.len() - in_prefix_len;
281 in_out_len - (in_out_len % BLOCK_LEN)
282 };
283 {
284 let mut chunk_len = CHUNK_BLOCKS * BLOCK_LEN;
285 let mut output = 0;
286 let mut input = in_prefix_len;
287 loop {
288 if whole_len - output < chunk_len {
289 chunk_len = whole_len - output;
290 }
291 if chunk_len == 0 {
292 break;
293 }
294
295 auth.update_blocks(&in_out[input..][..chunk_len]);
296 aes_key.ctr32_encrypt_within(
297 &mut in_out[output..][..(chunk_len + in_prefix_len)],
298 in_prefix_len..,
299 &mut ctr,
300 cpu_features,
301 );
302 output += chunk_len;
303 input += chunk_len;
304 }
305 }
306
307 let remainder = &mut in_out[whole_len..];
308 shift::shift_partial((in_prefix_len, remainder), |remainder| {
309 let mut input = Block::zero();
310 input.overwrite_part_at(0, remainder);
311 auth.update_block(input);
312 aes_key.encrypt_iv_xor_block(ctr.into(), input, cpu_features)
313 });
314
315 Ok(finish(aes_key, auth, tag_iv))
316}
317
318fn finish(aes_key: &aes::Key, gcm_ctx: gcm::Context, tag_iv: aes::Iv) -> Tag {
319 // Finalize the tag and return it.
320 gcm_ctx.pre_finish(|pre_tag: Block, cpu_features: Features| {
321 let encrypted_iv: Block = aes_key.encrypt_block(a:tag_iv.into_block_less_safe(), cpu_features);
322 let tag: Block = pre_tag ^ encrypted_iv;
323 Tag(*tag.as_ref())
324 })
325}
326
327pub(super) const MAX_IN_OUT_LEN: usize = super::max_input_len(BLOCK_LEN, overhead_blocks_per_nonce:2);
328
329// [NIST SP800-38D] Section 5.2.1.1. Note that [RFC 5116 Section 5.1] and
330// [RFC 5116 Section 5.2] have an off-by-one error in `P_MAX`.
331//
332// [NIST SP800-38D]:
333// http://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38d.pdf
334// [RFC 5116 Section 5.1]: https://tools.ietf.org/html/rfc5116#section-5.1
335// [RFC 5116 Section 5.2]: https://tools.ietf.org/html/rfc5116#section-5.2
336const _MAX_INPUT_LEN_BOUNDED_BY_NIST: () =
337 assert!(MAX_IN_OUT_LEN == usize_from_u64_saturated(((1u64 << 39) - 256) / 8));
338