1//! Autodetection support for hardware accelerated AES backends with fallback
2//! to the fixsliced "soft" implementation.
3
4use crate::soft;
5use cipher::{
6 consts::{U16, U24, U32},
7 AlgorithmName, BlockCipher, BlockClosure, BlockDecrypt, BlockEncrypt, BlockSizeUser, Key,
8 KeyInit, KeySizeUser,
9};
10use core::fmt;
11use core::mem::ManuallyDrop;
12
13#[cfg(all(target_arch = "aarch64", aes_armv8))]
14use crate::armv8 as intrinsics;
15
16#[cfg(any(target_arch = "x86_64", target_arch = "x86"))]
17use crate::ni as intrinsics;
18
19cpufeatures::new!(aes_intrinsics, "aes");
20
21macro_rules! define_aes_impl {
22 (
23 $name:ident,
24 $name_enc:ident,
25 $name_dec:ident,
26 $module:tt,
27 $key_size:ty,
28 $doc:expr $(,)?
29 ) => {
30 mod $module {
31 use super::{intrinsics, soft};
32 use core::mem::ManuallyDrop;
33
34 pub(super) union Inner {
35 pub(super) intrinsics: ManuallyDrop<intrinsics::$name>,
36 pub(super) soft: ManuallyDrop<soft::$name>,
37 }
38
39 pub(super) union InnerEnc {
40 pub(super) intrinsics: ManuallyDrop<intrinsics::$name_enc>,
41 pub(super) soft: ManuallyDrop<soft::$name_enc>,
42 }
43
44 pub(super) union InnerDec {
45 pub(super) intrinsics: ManuallyDrop<intrinsics::$name_dec>,
46 pub(super) soft: ManuallyDrop<soft::$name_dec>,
47 }
48 }
49
50 #[doc=$doc]
51 #[doc = "block cipher"]
52 pub struct $name {
53 inner: $module::Inner,
54 token: aes_intrinsics::InitToken,
55 }
56
57 impl KeySizeUser for $name {
58 type KeySize = $key_size;
59 }
60 impl From<$name_enc> for $name {
61 #[inline]
62 fn from(enc: $name_enc) -> $name {
63 Self::from(&enc)
64 }
65 }
66
67 impl From<&$name_enc> for $name {
68 fn from(enc: &$name_enc) -> $name {
69 use core::ops::Deref;
70 let inner = if enc.token.get() {
71 $module::Inner {
72 intrinsics: ManuallyDrop::new(unsafe {
73 enc.inner.intrinsics.deref().into()
74 }),
75 }
76 } else {
77 $module::Inner {
78 soft: ManuallyDrop::new(unsafe { enc.inner.soft.deref().into() }),
79 }
80 };
81
82 Self {
83 inner,
84 token: enc.token,
85 }
86 }
87 }
88
89 impl KeyInit for $name {
90 #[inline]
91 fn new(key: &Key<Self>) -> Self {
92 let (token, aesni_present) = aes_intrinsics::init_get();
93
94 let inner = if aesni_present {
95 $module::Inner {
96 intrinsics: ManuallyDrop::new(intrinsics::$name::new(key)),
97 }
98 } else {
99 $module::Inner {
100 soft: ManuallyDrop::new(soft::$name::new(key)),
101 }
102 };
103
104 Self { inner, token }
105 }
106 }
107
108 impl Clone for $name {
109 fn clone(&self) -> Self {
110 let inner = if self.token.get() {
111 $module::Inner {
112 intrinsics: unsafe { self.inner.intrinsics.clone() },
113 }
114 } else {
115 $module::Inner {
116 soft: unsafe { self.inner.soft.clone() },
117 }
118 };
119
120 Self {
121 inner,
122 token: self.token,
123 }
124 }
125 }
126
127 impl BlockSizeUser for $name {
128 type BlockSize = U16;
129 }
130
131 impl BlockCipher for $name {}
132
133 impl BlockEncrypt for $name {
134 fn encrypt_with_backend(&self, f: impl BlockClosure<BlockSize = U16>) {
135 unsafe {
136 if self.token.get() {
137 #[target_feature(enable = "aes")]
138 unsafe fn inner(
139 state: &intrinsics::$name,
140 f: impl BlockClosure<BlockSize = U16>,
141 ) {
142 f.call(&mut state.get_enc_backend());
143 }
144 inner(&self.inner.intrinsics, f);
145 } else {
146 f.call(&mut self.inner.soft.get_enc_backend());
147 }
148 }
149 }
150 }
151
152 impl BlockDecrypt for $name {
153 fn decrypt_with_backend(&self, f: impl BlockClosure<BlockSize = U16>) {
154 unsafe {
155 if self.token.get() {
156 #[target_feature(enable = "aes")]
157 unsafe fn inner(
158 state: &intrinsics::$name,
159 f: impl BlockClosure<BlockSize = U16>,
160 ) {
161 f.call(&mut state.get_dec_backend());
162 }
163 inner(&self.inner.intrinsics, f);
164 } else {
165 f.call(&mut self.inner.soft.get_dec_backend());
166 }
167 }
168 }
169 }
170
171 impl fmt::Debug for $name {
172 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
173 f.write_str(concat!(stringify!($name), " { .. }"))
174 }
175 }
176
177 impl AlgorithmName for $name {
178 fn write_alg_name(f: &mut fmt::Formatter<'_>) -> fmt::Result {
179 f.write_str(stringify!($name))
180 }
181 }
182
183 impl Drop for $name {
184 #[inline]
185 fn drop(&mut self) {
186 if self.token.get() {
187 unsafe { ManuallyDrop::drop(&mut self.inner.intrinsics) };
188 } else {
189 unsafe { ManuallyDrop::drop(&mut self.inner.soft) };
190 };
191 }
192 }
193
194 #[cfg(feature = "zeroize")]
195 impl zeroize::ZeroizeOnDrop for $name {}
196
197 #[doc=$doc]
198 #[doc = "block cipher (encrypt-only)"]
199 pub struct $name_enc {
200 inner: $module::InnerEnc,
201 token: aes_intrinsics::InitToken,
202 }
203
204 impl KeySizeUser for $name_enc {
205 type KeySize = $key_size;
206 }
207
208 impl KeyInit for $name_enc {
209 #[inline]
210 fn new(key: &Key<Self>) -> Self {
211 let (token, aesni_present) = aes_intrinsics::init_get();
212
213 let inner = if aesni_present {
214 $module::InnerEnc {
215 intrinsics: ManuallyDrop::new(intrinsics::$name_enc::new(key)),
216 }
217 } else {
218 $module::InnerEnc {
219 soft: ManuallyDrop::new(soft::$name_enc::new(key)),
220 }
221 };
222
223 Self { inner, token }
224 }
225 }
226
227 impl Clone for $name_enc {
228 fn clone(&self) -> Self {
229 let inner = if self.token.get() {
230 $module::InnerEnc {
231 intrinsics: unsafe { self.inner.intrinsics.clone() },
232 }
233 } else {
234 $module::InnerEnc {
235 soft: unsafe { self.inner.soft.clone() },
236 }
237 };
238
239 Self {
240 inner,
241 token: self.token,
242 }
243 }
244 }
245
246 impl BlockSizeUser for $name_enc {
247 type BlockSize = U16;
248 }
249
250 impl BlockCipher for $name_enc {}
251
252 impl BlockEncrypt for $name_enc {
253 fn encrypt_with_backend(&self, f: impl BlockClosure<BlockSize = U16>) {
254 unsafe {
255 if self.token.get() {
256 #[target_feature(enable = "aes")]
257 unsafe fn inner(
258 state: &intrinsics::$name_enc,
259 f: impl BlockClosure<BlockSize = U16>,
260 ) {
261 f.call(&mut state.get_enc_backend());
262 }
263 inner(&self.inner.intrinsics, f);
264 } else {
265 f.call(&mut self.inner.soft.get_enc_backend());
266 }
267 }
268 }
269 }
270
271 impl fmt::Debug for $name_enc {
272 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
273 f.write_str(concat!(stringify!($name_enc), " { .. }"))
274 }
275 }
276
277 impl AlgorithmName for $name_enc {
278 fn write_alg_name(f: &mut fmt::Formatter<'_>) -> fmt::Result {
279 f.write_str(stringify!($name_enc))
280 }
281 }
282
283 impl Drop for $name_enc {
284 #[inline]
285 fn drop(&mut self) {
286 if self.token.get() {
287 unsafe { ManuallyDrop::drop(&mut self.inner.intrinsics) };
288 } else {
289 unsafe { ManuallyDrop::drop(&mut self.inner.soft) };
290 };
291 }
292 }
293
294 #[cfg(feature = "zeroize")]
295 impl zeroize::ZeroizeOnDrop for $name_enc {}
296
297 #[doc=$doc]
298 #[doc = "block cipher (decrypt-only)"]
299 pub struct $name_dec {
300 inner: $module::InnerDec,
301 token: aes_intrinsics::InitToken,
302 }
303
304 impl KeySizeUser for $name_dec {
305 type KeySize = $key_size;
306 }
307
308 impl From<$name_enc> for $name_dec {
309 #[inline]
310 fn from(enc: $name_enc) -> $name_dec {
311 Self::from(&enc)
312 }
313 }
314
315 impl From<&$name_enc> for $name_dec {
316 fn from(enc: &$name_enc) -> $name_dec {
317 use core::ops::Deref;
318 let inner = if enc.token.get() {
319 $module::InnerDec {
320 intrinsics: ManuallyDrop::new(unsafe {
321 enc.inner.intrinsics.deref().into()
322 }),
323 }
324 } else {
325 $module::InnerDec {
326 soft: ManuallyDrop::new(unsafe { enc.inner.soft.deref().into() }),
327 }
328 };
329
330 Self {
331 inner,
332 token: enc.token,
333 }
334 }
335 }
336
337 impl KeyInit for $name_dec {
338 #[inline]
339 fn new(key: &Key<Self>) -> Self {
340 let (token, aesni_present) = aes_intrinsics::init_get();
341
342 let inner = if aesni_present {
343 $module::InnerDec {
344 intrinsics: ManuallyDrop::new(intrinsics::$name_dec::new(key)),
345 }
346 } else {
347 $module::InnerDec {
348 soft: ManuallyDrop::new(soft::$name_dec::new(key)),
349 }
350 };
351
352 Self { inner, token }
353 }
354 }
355
356 impl Clone for $name_dec {
357 fn clone(&self) -> Self {
358 let inner = if self.token.get() {
359 $module::InnerDec {
360 intrinsics: unsafe { self.inner.intrinsics.clone() },
361 }
362 } else {
363 $module::InnerDec {
364 soft: unsafe { self.inner.soft.clone() },
365 }
366 };
367
368 Self {
369 inner,
370 token: self.token,
371 }
372 }
373 }
374
375 impl BlockSizeUser for $name_dec {
376 type BlockSize = U16;
377 }
378
379 impl BlockCipher for $name_dec {}
380
381 impl BlockDecrypt for $name_dec {
382 fn decrypt_with_backend(&self, f: impl BlockClosure<BlockSize = U16>) {
383 unsafe {
384 if self.token.get() {
385 #[target_feature(enable = "aes")]
386 unsafe fn inner(
387 state: &intrinsics::$name_dec,
388 f: impl BlockClosure<BlockSize = U16>,
389 ) {
390 f.call(&mut state.get_dec_backend());
391 }
392 inner(&self.inner.intrinsics, f);
393 } else {
394 f.call(&mut self.inner.soft.get_dec_backend());
395 }
396 }
397 }
398 }
399
400 impl fmt::Debug for $name_dec {
401 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
402 f.write_str(concat!(stringify!($name_dec), " { .. }"))
403 }
404 }
405
406 impl AlgorithmName for $name_dec {
407 fn write_alg_name(f: &mut fmt::Formatter<'_>) -> fmt::Result {
408 f.write_str(stringify!($name_dec))
409 }
410 }
411
412 impl Drop for $name_dec {
413 #[inline]
414 fn drop(&mut self) {
415 if self.token.get() {
416 unsafe { ManuallyDrop::drop(&mut self.inner.intrinsics) };
417 } else {
418 unsafe { ManuallyDrop::drop(&mut self.inner.soft) };
419 };
420 }
421 }
422
423 #[cfg(feature = "zeroize")]
424 impl zeroize::ZeroizeOnDrop for $name_dec {}
425 };
426}
427
428define_aes_impl!(Aes128, Aes128Enc, Aes128Dec, aes128, U16, "AES-128");
429define_aes_impl!(Aes192, Aes192Enc, Aes192Dec, aes192, U24, "AES-192");
430define_aes_impl!(Aes256, Aes256Enc, Aes256Dec, aes256, U32, "AES-256");
431