1//! Vectorized AES Instructions (VAES)
2//!
3//! The intrinsics here correspond to those in the `immintrin.h` C header.
4//!
5//! The reference is [Intel 64 and IA-32 Architectures Software Developer's
6//! Manual Volume 2: Instruction Set Reference, A-Z][intel64_ref].
7//!
8//! [intel64_ref]: http://www.intel.de/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
9
10use crate::core_arch::x86::__m256i;
11use crate::core_arch::x86::__m512i;
12
13#[cfg(test)]
14use stdarch_test::assert_instr;
15
16#[allow(improper_ctypes)]
17unsafe extern "C" {
18 #[link_name = "llvm.x86.aesni.aesenc.256"]
19 unsafefn aesenc_256(a: __m256i, round_key: __m256i) -> __m256i;
20 #[link_name = "llvm.x86.aesni.aesenclast.256"]
21 unsafefn aesenclast_256(a: __m256i, round_key: __m256i) -> __m256i;
22 #[link_name = "llvm.x86.aesni.aesdec.256"]
23 unsafefn aesdec_256(a: __m256i, round_key: __m256i) -> __m256i;
24 #[link_name = "llvm.x86.aesni.aesdeclast.256"]
25 unsafefn aesdeclast_256(a: __m256i, round_key: __m256i) -> __m256i;
26 #[link_name = "llvm.x86.aesni.aesenc.512"]
27 unsafefn aesenc_512(a: __m512i, round_key: __m512i) -> __m512i;
28 #[link_name = "llvm.x86.aesni.aesenclast.512"]
29 unsafefn aesenclast_512(a: __m512i, round_key: __m512i) -> __m512i;
30 #[link_name = "llvm.x86.aesni.aesdec.512"]
31 unsafefn aesdec_512(a: __m512i, round_key: __m512i) -> __m512i;
32 #[link_name = "llvm.x86.aesni.aesdeclast.512"]
33 unsafefn aesdeclast_512(a: __m512i, round_key: __m512i) -> __m512i;
34}
35
36/// Performs one round of an AES encryption flow on each 128-bit word (state) in `a` using
37/// the corresponding 128-bit word (key) in `round_key`.
38///
39/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_aesenc_epi128)
40#[inline]
41#[target_feature(enable = "vaes")]
42#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
43#[cfg_attr(test, assert_instr(vaesenc))]
44pub fn _mm256_aesenc_epi128(a: __m256i, round_key: __m256i) -> __m256i {
45 unsafe { aesenc_256(a, round_key) }
46}
47
48/// Performs the last round of an AES encryption flow on each 128-bit word (state) in `a` using
49/// the corresponding 128-bit word (key) in `round_key`.
50///
51/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_aesenclast_epi128)
52#[inline]
53#[target_feature(enable = "vaes")]
54#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
55#[cfg_attr(test, assert_instr(vaesenclast))]
56pub fn _mm256_aesenclast_epi128(a: __m256i, round_key: __m256i) -> __m256i {
57 unsafe { aesenclast_256(a, round_key) }
58}
59
60/// Performs one round of an AES decryption flow on each 128-bit word (state) in `a` using
61/// the corresponding 128-bit word (key) in `round_key`.
62///
63/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_aesdec_epi128)
64#[inline]
65#[target_feature(enable = "vaes")]
66#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
67#[cfg_attr(test, assert_instr(vaesdec))]
68pub fn _mm256_aesdec_epi128(a: __m256i, round_key: __m256i) -> __m256i {
69 unsafe { aesdec_256(a, round_key) }
70}
71
72/// Performs the last round of an AES decryption flow on each 128-bit word (state) in `a` using
73/// the corresponding 128-bit word (key) in `round_key`.
74///
75/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_aesdeclast_epi128)
76#[inline]
77#[target_feature(enable = "vaes")]
78#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
79#[cfg_attr(test, assert_instr(vaesdeclast))]
80pub fn _mm256_aesdeclast_epi128(a: __m256i, round_key: __m256i) -> __m256i {
81 unsafe { aesdeclast_256(a, round_key) }
82}
83
84/// Performs one round of an AES encryption flow on each 128-bit word (state) in `a` using
85/// the corresponding 128-bit word (key) in `round_key`.
86///
87/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm512_aesenc_epi128)
88#[inline]
89#[target_feature(enable = "vaes,avx512f")]
90#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
91#[cfg_attr(test, assert_instr(vaesenc))]
92pub fn _mm512_aesenc_epi128(a: __m512i, round_key: __m512i) -> __m512i {
93 unsafe { aesenc_512(a, round_key) }
94}
95
96/// Performs the last round of an AES encryption flow on each 128-bit word (state) in `a` using
97/// the corresponding 128-bit word (key) in `round_key`.
98///
99/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm512_aesenclast_epi128)
100#[inline]
101#[target_feature(enable = "vaes,avx512f")]
102#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
103#[cfg_attr(test, assert_instr(vaesenclast))]
104pub fn _mm512_aesenclast_epi128(a: __m512i, round_key: __m512i) -> __m512i {
105 unsafe { aesenclast_512(a, round_key) }
106}
107
108/// Performs one round of an AES decryption flow on each 128-bit word (state) in `a` using
109/// the corresponding 128-bit word (key) in `round_key`.
110///
111/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm512_aesdec_epi128)
112#[inline]
113#[target_feature(enable = "vaes,avx512f")]
114#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
115#[cfg_attr(test, assert_instr(vaesdec))]
116pub fn _mm512_aesdec_epi128(a: __m512i, round_key: __m512i) -> __m512i {
117 unsafe { aesdec_512(a, round_key) }
118}
119
120/// Performs the last round of an AES decryption flow on each 128-bit word (state) in `a` using
121/// the corresponding 128-bit word (key) in `round_key`.
122///
123/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm512_aesdeclast_epi128)
124#[inline]
125#[target_feature(enable = "vaes,avx512f")]
126#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
127#[cfg_attr(test, assert_instr(vaesdeclast))]
128pub fn _mm512_aesdeclast_epi128(a: __m512i, round_key: __m512i) -> __m512i {
129 unsafe { aesdeclast_512(a, round_key) }
130}
131
132#[cfg(test)]
133mod tests {
134 // The constants in the tests below are just bit patterns. They should not
135 // be interpreted as integers; signedness does not make sense for them, but
136 // __mXXXi happens to be defined in terms of signed integers.
137 #![allow(overflowing_literals)]
138
139 use stdarch_test::simd_test;
140
141 use crate::core_arch::x86::*;
142
143 // the first parts of these tests are straight ports from the AES-NI tests
144 // the second parts directly compare the two, for inputs that are different across lanes
145 // and "more random" than the standard test vectors
146 // ideally we'd be using quickcheck here instead
147
148 #[target_feature(enable = "avx2")]
149 unsafe fn helper_for_256_vaes(
150 linear: unsafe fn(__m128i, __m128i) -> __m128i,
151 vectorized: unsafe fn(__m256i, __m256i) -> __m256i,
152 ) {
153 let a = _mm256_set_epi64x(
154 0xDCB4DB3657BF0B7D,
155 0x18DB0601068EDD9F,
156 0xB76B908233200DC5,
157 0xE478235FA8E22D5E,
158 );
159 let k = _mm256_set_epi64x(
160 0x672F6F105A94CEA7,
161 0x8298B8FFCA5F829C,
162 0xA3927047B3FB61D8,
163 0x978093862CDE7187,
164 );
165 let mut a_decomp = [_mm_setzero_si128(); 2];
166 a_decomp[0] = _mm256_extracti128_si256::<0>(a);
167 a_decomp[1] = _mm256_extracti128_si256::<1>(a);
168 let mut k_decomp = [_mm_setzero_si128(); 2];
169 k_decomp[0] = _mm256_extracti128_si256::<0>(k);
170 k_decomp[1] = _mm256_extracti128_si256::<1>(k);
171 let r = vectorized(a, k);
172 let mut e_decomp = [_mm_setzero_si128(); 2];
173 for i in 0..2 {
174 e_decomp[i] = linear(a_decomp[i], k_decomp[i]);
175 }
176 assert_eq_m128i(_mm256_extracti128_si256::<0>(r), e_decomp[0]);
177 assert_eq_m128i(_mm256_extracti128_si256::<1>(r), e_decomp[1]);
178 }
179
180 #[target_feature(enable = "sse2")]
181 unsafe fn setup_state_key<T>(broadcast: unsafe fn(__m128i) -> T) -> (T, T) {
182 // Constants taken from https://msdn.microsoft.com/en-us/library/cc664949.aspx.
183 let a = _mm_set_epi64x(0x0123456789abcdef, 0x8899aabbccddeeff);
184 let k = _mm_set_epi64x(0x1133557799bbddff, 0x0022446688aaccee);
185 (broadcast(a), broadcast(k))
186 }
187
188 #[target_feature(enable = "avx2")]
189 unsafe fn setup_state_key_256() -> (__m256i, __m256i) {
190 setup_state_key(_mm256_broadcastsi128_si256)
191 }
192
193 #[target_feature(enable = "avx512f")]
194 unsafe fn setup_state_key_512() -> (__m512i, __m512i) {
195 setup_state_key(_mm512_broadcast_i32x4)
196 }
197
198 #[simd_test(enable = "vaes,avx512vl")]
199 unsafe fn test_mm256_aesdec_epi128() {
200 // Constants taken from https://msdn.microsoft.com/en-us/library/cc664949.aspx.
201 let (a, k) = setup_state_key_256();
202 let e = _mm_set_epi64x(0x044e4f5176fec48f, 0xb57ecfa381da39ee);
203 let e = _mm256_broadcastsi128_si256(e);
204 let r = _mm256_aesdec_epi128(a, k);
205 assert_eq_m256i(r, e);
206
207 helper_for_256_vaes(_mm_aesdec_si128, _mm256_aesdec_epi128);
208 }
209
210 #[simd_test(enable = "vaes,avx512vl")]
211 unsafe fn test_mm256_aesdeclast_epi128() {
212 // Constants taken from https://msdn.microsoft.com/en-us/library/cc714178.aspx.
213 let (a, k) = setup_state_key_256();
214 let e = _mm_set_epi64x(0x36cad57d9072bf9e, 0xf210dd981fa4a493);
215 let e = _mm256_broadcastsi128_si256(e);
216 let r = _mm256_aesdeclast_epi128(a, k);
217 assert_eq_m256i(r, e);
218
219 helper_for_256_vaes(_mm_aesdeclast_si128, _mm256_aesdeclast_epi128);
220 }
221
222 #[simd_test(enable = "vaes,avx512vl")]
223 unsafe fn test_mm256_aesenc_epi128() {
224 // Constants taken from https://msdn.microsoft.com/en-us/library/cc664810.aspx.
225 // they are repeated appropriately
226 let (a, k) = setup_state_key_256();
227 let e = _mm_set_epi64x(0x16ab0e57dfc442ed, 0x28e4ee1884504333);
228 let e = _mm256_broadcastsi128_si256(e);
229 let r = _mm256_aesenc_epi128(a, k);
230 assert_eq_m256i(r, e);
231
232 helper_for_256_vaes(_mm_aesenc_si128, _mm256_aesenc_epi128);
233 }
234
235 #[simd_test(enable = "vaes,avx512vl")]
236 unsafe fn test_mm256_aesenclast_epi128() {
237 // Constants taken from https://msdn.microsoft.com/en-us/library/cc714136.aspx.
238 let (a, k) = setup_state_key_256();
239 let e = _mm_set_epi64x(0xb6dd7df25d7ab320, 0x4b04f98cf4c860f8);
240 let e = _mm256_broadcastsi128_si256(e);
241 let r = _mm256_aesenclast_epi128(a, k);
242 assert_eq_m256i(r, e);
243
244 helper_for_256_vaes(_mm_aesenclast_si128, _mm256_aesenclast_epi128);
245 }
246
247 #[target_feature(enable = "avx512f")]
248 unsafe fn helper_for_512_vaes(
249 linear: unsafe fn(__m128i, __m128i) -> __m128i,
250 vectorized: unsafe fn(__m512i, __m512i) -> __m512i,
251 ) {
252 let a = _mm512_set_epi64(
253 0xDCB4DB3657BF0B7D,
254 0x18DB0601068EDD9F,
255 0xB76B908233200DC5,
256 0xE478235FA8E22D5E,
257 0xAB05CFFA2621154C,
258 0x1171B47A186174C9,
259 0x8C6B6C0E7595CEC9,
260 0xBE3E7D4934E961BD,
261 );
262 let k = _mm512_set_epi64(
263 0x672F6F105A94CEA7,
264 0x8298B8FFCA5F829C,
265 0xA3927047B3FB61D8,
266 0x978093862CDE7187,
267 0xB1927AB22F31D0EC,
268 0xA9A5DA619BE4D7AF,
269 0xCA2590F56884FDC6,
270 0x19BE9F660038BDB5,
271 );
272 let mut a_decomp = [_mm_setzero_si128(); 4];
273 a_decomp[0] = _mm512_extracti32x4_epi32::<0>(a);
274 a_decomp[1] = _mm512_extracti32x4_epi32::<1>(a);
275 a_decomp[2] = _mm512_extracti32x4_epi32::<2>(a);
276 a_decomp[3] = _mm512_extracti32x4_epi32::<3>(a);
277 let mut k_decomp = [_mm_setzero_si128(); 4];
278 k_decomp[0] = _mm512_extracti32x4_epi32::<0>(k);
279 k_decomp[1] = _mm512_extracti32x4_epi32::<1>(k);
280 k_decomp[2] = _mm512_extracti32x4_epi32::<2>(k);
281 k_decomp[3] = _mm512_extracti32x4_epi32::<3>(k);
282 let r = vectorized(a, k);
283 let mut e_decomp = [_mm_setzero_si128(); 4];
284 for i in 0..4 {
285 e_decomp[i] = linear(a_decomp[i], k_decomp[i]);
286 }
287 assert_eq_m128i(_mm512_extracti32x4_epi32::<0>(r), e_decomp[0]);
288 assert_eq_m128i(_mm512_extracti32x4_epi32::<1>(r), e_decomp[1]);
289 assert_eq_m128i(_mm512_extracti32x4_epi32::<2>(r), e_decomp[2]);
290 assert_eq_m128i(_mm512_extracti32x4_epi32::<3>(r), e_decomp[3]);
291 }
292
293 #[simd_test(enable = "vaes,avx512f")]
294 unsafe fn test_mm512_aesdec_epi128() {
295 // Constants taken from https://msdn.microsoft.com/en-us/library/cc664949.aspx.
296 let (a, k) = setup_state_key_512();
297 let e = _mm_set_epi64x(0x044e4f5176fec48f, 0xb57ecfa381da39ee);
298 let e = _mm512_broadcast_i32x4(e);
299 let r = _mm512_aesdec_epi128(a, k);
300 assert_eq_m512i(r, e);
301
302 helper_for_512_vaes(_mm_aesdec_si128, _mm512_aesdec_epi128);
303 }
304
305 #[simd_test(enable = "vaes,avx512f")]
306 unsafe fn test_mm512_aesdeclast_epi128() {
307 // Constants taken from https://msdn.microsoft.com/en-us/library/cc714178.aspx.
308 let (a, k) = setup_state_key_512();
309 let e = _mm_set_epi64x(0x36cad57d9072bf9e, 0xf210dd981fa4a493);
310 let e = _mm512_broadcast_i32x4(e);
311 let r = _mm512_aesdeclast_epi128(a, k);
312 assert_eq_m512i(r, e);
313
314 helper_for_512_vaes(_mm_aesdeclast_si128, _mm512_aesdeclast_epi128);
315 }
316
317 #[simd_test(enable = "vaes,avx512f")]
318 unsafe fn test_mm512_aesenc_epi128() {
319 // Constants taken from https://msdn.microsoft.com/en-us/library/cc664810.aspx.
320 let (a, k) = setup_state_key_512();
321 let e = _mm_set_epi64x(0x16ab0e57dfc442ed, 0x28e4ee1884504333);
322 let e = _mm512_broadcast_i32x4(e);
323 let r = _mm512_aesenc_epi128(a, k);
324 assert_eq_m512i(r, e);
325
326 helper_for_512_vaes(_mm_aesenc_si128, _mm512_aesenc_epi128);
327 }
328
329 #[simd_test(enable = "vaes,avx512f")]
330 unsafe fn test_mm512_aesenclast_epi128() {
331 // Constants taken from https://msdn.microsoft.com/en-us/library/cc714136.aspx.
332 let (a, k) = setup_state_key_512();
333 let e = _mm_set_epi64x(0xb6dd7df25d7ab320, 0x4b04f98cf4c860f8);
334 let e = _mm512_broadcast_i32x4(e);
335 let r = _mm512_aesenclast_epi128(a, k);
336 assert_eq_m512i(r, e);
337
338 helper_for_512_vaes(_mm_aesenclast_si128, _mm512_aesenclast_epi128);
339 }
340}
341