1//! AES Key Locker Intrinsics
2//!
3//! The Intrinsics here correspond to those in the `keylockerintrin.h` C header.
4
5use crate::core_arch::x86::__m128i;
6use crate::ptr;
7
8#[cfg(test)]
9use stdarch_test::assert_instr;
10
11#[repr(C, packed)]
12struct EncodeKey128Output(u32, __m128i, __m128i, __m128i, __m128i, __m128i, __m128i);
13
14#[repr(C, packed)]
15struct EncodeKey256Output(
16 u32,
17 __m128i,
18 __m128i,
19 __m128i,
20 __m128i,
21 __m128i,
22 __m128i,
23 __m128i,
24);
25
26#[repr(C, packed)]
27struct AesOutput(u8, __m128i);
28
29#[repr(C, packed)]
30struct WideAesOutput(
31 u8,
32 __m128i,
33 __m128i,
34 __m128i,
35 __m128i,
36 __m128i,
37 __m128i,
38 __m128i,
39 __m128i,
40);
41
42#[allow(improper_ctypes)]
43unsafe extern "unadjusted" {
44 #[link_name = "llvm.x86.loadiwkey"]
45 unsafefn loadiwkey(integrity_key: __m128i, key_lo: __m128i, key_hi: __m128i, control: u32);
46
47 #[link_name = "llvm.x86.encodekey128"]
48 unsafefn encodekey128(key_metadata: u32, key: __m128i) -> EncodeKey128Output;
49 #[link_name = "llvm.x86.encodekey256"]
50 unsafefn encodekey256(key_metadata: u32, key_lo: __m128i, key_hi: __m128i) -> EncodeKey256Output;
51
52 #[link_name = "llvm.x86.aesenc128kl"]
53 unsafefn aesenc128kl(data: __m128i, handle: *const u8) -> AesOutput;
54 #[link_name = "llvm.x86.aesdec128kl"]
55 unsafefn aesdec128kl(data: __m128i, handle: *const u8) -> AesOutput;
56 #[link_name = "llvm.x86.aesenc256kl"]
57 unsafefn aesenc256kl(data: __m128i, handle: *const u8) -> AesOutput;
58 #[link_name = "llvm.x86.aesdec256kl"]
59 unsafefn aesdec256kl(data: __m128i, handle: *const u8) -> AesOutput;
60
61 #[link_name = "llvm.x86.aesencwide128kl"]
62 unsafefn aesencwide128kl(
63 handle: *const u8,
64 i0: __m128i,
65 i1: __m128i,
66 i2: __m128i,
67 i3: __m128i,
68 i4: __m128i,
69 i5: __m128i,
70 i6: __m128i,
71 i7: __m128i,
72 ) -> WideAesOutput;
73 #[link_name = "llvm.x86.aesdecwide128kl"]
74 unsafefn aesdecwide128kl(
75 handle: *const u8,
76 i0: __m128i,
77 i1: __m128i,
78 i2: __m128i,
79 i3: __m128i,
80 i4: __m128i,
81 i5: __m128i,
82 i6: __m128i,
83 i7: __m128i,
84 ) -> WideAesOutput;
85 #[link_name = "llvm.x86.aesencwide256kl"]
86 unsafefn aesencwide256kl(
87 handle: *const u8,
88 i0: __m128i,
89 i1: __m128i,
90 i2: __m128i,
91 i3: __m128i,
92 i4: __m128i,
93 i5: __m128i,
94 i6: __m128i,
95 i7: __m128i,
96 ) -> WideAesOutput;
97 #[link_name = "llvm.x86.aesdecwide256kl"]
98 unsafefn aesdecwide256kl(
99 handle: *const u8,
100 i0: __m128i,
101 i1: __m128i,
102 i2: __m128i,
103 i3: __m128i,
104 i4: __m128i,
105 i5: __m128i,
106 i6: __m128i,
107 i7: __m128i,
108 ) -> WideAesOutput;
109}
110
111/// Load internal wrapping key (IWKey). The 32-bit unsigned integer `control` specifies IWKey's KeySource
112/// and whether backing up the key is permitted. IWKey's 256-bit encryption key is loaded from `key_lo`
113/// and `key_hi`.
114///
115/// - `control[0]`: NoBackup bit. If set, the IWKey cannot be backed up.
116/// - `control[1:4]`: KeySource bits. These bits specify the encoding method of the IWKey. The only
117/// allowed values are `0` (AES GCM SIV wrapping algorithm with the specified key) and `1` (AES GCM
118/// SIV wrapping algorithm with random keys enforced by hardware). After calling `_mm_loadiwkey` with
119/// KeySource set to `1`, software must check `ZF` to ensure that the key was loaded successfully.
120/// Using any other value may result in a General Protection Exception.
121/// - `control[5:31]`: Reserved for future use, must be set to `0`.
122///
123/// Note that setting the NoBackup bit and using the KeySource value `1` requires hardware support. These
124/// permissions can be found by calling `__cpuid(0x19)` and checking the `ECX[0:1]` bits. Failing to follow
125/// these restrictions may result in a General Protection Exception.
126///
127/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_loadiwkey)
128#[inline]
129#[target_feature(enable = "kl")]
130#[stable(feature = "keylocker_x86", since = "CURRENT_RUSTC_VERSION")]
131#[cfg_attr(test, assert_instr(loadiwkey))]
132pub unsafe fn _mm_loadiwkey(
133 control: u32,
134 integrity_key: __m128i,
135 key_lo: __m128i,
136 key_hi: __m128i,
137) {
138 loadiwkey(integrity_key, key_lo, key_hi, control);
139}
140
141/// Wrap a 128-bit AES key into a 384-bit key handle and stores it in `handle`. Returns the `control`
142/// parameter used to create the IWKey.
143///
144/// - `key_params[0]`: If set, this key can only be used by the Kernel.
145/// - `key_params[1]`: If set, this key can not be used to encrypt.
146/// - `key_params[2]`: If set, this key can not be used to decrypt.
147/// - `key_params[31:3]`: Reserved for future use, must be set to `0`.
148///
149/// Note that these restrictions need hardware support, and the supported restrictions can be found by
150/// calling `__cpuid(0x19)` and checking the `EAX[0:2]` bits. Failing to follow these restrictions may
151/// result in a General Protection Exception.
152///
153/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_encodekey128_u32)
154#[inline]
155#[target_feature(enable = "kl")]
156#[stable(feature = "keylocker_x86", since = "CURRENT_RUSTC_VERSION")]
157#[cfg_attr(test, assert_instr(encodekey128))]
158pub unsafe fn _mm_encodekey128_u32(key_params: u32, key: __m128i, handle: *mut u8) -> u32 {
159 let EncodeKey128Output(control: u32, key0: __m128i, key1: __m128i, key2: __m128i, _, _, _) = encodekey128(key_metadata:key_params, key);
160 ptr::write_unaligned(dst:handle.cast(), [key0, key1, key2]);
161 control
162}
163
164/// Wrap a 256-bit AES key into a 512-bit key handle and stores it in `handle`. Returns the `control`
165/// parameter used to create the IWKey.
166///
167/// - `key_params[0]`: If set, this key can only be used by the Kernel.
168/// - `key_params[1]`: If set, this key can not be used to encrypt.
169/// - `key_params[2]`: If set, this key can not be used to decrypt.
170/// - `key_params[31:3]`: Reserved for future use, must be set to `0`.
171///
172/// Note that these restrictions need hardware support, and the supported restrictions can be found by
173/// calling `__cpuid(0x19)` and checking the `EAX[0:2]` bits. Failing to follow these restrictions may
174/// result in a General Protection Exception.
175///
176/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_encodekey256_u32)
177#[inline]
178#[target_feature(enable = "kl")]
179#[stable(feature = "keylocker_x86", since = "CURRENT_RUSTC_VERSION")]
180#[cfg_attr(test, assert_instr(encodekey256))]
181pub unsafe fn _mm_encodekey256_u32(
182 key_params: u32,
183 key_lo: __m128i,
184 key_hi: __m128i,
185 handle: *mut u8,
186) -> u32 {
187 let EncodeKey256Output(control: u32, key0: __m128i, key1: __m128i, key2: __m128i, key3: __m128i, _, _, _) =
188 encodekey256(key_metadata:key_params, key_lo, key_hi);
189 ptr::write_unaligned(dst:handle.cast(), [key0, key1, key2, key3]);
190 control
191}
192
193/// Encrypt 10 rounds of unsigned 8-bit integers in `input` using 128-bit AES key specified in the
194/// 384-bit key handle `handle`. Store the resulting unsigned 8-bit integers into the corresponding
195/// elements of `output`. Returns `0` if the operation was successful, and `1` if the operation failed
196/// due to a handle violation.
197///
198/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesenc128kl_u8)
199#[inline]
200#[target_feature(enable = "kl")]
201#[stable(feature = "keylocker_x86", since = "CURRENT_RUSTC_VERSION")]
202#[cfg_attr(test, assert_instr(aesenc128kl))]
203pub unsafe fn _mm_aesenc128kl_u8(output: *mut __m128i, input: __m128i, handle: *const u8) -> u8 {
204 let AesOutput(status: u8, result: __m128i) = aesenc128kl(data:input, handle);
205 *output = result;
206 status
207}
208
209/// Decrypt 10 rounds of unsigned 8-bit integers in `input` using 128-bit AES key specified in the
210/// 384-bit key handle `handle`. Store the resulting unsigned 8-bit integers into the corresponding
211/// elements of `output`. Returns `0` if the operation was successful, and `1` if the operation failed
212/// due to a handle violation.
213///
214/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesdec128kl_u8)
215#[inline]
216#[target_feature(enable = "kl")]
217#[stable(feature = "keylocker_x86", since = "CURRENT_RUSTC_VERSION")]
218#[cfg_attr(test, assert_instr(aesdec128kl))]
219pub unsafe fn _mm_aesdec128kl_u8(output: *mut __m128i, input: __m128i, handle: *const u8) -> u8 {
220 let AesOutput(status: u8, result: __m128i) = aesdec128kl(data:input, handle);
221 *output = result;
222 status
223}
224
225/// Encrypt 14 rounds of unsigned 8-bit integers in `input` using 256-bit AES key specified in the
226/// 512-bit key handle `handle`. Store the resulting unsigned 8-bit integers into the corresponding
227/// elements of `output`. Returns `0` if the operation was successful, and `1` if the operation failed
228/// due to a handle violation.
229///
230/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesenc256kl_u8)
231#[inline]
232#[target_feature(enable = "kl")]
233#[stable(feature = "keylocker_x86", since = "CURRENT_RUSTC_VERSION")]
234#[cfg_attr(test, assert_instr(aesenc256kl))]
235pub unsafe fn _mm_aesenc256kl_u8(output: *mut __m128i, input: __m128i, handle: *const u8) -> u8 {
236 let AesOutput(status: u8, result: __m128i) = aesenc256kl(data:input, handle);
237 *output = result;
238 status
239}
240
241/// Decrypt 14 rounds of unsigned 8-bit integers in `input` using 256-bit AES key specified in the
242/// 512-bit key handle `handle`. Store the resulting unsigned 8-bit integers into the corresponding
243/// elements of `output`. Returns `0` if the operation was successful, and `1` if the operation failed
244/// due to a handle violation.
245///
246/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesdec256kl_u8)
247#[inline]
248#[target_feature(enable = "kl")]
249#[stable(feature = "keylocker_x86", since = "CURRENT_RUSTC_VERSION")]
250#[cfg_attr(test, assert_instr(aesdec256kl))]
251pub unsafe fn _mm_aesdec256kl_u8(output: *mut __m128i, input: __m128i, handle: *const u8) -> u8 {
252 let AesOutput(status: u8, result: __m128i) = aesdec256kl(data:input, handle);
253 *output = result;
254 status
255}
256
257/// Encrypt 10 rounds of 8 groups of unsigned 8-bit integers in `input` using 128-bit AES key specified
258/// in the 384-bit key handle `handle`. Store the resulting unsigned 8-bit integers into the corresponding
259/// elements of `output`. Returns `0` if the operation was successful, and `1` if the operation failed
260/// due to a handle violation.
261///
262/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesencwide128kl_u8)
263#[inline]
264#[target_feature(enable = "widekl")]
265#[stable(feature = "keylocker_x86", since = "CURRENT_RUSTC_VERSION")]
266#[cfg_attr(test, assert_instr(aesencwide128kl))]
267pub unsafe fn _mm_aesencwide128kl_u8(
268 output: *mut __m128i,
269 input: *const __m128i,
270 handle: *const u8,
271) -> u8 {
272 let input: &[__m128i] = &*ptr::slice_from_raw_parts(data:input, len:8);
273 let WideAesOutput(status: u8, out0: __m128i, out1: __m128i, out2: __m128i, out3: __m128i, out4: __m128i, out5: __m128i, out6: __m128i, out7: __m128i) = aesencwide128kl(
274 handle, i0:input[0], i1:input[1], i2:input[2], i3:input[3], i4:input[4], i5:input[5], i6:input[6], i7:input[7],
275 );
276 *output.cast() = [out0, out1, out2, out3, out4, out5, out6, out7];
277 status
278}
279
280/// Decrypt 10 rounds of 8 groups of unsigned 8-bit integers in `input` using 128-bit AES key specified
281/// in the 384-bit key handle `handle`. Store the resulting unsigned 8-bit integers into the corresponding
282/// elements of `output`. Returns `0` if the operation was successful, and `1` if the operation failed
283/// due to a handle violation.
284///
285/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesdecwide128kl_u8)
286#[inline]
287#[target_feature(enable = "widekl")]
288#[stable(feature = "keylocker_x86", since = "CURRENT_RUSTC_VERSION")]
289#[cfg_attr(test, assert_instr(aesdecwide128kl))]
290pub unsafe fn _mm_aesdecwide128kl_u8(
291 output: *mut __m128i,
292 input: *const __m128i,
293 handle: *const u8,
294) -> u8 {
295 let input: &[__m128i] = &*ptr::slice_from_raw_parts(data:input, len:8);
296 let WideAesOutput(status: u8, out0: __m128i, out1: __m128i, out2: __m128i, out3: __m128i, out4: __m128i, out5: __m128i, out6: __m128i, out7: __m128i) = aesdecwide128kl(
297 handle, i0:input[0], i1:input[1], i2:input[2], i3:input[3], i4:input[4], i5:input[5], i6:input[6], i7:input[7],
298 );
299 *output.cast() = [out0, out1, out2, out3, out4, out5, out6, out7];
300 status
301}
302
303/// Encrypt 14 rounds of 8 groups of unsigned 8-bit integers in `input` using 256-bit AES key specified
304/// in the 512-bit key handle `handle`. Store the resulting unsigned 8-bit integers into the corresponding
305/// elements of `output`. Returns `0` if the operation was successful, and `1` if the operation failed
306/// due to a handle violation.
307///
308/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesencwide256kl_u8)
309#[inline]
310#[target_feature(enable = "widekl")]
311#[stable(feature = "keylocker_x86", since = "CURRENT_RUSTC_VERSION")]
312#[cfg_attr(test, assert_instr(aesencwide256kl))]
313pub unsafe fn _mm_aesencwide256kl_u8(
314 output: *mut __m128i,
315 input: *const __m128i,
316 handle: *const u8,
317) -> u8 {
318 let input: &[__m128i] = &*ptr::slice_from_raw_parts(data:input, len:8);
319 let WideAesOutput(status: u8, out0: __m128i, out1: __m128i, out2: __m128i, out3: __m128i, out4: __m128i, out5: __m128i, out6: __m128i, out7: __m128i) = aesencwide256kl(
320 handle, i0:input[0], i1:input[1], i2:input[2], i3:input[3], i4:input[4], i5:input[5], i6:input[6], i7:input[7],
321 );
322 *output.cast() = [out0, out1, out2, out3, out4, out5, out6, out7];
323 status
324}
325
326/// Decrypt 14 rounds of 8 groups of unsigned 8-bit integers in `input` using 256-bit AES key specified
327/// in the 512-bit key handle `handle`. Store the resulting unsigned 8-bit integers into the corresponding
328/// elements of `output`. Returns `0` if the operation was successful, and `1` if the operation failed
329/// due to a handle violation.
330///
331/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesdecwide256kl_u8)
332#[inline]
333#[target_feature(enable = "widekl")]
334#[stable(feature = "keylocker_x86", since = "CURRENT_RUSTC_VERSION")]
335#[cfg_attr(test, assert_instr(aesdecwide256kl))]
336pub unsafe fn _mm_aesdecwide256kl_u8(
337 output: *mut __m128i,
338 input: *const __m128i,
339 handle: *const u8,
340) -> u8 {
341 let input: &[__m128i] = &*ptr::slice_from_raw_parts(data:input, len:8);
342 let WideAesOutput(status: u8, out0: __m128i, out1: __m128i, out2: __m128i, out3: __m128i, out4: __m128i, out5: __m128i, out6: __m128i, out7: __m128i) = aesdecwide256kl(
343 handle, i0:input[0], i1:input[1], i2:input[2], i3:input[3], i4:input[4], i5:input[5], i6:input[6], i7:input[7],
344 );
345 *output.cast() = [out0, out1, out2, out3, out4, out5, out6, out7];
346 status
347}
348
349#[cfg(test)]
350mod tests {
351 use crate::core_arch::x86::*;
352 use stdarch_test::simd_test;
353
354 #[target_feature(enable = "kl")]
355 unsafe fn encodekey128() -> [u8; 48] {
356 let mut handle = [0; 48];
357 let _ = _mm_encodekey128_u32(0, _mm_setzero_si128(), handle.as_mut_ptr());
358 handle
359 }
360
361 #[target_feature(enable = "kl")]
362 unsafe fn encodekey256() -> [u8; 64] {
363 let mut handle = [0; 64];
364 let _ = _mm_encodekey256_u32(
365 0,
366 _mm_setzero_si128(),
367 _mm_setzero_si128(),
368 handle.as_mut_ptr(),
369 );
370 handle
371 }
372
373 #[simd_test(enable = "kl")]
374 unsafe fn test_mm_encodekey128_u32() {
375 encodekey128();
376 }
377
378 #[simd_test(enable = "kl")]
379 unsafe fn test_mm_encodekey256_u32() {
380 encodekey256();
381 }
382
383 #[simd_test(enable = "kl")]
384 unsafe fn test_mm_aesenc128kl_u8() {
385 let mut buffer = _mm_setzero_si128();
386 let key = encodekey128();
387
388 for _ in 0..100 {
389 let status = _mm_aesenc128kl_u8(&mut buffer, buffer, key.as_ptr());
390 assert_eq!(status, 0);
391 }
392 for _ in 0..100 {
393 let status = _mm_aesdec128kl_u8(&mut buffer, buffer, key.as_ptr());
394 assert_eq!(status, 0);
395 }
396
397 assert_eq_m128i(buffer, _mm_setzero_si128());
398 }
399
400 #[simd_test(enable = "kl")]
401 unsafe fn test_mm_aesdec128kl_u8() {
402 let mut buffer = _mm_setzero_si128();
403 let key = encodekey128();
404
405 for _ in 0..100 {
406 let status = _mm_aesdec128kl_u8(&mut buffer, buffer, key.as_ptr());
407 assert_eq!(status, 0);
408 }
409 for _ in 0..100 {
410 let status = _mm_aesenc128kl_u8(&mut buffer, buffer, key.as_ptr());
411 assert_eq!(status, 0);
412 }
413
414 assert_eq_m128i(buffer, _mm_setzero_si128());
415 }
416
417 #[simd_test(enable = "kl")]
418 unsafe fn test_mm_aesenc256kl_u8() {
419 let mut buffer = _mm_setzero_si128();
420 let key = encodekey256();
421
422 for _ in 0..100 {
423 let status = _mm_aesenc256kl_u8(&mut buffer, buffer, key.as_ptr());
424 assert_eq!(status, 0);
425 }
426 for _ in 0..100 {
427 let status = _mm_aesdec256kl_u8(&mut buffer, buffer, key.as_ptr());
428 assert_eq!(status, 0);
429 }
430
431 assert_eq_m128i(buffer, _mm_setzero_si128());
432 }
433
434 #[simd_test(enable = "kl")]
435 unsafe fn test_mm_aesdec256kl_u8() {
436 let mut buffer = _mm_setzero_si128();
437 let key = encodekey256();
438
439 for _ in 0..100 {
440 let status = _mm_aesdec256kl_u8(&mut buffer, buffer, key.as_ptr());
441 assert_eq!(status, 0);
442 }
443 for _ in 0..100 {
444 let status = _mm_aesenc256kl_u8(&mut buffer, buffer, key.as_ptr());
445 assert_eq!(status, 0);
446 }
447
448 assert_eq_m128i(buffer, _mm_setzero_si128());
449 }
450
451 #[simd_test(enable = "widekl")]
452 unsafe fn test_mm_aesencwide128kl_u8() {
453 let mut buffer = [_mm_setzero_si128(); 8];
454 let key = encodekey128();
455
456 for _ in 0..100 {
457 let status = _mm_aesencwide128kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr());
458 assert_eq!(status, 0);
459 }
460 for _ in 0..100 {
461 let status = _mm_aesdecwide128kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr());
462 assert_eq!(status, 0);
463 }
464
465 for elem in buffer {
466 assert_eq_m128i(elem, _mm_setzero_si128());
467 }
468 }
469
470 #[simd_test(enable = "widekl")]
471 unsafe fn test_mm_aesdecwide128kl_u8() {
472 let mut buffer = [_mm_setzero_si128(); 8];
473 let key = encodekey128();
474
475 for _ in 0..100 {
476 let status = _mm_aesdecwide128kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr());
477 assert_eq!(status, 0);
478 }
479 for _ in 0..100 {
480 let status = _mm_aesencwide128kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr());
481 assert_eq!(status, 0);
482 }
483
484 for elem in buffer {
485 assert_eq_m128i(elem, _mm_setzero_si128());
486 }
487 }
488
489 #[simd_test(enable = "widekl")]
490 unsafe fn test_mm_aesencwide256kl_u8() {
491 let mut buffer = [_mm_setzero_si128(); 8];
492 let key = encodekey256();
493
494 for _ in 0..100 {
495 let status = _mm_aesencwide256kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr());
496 assert_eq!(status, 0);
497 }
498 for _ in 0..100 {
499 let status = _mm_aesdecwide256kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr());
500 assert_eq!(status, 0);
501 }
502
503 for elem in buffer {
504 assert_eq_m128i(elem, _mm_setzero_si128());
505 }
506 }
507
508 #[simd_test(enable = "widekl")]
509 unsafe fn test_mm_aesdecwide256kl_u8() {
510 let mut buffer = [_mm_setzero_si128(); 8];
511 let key = encodekey256();
512
513 for _ in 0..100 {
514 let status = _mm_aesdecwide256kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr());
515 assert_eq!(status, 0);
516 }
517 for _ in 0..100 {
518 let status = _mm_aesencwide256kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr());
519 assert_eq!(status, 0);
520 }
521
522 for elem in buffer {
523 assert_eq_m128i(elem, _mm_setzero_si128());
524 }
525 }
526}
527