1//! AES Key Locker Intrinsics
2//!
3//! The Intrinsics here correspond to those in the `keylockerintrin.h` C header.
4
5use crate::core_arch::x86::__m128i;
6use crate::ptr;
7
8#[cfg(test)]
9use stdarch_test::assert_instr;
10
11#[repr(C, packed)]
12struct EncodeKey128Output(u32, __m128i, __m128i, __m128i, __m128i, __m128i, __m128i);
13
14#[repr(C, packed)]
15struct EncodeKey256Output(
16 u32,
17 __m128i,
18 __m128i,
19 __m128i,
20 __m128i,
21 __m128i,
22 __m128i,
23 __m128i,
24);
25
26#[repr(C, packed)]
27struct AesOutput(u8, __m128i);
28
29#[repr(C, packed)]
30struct WideAesOutput(
31 u8,
32 __m128i,
33 __m128i,
34 __m128i,
35 __m128i,
36 __m128i,
37 __m128i,
38 __m128i,
39 __m128i,
40);
41
42#[allow(improper_ctypes)]
43unsafe extern "unadjusted" {
44 #[link_name = "llvm.x86.loadiwkey"]
45 unsafefn loadiwkey(integrity_key: __m128i, key_lo: __m128i, key_hi: __m128i, control: u32);
46
47 #[link_name = "llvm.x86.encodekey128"]
48 unsafefn encodekey128(key_metadata: u32, key: __m128i) -> EncodeKey128Output;
49 #[link_name = "llvm.x86.encodekey256"]
50 unsafefn encodekey256(key_metadata: u32, key_lo: __m128i, key_hi: __m128i) -> EncodeKey256Output;
51
52 #[link_name = "llvm.x86.aesenc128kl"]
53 unsafefn aesenc128kl(data: __m128i, handle: *const u8) -> AesOutput;
54 #[link_name = "llvm.x86.aesdec128kl"]
55 unsafefn aesdec128kl(data: __m128i, handle: *const u8) -> AesOutput;
56 #[link_name = "llvm.x86.aesenc256kl"]
57 unsafefn aesenc256kl(data: __m128i, handle: *const u8) -> AesOutput;
58 #[link_name = "llvm.x86.aesdec256kl"]
59 unsafefn aesdec256kl(data: __m128i, handle: *const u8) -> AesOutput;
60
61 #[link_name = "llvm.x86.aesencwide128kl"]
62 unsafefn aesencwide128kl(
63 handle: *const u8,
64 i0: __m128i,
65 i1: __m128i,
66 i2: __m128i,
67 i3: __m128i,
68 i4: __m128i,
69 i5: __m128i,
70 i6: __m128i,
71 i7: __m128i,
72 ) -> WideAesOutput;
73 #[link_name = "llvm.x86.aesdecwide128kl"]
74 unsafefn aesdecwide128kl(
75 handle: *const u8,
76 i0: __m128i,
77 i1: __m128i,
78 i2: __m128i,
79 i3: __m128i,
80 i4: __m128i,
81 i5: __m128i,
82 i6: __m128i,
83 i7: __m128i,
84 ) -> WideAesOutput;
85 #[link_name = "llvm.x86.aesencwide256kl"]
86 unsafefn aesencwide256kl(
87 handle: *const u8,
88 i0: __m128i,
89 i1: __m128i,
90 i2: __m128i,
91 i3: __m128i,
92 i4: __m128i,
93 i5: __m128i,
94 i6: __m128i,
95 i7: __m128i,
96 ) -> WideAesOutput;
97 #[link_name = "llvm.x86.aesdecwide256kl"]
98 unsafefn aesdecwide256kl(
99 handle: *const u8,
100 i0: __m128i,
101 i1: __m128i,
102 i2: __m128i,
103 i3: __m128i,
104 i4: __m128i,
105 i5: __m128i,
106 i6: __m128i,
107 i7: __m128i,
108 ) -> WideAesOutput;
109}
110
111/// Load internal wrapping key (IWKey). The 32-bit unsigned integer `control` specifies IWKey's KeySource
112/// and whether backing up the key is permitted. IWKey's 256-bit encryption key is loaded from `key_lo`
113/// and `key_hi`.
114///
115/// - `control[0]`: NoBackup bit. If set, the IWKey cannot be backed up.
116/// - `control[1:4]`: KeySource bits. These bits specify the encoding method of the IWKey. The only
117/// allowed values are `0` (AES GCM SIV wrapping algorithm with the specified key) and `1` (AES GCM
118/// SIV wrapping algorithm with random keys enforced by hardware). After calling `_mm_loadiwkey` with
119/// KeySource set to `1`, software must check `ZF` to ensure that the key was loaded successfully.
120/// Using any other value may result in a General Protection Exception.
121/// - `control[5:31]`: Reserved for future use, must be set to `0`.
122///
123/// Note that setting the NoBackup bit and using the KeySource value `1` requires hardware support. These
124/// permissions can be found by calling `__cpuid(0x19)` and checking the `ECX[0:1]` bits. Failing to follow
125/// these restrictions may result in a General Protection Exception.
126///
127/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_loadiwkey)
128#[inline]
129#[target_feature(enable = "kl")]
130#[stable(feature = "keylocker_x86", since = "1.89.0")]
131#[cfg_attr(test, assert_instr(loadiwkey))]
132pub unsafe fn _mm_loadiwkey(
133 control: u32,
134 integrity_key: __m128i,
135 key_lo: __m128i,
136 key_hi: __m128i,
137) {
138 loadiwkey(integrity_key, key_lo, key_hi, control);
139}
140
141/// Wrap a 128-bit AES key into a 384-bit key handle and stores it in `handle`. Returns the `control`
142/// parameter used to create the IWKey.
143///
144/// - `key_params[0]`: If set, this key can only be used by the Kernel.
145/// - `key_params[1]`: If set, this key can not be used to encrypt.
146/// - `key_params[2]`: If set, this key can not be used to decrypt.
147/// - `key_params[31:3]`: Reserved for future use, must be set to `0`.
148///
149/// Note that these restrictions need hardware support, and the supported restrictions can be found by
150/// calling `__cpuid(0x19)` and checking the `EAX[0:2]` bits. Failing to follow these restrictions may
151/// result in a General Protection Exception.
152///
153/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_encodekey128_u32)
154#[inline]
155#[target_feature(enable = "kl")]
156#[stable(feature = "keylocker_x86", since = "1.89.0")]
157#[cfg_attr(test, assert_instr(encodekey128))]
158pub unsafe fn _mm_encodekey128_u32(key_params: u32, key: __m128i, handle: *mut u8) -> u32 {
159 let EncodeKey128Output(control: u32, key0: __m128i, key1: __m128i, key2: __m128i, _, _, _) = encodekey128(key_metadata:key_params, key);
160 ptr::write_unaligned(dst:handle.cast(), [key0, key1, key2]);
161 control
162}
163
164/// Wrap a 256-bit AES key into a 512-bit key handle and stores it in `handle`. Returns the `control`
165/// parameter used to create the IWKey.
166///
167/// - `key_params[0]`: If set, this key can only be used by the Kernel.
168/// - `key_params[1]`: If set, this key can not be used to encrypt.
169/// - `key_params[2]`: If set, this key can not be used to decrypt.
170/// - `key_params[31:3]`: Reserved for future use, must be set to `0`.
171///
172/// Note that these restrictions need hardware support, and the supported restrictions can be found by
173/// calling `__cpuid(0x19)` and checking the `EAX[0:2]` bits. Failing to follow these restrictions may
174/// result in a General Protection Exception.
175///
176/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_encodekey256_u32)
177#[inline]
178#[target_feature(enable = "kl")]
179#[stable(feature = "keylocker_x86", since = "1.89.0")]
180#[cfg_attr(test, assert_instr(encodekey256))]
181pub unsafe fn _mm_encodekey256_u32(
182 key_params: u32,
183 key_lo: __m128i,
184 key_hi: __m128i,
185 handle: *mut u8,
186) -> u32 {
187 let EncodeKey256Output(control: u32, key0: __m128i, key1: __m128i, key2: __m128i, key3: __m128i, _, _, _) =
188 encodekey256(key_metadata:key_params, key_lo, key_hi);
189 ptr::write_unaligned(dst:handle.cast(), [key0, key1, key2, key3]);
190 control
191}
192
193/// Encrypt 10 rounds of unsigned 8-bit integers in `input` using 128-bit AES key specified in the
194/// 384-bit key handle `handle`. Store the resulting unsigned 8-bit integers into the corresponding
195/// elements of `output`. Returns `0` if the operation was successful, and `1` if the operation failed
196/// due to a handle violation.
197///
198/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesenc128kl_u8)
199#[inline]
200#[target_feature(enable = "kl")]
201#[stable(feature = "keylocker_x86", since = "1.89.0")]
202#[cfg_attr(test, assert_instr(aesenc128kl))]
203pub unsafe fn _mm_aesenc128kl_u8(output: *mut __m128i, input: __m128i, handle: *const u8) -> u8 {
204 let AesOutput(status: u8, result: __m128i) = aesenc128kl(data:input, handle);
205 *output = result;
206 status
207}
208
209/// Decrypt 10 rounds of unsigned 8-bit integers in `input` using 128-bit AES key specified in the
210/// 384-bit key handle `handle`. Store the resulting unsigned 8-bit integers into the corresponding
211/// elements of `output`. Returns `0` if the operation was successful, and `1` if the operation failed
212/// due to a handle violation.
213///
214/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesdec128kl_u8)
215#[inline]
216#[target_feature(enable = "kl")]
217#[stable(feature = "keylocker_x86", since = "1.89.0")]
218#[cfg_attr(test, assert_instr(aesdec128kl))]
219pub unsafe fn _mm_aesdec128kl_u8(output: *mut __m128i, input: __m128i, handle: *const u8) -> u8 {
220 let AesOutput(status: u8, result: __m128i) = aesdec128kl(data:input, handle);
221 *output = result;
222 status
223}
224
225/// Encrypt 14 rounds of unsigned 8-bit integers in `input` using 256-bit AES key specified in the
226/// 512-bit key handle `handle`. Store the resulting unsigned 8-bit integers into the corresponding
227/// elements of `output`. Returns `0` if the operation was successful, and `1` if the operation failed
228/// due to a handle violation.
229///
230/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesenc256kl_u8)
231#[inline]
232#[target_feature(enable = "kl")]
233#[stable(feature = "keylocker_x86", since = "1.89.0")]
234#[cfg_attr(test, assert_instr(aesenc256kl))]
235pub unsafe fn _mm_aesenc256kl_u8(output: *mut __m128i, input: __m128i, handle: *const u8) -> u8 {
236 let AesOutput(status: u8, result: __m128i) = aesenc256kl(data:input, handle);
237 *output = result;
238 status
239}
240
241/// Decrypt 14 rounds of unsigned 8-bit integers in `input` using 256-bit AES key specified in the
242/// 512-bit key handle `handle`. Store the resulting unsigned 8-bit integers into the corresponding
243/// elements of `output`. Returns `0` if the operation was successful, and `1` if the operation failed
244/// due to a handle violation.
245///
246/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesdec256kl_u8)
247#[inline]
248#[target_feature(enable = "kl")]
249#[stable(feature = "keylocker_x86", since = "1.89.0")]
250#[cfg_attr(test, assert_instr(aesdec256kl))]
251pub unsafe fn _mm_aesdec256kl_u8(output: *mut __m128i, input: __m128i, handle: *const u8) -> u8 {
252 let AesOutput(status: u8, result: __m128i) = aesdec256kl(data:input, handle);
253 *output = result;
254 status
255}
256
257/// Encrypt 10 rounds of 8 groups of unsigned 8-bit integers in `input` using 128-bit AES key specified
258/// in the 384-bit key handle `handle`. Store the resulting unsigned 8-bit integers into the corresponding
259/// elements of `output`. Returns `0` if the operation was successful, and `1` if the operation failed
260/// due to a handle violation.
261///
262/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesencwide128kl_u8)
263#[inline]
264#[target_feature(enable = "widekl")]
265#[stable(feature = "keylocker_x86", since = "1.89.0")]
266#[cfg_attr(test, assert_instr(aesencwide128kl))]
267pub unsafe fn _mm_aesencwide128kl_u8(
268 output: *mut __m128i,
269 input: *const __m128i,
270 handle: *const u8,
271) -> u8 {
272 let input: &[__m128i] = &*ptr::slice_from_raw_parts(data:input, len:8);
273 let WideAesOutput(status: u8, out0: __m128i, out1: __m128i, out2: __m128i, out3: __m128i, out4: __m128i, out5: __m128i, out6: __m128i, out7: __m128i) = aesencwide128kl(
274 handle, i0:input[0], i1:input[1], i2:input[2], i3:input[3], i4:input[4], i5:input[5], i6:input[6], i7:input[7],
275 );
276 *output.cast() = [out0, out1, out2, out3, out4, out5, out6, out7];
277 status
278}
279
280/// Decrypt 10 rounds of 8 groups of unsigned 8-bit integers in `input` using 128-bit AES key specified
281/// in the 384-bit key handle `handle`. Store the resulting unsigned 8-bit integers into the corresponding
282/// elements of `output`. Returns `0` if the operation was successful, and `1` if the operation failed
283/// due to a handle violation.
284///
285/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesdecwide128kl_u8)
286#[inline]
287#[target_feature(enable = "widekl")]
288#[stable(feature = "keylocker_x86", since = "1.89.0")]
289#[cfg_attr(test, assert_instr(aesdecwide128kl))]
290pub unsafe fn _mm_aesdecwide128kl_u8(
291 output: *mut __m128i,
292 input: *const __m128i,
293 handle: *const u8,
294) -> u8 {
295 let input: &[__m128i] = &*ptr::slice_from_raw_parts(data:input, len:8);
296 let WideAesOutput(status: u8, out0: __m128i, out1: __m128i, out2: __m128i, out3: __m128i, out4: __m128i, out5: __m128i, out6: __m128i, out7: __m128i) = aesdecwide128kl(
297 handle, i0:input[0], i1:input[1], i2:input[2], i3:input[3], i4:input[4], i5:input[5], i6:input[6], i7:input[7],
298 );
299 *output.cast() = [out0, out1, out2, out3, out4, out5, out6, out7];
300 status
301}
302
303/// Encrypt 14 rounds of 8 groups of unsigned 8-bit integers in `input` using 256-bit AES key specified
304/// in the 512-bit key handle `handle`. Store the resulting unsigned 8-bit integers into the corresponding
305/// elements of `output`. Returns `0` if the operation was successful, and `1` if the operation failed
306/// due to a handle violation.
307///
308/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesencwide256kl_u8)
309#[inline]
310#[target_feature(enable = "widekl")]
311#[stable(feature = "keylocker_x86", since = "1.89.0")]
312#[cfg_attr(test, assert_instr(aesencwide256kl))]
313pub unsafe fn _mm_aesencwide256kl_u8(
314 output: *mut __m128i,
315 input: *const __m128i,
316 handle: *const u8,
317) -> u8 {
318 let input: &[__m128i] = &*ptr::slice_from_raw_parts(data:input, len:8);
319 let WideAesOutput(status: u8, out0: __m128i, out1: __m128i, out2: __m128i, out3: __m128i, out4: __m128i, out5: __m128i, out6: __m128i, out7: __m128i) = aesencwide256kl(
320 handle, i0:input[0], i1:input[1], i2:input[2], i3:input[3], i4:input[4], i5:input[5], i6:input[6], i7:input[7],
321 );
322 *output.cast() = [out0, out1, out2, out3, out4, out5, out6, out7];
323 status
324}
325
326/// Decrypt 14 rounds of 8 groups of unsigned 8-bit integers in `input` using 256-bit AES key specified
327/// in the 512-bit key handle `handle`. Store the resulting unsigned 8-bit integers into the corresponding
328/// elements of `output`. Returns `0` if the operation was successful, and `1` if the operation failed
329/// due to a handle violation.
330///
331/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesdecwide256kl_u8)
332#[inline]
333#[target_feature(enable = "widekl")]
334#[stable(feature = "keylocker_x86", since = "1.89.0")]
335#[cfg_attr(test, assert_instr(aesdecwide256kl))]
336pub unsafe fn _mm_aesdecwide256kl_u8(
337 output: *mut __m128i,
338 input: *const __m128i,
339 handle: *const u8,
340) -> u8 {
341 let input: &[__m128i] = &*ptr::slice_from_raw_parts(data:input, len:8);
342 let WideAesOutput(status: u8, out0: __m128i, out1: __m128i, out2: __m128i, out3: __m128i, out4: __m128i, out5: __m128i, out6: __m128i, out7: __m128i) = aesdecwide256kl(
343 handle, i0:input[0], i1:input[1], i2:input[2], i3:input[3], i4:input[4], i5:input[5], i6:input[6], i7:input[7],
344 );
345 *output.cast() = [out0, out1, out2, out3, out4, out5, out6, out7];
346 status
347}
348
349#[cfg(test)]
350mod tests {
351 use crate::core_arch::x86::*;
352 use stdarch_test::simd_test;
353
354 #[target_feature(enable = "kl")]
355 fn encodekey128() -> [u8; 48] {
356 let mut handle = [0; 48];
357 let _ = unsafe { _mm_encodekey128_u32(0, _mm_setzero_si128(), handle.as_mut_ptr()) };
358 handle
359 }
360
361 #[target_feature(enable = "kl")]
362 fn encodekey256() -> [u8; 64] {
363 let mut handle = [0; 64];
364 let _ = unsafe {
365 _mm_encodekey256_u32(
366 0,
367 _mm_setzero_si128(),
368 _mm_setzero_si128(),
369 handle.as_mut_ptr(),
370 )
371 };
372 handle
373 }
374
375 #[simd_test(enable = "kl")]
376 fn test_mm_encodekey128_u32() {
377 encodekey128();
378 }
379
380 #[simd_test(enable = "kl")]
381 fn test_mm_encodekey256_u32() {
382 encodekey256();
383 }
384
385 #[simd_test(enable = "kl")]
386 fn test_mm_aesenc128kl_u8() {
387 let mut buffer = _mm_setzero_si128();
388 let key = encodekey128();
389
390 for _ in 0..100 {
391 let status = unsafe { _mm_aesenc128kl_u8(&mut buffer, buffer, key.as_ptr()) };
392 assert_eq!(status, 0);
393 }
394 for _ in 0..100 {
395 let status = unsafe { _mm_aesdec128kl_u8(&mut buffer, buffer, key.as_ptr()) };
396 assert_eq!(status, 0);
397 }
398
399 assert_eq_m128i(buffer, _mm_setzero_si128());
400 }
401
402 #[simd_test(enable = "kl")]
403 fn test_mm_aesdec128kl_u8() {
404 let mut buffer = _mm_setzero_si128();
405 let key = encodekey128();
406
407 for _ in 0..100 {
408 let status = unsafe { _mm_aesdec128kl_u8(&mut buffer, buffer, key.as_ptr()) };
409 assert_eq!(status, 0);
410 }
411 for _ in 0..100 {
412 let status = unsafe { _mm_aesenc128kl_u8(&mut buffer, buffer, key.as_ptr()) };
413 assert_eq!(status, 0);
414 }
415
416 assert_eq_m128i(buffer, _mm_setzero_si128());
417 }
418
419 #[simd_test(enable = "kl")]
420 fn test_mm_aesenc256kl_u8() {
421 let mut buffer = _mm_setzero_si128();
422 let key = encodekey256();
423
424 for _ in 0..100 {
425 let status = unsafe { _mm_aesenc256kl_u8(&mut buffer, buffer, key.as_ptr()) };
426 assert_eq!(status, 0);
427 }
428 for _ in 0..100 {
429 let status = unsafe { _mm_aesdec256kl_u8(&mut buffer, buffer, key.as_ptr()) };
430 assert_eq!(status, 0);
431 }
432
433 assert_eq_m128i(buffer, _mm_setzero_si128());
434 }
435
436 #[simd_test(enable = "kl")]
437 fn test_mm_aesdec256kl_u8() {
438 let mut buffer = _mm_setzero_si128();
439 let key = encodekey256();
440
441 for _ in 0..100 {
442 let status = unsafe { _mm_aesdec256kl_u8(&mut buffer, buffer, key.as_ptr()) };
443 assert_eq!(status, 0);
444 }
445 for _ in 0..100 {
446 let status = unsafe { _mm_aesenc256kl_u8(&mut buffer, buffer, key.as_ptr()) };
447 assert_eq!(status, 0);
448 }
449
450 assert_eq_m128i(buffer, _mm_setzero_si128());
451 }
452
453 #[simd_test(enable = "widekl")]
454 fn test_mm_aesencwide128kl_u8() {
455 let mut buffer = [_mm_setzero_si128(); 8];
456 let key = encodekey128();
457
458 for _ in 0..100 {
459 let status = unsafe {
460 _mm_aesencwide128kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr())
461 };
462 assert_eq!(status, 0);
463 }
464 for _ in 0..100 {
465 let status = unsafe {
466 _mm_aesdecwide128kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr())
467 };
468 assert_eq!(status, 0);
469 }
470
471 for elem in buffer {
472 assert_eq_m128i(elem, _mm_setzero_si128());
473 }
474 }
475
476 #[simd_test(enable = "widekl")]
477 fn test_mm_aesdecwide128kl_u8() {
478 let mut buffer = [_mm_setzero_si128(); 8];
479 let key = encodekey128();
480
481 for _ in 0..100 {
482 let status = unsafe {
483 _mm_aesdecwide128kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr())
484 };
485 assert_eq!(status, 0);
486 }
487 for _ in 0..100 {
488 let status = unsafe {
489 _mm_aesencwide128kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr())
490 };
491 assert_eq!(status, 0);
492 }
493
494 for elem in buffer {
495 assert_eq_m128i(elem, _mm_setzero_si128());
496 }
497 }
498
499 #[simd_test(enable = "widekl")]
500 fn test_mm_aesencwide256kl_u8() {
501 let mut buffer = [_mm_setzero_si128(); 8];
502 let key = encodekey256();
503
504 for _ in 0..100 {
505 let status = unsafe {
506 _mm_aesencwide256kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr())
507 };
508 assert_eq!(status, 0);
509 }
510 for _ in 0..100 {
511 let status = unsafe {
512 _mm_aesdecwide256kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr())
513 };
514 assert_eq!(status, 0);
515 }
516
517 for elem in buffer {
518 assert_eq_m128i(elem, _mm_setzero_si128());
519 }
520 }
521
522 #[simd_test(enable = "widekl")]
523 fn test_mm_aesdecwide256kl_u8() {
524 let mut buffer = [_mm_setzero_si128(); 8];
525 let key = encodekey256();
526
527 for _ in 0..100 {
528 let status = unsafe {
529 _mm_aesdecwide256kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr())
530 };
531 assert_eq!(status, 0);
532 }
533 for _ in 0..100 {
534 let status = unsafe {
535 _mm_aesencwide256kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr())
536 };
537 assert_eq!(status, 0);
538 }
539
540 for elem in buffer {
541 assert_eq_m128i(elem, _mm_setzero_si128());
542 }
543 }
544}
545