1//! Trailing Bit Manipulation (TBM) instruction set.
2//!
3//! The reference is [AMD64 Architecture Programmer's Manual, Volume 3:
4//! General-Purpose and System Instructions][amd64_ref].
5//!
6//! [Wikipedia][wikipedia_bmi] provides a quick overview of the available
7//! instructions.
8//!
9//! [amd64_ref]: https://docs.amd.com/v/u/en-US/24594_3.37
10//! [wikipedia_bmi]:
11//! https://en.wikipedia.org/wiki/Bit_Manipulation_Instruction_Sets#ABM_.28Advanced_Bit_Manipulation.29
12
13#[cfg(test)]
14use stdarch_test::assert_instr;
15
16unsafe extern "C" {
17 #[link_name = "llvm.x86.tbm.bextri.u64"]
18 unsafefn bextri_u64(a: u64, control: u64) -> u64;
19}
20
21/// Extracts bits of `a` specified by `control` into
22/// the least significant bits of the result.
23///
24/// Bits `[7,0]` of `control` specify the index to the first bit in the range to
25/// be extracted, and bits `[15,8]` specify the length of the range. For any bit
26/// position in the specified range that lie beyond the MSB of the source operand,
27/// zeroes will be written. If the range is empty, the result is zero.
28#[inline]
29#[target_feature(enable = "tbm")]
30#[cfg_attr(test, assert_instr(bextr, CONTROL = 0x0404))]
31#[rustc_legacy_const_generics(1)]
32#[stable(feature = "simd_x86_updates", since = "1.82.0")]
33pub fn _bextri_u64<const CONTROL: u64>(a: u64) -> u64 {
34 static_assert_uimm_bits!(CONTROL, 16);
35 unsafe { bextri_u64(a, CONTROL) }
36}
37
38/// Clears all bits below the least significant zero bit of `x`.
39///
40/// If there is no zero bit in `x`, it returns zero.
41#[inline]
42#[target_feature(enable = "tbm")]
43#[cfg_attr(test, assert_instr(blcfill))]
44#[stable(feature = "simd_x86", since = "1.27.0")]
45#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
46pub const fn _blcfill_u64(x: u64) -> u64 {
47 x & x.wrapping_add(1)
48}
49
50/// Sets all bits of `x` to 1 except for the least significant zero bit.
51///
52/// If there is no zero bit in `x`, it sets all bits.
53#[inline]
54#[target_feature(enable = "tbm")]
55#[cfg_attr(test, assert_instr(blci))]
56#[stable(feature = "simd_x86", since = "1.27.0")]
57#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
58pub const fn _blci_u64(x: u64) -> u64 {
59 x | !x.wrapping_add(1)
60}
61
62/// Sets the least significant zero bit of `x` and clears all other bits.
63///
64/// If there is no zero bit in `x`, it returns zero.
65#[inline]
66#[target_feature(enable = "tbm")]
67#[cfg_attr(test, assert_instr(blcic))]
68#[stable(feature = "simd_x86", since = "1.27.0")]
69#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
70pub const fn _blcic_u64(x: u64) -> u64 {
71 !x & x.wrapping_add(1)
72}
73
74/// Sets the least significant zero bit of `x` and clears all bits above
75/// that bit.
76///
77/// If there is no zero bit in `x`, it sets all the bits.
78#[inline]
79#[target_feature(enable = "tbm")]
80#[cfg_attr(test, assert_instr(blcmsk))]
81#[stable(feature = "simd_x86", since = "1.27.0")]
82#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
83pub const fn _blcmsk_u64(x: u64) -> u64 {
84 x ^ x.wrapping_add(1)
85}
86
87/// Sets the least significant zero bit of `x`.
88///
89/// If there is no zero bit in `x`, it returns `x`.
90#[inline]
91#[target_feature(enable = "tbm")]
92#[cfg_attr(test, assert_instr(blcs))]
93#[stable(feature = "simd_x86", since = "1.27.0")]
94#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
95pub const fn _blcs_u64(x: u64) -> u64 {
96 x | x.wrapping_add(1)
97}
98
99/// Sets all bits of `x` below the least significant one.
100///
101/// If there is no set bit in `x`, it sets all the bits.
102#[inline]
103#[target_feature(enable = "tbm")]
104#[cfg_attr(test, assert_instr(blsfill))]
105#[stable(feature = "simd_x86", since = "1.27.0")]
106#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
107pub const fn _blsfill_u64(x: u64) -> u64 {
108 x | x.wrapping_sub(1)
109}
110
111/// Clears least significant bit and sets all other bits.
112///
113/// If there is no set bit in `x`, it sets all the bits.
114#[inline]
115#[target_feature(enable = "tbm")]
116#[cfg_attr(test, assert_instr(blsic))]
117#[stable(feature = "simd_x86", since = "1.27.0")]
118#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
119pub const fn _blsic_u64(x: u64) -> u64 {
120 !x | x.wrapping_sub(1)
121}
122
123/// Clears all bits below the least significant zero of `x` and sets all other
124/// bits.
125///
126/// If the least significant bit of `x` is `0`, it sets all bits.
127#[inline]
128#[target_feature(enable = "tbm")]
129#[cfg_attr(test, assert_instr(t1mskc))]
130#[stable(feature = "simd_x86", since = "1.27.0")]
131#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
132pub const fn _t1mskc_u64(x: u64) -> u64 {
133 !x | x.wrapping_add(1)
134}
135
136/// Sets all bits below the least significant one of `x` and clears all other
137/// bits.
138///
139/// If the least significant bit of `x` is 1, it returns zero.
140#[inline]
141#[target_feature(enable = "tbm")]
142#[cfg_attr(test, assert_instr(tzmsk))]
143#[stable(feature = "simd_x86", since = "1.27.0")]
144#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
145pub const fn _tzmsk_u64(x: u64) -> u64 {
146 !x & x.wrapping_sub(1)
147}
148
149#[cfg(test)]
150mod tests {
151 use crate::core_arch::assert_eq_const as assert_eq;
152 use stdarch_test::simd_test;
153
154 use crate::core_arch::x86_64::*;
155
156 #[simd_test(enable = "tbm")]
157 fn test_bextri_u64() {
158 assert_eq!(_bextri_u64::<0x0404>(0b0101_0000u64), 0b0000_0101u64);
159 }
160
161 #[simd_test(enable = "tbm")]
162 const fn test_blcfill_u64() {
163 assert_eq!(_blcfill_u64(0b0101_0111u64), 0b0101_0000u64);
164 assert_eq!(_blcfill_u64(0b1111_1111u64), 0u64);
165 }
166
167 #[simd_test(enable = "tbm")]
168 const fn test_blci_u64() {
169 assert_eq!(
170 _blci_u64(0b0101_0000u64),
171 0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1110u64
172 );
173 assert_eq!(
174 _blci_u64(0b1111_1111u64),
175 0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1110_1111_1111u64
176 );
177 }
178
179 #[simd_test(enable = "tbm")]
180 const fn test_blcic_u64() {
181 assert_eq!(_blcic_u64(0b0101_0001u64), 0b0000_0010u64);
182 assert_eq!(_blcic_u64(0b1111_1111u64), 0b1_0000_0000u64);
183 }
184
185 #[simd_test(enable = "tbm")]
186 const fn test_blcmsk_u64() {
187 assert_eq!(_blcmsk_u64(0b0101_0001u64), 0b0000_0011u64);
188 assert_eq!(_blcmsk_u64(0b1111_1111u64), 0b1_1111_1111u64);
189 }
190
191 #[simd_test(enable = "tbm")]
192 const fn test_blcs_u64() {
193 assert_eq!(_blcs_u64(0b0101_0001u64), 0b0101_0011u64);
194 assert_eq!(_blcs_u64(0b1111_1111u64), 0b1_1111_1111u64);
195 }
196
197 #[simd_test(enable = "tbm")]
198 const fn test_blsfill_u64() {
199 assert_eq!(_blsfill_u64(0b0101_0100u64), 0b0101_0111u64);
200 assert_eq!(
201 _blsfill_u64(0u64),
202 0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111u64
203 );
204 }
205
206 #[simd_test(enable = "tbm")]
207 const fn test_blsic_u64() {
208 assert_eq!(
209 _blsic_u64(0b0101_0100u64),
210 0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1011u64
211 );
212 assert_eq!(
213 _blsic_u64(0u64),
214 0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111u64
215 );
216 }
217
218 #[simd_test(enable = "tbm")]
219 const fn test_t1mskc_u64() {
220 assert_eq!(
221 _t1mskc_u64(0b0101_0111u64),
222 0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1000u64
223 );
224 assert_eq!(
225 _t1mskc_u64(0u64),
226 0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111u64
227 );
228 }
229
230 #[simd_test(enable = "tbm")]
231 const fn test_tzmsk_u64() {
232 assert_eq!(_tzmsk_u64(0b0101_1000u64), 0b0000_0111u64);
233 assert_eq!(_tzmsk_u64(0b0101_1001u64), 0b0000_0000u64);
234 }
235}
236