1//! Trailing Bit Manipulation (TBM) instruction set.
2//!
3//! The reference is [AMD64 Architecture Programmer's Manual, Volume 3:
4//! General-Purpose and System Instructions][amd64_ref].
5//!
6//! [Wikipedia][wikipedia_bmi] provides a quick overview of the available
7//! instructions.
8//!
9//! [amd64_ref]: http://support.amd.com/TechDocs/24594.pdf
10//! [wikipedia_bmi]:
11//! https://en.wikipedia.org/wiki/Bit_Manipulation_Instruction_Sets#ABM_.28Advanced_Bit_Manipulation.29
12
13#[cfg(test)]
14use stdarch_test::assert_instr;
15
16unsafe extern "C" {
17 #[link_name = "llvm.x86.tbm.bextri.u64"]
18 unsafefn bextri_u64(a: u64, control: u64) -> u64;
19}
20
21/// Extracts bits of `a` specified by `control` into
22/// the least significant bits of the result.
23///
24/// Bits `[7,0]` of `control` specify the index to the first bit in the range to
25/// be extracted, and bits `[15,8]` specify the length of the range. For any bit
26/// position in the specified range that lie beyond the MSB of the source operand,
27/// zeroes will be written. If the range is empty, the result is zero.
28#[inline]
29#[target_feature(enable = "tbm")]
30#[cfg_attr(test, assert_instr(bextr, CONTROL = 0x0404))]
31#[rustc_legacy_const_generics(1)]
32#[stable(feature = "simd_x86_updates", since = "1.82.0")]
33pub unsafe fn _bextri_u64<const CONTROL: u64>(a: u64) -> u64 {
34 static_assert_uimm_bits!(CONTROL, 16);
35 unsafe { bextri_u64(a, CONTROL) }
36}
37
38/// Clears all bits below the least significant zero bit of `x`.
39///
40/// If there is no zero bit in `x`, it returns zero.
41#[inline]
42#[target_feature(enable = "tbm")]
43#[cfg_attr(test, assert_instr(blcfill))]
44#[stable(feature = "simd_x86", since = "1.27.0")]
45pub unsafe fn _blcfill_u64(x: u64) -> u64 {
46 x & x.wrapping_add(1)
47}
48
49/// Sets all bits of `x` to 1 except for the least significant zero bit.
50///
51/// If there is no zero bit in `x`, it sets all bits.
52#[inline]
53#[target_feature(enable = "tbm")]
54#[cfg_attr(test, assert_instr(blci))]
55#[stable(feature = "simd_x86", since = "1.27.0")]
56pub unsafe fn _blci_u64(x: u64) -> u64 {
57 x | !x.wrapping_add(1)
58}
59
60/// Sets the least significant zero bit of `x` and clears all other bits.
61///
62/// If there is no zero bit in `x`, it returns zero.
63#[inline]
64#[target_feature(enable = "tbm")]
65#[cfg_attr(test, assert_instr(blcic))]
66#[stable(feature = "simd_x86", since = "1.27.0")]
67pub unsafe fn _blcic_u64(x: u64) -> u64 {
68 !x & x.wrapping_add(1)
69}
70
71/// Sets the least significant zero bit of `x` and clears all bits above
72/// that bit.
73///
74/// If there is no zero bit in `x`, it sets all the bits.
75#[inline]
76#[target_feature(enable = "tbm")]
77#[cfg_attr(test, assert_instr(blcmsk))]
78#[stable(feature = "simd_x86", since = "1.27.0")]
79pub unsafe fn _blcmsk_u64(x: u64) -> u64 {
80 x ^ x.wrapping_add(1)
81}
82
83/// Sets the least significant zero bit of `x`.
84///
85/// If there is no zero bit in `x`, it returns `x`.
86#[inline]
87#[target_feature(enable = "tbm")]
88#[cfg_attr(test, assert_instr(blcs))]
89#[stable(feature = "simd_x86", since = "1.27.0")]
90pub unsafe fn _blcs_u64(x: u64) -> u64 {
91 x | x.wrapping_add(1)
92}
93
94/// Sets all bits of `x` below the least significant one.
95///
96/// If there is no set bit in `x`, it sets all the bits.
97#[inline]
98#[target_feature(enable = "tbm")]
99#[cfg_attr(test, assert_instr(blsfill))]
100#[stable(feature = "simd_x86", since = "1.27.0")]
101pub unsafe fn _blsfill_u64(x: u64) -> u64 {
102 x | x.wrapping_sub(1)
103}
104
105/// Clears least significant bit and sets all other bits.
106///
107/// If there is no set bit in `x`, it sets all the bits.
108#[inline]
109#[target_feature(enable = "tbm")]
110#[cfg_attr(test, assert_instr(blsic))]
111#[stable(feature = "simd_x86", since = "1.27.0")]
112pub unsafe fn _blsic_u64(x: u64) -> u64 {
113 !x | x.wrapping_sub(1)
114}
115
116/// Clears all bits below the least significant zero of `x` and sets all other
117/// bits.
118///
119/// If the least significant bit of `x` is `0`, it sets all bits.
120#[inline]
121#[target_feature(enable = "tbm")]
122#[cfg_attr(test, assert_instr(t1mskc))]
123#[stable(feature = "simd_x86", since = "1.27.0")]
124pub unsafe fn _t1mskc_u64(x: u64) -> u64 {
125 !x | x.wrapping_add(1)
126}
127
128/// Sets all bits below the least significant one of `x` and clears all other
129/// bits.
130///
131/// If the least significant bit of `x` is 1, it returns zero.
132#[inline]
133#[target_feature(enable = "tbm")]
134#[cfg_attr(test, assert_instr(tzmsk))]
135#[stable(feature = "simd_x86", since = "1.27.0")]
136pub unsafe fn _tzmsk_u64(x: u64) -> u64 {
137 !x & x.wrapping_sub(1)
138}
139
140#[cfg(test)]
141mod tests {
142 use stdarch_test::simd_test;
143
144 use crate::core_arch::x86_64::*;
145
146 #[simd_test(enable = "tbm")]
147 unsafe fn test_bextri_u64() {
148 assert_eq!(_bextri_u64::<0x0404>(0b0101_0000u64), 0b0000_0101u64);
149 }
150
151 #[simd_test(enable = "tbm")]
152 unsafe fn test_blcfill_u64() {
153 assert_eq!(_blcfill_u64(0b0101_0111u64), 0b0101_0000u64);
154 assert_eq!(_blcfill_u64(0b1111_1111u64), 0u64);
155 }
156
157 #[simd_test(enable = "tbm")]
158 unsafe fn test_blci_u64() {
159 assert_eq!(
160 _blci_u64(0b0101_0000u64),
161 0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1110u64
162 );
163 assert_eq!(
164 _blci_u64(0b1111_1111u64),
165 0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1110_1111_1111u64
166 );
167 }
168
169 #[simd_test(enable = "tbm")]
170 unsafe fn test_blcic_u64() {
171 assert_eq!(_blcic_u64(0b0101_0001u64), 0b0000_0010u64);
172 assert_eq!(_blcic_u64(0b1111_1111u64), 0b1_0000_0000u64);
173 }
174
175 #[simd_test(enable = "tbm")]
176 unsafe fn test_blcmsk_u64() {
177 assert_eq!(_blcmsk_u64(0b0101_0001u64), 0b0000_0011u64);
178 assert_eq!(_blcmsk_u64(0b1111_1111u64), 0b1_1111_1111u64);
179 }
180
181 #[simd_test(enable = "tbm")]
182 unsafe fn test_blcs_u64() {
183 assert_eq!(_blcs_u64(0b0101_0001u64), 0b0101_0011u64);
184 assert_eq!(_blcs_u64(0b1111_1111u64), 0b1_1111_1111u64);
185 }
186
187 #[simd_test(enable = "tbm")]
188 unsafe fn test_blsfill_u64() {
189 assert_eq!(_blsfill_u64(0b0101_0100u64), 0b0101_0111u64);
190 assert_eq!(
191 _blsfill_u64(0u64),
192 0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111u64
193 );
194 }
195
196 #[simd_test(enable = "tbm")]
197 unsafe fn test_blsic_u64() {
198 assert_eq!(
199 _blsic_u64(0b0101_0100u64),
200 0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1011u64
201 );
202 assert_eq!(
203 _blsic_u64(0u64),
204 0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111u64
205 );
206 }
207
208 #[simd_test(enable = "tbm")]
209 unsafe fn test_t1mskc_u64() {
210 assert_eq!(
211 _t1mskc_u64(0b0101_0111u64),
212 0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1000u64
213 );
214 assert_eq!(
215 _t1mskc_u64(0u64),
216 0b1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111_1111u64
217 );
218 }
219
220 #[simd_test(enable = "tbm")]
221 unsafe fn test_tzmsk_u64() {
222 assert_eq!(_tzmsk_u64(0b0101_1000u64), 0b0000_0111u64);
223 assert_eq!(_tzmsk_u64(0b0101_1001u64), 0b0000_0000u64);
224 }
225}
226