core/stdarch/crates/core_arch/src/x86/
mod.rs

1//! `x86` and `x86_64` intrinsics.
2
3#[allow(unused_imports)]
4use crate::marker::Sized;
5use crate::mem::transmute;
6
7#[macro_use]
8mod macros;
9
10types! {
11    #![stable(feature = "simd_x86", since = "1.27.0")]
12
13    /// 128-bit wide integer vector type, x86-specific
14    ///
15    /// This type is the same as the `__m128i` type defined by Intel,
16    /// representing a 128-bit SIMD register. Usage of this type typically
17    /// corresponds to the `sse` and up target features for x86/x86_64.
18    ///
19    /// Internally this type may be viewed as:
20    ///
21    /// * `i8x16` - sixteen `i8` variables packed together
22    /// * `i16x8` - eight `i16` variables packed together
23    /// * `i32x4` - four `i32` variables packed together
24    /// * `i64x2` - two `i64` variables packed together
25    ///
26    /// (as well as unsigned versions). Each intrinsic may interpret the
27    /// internal bits differently, check the documentation of the intrinsic
28    /// to see how it's being used.
29    ///
30    /// The in-memory representation of this type is the same as the one of an
31    /// equivalent array (i.e. the in-memory order of elements is the same, and
32    /// there is no padding); however, the alignment is different and equal to
33    /// the size of the type. Note that the ABI for function calls may *not* be
34    /// the same.
35    ///
36    /// Note that this means that an instance of `__m128i` typically just means
37    /// a "bag of bits" which is left up to interpretation at the point of use.
38    ///
39    /// Most intrinsics using `__m128i` are prefixed with `_mm_` and the
40    /// integer types tend to correspond to suffixes like "epi8" or "epi32".
41    ///
42    /// # Examples
43    ///
44    /// ```
45    /// #[cfg(target_arch = "x86")]
46    /// use std::arch::x86::*;
47    /// #[cfg(target_arch = "x86_64")]
48    /// use std::arch::x86_64::*;
49    ///
50    /// # fn main() {
51    /// # #[target_feature(enable = "sse2")]
52    /// # unsafe fn foo() {
53    /// let all_bytes_zero = _mm_setzero_si128();
54    /// let all_bytes_one = _mm_set1_epi8(1);
55    /// let four_i32 = _mm_set_epi32(1, 2, 3, 4);
56    /// # }
57    /// # if is_x86_feature_detected!("sse2") { unsafe { foo() } }
58    /// # }
59    /// ```
60    pub struct __m128i(2 x i64);
61
62    /// 128-bit wide set of four `f32` types, x86-specific
63    ///
64    /// This type is the same as the `__m128` type defined by Intel,
65    /// representing a 128-bit SIMD register which internally is consisted of
66    /// four packed `f32` instances. Usage of this type typically corresponds
67    /// to the `sse` and up target features for x86/x86_64.
68    ///
69    /// Note that unlike `__m128i`, the integer version of the 128-bit
70    /// registers, this `__m128` type has *one* interpretation. Each instance
71    /// of `__m128` always corresponds to `f32x4`, or four `f32` types packed
72    /// together.
73    ///
74    /// The in-memory representation of this type is the same as the one of an
75    /// equivalent array (i.e. the in-memory order of elements is the same, and
76    /// there is no padding); however, the alignment is different and equal to
77    /// the size of the type. Note that the ABI for function calls may *not* be
78    /// the same.
79    ///
80    /// Most intrinsics using `__m128` are prefixed with `_mm_` and are
81    /// suffixed with "ps" (or otherwise contain "ps"). Not to be confused with
82    /// "pd" which is used for `__m128d`.
83    ///
84    /// # Examples
85    ///
86    /// ```
87    /// #[cfg(target_arch = "x86")]
88    /// use std::arch::x86::*;
89    /// #[cfg(target_arch = "x86_64")]
90    /// use std::arch::x86_64::*;
91    ///
92    /// # fn main() {
93    /// # #[target_feature(enable = "sse")]
94    /// # unsafe fn foo() {
95    /// let four_zeros = _mm_setzero_ps();
96    /// let four_ones = _mm_set1_ps(1.0);
97    /// let four_floats = _mm_set_ps(1.0, 2.0, 3.0, 4.0);
98    /// # }
99    /// # if is_x86_feature_detected!("sse") { unsafe { foo() } }
100    /// # }
101    /// ```
102    pub struct __m128(4 x f32);
103
104    /// 128-bit wide set of two `f64` types, x86-specific
105    ///
106    /// This type is the same as the `__m128d` type defined by Intel,
107    /// representing a 128-bit SIMD register which internally is consisted of
108    /// two packed `f64` instances. Usage of this type typically corresponds
109    /// to the `sse` and up target features for x86/x86_64.
110    ///
111    /// Note that unlike `__m128i`, the integer version of the 128-bit
112    /// registers, this `__m128d` type has *one* interpretation. Each instance
113    /// of `__m128d` always corresponds to `f64x2`, or two `f64` types packed
114    /// together.
115    ///
116    /// The in-memory representation of this type is the same as the one of an
117    /// equivalent array (i.e. the in-memory order of elements is the same, and
118    /// there is no padding); however, the alignment is different and equal to
119    /// the size of the type. Note that the ABI for function calls may *not* be
120    /// the same.
121    ///
122    /// Most intrinsics using `__m128d` are prefixed with `_mm_` and are
123    /// suffixed with "pd" (or otherwise contain "pd"). Not to be confused with
124    /// "ps" which is used for `__m128`.
125    ///
126    /// # Examples
127    ///
128    /// ```
129    /// #[cfg(target_arch = "x86")]
130    /// use std::arch::x86::*;
131    /// #[cfg(target_arch = "x86_64")]
132    /// use std::arch::x86_64::*;
133    ///
134    /// # fn main() {
135    /// # #[target_feature(enable = "sse")]
136    /// # unsafe fn foo() {
137    /// let two_zeros = _mm_setzero_pd();
138    /// let two_ones = _mm_set1_pd(1.0);
139    /// let two_floats = _mm_set_pd(1.0, 2.0);
140    /// # }
141    /// # if is_x86_feature_detected!("sse") { unsafe { foo() } }
142    /// # }
143    /// ```
144    pub struct __m128d(2 x f64);
145
146    /// 256-bit wide integer vector type, x86-specific
147    ///
148    /// This type is the same as the `__m256i` type defined by Intel,
149    /// representing a 256-bit SIMD register. Usage of this type typically
150    /// corresponds to the `avx` and up target features for x86/x86_64.
151    ///
152    /// Internally this type may be viewed as:
153    ///
154    /// * `i8x32` - thirty two `i8` variables packed together
155    /// * `i16x16` - sixteen `i16` variables packed together
156    /// * `i32x8` - eight `i32` variables packed together
157    /// * `i64x4` - four `i64` variables packed together
158    ///
159    /// (as well as unsigned versions). Each intrinsic may interpret the
160    /// internal bits differently, check the documentation of the intrinsic
161    /// to see how it's being used.
162    ///
163    /// The in-memory representation of this type is the same as the one of an
164    /// equivalent array (i.e. the in-memory order of elements is the same, and
165    /// there is no padding); however, the alignment is different and equal to
166    /// the size of the type. Note that the ABI for function calls may *not* be
167    /// the same.
168    ///
169    /// Note that this means that an instance of `__m256i` typically just means
170    /// a "bag of bits" which is left up to interpretation at the point of use.
171    ///
172    /// # Examples
173    ///
174    /// ```
175    /// #[cfg(target_arch = "x86")]
176    /// use std::arch::x86::*;
177    /// #[cfg(target_arch = "x86_64")]
178    /// use std::arch::x86_64::*;
179    ///
180    /// # fn main() {
181    /// # #[target_feature(enable = "avx")]
182    /// # unsafe fn foo() {
183    /// let all_bytes_zero = _mm256_setzero_si256();
184    /// let all_bytes_one = _mm256_set1_epi8(1);
185    /// let eight_i32 = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8);
186    /// # }
187    /// # if is_x86_feature_detected!("avx") { unsafe { foo() } }
188    /// # }
189    /// ```
190    pub struct __m256i(4 x i64);
191
192    /// 256-bit wide set of eight `f32` types, x86-specific
193    ///
194    /// This type is the same as the `__m256` type defined by Intel,
195    /// representing a 256-bit SIMD register which internally is consisted of
196    /// eight packed `f32` instances. Usage of this type typically corresponds
197    /// to the `avx` and up target features for x86/x86_64.
198    ///
199    /// Note that unlike `__m256i`, the integer version of the 256-bit
200    /// registers, this `__m256` type has *one* interpretation. Each instance
201    /// of `__m256` always corresponds to `f32x8`, or eight `f32` types packed
202    /// together.
203    ///
204    /// The in-memory representation of this type is the same as the one of an
205    /// equivalent array (i.e. the in-memory order of elements is the same, and
206    /// there is no padding  between two consecutive elements); however, the
207    /// alignment is different and equal to the size of the type. Note that the
208    /// ABI for function calls may *not* be the same.
209    ///
210    /// Most intrinsics using `__m256` are prefixed with `_mm256_` and are
211    /// suffixed with "ps" (or otherwise contain "ps"). Not to be confused with
212    /// "pd" which is used for `__m256d`.
213    ///
214    /// # Examples
215    ///
216    /// ```
217    /// #[cfg(target_arch = "x86")]
218    /// use std::arch::x86::*;
219    /// #[cfg(target_arch = "x86_64")]
220    /// use std::arch::x86_64::*;
221    ///
222    /// # fn main() {
223    /// # #[target_feature(enable = "avx")]
224    /// # unsafe fn foo() {
225    /// let eight_zeros = _mm256_setzero_ps();
226    /// let eight_ones = _mm256_set1_ps(1.0);
227    /// let eight_floats = _mm256_set_ps(1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0);
228    /// # }
229    /// # if is_x86_feature_detected!("avx") { unsafe { foo() } }
230    /// # }
231    /// ```
232    pub struct __m256(8 x f32);
233
234    /// 256-bit wide set of four `f64` types, x86-specific
235    ///
236    /// This type is the same as the `__m256d` type defined by Intel,
237    /// representing a 256-bit SIMD register which internally is consisted of
238    /// four packed `f64` instances. Usage of this type typically corresponds
239    /// to the `avx` and up target features for x86/x86_64.
240    ///
241    /// Note that unlike `__m256i`, the integer version of the 256-bit
242    /// registers, this `__m256d` type has *one* interpretation. Each instance
243    /// of `__m256d` always corresponds to `f64x4`, or four `f64` types packed
244    /// together.
245    ///
246    /// The in-memory representation of this type is the same as the one of an
247    /// equivalent array (i.e. the in-memory order of elements is the same, and
248    /// there is no padding); however, the alignment is different and equal to
249    /// the size of the type. Note that the ABI for function calls may *not* be
250    /// the same.
251    ///
252    /// Most intrinsics using `__m256d` are prefixed with `_mm256_` and are
253    /// suffixed with "pd" (or otherwise contain "pd"). Not to be confused with
254    /// "ps" which is used for `__m256`.
255    ///
256    /// # Examples
257    ///
258    /// ```
259    /// #[cfg(target_arch = "x86")]
260    /// use std::arch::x86::*;
261    /// #[cfg(target_arch = "x86_64")]
262    /// use std::arch::x86_64::*;
263    ///
264    /// # fn main() {
265    /// # #[target_feature(enable = "avx")]
266    /// # unsafe fn foo() {
267    /// let four_zeros = _mm256_setzero_pd();
268    /// let four_ones = _mm256_set1_pd(1.0);
269    /// let four_floats = _mm256_set_pd(1.0, 2.0, 3.0, 4.0);
270    /// # }
271    /// # if is_x86_feature_detected!("avx") { unsafe { foo() } }
272    /// # }
273    /// ```
274    pub struct __m256d(4 x f64);
275}
276
277types! {
278    #![stable(feature = "simd_avx512_types", since = "1.72.0")]
279
280    /// 512-bit wide integer vector type, x86-specific
281    ///
282    /// This type is the same as the `__m512i` type defined by Intel,
283    /// representing a 512-bit SIMD register. Usage of this type typically
284    /// corresponds to the `avx512*` and up target features for x86/x86_64.
285    ///
286    /// Internally this type may be viewed as:
287    ///
288    /// * `i8x64` - sixty-four `i8` variables packed together
289    /// * `i16x32` - thirty-two `i16` variables packed together
290    /// * `i32x16` - sixteen `i32` variables packed together
291    /// * `i64x8` - eight `i64` variables packed together
292    ///
293    /// (as well as unsigned versions). Each intrinsic may interpret the
294    /// internal bits differently, check the documentation of the intrinsic
295    /// to see how it's being used.
296    ///
297    /// The in-memory representation of this type is the same as the one of an
298    /// equivalent array (i.e. the in-memory order of elements is the same, and
299    /// there is no padding); however, the alignment is different and equal to
300    /// the size of the type. Note that the ABI for function calls may *not* be
301    /// the same.
302    ///
303    /// Note that this means that an instance of `__m512i` typically just means
304    /// a "bag of bits" which is left up to interpretation at the point of use.
305    pub struct __m512i(8 x i64);
306
307    /// 512-bit wide set of sixteen `f32` types, x86-specific
308    ///
309    /// This type is the same as the `__m512` type defined by Intel,
310    /// representing a 512-bit SIMD register which internally is consisted of
311    /// eight packed `f32` instances. Usage of this type typically corresponds
312    /// to the `avx512*` and up target features for x86/x86_64.
313    ///
314    /// Note that unlike `__m512i`, the integer version of the 512-bit
315    /// registers, this `__m512` type has *one* interpretation. Each instance
316    /// of `__m512` always corresponds to `f32x16`, or sixteen `f32` types
317    /// packed together.
318    ///
319    /// The in-memory representation of this type is the same as the one of an
320    /// equivalent array (i.e. the in-memory order of elements is the same, and
321    /// there is no padding  between two consecutive elements); however, the
322    /// alignment is different and equal to the size of the type. Note that the
323    /// ABI for function calls may *not* be the same.
324    ///
325    /// Most intrinsics using `__m512` are prefixed with `_mm512_` and are
326    /// suffixed with "ps" (or otherwise contain "ps"). Not to be confused with
327    /// "pd" which is used for `__m512d`.
328    pub struct __m512(16 x f32);
329
330    /// 512-bit wide set of eight `f64` types, x86-specific
331    ///
332    /// This type is the same as the `__m512d` type defined by Intel,
333    /// representing a 512-bit SIMD register which internally is consisted of
334    /// eight packed `f64` instances. Usage of this type typically corresponds
335    /// to the `avx` and up target features for x86/x86_64.
336    ///
337    /// Note that unlike `__m512i`, the integer version of the 512-bit
338    /// registers, this `__m512d` type has *one* interpretation. Each instance
339    /// of `__m512d` always corresponds to `f64x4`, or eight `f64` types packed
340    /// together.
341    ///
342    /// The in-memory representation of this type is the same as the one of an
343    /// equivalent array (i.e. the in-memory order of elements is the same, and
344    /// there is no padding  between two consecutive elements); however, the
345    /// alignment is different and equal to the size of the type. Note that the
346    /// ABI for function calls may *not* be the same.
347    ///
348    /// Most intrinsics using `__m512d` are prefixed with `_mm512_` and are
349    /// suffixed with "pd" (or otherwise contain "pd"). Not to be confused with
350    /// "ps" which is used for `__m512`.
351    pub struct __m512d(8 x f64);
352}
353
354types! {
355    #![unstable(feature = "stdarch_x86_avx512", issue = "111137")]
356
357    /// 128-bit wide set of eight `u16` types, x86-specific
358    ///
359    /// This type is representing a 128-bit SIMD register which internally is consisted of
360    /// eight packed `u16` instances. Its purpose is for bf16 related intrinsic
361    /// implementations.
362    ///
363    /// The in-memory representation of this type is the same as the one of an
364    /// equivalent array (i.e. the in-memory order of elements is the same, and
365    /// there is no padding); however, the alignment is different and equal to
366    /// the size of the type. Note that the ABI for function calls may *not* be
367    /// the same.
368    pub struct __m128bh(8 x u16);
369
370    /// 256-bit wide set of 16 `u16` types, x86-specific
371    ///
372    /// This type is the same as the `__m256bh` type defined by Intel,
373    /// representing a 256-bit SIMD register which internally is consisted of
374    /// 16 packed `u16` instances. Its purpose is for bf16 related intrinsic
375    /// implementations.
376    ///
377    /// The in-memory representation of this type is the same as the one of an
378    /// equivalent array (i.e. the in-memory order of elements is the same, and
379    /// there is no padding); however, the alignment is different and equal to
380    /// the size of the type. Note that the ABI for function calls may *not* be
381    /// the same.
382    pub struct __m256bh(16 x u16);
383
384    /// 512-bit wide set of 32 `u16` types, x86-specific
385    ///
386    /// This type is the same as the `__m512bh` type defined by Intel,
387    /// representing a 512-bit SIMD register which internally is consisted of
388    /// 32 packed `u16` instances. Its purpose is for bf16 related intrinsic
389    /// implementations.
390    ///
391    /// The in-memory representation of this type is the same as the one of an
392    /// equivalent array (i.e. the in-memory order of elements is the same, and
393    /// there is no padding); however, the alignment is different and equal to
394    /// the size of the type. Note that the ABI for function calls may *not* be
395    /// the same.
396    pub struct __m512bh(32 x u16);
397}
398
399types! {
400    #![unstable(feature = "stdarch_x86_avx512_f16", issue = "127213")]
401
402    /// 128-bit wide set of 8 `f16` types, x86-specific
403    ///
404    /// This type is the same as the `__m128h` type defined by Intel,
405    /// representing a 128-bit SIMD register which internally is consisted of
406    /// 8 packed `f16` instances. its purpose is for f16 related intrinsic
407    /// implementations.
408    ///
409    /// The in-memory representation of this type is the same as the one of an
410    /// equivalent array (i.e. the in-memory order of elements is the same, and
411    /// there is no padding); however, the alignment is different and equal to
412    /// the size of the type. Note that the ABI for function calls may *not* be
413    /// the same.
414    pub struct __m128h(8 x f16);
415
416    /// 256-bit wide set of 16 `f16` types, x86-specific
417    ///
418    /// This type is the same as the `__m256h` type defined by Intel,
419    /// representing a 256-bit SIMD register which internally is consisted of
420    /// 16 packed `f16` instances. its purpose is for f16 related intrinsic
421    /// implementations.
422    ///
423    /// The in-memory representation of this type is the same as the one of an
424    /// equivalent array (i.e. the in-memory order of elements is the same, and
425    /// there is no padding); however, the alignment is different and equal to
426    /// the size of the type. Note that the ABI for function calls may *not* be
427    /// the same.
428    pub struct __m256h(16 x f16);
429
430    /// 512-bit wide set of 32 `f16` types, x86-specific
431    ///
432    /// This type is the same as the `__m512h` type defined by Intel,
433    /// representing a 512-bit SIMD register which internally is consisted of
434    /// 32 packed `f16` instances. its purpose is for f16 related intrinsic
435    /// implementations.
436    ///
437    /// The in-memory representation of this type is the same as the one of an
438    /// equivalent array (i.e. the in-memory order of elements is the same, and
439    /// there is no padding); however, the alignment is different and equal to
440    /// the size of the type. Note that the ABI for function calls may *not* be
441    /// the same.
442    pub struct __m512h(32 x f16);
443}
444
445/// The BFloat16 type used in AVX-512 intrinsics.
446#[repr(transparent)]
447#[derive(Copy, Clone, Debug)]
448#[allow(non_camel_case_types)]
449#[unstable(feature = "stdarch_x86_avx512_bf16", issue = "127356")]
450pub struct bf16(u16);
451
452impl bf16 {
453    /// Raw transmutation from `u16`
454    #[inline]
455    #[must_use]
456    #[unstable(feature = "stdarch_x86_avx512_bf16", issue = "127356")]
457    pub const fn from_bits(bits: u16) -> bf16 {
458        bf16(bits)
459    }
460
461    /// Raw transmutation to `u16`
462    #[inline]
463    #[must_use = "this returns the result of the operation, without modifying the original"]
464    #[unstable(feature = "stdarch_x86_avx512_bf16", issue = "127356")]
465    pub const fn to_bits(self) -> u16 {
466        self.0
467    }
468}
469
470/// The `__mmask64` type used in AVX-512 intrinsics, a 64-bit integer
471#[allow(non_camel_case_types)]
472#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
473pub type __mmask64 = u64;
474
475/// The `__mmask32` type used in AVX-512 intrinsics, a 32-bit integer
476#[allow(non_camel_case_types)]
477#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
478pub type __mmask32 = u32;
479
480/// The `__mmask16` type used in AVX-512 intrinsics, a 16-bit integer
481#[allow(non_camel_case_types)]
482#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
483pub type __mmask16 = u16;
484
485/// The `__mmask8` type used in AVX-512 intrinsics, a 8-bit integer
486#[allow(non_camel_case_types)]
487#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
488pub type __mmask8 = u8;
489
490/// The `_MM_CMPINT_ENUM` type used to specify comparison operations in AVX-512 intrinsics.
491#[allow(non_camel_case_types)]
492#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
493pub type _MM_CMPINT_ENUM = i32;
494
495/// The `MM_MANTISSA_NORM_ENUM` type used to specify mantissa normalized operations in AVX-512 intrinsics.
496#[allow(non_camel_case_types)]
497#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
498pub type _MM_MANTISSA_NORM_ENUM = i32;
499
500/// The `MM_MANTISSA_SIGN_ENUM` type used to specify mantissa signed operations in AVX-512 intrinsics.
501#[allow(non_camel_case_types)]
502#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
503pub type _MM_MANTISSA_SIGN_ENUM = i32;
504
505/// The `MM_PERM_ENUM` type used to specify shuffle operations in AVX-512 intrinsics.
506#[allow(non_camel_case_types)]
507#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
508pub type _MM_PERM_ENUM = i32;
509
510#[cfg(test)]
511mod test;
512#[cfg(test)]
513pub use self::test::*;
514
515#[allow(non_camel_case_types)]
516pub(crate) trait m128iExt: Sized {
517    fn as_m128i(self) -> __m128i;
518
519    #[inline]
520    fn as_u8x16(self) -> crate::core_arch::simd::u8x16 {
521        unsafe { transmute(self.as_m128i()) }
522    }
523
524    #[inline]
525    fn as_u16x8(self) -> crate::core_arch::simd::u16x8 {
526        unsafe { transmute(self.as_m128i()) }
527    }
528
529    #[inline]
530    fn as_u32x4(self) -> crate::core_arch::simd::u32x4 {
531        unsafe { transmute(self.as_m128i()) }
532    }
533
534    #[inline]
535    fn as_u64x2(self) -> crate::core_arch::simd::u64x2 {
536        unsafe { transmute(self.as_m128i()) }
537    }
538
539    #[inline]
540    fn as_i8x16(self) -> crate::core_arch::simd::i8x16 {
541        unsafe { transmute(self.as_m128i()) }
542    }
543
544    #[inline]
545    fn as_i16x8(self) -> crate::core_arch::simd::i16x8 {
546        unsafe { transmute(self.as_m128i()) }
547    }
548
549    #[inline]
550    fn as_i32x4(self) -> crate::core_arch::simd::i32x4 {
551        unsafe { transmute(self.as_m128i()) }
552    }
553
554    #[inline]
555    fn as_i64x2(self) -> crate::core_arch::simd::i64x2 {
556        unsafe { transmute(self.as_m128i()) }
557    }
558}
559
560impl m128iExt for __m128i {
561    #[inline]
562    fn as_m128i(self) -> Self {
563        self
564    }
565}
566
567#[allow(non_camel_case_types)]
568pub(crate) trait m256iExt: Sized {
569    fn as_m256i(self) -> __m256i;
570
571    #[inline]
572    fn as_u8x32(self) -> crate::core_arch::simd::u8x32 {
573        unsafe { transmute(self.as_m256i()) }
574    }
575
576    #[inline]
577    fn as_u16x16(self) -> crate::core_arch::simd::u16x16 {
578        unsafe { transmute(self.as_m256i()) }
579    }
580
581    #[inline]
582    fn as_u32x8(self) -> crate::core_arch::simd::u32x8 {
583        unsafe { transmute(self.as_m256i()) }
584    }
585
586    #[inline]
587    fn as_u64x4(self) -> crate::core_arch::simd::u64x4 {
588        unsafe { transmute(self.as_m256i()) }
589    }
590
591    #[inline]
592    fn as_i8x32(self) -> crate::core_arch::simd::i8x32 {
593        unsafe { transmute(self.as_m256i()) }
594    }
595
596    #[inline]
597    fn as_i16x16(self) -> crate::core_arch::simd::i16x16 {
598        unsafe { transmute(self.as_m256i()) }
599    }
600
601    #[inline]
602    fn as_i32x8(self) -> crate::core_arch::simd::i32x8 {
603        unsafe { transmute(self.as_m256i()) }
604    }
605
606    #[inline]
607    fn as_i64x4(self) -> crate::core_arch::simd::i64x4 {
608        unsafe { transmute(self.as_m256i()) }
609    }
610}
611
612impl m256iExt for __m256i {
613    #[inline]
614    fn as_m256i(self) -> Self {
615        self
616    }
617}
618
619#[allow(non_camel_case_types)]
620pub(crate) trait m128Ext: Sized {
621    fn as_m128(self) -> __m128;
622
623    #[inline]
624    fn as_f32x4(self) -> crate::core_arch::simd::f32x4 {
625        unsafe { transmute(self.as_m128()) }
626    }
627}
628
629impl m128Ext for __m128 {
630    #[inline]
631    fn as_m128(self) -> Self {
632        self
633    }
634}
635
636#[allow(non_camel_case_types)]
637pub(crate) trait m128dExt: Sized {
638    fn as_m128d(self) -> __m128d;
639
640    #[inline]
641    fn as_f64x2(self) -> crate::core_arch::simd::f64x2 {
642        unsafe { transmute(self.as_m128d()) }
643    }
644}
645
646impl m128dExt for __m128d {
647    #[inline]
648    fn as_m128d(self) -> Self {
649        self
650    }
651}
652
653#[allow(non_camel_case_types)]
654pub(crate) trait m256Ext: Sized {
655    fn as_m256(self) -> __m256;
656
657    #[inline]
658    fn as_f32x8(self) -> crate::core_arch::simd::f32x8 {
659        unsafe { transmute(self.as_m256()) }
660    }
661}
662
663impl m256Ext for __m256 {
664    #[inline]
665    fn as_m256(self) -> Self {
666        self
667    }
668}
669
670#[allow(non_camel_case_types)]
671pub(crate) trait m256dExt: Sized {
672    fn as_m256d(self) -> __m256d;
673
674    #[inline]
675    fn as_f64x4(self) -> crate::core_arch::simd::f64x4 {
676        unsafe { transmute(self.as_m256d()) }
677    }
678}
679
680impl m256dExt for __m256d {
681    #[inline]
682    fn as_m256d(self) -> Self {
683        self
684    }
685}
686
687#[allow(non_camel_case_types)]
688pub(crate) trait m512iExt: Sized {
689    fn as_m512i(self) -> __m512i;
690
691    #[inline]
692    fn as_u8x64(self) -> crate::core_arch::simd::u8x64 {
693        unsafe { transmute(self.as_m512i()) }
694    }
695
696    #[inline]
697    fn as_i8x64(self) -> crate::core_arch::simd::i8x64 {
698        unsafe { transmute(self.as_m512i()) }
699    }
700
701    #[inline]
702    fn as_u16x32(self) -> crate::core_arch::simd::u16x32 {
703        unsafe { transmute(self.as_m512i()) }
704    }
705
706    #[inline]
707    fn as_i16x32(self) -> crate::core_arch::simd::i16x32 {
708        unsafe { transmute(self.as_m512i()) }
709    }
710
711    #[inline]
712    fn as_u32x16(self) -> crate::core_arch::simd::u32x16 {
713        unsafe { transmute(self.as_m512i()) }
714    }
715
716    #[inline]
717    fn as_i32x16(self) -> crate::core_arch::simd::i32x16 {
718        unsafe { transmute(self.as_m512i()) }
719    }
720
721    #[inline]
722    fn as_u64x8(self) -> crate::core_arch::simd::u64x8 {
723        unsafe { transmute(self.as_m512i()) }
724    }
725
726    #[inline]
727    fn as_i64x8(self) -> crate::core_arch::simd::i64x8 {
728        unsafe { transmute(self.as_m512i()) }
729    }
730}
731
732impl m512iExt for __m512i {
733    #[inline]
734    fn as_m512i(self) -> Self {
735        self
736    }
737}
738
739#[allow(non_camel_case_types)]
740pub(crate) trait m512Ext: Sized {
741    fn as_m512(self) -> __m512;
742
743    #[inline]
744    fn as_f32x16(self) -> crate::core_arch::simd::f32x16 {
745        unsafe { transmute(self.as_m512()) }
746    }
747}
748
749impl m512Ext for __m512 {
750    #[inline]
751    fn as_m512(self) -> Self {
752        self
753    }
754}
755
756#[allow(non_camel_case_types)]
757pub(crate) trait m512dExt: Sized {
758    fn as_m512d(self) -> __m512d;
759
760    #[inline]
761    fn as_f64x8(self) -> crate::core_arch::simd::f64x8 {
762        unsafe { transmute(self.as_m512d()) }
763    }
764}
765
766impl m512dExt for __m512d {
767    #[inline]
768    fn as_m512d(self) -> Self {
769        self
770    }
771}
772
773#[allow(non_camel_case_types)]
774pub(crate) trait m128bhExt: Sized {
775    fn as_m128bh(self) -> __m128bh;
776
777    #[inline]
778    fn as_u16x8(self) -> crate::core_arch::simd::u16x8 {
779        unsafe { transmute(self.as_m128bh()) }
780    }
781
782    #[inline]
783    fn as_i16x8(self) -> crate::core_arch::simd::i16x8 {
784        unsafe { transmute(self.as_m128bh()) }
785    }
786
787    #[inline]
788    fn as_u32x4(self) -> crate::core_arch::simd::u32x4 {
789        unsafe { transmute(self.as_m128bh()) }
790    }
791
792    #[inline]
793    fn as_i32x4(self) -> crate::core_arch::simd::i32x4 {
794        unsafe { transmute(self.as_m128bh()) }
795    }
796}
797
798impl m128bhExt for __m128bh {
799    #[inline]
800    fn as_m128bh(self) -> Self {
801        self
802    }
803}
804
805#[allow(non_camel_case_types)]
806pub(crate) trait m256bhExt: Sized {
807    fn as_m256bh(self) -> __m256bh;
808
809    #[inline]
810    fn as_u16x16(self) -> crate::core_arch::simd::u16x16 {
811        unsafe { transmute(self.as_m256bh()) }
812    }
813
814    #[inline]
815    fn as_i16x16(self) -> crate::core_arch::simd::i16x16 {
816        unsafe { transmute(self.as_m256bh()) }
817    }
818
819    #[inline]
820    fn as_u32x8(self) -> crate::core_arch::simd::u32x8 {
821        unsafe { transmute(self.as_m256bh()) }
822    }
823
824    #[inline]
825    fn as_i32x8(self) -> crate::core_arch::simd::i32x8 {
826        unsafe { transmute(self.as_m256bh()) }
827    }
828}
829
830impl m256bhExt for __m256bh {
831    #[inline]
832    fn as_m256bh(self) -> Self {
833        self
834    }
835}
836
837#[allow(non_camel_case_types)]
838pub(crate) trait m512bhExt: Sized {
839    fn as_m512bh(self) -> __m512bh;
840
841    #[inline]
842    fn as_u16x32(self) -> crate::core_arch::simd::u16x32 {
843        unsafe { transmute(self.as_m512bh()) }
844    }
845
846    #[inline]
847    fn as_i16x32(self) -> crate::core_arch::simd::i16x32 {
848        unsafe { transmute(self.as_m512bh()) }
849    }
850
851    #[inline]
852    fn as_u32x16(self) -> crate::core_arch::simd::u32x16 {
853        unsafe { transmute(self.as_m512bh()) }
854    }
855
856    #[inline]
857    fn as_i32x16(self) -> crate::core_arch::simd::i32x16 {
858        unsafe { transmute(self.as_m512bh()) }
859    }
860}
861
862impl m512bhExt for __m512bh {
863    #[inline]
864    fn as_m512bh(self) -> Self {
865        self
866    }
867}
868
869#[allow(non_camel_case_types)]
870pub(crate) trait m128hExt: Sized {
871    fn as_m128h(self) -> __m128h;
872
873    #[inline]
874    fn as_f16x8(self) -> crate::core_arch::simd::f16x8 {
875        unsafe { transmute(self.as_m128h()) }
876    }
877}
878
879impl m128hExt for __m128h {
880    #[inline]
881    fn as_m128h(self) -> Self {
882        self
883    }
884}
885
886#[allow(non_camel_case_types)]
887pub(crate) trait m256hExt: Sized {
888    fn as_m256h(self) -> __m256h;
889
890    #[inline]
891    fn as_f16x16(self) -> crate::core_arch::simd::f16x16 {
892        unsafe { transmute(self.as_m256h()) }
893    }
894}
895
896impl m256hExt for __m256h {
897    #[inline]
898    fn as_m256h(self) -> Self {
899        self
900    }
901}
902
903#[allow(non_camel_case_types)]
904pub(crate) trait m512hExt: Sized {
905    fn as_m512h(self) -> __m512h;
906
907    #[inline]
908    fn as_f16x32(self) -> crate::core_arch::simd::f16x32 {
909        unsafe { transmute(self.as_m512h()) }
910    }
911}
912
913mod eflags;
914#[stable(feature = "simd_x86", since = "1.27.0")]
915pub use self::eflags::*;
916
917mod fxsr;
918#[stable(feature = "simd_x86", since = "1.27.0")]
919pub use self::fxsr::*;
920
921mod bswap;
922#[stable(feature = "simd_x86", since = "1.27.0")]
923pub use self::bswap::*;
924
925mod rdtsc;
926#[stable(feature = "simd_x86", since = "1.27.0")]
927pub use self::rdtsc::*;
928
929mod cpuid;
930#[stable(feature = "simd_x86", since = "1.27.0")]
931pub use self::cpuid::*;
932mod xsave;
933#[stable(feature = "simd_x86", since = "1.27.0")]
934pub use self::xsave::*;
935
936mod sse;
937#[stable(feature = "simd_x86", since = "1.27.0")]
938pub use self::sse::*;
939mod sse2;
940#[stable(feature = "simd_x86", since = "1.27.0")]
941pub use self::sse2::*;
942mod sse3;
943#[stable(feature = "simd_x86", since = "1.27.0")]
944pub use self::sse3::*;
945mod ssse3;
946#[stable(feature = "simd_x86", since = "1.27.0")]
947pub use self::ssse3::*;
948mod sse41;
949#[stable(feature = "simd_x86", since = "1.27.0")]
950pub use self::sse41::*;
951mod sse42;
952#[stable(feature = "simd_x86", since = "1.27.0")]
953pub use self::sse42::*;
954mod avx;
955#[stable(feature = "simd_x86", since = "1.27.0")]
956pub use self::avx::*;
957mod avx2;
958#[stable(feature = "simd_x86", since = "1.27.0")]
959pub use self::avx2::*;
960mod fma;
961#[stable(feature = "simd_x86", since = "1.27.0")]
962pub use self::fma::*;
963
964mod abm;
965#[stable(feature = "simd_x86", since = "1.27.0")]
966pub use self::abm::*;
967mod bmi1;
968#[stable(feature = "simd_x86", since = "1.27.0")]
969pub use self::bmi1::*;
970
971mod bmi2;
972#[stable(feature = "simd_x86", since = "1.27.0")]
973pub use self::bmi2::*;
974
975mod sse4a;
976#[stable(feature = "simd_x86", since = "1.27.0")]
977pub use self::sse4a::*;
978
979mod tbm;
980#[stable(feature = "simd_x86", since = "1.27.0")]
981pub use self::tbm::*;
982
983mod pclmulqdq;
984#[stable(feature = "simd_x86", since = "1.27.0")]
985pub use self::pclmulqdq::*;
986
987mod aes;
988#[stable(feature = "simd_x86", since = "1.27.0")]
989pub use self::aes::*;
990
991mod rdrand;
992#[stable(feature = "simd_x86", since = "1.27.0")]
993pub use self::rdrand::*;
994
995mod sha;
996#[stable(feature = "simd_x86", since = "1.27.0")]
997pub use self::sha::*;
998
999mod adx;
1000#[stable(feature = "simd_x86_adx", since = "1.33.0")]
1001pub use self::adx::*;
1002
1003#[cfg(test)]
1004use stdarch_test::assert_instr;
1005
1006mod avx512f;
1007#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
1008pub use self::avx512f::*;
1009
1010mod avx512bw;
1011#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
1012pub use self::avx512bw::*;
1013
1014mod avx512cd;
1015#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
1016pub use self::avx512cd::*;
1017
1018mod avx512dq;
1019#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
1020pub use self::avx512dq::*;
1021
1022mod avx512ifma;
1023#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
1024pub use self::avx512ifma::*;
1025
1026mod avx512vbmi;
1027#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
1028pub use self::avx512vbmi::*;
1029
1030mod avx512vbmi2;
1031#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
1032pub use self::avx512vbmi2::*;
1033
1034mod avx512vnni;
1035#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
1036pub use self::avx512vnni::*;
1037
1038mod avx512bitalg;
1039#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
1040pub use self::avx512bitalg::*;
1041
1042mod gfni;
1043#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
1044pub use self::gfni::*;
1045
1046mod avx512vpopcntdq;
1047#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
1048pub use self::avx512vpopcntdq::*;
1049
1050mod vaes;
1051#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
1052pub use self::vaes::*;
1053
1054mod vpclmulqdq;
1055#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
1056pub use self::vpclmulqdq::*;
1057
1058mod bt;
1059#[stable(feature = "simd_x86_bittest", since = "1.55.0")]
1060pub use self::bt::*;
1061
1062mod rtm;
1063#[unstable(feature = "stdarch_x86_rtm", issue = "111138")]
1064pub use self::rtm::*;
1065
1066mod f16c;
1067#[stable(feature = "x86_f16c_intrinsics", since = "1.68.0")]
1068pub use self::f16c::*;
1069
1070mod avx512bf16;
1071#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
1072pub use self::avx512bf16::*;
1073
1074mod avxneconvert;
1075#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
1076pub use self::avxneconvert::*;
1077
1078mod avx512fp16;
1079#[unstable(feature = "stdarch_x86_avx512_f16", issue = "127213")]
1080pub use self::avx512fp16::*;