core/stdarch/crates/core_arch/src/x86/mod.rs
1//! `x86` and `x86_64` intrinsics.
2
3#[allow(unused_imports)]
4use crate::marker::Sized;
5use crate::mem::transmute;
6
7#[macro_use]
8mod macros;
9
10types! {
11 #![stable(feature = "simd_x86", since = "1.27.0")]
12
13 /// 128-bit wide integer vector type, x86-specific
14 ///
15 /// This type is the same as the `__m128i` type defined by Intel,
16 /// representing a 128-bit SIMD register. Usage of this type typically
17 /// corresponds to the `sse` and up target features for x86/x86_64.
18 ///
19 /// Internally this type may be viewed as:
20 ///
21 /// * `i8x16` - sixteen `i8` variables packed together
22 /// * `i16x8` - eight `i16` variables packed together
23 /// * `i32x4` - four `i32` variables packed together
24 /// * `i64x2` - two `i64` variables packed together
25 ///
26 /// (as well as unsigned versions). Each intrinsic may interpret the
27 /// internal bits differently, check the documentation of the intrinsic
28 /// to see how it's being used.
29 ///
30 /// The in-memory representation of this type is the same as the one of an
31 /// equivalent array (i.e. the in-memory order of elements is the same, and
32 /// there is no padding); however, the alignment is different and equal to
33 /// the size of the type. Note that the ABI for function calls may *not* be
34 /// the same.
35 ///
36 /// Note that this means that an instance of `__m128i` typically just means
37 /// a "bag of bits" which is left up to interpretation at the point of use.
38 ///
39 /// Most intrinsics using `__m128i` are prefixed with `_mm_` and the
40 /// integer types tend to correspond to suffixes like "epi8" or "epi32".
41 ///
42 /// # Examples
43 ///
44 /// ```
45 /// #[cfg(target_arch = "x86")]
46 /// use std::arch::x86::*;
47 /// #[cfg(target_arch = "x86_64")]
48 /// use std::arch::x86_64::*;
49 ///
50 /// # fn main() {
51 /// # #[target_feature(enable = "sse2")]
52 /// # unsafe fn foo() {
53 /// let all_bytes_zero = _mm_setzero_si128();
54 /// let all_bytes_one = _mm_set1_epi8(1);
55 /// let four_i32 = _mm_set_epi32(1, 2, 3, 4);
56 /// # }
57 /// # if is_x86_feature_detected!("sse2") { unsafe { foo() } }
58 /// # }
59 /// ```
60 pub struct __m128i(2 x i64);
61
62 /// 128-bit wide set of four `f32` types, x86-specific
63 ///
64 /// This type is the same as the `__m128` type defined by Intel,
65 /// representing a 128-bit SIMD register which internally is consisted of
66 /// four packed `f32` instances. Usage of this type typically corresponds
67 /// to the `sse` and up target features for x86/x86_64.
68 ///
69 /// Note that unlike `__m128i`, the integer version of the 128-bit
70 /// registers, this `__m128` type has *one* interpretation. Each instance
71 /// of `__m128` always corresponds to `f32x4`, or four `f32` types packed
72 /// together.
73 ///
74 /// The in-memory representation of this type is the same as the one of an
75 /// equivalent array (i.e. the in-memory order of elements is the same, and
76 /// there is no padding); however, the alignment is different and equal to
77 /// the size of the type. Note that the ABI for function calls may *not* be
78 /// the same.
79 ///
80 /// Most intrinsics using `__m128` are prefixed with `_mm_` and are
81 /// suffixed with "ps" (or otherwise contain "ps"). Not to be confused with
82 /// "pd" which is used for `__m128d`.
83 ///
84 /// # Examples
85 ///
86 /// ```
87 /// #[cfg(target_arch = "x86")]
88 /// use std::arch::x86::*;
89 /// #[cfg(target_arch = "x86_64")]
90 /// use std::arch::x86_64::*;
91 ///
92 /// # fn main() {
93 /// # #[target_feature(enable = "sse")]
94 /// # unsafe fn foo() {
95 /// let four_zeros = _mm_setzero_ps();
96 /// let four_ones = _mm_set1_ps(1.0);
97 /// let four_floats = _mm_set_ps(1.0, 2.0, 3.0, 4.0);
98 /// # }
99 /// # if is_x86_feature_detected!("sse") { unsafe { foo() } }
100 /// # }
101 /// ```
102 pub struct __m128(4 x f32);
103
104 /// 128-bit wide set of two `f64` types, x86-specific
105 ///
106 /// This type is the same as the `__m128d` type defined by Intel,
107 /// representing a 128-bit SIMD register which internally is consisted of
108 /// two packed `f64` instances. Usage of this type typically corresponds
109 /// to the `sse` and up target features for x86/x86_64.
110 ///
111 /// Note that unlike `__m128i`, the integer version of the 128-bit
112 /// registers, this `__m128d` type has *one* interpretation. Each instance
113 /// of `__m128d` always corresponds to `f64x2`, or two `f64` types packed
114 /// together.
115 ///
116 /// The in-memory representation of this type is the same as the one of an
117 /// equivalent array (i.e. the in-memory order of elements is the same, and
118 /// there is no padding); however, the alignment is different and equal to
119 /// the size of the type. Note that the ABI for function calls may *not* be
120 /// the same.
121 ///
122 /// Most intrinsics using `__m128d` are prefixed with `_mm_` and are
123 /// suffixed with "pd" (or otherwise contain "pd"). Not to be confused with
124 /// "ps" which is used for `__m128`.
125 ///
126 /// # Examples
127 ///
128 /// ```
129 /// #[cfg(target_arch = "x86")]
130 /// use std::arch::x86::*;
131 /// #[cfg(target_arch = "x86_64")]
132 /// use std::arch::x86_64::*;
133 ///
134 /// # fn main() {
135 /// # #[target_feature(enable = "sse")]
136 /// # unsafe fn foo() {
137 /// let two_zeros = _mm_setzero_pd();
138 /// let two_ones = _mm_set1_pd(1.0);
139 /// let two_floats = _mm_set_pd(1.0, 2.0);
140 /// # }
141 /// # if is_x86_feature_detected!("sse") { unsafe { foo() } }
142 /// # }
143 /// ```
144 pub struct __m128d(2 x f64);
145
146 /// 256-bit wide integer vector type, x86-specific
147 ///
148 /// This type is the same as the `__m256i` type defined by Intel,
149 /// representing a 256-bit SIMD register. Usage of this type typically
150 /// corresponds to the `avx` and up target features for x86/x86_64.
151 ///
152 /// Internally this type may be viewed as:
153 ///
154 /// * `i8x32` - thirty two `i8` variables packed together
155 /// * `i16x16` - sixteen `i16` variables packed together
156 /// * `i32x8` - eight `i32` variables packed together
157 /// * `i64x4` - four `i64` variables packed together
158 ///
159 /// (as well as unsigned versions). Each intrinsic may interpret the
160 /// internal bits differently, check the documentation of the intrinsic
161 /// to see how it's being used.
162 ///
163 /// The in-memory representation of this type is the same as the one of an
164 /// equivalent array (i.e. the in-memory order of elements is the same, and
165 /// there is no padding); however, the alignment is different and equal to
166 /// the size of the type. Note that the ABI for function calls may *not* be
167 /// the same.
168 ///
169 /// Note that this means that an instance of `__m256i` typically just means
170 /// a "bag of bits" which is left up to interpretation at the point of use.
171 ///
172 /// # Examples
173 ///
174 /// ```
175 /// #[cfg(target_arch = "x86")]
176 /// use std::arch::x86::*;
177 /// #[cfg(target_arch = "x86_64")]
178 /// use std::arch::x86_64::*;
179 ///
180 /// # fn main() {
181 /// # #[target_feature(enable = "avx")]
182 /// # unsafe fn foo() {
183 /// let all_bytes_zero = _mm256_setzero_si256();
184 /// let all_bytes_one = _mm256_set1_epi8(1);
185 /// let eight_i32 = _mm256_set_epi32(1, 2, 3, 4, 5, 6, 7, 8);
186 /// # }
187 /// # if is_x86_feature_detected!("avx") { unsafe { foo() } }
188 /// # }
189 /// ```
190 pub struct __m256i(4 x i64);
191
192 /// 256-bit wide set of eight `f32` types, x86-specific
193 ///
194 /// This type is the same as the `__m256` type defined by Intel,
195 /// representing a 256-bit SIMD register which internally is consisted of
196 /// eight packed `f32` instances. Usage of this type typically corresponds
197 /// to the `avx` and up target features for x86/x86_64.
198 ///
199 /// Note that unlike `__m256i`, the integer version of the 256-bit
200 /// registers, this `__m256` type has *one* interpretation. Each instance
201 /// of `__m256` always corresponds to `f32x8`, or eight `f32` types packed
202 /// together.
203 ///
204 /// The in-memory representation of this type is the same as the one of an
205 /// equivalent array (i.e. the in-memory order of elements is the same, and
206 /// there is no padding between two consecutive elements); however, the
207 /// alignment is different and equal to the size of the type. Note that the
208 /// ABI for function calls may *not* be the same.
209 ///
210 /// Most intrinsics using `__m256` are prefixed with `_mm256_` and are
211 /// suffixed with "ps" (or otherwise contain "ps"). Not to be confused with
212 /// "pd" which is used for `__m256d`.
213 ///
214 /// # Examples
215 ///
216 /// ```
217 /// #[cfg(target_arch = "x86")]
218 /// use std::arch::x86::*;
219 /// #[cfg(target_arch = "x86_64")]
220 /// use std::arch::x86_64::*;
221 ///
222 /// # fn main() {
223 /// # #[target_feature(enable = "avx")]
224 /// # unsafe fn foo() {
225 /// let eight_zeros = _mm256_setzero_ps();
226 /// let eight_ones = _mm256_set1_ps(1.0);
227 /// let eight_floats = _mm256_set_ps(1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0);
228 /// # }
229 /// # if is_x86_feature_detected!("avx") { unsafe { foo() } }
230 /// # }
231 /// ```
232 pub struct __m256(8 x f32);
233
234 /// 256-bit wide set of four `f64` types, x86-specific
235 ///
236 /// This type is the same as the `__m256d` type defined by Intel,
237 /// representing a 256-bit SIMD register which internally is consisted of
238 /// four packed `f64` instances. Usage of this type typically corresponds
239 /// to the `avx` and up target features for x86/x86_64.
240 ///
241 /// Note that unlike `__m256i`, the integer version of the 256-bit
242 /// registers, this `__m256d` type has *one* interpretation. Each instance
243 /// of `__m256d` always corresponds to `f64x4`, or four `f64` types packed
244 /// together.
245 ///
246 /// The in-memory representation of this type is the same as the one of an
247 /// equivalent array (i.e. the in-memory order of elements is the same, and
248 /// there is no padding); however, the alignment is different and equal to
249 /// the size of the type. Note that the ABI for function calls may *not* be
250 /// the same.
251 ///
252 /// Most intrinsics using `__m256d` are prefixed with `_mm256_` and are
253 /// suffixed with "pd" (or otherwise contain "pd"). Not to be confused with
254 /// "ps" which is used for `__m256`.
255 ///
256 /// # Examples
257 ///
258 /// ```
259 /// #[cfg(target_arch = "x86")]
260 /// use std::arch::x86::*;
261 /// #[cfg(target_arch = "x86_64")]
262 /// use std::arch::x86_64::*;
263 ///
264 /// # fn main() {
265 /// # #[target_feature(enable = "avx")]
266 /// # unsafe fn foo() {
267 /// let four_zeros = _mm256_setzero_pd();
268 /// let four_ones = _mm256_set1_pd(1.0);
269 /// let four_floats = _mm256_set_pd(1.0, 2.0, 3.0, 4.0);
270 /// # }
271 /// # if is_x86_feature_detected!("avx") { unsafe { foo() } }
272 /// # }
273 /// ```
274 pub struct __m256d(4 x f64);
275}
276
277types! {
278 #![stable(feature = "simd_avx512_types", since = "1.72.0")]
279
280 /// 512-bit wide integer vector type, x86-specific
281 ///
282 /// This type is the same as the `__m512i` type defined by Intel,
283 /// representing a 512-bit SIMD register. Usage of this type typically
284 /// corresponds to the `avx512*` and up target features for x86/x86_64.
285 ///
286 /// Internally this type may be viewed as:
287 ///
288 /// * `i8x64` - sixty-four `i8` variables packed together
289 /// * `i16x32` - thirty-two `i16` variables packed together
290 /// * `i32x16` - sixteen `i32` variables packed together
291 /// * `i64x8` - eight `i64` variables packed together
292 ///
293 /// (as well as unsigned versions). Each intrinsic may interpret the
294 /// internal bits differently, check the documentation of the intrinsic
295 /// to see how it's being used.
296 ///
297 /// The in-memory representation of this type is the same as the one of an
298 /// equivalent array (i.e. the in-memory order of elements is the same, and
299 /// there is no padding); however, the alignment is different and equal to
300 /// the size of the type. Note that the ABI for function calls may *not* be
301 /// the same.
302 ///
303 /// Note that this means that an instance of `__m512i` typically just means
304 /// a "bag of bits" which is left up to interpretation at the point of use.
305 pub struct __m512i(8 x i64);
306
307 /// 512-bit wide set of sixteen `f32` types, x86-specific
308 ///
309 /// This type is the same as the `__m512` type defined by Intel,
310 /// representing a 512-bit SIMD register which internally is consisted of
311 /// eight packed `f32` instances. Usage of this type typically corresponds
312 /// to the `avx512*` and up target features for x86/x86_64.
313 ///
314 /// Note that unlike `__m512i`, the integer version of the 512-bit
315 /// registers, this `__m512` type has *one* interpretation. Each instance
316 /// of `__m512` always corresponds to `f32x16`, or sixteen `f32` types
317 /// packed together.
318 ///
319 /// The in-memory representation of this type is the same as the one of an
320 /// equivalent array (i.e. the in-memory order of elements is the same, and
321 /// there is no padding between two consecutive elements); however, the
322 /// alignment is different and equal to the size of the type. Note that the
323 /// ABI for function calls may *not* be the same.
324 ///
325 /// Most intrinsics using `__m512` are prefixed with `_mm512_` and are
326 /// suffixed with "ps" (or otherwise contain "ps"). Not to be confused with
327 /// "pd" which is used for `__m512d`.
328 pub struct __m512(16 x f32);
329
330 /// 512-bit wide set of eight `f64` types, x86-specific
331 ///
332 /// This type is the same as the `__m512d` type defined by Intel,
333 /// representing a 512-bit SIMD register which internally is consisted of
334 /// eight packed `f64` instances. Usage of this type typically corresponds
335 /// to the `avx` and up target features for x86/x86_64.
336 ///
337 /// Note that unlike `__m512i`, the integer version of the 512-bit
338 /// registers, this `__m512d` type has *one* interpretation. Each instance
339 /// of `__m512d` always corresponds to `f64x4`, or eight `f64` types packed
340 /// together.
341 ///
342 /// The in-memory representation of this type is the same as the one of an
343 /// equivalent array (i.e. the in-memory order of elements is the same, and
344 /// there is no padding between two consecutive elements); however, the
345 /// alignment is different and equal to the size of the type. Note that the
346 /// ABI for function calls may *not* be the same.
347 ///
348 /// Most intrinsics using `__m512d` are prefixed with `_mm512_` and are
349 /// suffixed with "pd" (or otherwise contain "pd"). Not to be confused with
350 /// "ps" which is used for `__m512`.
351 pub struct __m512d(8 x f64);
352}
353
354types! {
355 #![unstable(feature = "stdarch_x86_avx512", issue = "111137")]
356
357 /// 128-bit wide set of eight `u16` types, x86-specific
358 ///
359 /// This type is representing a 128-bit SIMD register which internally is consisted of
360 /// eight packed `u16` instances. Its purpose is for bf16 related intrinsic
361 /// implementations.
362 ///
363 /// The in-memory representation of this type is the same as the one of an
364 /// equivalent array (i.e. the in-memory order of elements is the same, and
365 /// there is no padding); however, the alignment is different and equal to
366 /// the size of the type. Note that the ABI for function calls may *not* be
367 /// the same.
368 pub struct __m128bh(8 x u16);
369
370 /// 256-bit wide set of 16 `u16` types, x86-specific
371 ///
372 /// This type is the same as the `__m256bh` type defined by Intel,
373 /// representing a 256-bit SIMD register which internally is consisted of
374 /// 16 packed `u16` instances. Its purpose is for bf16 related intrinsic
375 /// implementations.
376 ///
377 /// The in-memory representation of this type is the same as the one of an
378 /// equivalent array (i.e. the in-memory order of elements is the same, and
379 /// there is no padding); however, the alignment is different and equal to
380 /// the size of the type. Note that the ABI for function calls may *not* be
381 /// the same.
382 pub struct __m256bh(16 x u16);
383
384 /// 512-bit wide set of 32 `u16` types, x86-specific
385 ///
386 /// This type is the same as the `__m512bh` type defined by Intel,
387 /// representing a 512-bit SIMD register which internally is consisted of
388 /// 32 packed `u16` instances. Its purpose is for bf16 related intrinsic
389 /// implementations.
390 ///
391 /// The in-memory representation of this type is the same as the one of an
392 /// equivalent array (i.e. the in-memory order of elements is the same, and
393 /// there is no padding); however, the alignment is different and equal to
394 /// the size of the type. Note that the ABI for function calls may *not* be
395 /// the same.
396 pub struct __m512bh(32 x u16);
397}
398
399types! {
400 #![unstable(feature = "stdarch_x86_avx512_f16", issue = "127213")]
401
402 /// 128-bit wide set of 8 `f16` types, x86-specific
403 ///
404 /// This type is the same as the `__m128h` type defined by Intel,
405 /// representing a 128-bit SIMD register which internally is consisted of
406 /// 8 packed `f16` instances. its purpose is for f16 related intrinsic
407 /// implementations.
408 ///
409 /// The in-memory representation of this type is the same as the one of an
410 /// equivalent array (i.e. the in-memory order of elements is the same, and
411 /// there is no padding); however, the alignment is different and equal to
412 /// the size of the type. Note that the ABI for function calls may *not* be
413 /// the same.
414 pub struct __m128h(8 x f16);
415
416 /// 256-bit wide set of 16 `f16` types, x86-specific
417 ///
418 /// This type is the same as the `__m256h` type defined by Intel,
419 /// representing a 256-bit SIMD register which internally is consisted of
420 /// 16 packed `f16` instances. its purpose is for f16 related intrinsic
421 /// implementations.
422 ///
423 /// The in-memory representation of this type is the same as the one of an
424 /// equivalent array (i.e. the in-memory order of elements is the same, and
425 /// there is no padding); however, the alignment is different and equal to
426 /// the size of the type. Note that the ABI for function calls may *not* be
427 /// the same.
428 pub struct __m256h(16 x f16);
429
430 /// 512-bit wide set of 32 `f16` types, x86-specific
431 ///
432 /// This type is the same as the `__m512h` type defined by Intel,
433 /// representing a 512-bit SIMD register which internally is consisted of
434 /// 32 packed `f16` instances. its purpose is for f16 related intrinsic
435 /// implementations.
436 ///
437 /// The in-memory representation of this type is the same as the one of an
438 /// equivalent array (i.e. the in-memory order of elements is the same, and
439 /// there is no padding); however, the alignment is different and equal to
440 /// the size of the type. Note that the ABI for function calls may *not* be
441 /// the same.
442 pub struct __m512h(32 x f16);
443}
444
445/// The BFloat16 type used in AVX-512 intrinsics.
446#[repr(transparent)]
447#[derive(Copy, Clone, Debug)]
448#[allow(non_camel_case_types)]
449#[unstable(feature = "stdarch_x86_avx512_bf16", issue = "127356")]
450pub struct bf16(u16);
451
452impl bf16 {
453 /// Raw transmutation from `u16`
454 #[inline]
455 #[must_use]
456 #[unstable(feature = "stdarch_x86_avx512_bf16", issue = "127356")]
457 pub const fn from_bits(bits: u16) -> bf16 {
458 bf16(bits)
459 }
460
461 /// Raw transmutation to `u16`
462 #[inline]
463 #[must_use = "this returns the result of the operation, without modifying the original"]
464 #[unstable(feature = "stdarch_x86_avx512_bf16", issue = "127356")]
465 pub const fn to_bits(self) -> u16 {
466 self.0
467 }
468}
469
470/// The `__mmask64` type used in AVX-512 intrinsics, a 64-bit integer
471#[allow(non_camel_case_types)]
472#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
473pub type __mmask64 = u64;
474
475/// The `__mmask32` type used in AVX-512 intrinsics, a 32-bit integer
476#[allow(non_camel_case_types)]
477#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
478pub type __mmask32 = u32;
479
480/// The `__mmask16` type used in AVX-512 intrinsics, a 16-bit integer
481#[allow(non_camel_case_types)]
482#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
483pub type __mmask16 = u16;
484
485/// The `__mmask8` type used in AVX-512 intrinsics, a 8-bit integer
486#[allow(non_camel_case_types)]
487#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
488pub type __mmask8 = u8;
489
490/// The `_MM_CMPINT_ENUM` type used to specify comparison operations in AVX-512 intrinsics.
491#[allow(non_camel_case_types)]
492#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
493pub type _MM_CMPINT_ENUM = i32;
494
495/// The `MM_MANTISSA_NORM_ENUM` type used to specify mantissa normalized operations in AVX-512 intrinsics.
496#[allow(non_camel_case_types)]
497#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
498pub type _MM_MANTISSA_NORM_ENUM = i32;
499
500/// The `MM_MANTISSA_SIGN_ENUM` type used to specify mantissa signed operations in AVX-512 intrinsics.
501#[allow(non_camel_case_types)]
502#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
503pub type _MM_MANTISSA_SIGN_ENUM = i32;
504
505/// The `MM_PERM_ENUM` type used to specify shuffle operations in AVX-512 intrinsics.
506#[allow(non_camel_case_types)]
507#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
508pub type _MM_PERM_ENUM = i32;
509
510#[cfg(test)]
511mod test;
512#[cfg(test)]
513pub use self::test::*;
514
515#[allow(non_camel_case_types)]
516pub(crate) trait m128iExt: Sized {
517 fn as_m128i(self) -> __m128i;
518
519 #[inline]
520 fn as_u8x16(self) -> crate::core_arch::simd::u8x16 {
521 unsafe { transmute(self.as_m128i()) }
522 }
523
524 #[inline]
525 fn as_u16x8(self) -> crate::core_arch::simd::u16x8 {
526 unsafe { transmute(self.as_m128i()) }
527 }
528
529 #[inline]
530 fn as_u32x4(self) -> crate::core_arch::simd::u32x4 {
531 unsafe { transmute(self.as_m128i()) }
532 }
533
534 #[inline]
535 fn as_u64x2(self) -> crate::core_arch::simd::u64x2 {
536 unsafe { transmute(self.as_m128i()) }
537 }
538
539 #[inline]
540 fn as_i8x16(self) -> crate::core_arch::simd::i8x16 {
541 unsafe { transmute(self.as_m128i()) }
542 }
543
544 #[inline]
545 fn as_i16x8(self) -> crate::core_arch::simd::i16x8 {
546 unsafe { transmute(self.as_m128i()) }
547 }
548
549 #[inline]
550 fn as_i32x4(self) -> crate::core_arch::simd::i32x4 {
551 unsafe { transmute(self.as_m128i()) }
552 }
553
554 #[inline]
555 fn as_i64x2(self) -> crate::core_arch::simd::i64x2 {
556 unsafe { transmute(self.as_m128i()) }
557 }
558}
559
560impl m128iExt for __m128i {
561 #[inline]
562 fn as_m128i(self) -> Self {
563 self
564 }
565}
566
567#[allow(non_camel_case_types)]
568pub(crate) trait m256iExt: Sized {
569 fn as_m256i(self) -> __m256i;
570
571 #[inline]
572 fn as_u8x32(self) -> crate::core_arch::simd::u8x32 {
573 unsafe { transmute(self.as_m256i()) }
574 }
575
576 #[inline]
577 fn as_u16x16(self) -> crate::core_arch::simd::u16x16 {
578 unsafe { transmute(self.as_m256i()) }
579 }
580
581 #[inline]
582 fn as_u32x8(self) -> crate::core_arch::simd::u32x8 {
583 unsafe { transmute(self.as_m256i()) }
584 }
585
586 #[inline]
587 fn as_u64x4(self) -> crate::core_arch::simd::u64x4 {
588 unsafe { transmute(self.as_m256i()) }
589 }
590
591 #[inline]
592 fn as_i8x32(self) -> crate::core_arch::simd::i8x32 {
593 unsafe { transmute(self.as_m256i()) }
594 }
595
596 #[inline]
597 fn as_i16x16(self) -> crate::core_arch::simd::i16x16 {
598 unsafe { transmute(self.as_m256i()) }
599 }
600
601 #[inline]
602 fn as_i32x8(self) -> crate::core_arch::simd::i32x8 {
603 unsafe { transmute(self.as_m256i()) }
604 }
605
606 #[inline]
607 fn as_i64x4(self) -> crate::core_arch::simd::i64x4 {
608 unsafe { transmute(self.as_m256i()) }
609 }
610}
611
612impl m256iExt for __m256i {
613 #[inline]
614 fn as_m256i(self) -> Self {
615 self
616 }
617}
618
619#[allow(non_camel_case_types)]
620pub(crate) trait m128Ext: Sized {
621 fn as_m128(self) -> __m128;
622
623 #[inline]
624 fn as_f32x4(self) -> crate::core_arch::simd::f32x4 {
625 unsafe { transmute(self.as_m128()) }
626 }
627}
628
629impl m128Ext for __m128 {
630 #[inline]
631 fn as_m128(self) -> Self {
632 self
633 }
634}
635
636#[allow(non_camel_case_types)]
637pub(crate) trait m128dExt: Sized {
638 fn as_m128d(self) -> __m128d;
639
640 #[inline]
641 fn as_f64x2(self) -> crate::core_arch::simd::f64x2 {
642 unsafe { transmute(self.as_m128d()) }
643 }
644}
645
646impl m128dExt for __m128d {
647 #[inline]
648 fn as_m128d(self) -> Self {
649 self
650 }
651}
652
653#[allow(non_camel_case_types)]
654pub(crate) trait m256Ext: Sized {
655 fn as_m256(self) -> __m256;
656
657 #[inline]
658 fn as_f32x8(self) -> crate::core_arch::simd::f32x8 {
659 unsafe { transmute(self.as_m256()) }
660 }
661}
662
663impl m256Ext for __m256 {
664 #[inline]
665 fn as_m256(self) -> Self {
666 self
667 }
668}
669
670#[allow(non_camel_case_types)]
671pub(crate) trait m256dExt: Sized {
672 fn as_m256d(self) -> __m256d;
673
674 #[inline]
675 fn as_f64x4(self) -> crate::core_arch::simd::f64x4 {
676 unsafe { transmute(self.as_m256d()) }
677 }
678}
679
680impl m256dExt for __m256d {
681 #[inline]
682 fn as_m256d(self) -> Self {
683 self
684 }
685}
686
687#[allow(non_camel_case_types)]
688pub(crate) trait m512iExt: Sized {
689 fn as_m512i(self) -> __m512i;
690
691 #[inline]
692 fn as_u8x64(self) -> crate::core_arch::simd::u8x64 {
693 unsafe { transmute(self.as_m512i()) }
694 }
695
696 #[inline]
697 fn as_i8x64(self) -> crate::core_arch::simd::i8x64 {
698 unsafe { transmute(self.as_m512i()) }
699 }
700
701 #[inline]
702 fn as_u16x32(self) -> crate::core_arch::simd::u16x32 {
703 unsafe { transmute(self.as_m512i()) }
704 }
705
706 #[inline]
707 fn as_i16x32(self) -> crate::core_arch::simd::i16x32 {
708 unsafe { transmute(self.as_m512i()) }
709 }
710
711 #[inline]
712 fn as_u32x16(self) -> crate::core_arch::simd::u32x16 {
713 unsafe { transmute(self.as_m512i()) }
714 }
715
716 #[inline]
717 fn as_i32x16(self) -> crate::core_arch::simd::i32x16 {
718 unsafe { transmute(self.as_m512i()) }
719 }
720
721 #[inline]
722 fn as_u64x8(self) -> crate::core_arch::simd::u64x8 {
723 unsafe { transmute(self.as_m512i()) }
724 }
725
726 #[inline]
727 fn as_i64x8(self) -> crate::core_arch::simd::i64x8 {
728 unsafe { transmute(self.as_m512i()) }
729 }
730}
731
732impl m512iExt for __m512i {
733 #[inline]
734 fn as_m512i(self) -> Self {
735 self
736 }
737}
738
739#[allow(non_camel_case_types)]
740pub(crate) trait m512Ext: Sized {
741 fn as_m512(self) -> __m512;
742
743 #[inline]
744 fn as_f32x16(self) -> crate::core_arch::simd::f32x16 {
745 unsafe { transmute(self.as_m512()) }
746 }
747}
748
749impl m512Ext for __m512 {
750 #[inline]
751 fn as_m512(self) -> Self {
752 self
753 }
754}
755
756#[allow(non_camel_case_types)]
757pub(crate) trait m512dExt: Sized {
758 fn as_m512d(self) -> __m512d;
759
760 #[inline]
761 fn as_f64x8(self) -> crate::core_arch::simd::f64x8 {
762 unsafe { transmute(self.as_m512d()) }
763 }
764}
765
766impl m512dExt for __m512d {
767 #[inline]
768 fn as_m512d(self) -> Self {
769 self
770 }
771}
772
773#[allow(non_camel_case_types)]
774pub(crate) trait m128bhExt: Sized {
775 fn as_m128bh(self) -> __m128bh;
776
777 #[inline]
778 fn as_u16x8(self) -> crate::core_arch::simd::u16x8 {
779 unsafe { transmute(self.as_m128bh()) }
780 }
781
782 #[inline]
783 fn as_i16x8(self) -> crate::core_arch::simd::i16x8 {
784 unsafe { transmute(self.as_m128bh()) }
785 }
786
787 #[inline]
788 fn as_u32x4(self) -> crate::core_arch::simd::u32x4 {
789 unsafe { transmute(self.as_m128bh()) }
790 }
791
792 #[inline]
793 fn as_i32x4(self) -> crate::core_arch::simd::i32x4 {
794 unsafe { transmute(self.as_m128bh()) }
795 }
796}
797
798impl m128bhExt for __m128bh {
799 #[inline]
800 fn as_m128bh(self) -> Self {
801 self
802 }
803}
804
805#[allow(non_camel_case_types)]
806pub(crate) trait m256bhExt: Sized {
807 fn as_m256bh(self) -> __m256bh;
808
809 #[inline]
810 fn as_u16x16(self) -> crate::core_arch::simd::u16x16 {
811 unsafe { transmute(self.as_m256bh()) }
812 }
813
814 #[inline]
815 fn as_i16x16(self) -> crate::core_arch::simd::i16x16 {
816 unsafe { transmute(self.as_m256bh()) }
817 }
818
819 #[inline]
820 fn as_u32x8(self) -> crate::core_arch::simd::u32x8 {
821 unsafe { transmute(self.as_m256bh()) }
822 }
823
824 #[inline]
825 fn as_i32x8(self) -> crate::core_arch::simd::i32x8 {
826 unsafe { transmute(self.as_m256bh()) }
827 }
828}
829
830impl m256bhExt for __m256bh {
831 #[inline]
832 fn as_m256bh(self) -> Self {
833 self
834 }
835}
836
837#[allow(non_camel_case_types)]
838pub(crate) trait m512bhExt: Sized {
839 fn as_m512bh(self) -> __m512bh;
840
841 #[inline]
842 fn as_u16x32(self) -> crate::core_arch::simd::u16x32 {
843 unsafe { transmute(self.as_m512bh()) }
844 }
845
846 #[inline]
847 fn as_i16x32(self) -> crate::core_arch::simd::i16x32 {
848 unsafe { transmute(self.as_m512bh()) }
849 }
850
851 #[inline]
852 fn as_u32x16(self) -> crate::core_arch::simd::u32x16 {
853 unsafe { transmute(self.as_m512bh()) }
854 }
855
856 #[inline]
857 fn as_i32x16(self) -> crate::core_arch::simd::i32x16 {
858 unsafe { transmute(self.as_m512bh()) }
859 }
860}
861
862impl m512bhExt for __m512bh {
863 #[inline]
864 fn as_m512bh(self) -> Self {
865 self
866 }
867}
868
869#[allow(non_camel_case_types)]
870pub(crate) trait m128hExt: Sized {
871 fn as_m128h(self) -> __m128h;
872
873 #[inline]
874 fn as_f16x8(self) -> crate::core_arch::simd::f16x8 {
875 unsafe { transmute(self.as_m128h()) }
876 }
877}
878
879impl m128hExt for __m128h {
880 #[inline]
881 fn as_m128h(self) -> Self {
882 self
883 }
884}
885
886#[allow(non_camel_case_types)]
887pub(crate) trait m256hExt: Sized {
888 fn as_m256h(self) -> __m256h;
889
890 #[inline]
891 fn as_f16x16(self) -> crate::core_arch::simd::f16x16 {
892 unsafe { transmute(self.as_m256h()) }
893 }
894}
895
896impl m256hExt for __m256h {
897 #[inline]
898 fn as_m256h(self) -> Self {
899 self
900 }
901}
902
903#[allow(non_camel_case_types)]
904pub(crate) trait m512hExt: Sized {
905 fn as_m512h(self) -> __m512h;
906
907 #[inline]
908 fn as_f16x32(self) -> crate::core_arch::simd::f16x32 {
909 unsafe { transmute(self.as_m512h()) }
910 }
911}
912
913mod eflags;
914#[stable(feature = "simd_x86", since = "1.27.0")]
915pub use self::eflags::*;
916
917mod fxsr;
918#[stable(feature = "simd_x86", since = "1.27.0")]
919pub use self::fxsr::*;
920
921mod bswap;
922#[stable(feature = "simd_x86", since = "1.27.0")]
923pub use self::bswap::*;
924
925mod rdtsc;
926#[stable(feature = "simd_x86", since = "1.27.0")]
927pub use self::rdtsc::*;
928
929mod cpuid;
930#[stable(feature = "simd_x86", since = "1.27.0")]
931pub use self::cpuid::*;
932mod xsave;
933#[stable(feature = "simd_x86", since = "1.27.0")]
934pub use self::xsave::*;
935
936mod sse;
937#[stable(feature = "simd_x86", since = "1.27.0")]
938pub use self::sse::*;
939mod sse2;
940#[stable(feature = "simd_x86", since = "1.27.0")]
941pub use self::sse2::*;
942mod sse3;
943#[stable(feature = "simd_x86", since = "1.27.0")]
944pub use self::sse3::*;
945mod ssse3;
946#[stable(feature = "simd_x86", since = "1.27.0")]
947pub use self::ssse3::*;
948mod sse41;
949#[stable(feature = "simd_x86", since = "1.27.0")]
950pub use self::sse41::*;
951mod sse42;
952#[stable(feature = "simd_x86", since = "1.27.0")]
953pub use self::sse42::*;
954mod avx;
955#[stable(feature = "simd_x86", since = "1.27.0")]
956pub use self::avx::*;
957mod avx2;
958#[stable(feature = "simd_x86", since = "1.27.0")]
959pub use self::avx2::*;
960mod fma;
961#[stable(feature = "simd_x86", since = "1.27.0")]
962pub use self::fma::*;
963
964mod abm;
965#[stable(feature = "simd_x86", since = "1.27.0")]
966pub use self::abm::*;
967mod bmi1;
968#[stable(feature = "simd_x86", since = "1.27.0")]
969pub use self::bmi1::*;
970
971mod bmi2;
972#[stable(feature = "simd_x86", since = "1.27.0")]
973pub use self::bmi2::*;
974
975mod sse4a;
976#[stable(feature = "simd_x86", since = "1.27.0")]
977pub use self::sse4a::*;
978
979mod tbm;
980#[stable(feature = "simd_x86", since = "1.27.0")]
981pub use self::tbm::*;
982
983mod pclmulqdq;
984#[stable(feature = "simd_x86", since = "1.27.0")]
985pub use self::pclmulqdq::*;
986
987mod aes;
988#[stable(feature = "simd_x86", since = "1.27.0")]
989pub use self::aes::*;
990
991mod rdrand;
992#[stable(feature = "simd_x86", since = "1.27.0")]
993pub use self::rdrand::*;
994
995mod sha;
996#[stable(feature = "simd_x86", since = "1.27.0")]
997pub use self::sha::*;
998
999mod adx;
1000#[stable(feature = "simd_x86_adx", since = "1.33.0")]
1001pub use self::adx::*;
1002
1003#[cfg(test)]
1004use stdarch_test::assert_instr;
1005
1006mod avx512f;
1007#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
1008pub use self::avx512f::*;
1009
1010mod avx512bw;
1011#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
1012pub use self::avx512bw::*;
1013
1014mod avx512cd;
1015#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
1016pub use self::avx512cd::*;
1017
1018mod avx512dq;
1019#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
1020pub use self::avx512dq::*;
1021
1022mod avx512ifma;
1023#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
1024pub use self::avx512ifma::*;
1025
1026mod avx512vbmi;
1027#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
1028pub use self::avx512vbmi::*;
1029
1030mod avx512vbmi2;
1031#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
1032pub use self::avx512vbmi2::*;
1033
1034mod avx512vnni;
1035#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
1036pub use self::avx512vnni::*;
1037
1038mod avx512bitalg;
1039#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
1040pub use self::avx512bitalg::*;
1041
1042mod gfni;
1043#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
1044pub use self::gfni::*;
1045
1046mod avx512vpopcntdq;
1047#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
1048pub use self::avx512vpopcntdq::*;
1049
1050mod vaes;
1051#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
1052pub use self::vaes::*;
1053
1054mod vpclmulqdq;
1055#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
1056pub use self::vpclmulqdq::*;
1057
1058mod bt;
1059#[stable(feature = "simd_x86_bittest", since = "1.55.0")]
1060pub use self::bt::*;
1061
1062mod rtm;
1063#[unstable(feature = "stdarch_x86_rtm", issue = "111138")]
1064pub use self::rtm::*;
1065
1066mod f16c;
1067#[stable(feature = "x86_f16c_intrinsics", since = "1.68.0")]
1068pub use self::f16c::*;
1069
1070mod avx512bf16;
1071#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
1072pub use self::avx512bf16::*;
1073
1074mod avxneconvert;
1075#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
1076pub use self::avxneconvert::*;
1077
1078mod avx512fp16;
1079#[unstable(feature = "stdarch_x86_avx512_f16", issue = "127213")]
1080pub use self::avx512fp16::*;