1use crate::core_arch::simd::*;
11use crate::core_arch::x86::__m128i;
12use crate::core_arch::x86::__m256i;
13use crate::core_arch::x86::__m512i;
14use crate::core_arch::x86::__mmask16;
15use crate::core_arch::x86::__mmask8;
16use crate::core_arch::x86::m128iExt;
17use crate::core_arch::x86::m256iExt;
18use crate::core_arch::x86::m512iExt;
19use crate::intrinsics::simd::{simd_ctpop, simd_select_bitmask};
20use crate::mem::transmute;
21
22#[cfg(test)]
23use stdarch_test::assert_instr;
24
25#[inline]
29#[target_feature(enable = "avx512vpopcntdq")]
30#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
31#[cfg_attr(test, assert_instr(vpopcntd))]
32pub unsafe fn _mm512_popcnt_epi32(a: __m512i) -> __m512i {
33 transmute(simd_ctpop(a.as_i32x16()))
34}
35
36#[inline]
43#[target_feature(enable = "avx512vpopcntdq")]
44#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
45#[cfg_attr(test, assert_instr(vpopcntd))]
46pub unsafe fn _mm512_maskz_popcnt_epi32(k: __mmask16, a: __m512i) -> __m512i {
47 transmute(simd_select_bitmask(
48 k,
49 simd_ctpop(a.as_i32x16()),
50 i32x16::ZERO,
51 ))
52}
53
54#[inline]
61#[target_feature(enable = "avx512vpopcntdq")]
62#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
63#[cfg_attr(test, assert_instr(vpopcntd))]
64pub unsafe fn _mm512_mask_popcnt_epi32(src: __m512i, k: __mmask16, a: __m512i) -> __m512i {
65 transmute(simd_select_bitmask(
66 k,
67 simd_ctpop(a.as_i32x16()),
68 src.as_i32x16(),
69 ))
70}
71
72#[inline]
76#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
77#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
78#[cfg_attr(test, assert_instr(vpopcntd))]
79pub unsafe fn _mm256_popcnt_epi32(a: __m256i) -> __m256i {
80 transmute(simd_ctpop(a.as_i32x8()))
81}
82
83#[inline]
90#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
91#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
92#[cfg_attr(test, assert_instr(vpopcntd))]
93pub unsafe fn _mm256_maskz_popcnt_epi32(k: __mmask8, a: __m256i) -> __m256i {
94 transmute(simd_select_bitmask(
95 k,
96 simd_ctpop(a.as_i32x8()),
97 i32x8::ZERO,
98 ))
99}
100
101#[inline]
108#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
109#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
110#[cfg_attr(test, assert_instr(vpopcntd))]
111pub unsafe fn _mm256_mask_popcnt_epi32(src: __m256i, k: __mmask8, a: __m256i) -> __m256i {
112 transmute(simd_select_bitmask(
113 k,
114 simd_ctpop(a.as_i32x8()),
115 src.as_i32x8(),
116 ))
117}
118
119#[inline]
123#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
124#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
125#[cfg_attr(test, assert_instr(vpopcntd))]
126pub unsafe fn _mm_popcnt_epi32(a: __m128i) -> __m128i {
127 transmute(simd_ctpop(a.as_i32x4()))
128}
129
130#[inline]
137#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
138#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
139#[cfg_attr(test, assert_instr(vpopcntd))]
140pub unsafe fn _mm_maskz_popcnt_epi32(k: __mmask8, a: __m128i) -> __m128i {
141 transmute(simd_select_bitmask(
142 k,
143 simd_ctpop(a.as_i32x4()),
144 i32x4::ZERO,
145 ))
146}
147
148#[inline]
155#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
156#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
157#[cfg_attr(test, assert_instr(vpopcntd))]
158pub unsafe fn _mm_mask_popcnt_epi32(src: __m128i, k: __mmask8, a: __m128i) -> __m128i {
159 transmute(simd_select_bitmask(
160 k,
161 simd_ctpop(a.as_i32x4()),
162 src.as_i32x4(),
163 ))
164}
165
166#[inline]
170#[target_feature(enable = "avx512vpopcntdq")]
171#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
172#[cfg_attr(test, assert_instr(vpopcntq))]
173pub unsafe fn _mm512_popcnt_epi64(a: __m512i) -> __m512i {
174 transmute(simd_ctpop(a.as_i64x8()))
175}
176
177#[inline]
184#[target_feature(enable = "avx512vpopcntdq")]
185#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
186#[cfg_attr(test, assert_instr(vpopcntq))]
187pub unsafe fn _mm512_maskz_popcnt_epi64(k: __mmask8, a: __m512i) -> __m512i {
188 transmute(simd_select_bitmask(
189 k,
190 simd_ctpop(a.as_i64x8()),
191 i64x8::ZERO,
192 ))
193}
194
195#[inline]
202#[target_feature(enable = "avx512vpopcntdq")]
203#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
204#[cfg_attr(test, assert_instr(vpopcntq))]
205pub unsafe fn _mm512_mask_popcnt_epi64(src: __m512i, k: __mmask8, a: __m512i) -> __m512i {
206 transmute(simd_select_bitmask(
207 k,
208 simd_ctpop(a.as_i64x8()),
209 src.as_i64x8(),
210 ))
211}
212
213#[inline]
217#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
218#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
219#[cfg_attr(test, assert_instr(vpopcntq))]
220pub unsafe fn _mm256_popcnt_epi64(a: __m256i) -> __m256i {
221 transmute(simd_ctpop(a.as_i64x4()))
222}
223
224#[inline]
231#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
232#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
233#[cfg_attr(test, assert_instr(vpopcntq))]
234pub unsafe fn _mm256_maskz_popcnt_epi64(k: __mmask8, a: __m256i) -> __m256i {
235 transmute(simd_select_bitmask(
236 k,
237 simd_ctpop(a.as_i64x4()),
238 i64x4::ZERO,
239 ))
240}
241
242#[inline]
249#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
250#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
251#[cfg_attr(test, assert_instr(vpopcntq))]
252pub unsafe fn _mm256_mask_popcnt_epi64(src: __m256i, k: __mmask8, a: __m256i) -> __m256i {
253 transmute(simd_select_bitmask(
254 k,
255 simd_ctpop(a.as_i64x4()),
256 src.as_i64x4(),
257 ))
258}
259
260#[inline]
264#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
265#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
266#[cfg_attr(test, assert_instr(vpopcntq))]
267pub unsafe fn _mm_popcnt_epi64(a: __m128i) -> __m128i {
268 transmute(simd_ctpop(a.as_i64x2()))
269}
270
271#[inline]
278#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
279#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
280#[cfg_attr(test, assert_instr(vpopcntq))]
281pub unsafe fn _mm_maskz_popcnt_epi64(k: __mmask8, a: __m128i) -> __m128i {
282 transmute(simd_select_bitmask(
283 k,
284 simd_ctpop(a.as_i64x2()),
285 i64x2::ZERO,
286 ))
287}
288
289#[inline]
296#[target_feature(enable = "avx512vpopcntdq,avx512vl")]
297#[unstable(feature = "stdarch_x86_avx512", issue = "111137")]
298#[cfg_attr(test, assert_instr(vpopcntq))]
299pub unsafe fn _mm_mask_popcnt_epi64(src: __m128i, k: __mmask8, a: __m128i) -> __m128i {
300 transmute(simd_select_bitmask(
301 k,
302 simd_ctpop(a.as_i64x2()),
303 src.as_i64x2(),
304 ))
305}
306
307#[cfg(test)]
308mod tests {
309 use stdarch_test::simd_test;
310
311 use crate::core_arch::x86::*;
312
313 #[simd_test(enable = "avx512vpopcntdq,avx512f")]
314 unsafe fn test_mm512_popcnt_epi32() {
315 let test_data = _mm512_set_epi32(
316 0,
317 1,
318 -1,
319 2,
320 7,
321 0xFF_FE,
322 0x7F_FF_FF_FF,
323 -100,
324 0x40_00_00_00,
325 103,
326 371,
327 552,
328 432_948,
329 818_826_998,
330 255,
331 256,
332 );
333 let actual_result = _mm512_popcnt_epi32(test_data);
334 let reference_result =
335 _mm512_set_epi32(0, 1, 32, 1, 3, 15, 31, 28, 1, 5, 6, 3, 10, 17, 8, 1);
336 assert_eq_m512i(actual_result, reference_result);
337 }
338
339 #[simd_test(enable = "avx512vpopcntdq,avx512f")]
340 unsafe fn test_mm512_mask_popcnt_epi32() {
341 let test_data = _mm512_set_epi32(
342 0,
343 1,
344 -1,
345 2,
346 7,
347 0xFF_FE,
348 0x7F_FF_FF_FF,
349 -100,
350 0x40_00_00_00,
351 103,
352 371,
353 552,
354 432_948,
355 818_826_998,
356 255,
357 256,
358 );
359 let mask = 0xFF_00;
360 let actual_result = _mm512_mask_popcnt_epi32(test_data, mask, test_data);
361 let reference_result = _mm512_set_epi32(
362 0,
363 1,
364 32,
365 1,
366 3,
367 15,
368 31,
369 28,
370 0x40_00_00_00,
371 103,
372 371,
373 552,
374 432_948,
375 818_826_998,
376 255,
377 256,
378 );
379 assert_eq_m512i(actual_result, reference_result);
380 }
381
382 #[simd_test(enable = "avx512vpopcntdq,avx512f")]
383 unsafe fn test_mm512_maskz_popcnt_epi32() {
384 let test_data = _mm512_set_epi32(
385 0,
386 1,
387 -1,
388 2,
389 7,
390 0xFF_FE,
391 0x7F_FF_FF_FF,
392 -100,
393 0x40_00_00_00,
394 103,
395 371,
396 552,
397 432_948,
398 818_826_998,
399 255,
400 256,
401 );
402 let mask = 0xFF_00;
403 let actual_result = _mm512_maskz_popcnt_epi32(mask, test_data);
404 let reference_result = _mm512_set_epi32(0, 1, 32, 1, 3, 15, 31, 28, 0, 0, 0, 0, 0, 0, 0, 0);
405 assert_eq_m512i(actual_result, reference_result);
406 }
407
408 #[simd_test(enable = "avx512vpopcntdq,avx512f,avx512vl")]
409 unsafe fn test_mm256_popcnt_epi32() {
410 let test_data = _mm256_set_epi32(0, 1, -1, 2, 7, 0xFF_FE, 0x7F_FF_FF_FF, -100);
411 let actual_result = _mm256_popcnt_epi32(test_data);
412 let reference_result = _mm256_set_epi32(0, 1, 32, 1, 3, 15, 31, 28);
413 assert_eq_m256i(actual_result, reference_result);
414 }
415
416 #[simd_test(enable = "avx512vpopcntdq,avx512f,avx512vl")]
417 unsafe fn test_mm256_mask_popcnt_epi32() {
418 let test_data = _mm256_set_epi32(0, 1, -1, 2, 7, 0xFF_FE, 0x7F_FF_FF_FF, -100);
419 let mask = 0xF0;
420 let actual_result = _mm256_mask_popcnt_epi32(test_data, mask, test_data);
421 let reference_result = _mm256_set_epi32(0, 1, 32, 1, 7, 0xFF_FE, 0x7F_FF_FF_FF, -100);
422 assert_eq_m256i(actual_result, reference_result);
423 }
424
425 #[simd_test(enable = "avx512vpopcntdq,avx512f,avx512vl")]
426 unsafe fn test_mm256_maskz_popcnt_epi32() {
427 let test_data = _mm256_set_epi32(0, 1, -1, 2, 7, 0xFF_FE, 0x7F_FF_FF_FF, -100);
428 let mask = 0xF0;
429 let actual_result = _mm256_maskz_popcnt_epi32(mask, test_data);
430 let reference_result = _mm256_set_epi32(0, 1, 32, 1, 0, 0, 0, 0);
431 assert_eq_m256i(actual_result, reference_result);
432 }
433
434 #[simd_test(enable = "avx512vpopcntdq,avx512f,avx512vl")]
435 unsafe fn test_mm_popcnt_epi32() {
436 let test_data = _mm_set_epi32(0, 1, -1, -100);
437 let actual_result = _mm_popcnt_epi32(test_data);
438 let reference_result = _mm_set_epi32(0, 1, 32, 28);
439 assert_eq_m128i(actual_result, reference_result);
440 }
441
442 #[simd_test(enable = "avx512vpopcntdq,avx512f,avx512vl")]
443 unsafe fn test_mm_mask_popcnt_epi32() {
444 let test_data = _mm_set_epi32(0, 1, -1, -100);
445 let mask = 0xE;
446 let actual_result = _mm_mask_popcnt_epi32(test_data, mask, test_data);
447 let reference_result = _mm_set_epi32(0, 1, 32, -100);
448 assert_eq_m128i(actual_result, reference_result);
449 }
450
451 #[simd_test(enable = "avx512vpopcntdq,avx512f,avx512vl")]
452 unsafe fn test_mm_maskz_popcnt_epi32() {
453 let test_data = _mm_set_epi32(0, 1, -1, -100);
454 let mask = 0xE;
455 let actual_result = _mm_maskz_popcnt_epi32(mask, test_data);
456 let reference_result = _mm_set_epi32(0, 1, 32, 0);
457 assert_eq_m128i(actual_result, reference_result);
458 }
459
460 #[simd_test(enable = "avx512vpopcntdq,avx512f")]
461 unsafe fn test_mm512_popcnt_epi64() {
462 let test_data = _mm512_set_epi64(0, 1, -1, 2, 7, 0xFF_FE, 0x7F_FF_FF_FF_FF_FF_FF_FF, -100);
463 let actual_result = _mm512_popcnt_epi64(test_data);
464 let reference_result = _mm512_set_epi64(0, 1, 64, 1, 3, 15, 63, 60);
465 assert_eq_m512i(actual_result, reference_result);
466 }
467
468 #[simd_test(enable = "avx512vpopcntdq,avx512f")]
469 unsafe fn test_mm512_mask_popcnt_epi64() {
470 let test_data = _mm512_set_epi64(0, 1, -1, 2, 7, 0xFF_FE, 0x7F_FF_FF_FF_FF_FF_FF_FF, -100);
471 let mask = 0xF0;
472 let actual_result = _mm512_mask_popcnt_epi64(test_data, mask, test_data);
473 let reference_result =
474 _mm512_set_epi64(0, 1, 64, 1, 7, 0xFF_FE, 0x7F_FF_FF_FF_FF_FF_FF_FF, -100);
475 assert_eq_m512i(actual_result, reference_result);
476 }
477
478 #[simd_test(enable = "avx512vpopcntdq,avx512f")]
479 unsafe fn test_mm512_maskz_popcnt_epi64() {
480 let test_data = _mm512_set_epi64(0, 1, -1, 2, 7, 0xFF_FE, 0x7F_FF_FF_FF_FF_FF_FF_FF, -100);
481 let mask = 0xF0;
482 let actual_result = _mm512_maskz_popcnt_epi64(mask, test_data);
483 let reference_result = _mm512_set_epi64(0, 1, 64, 1, 0, 0, 0, 0);
484 assert_eq_m512i(actual_result, reference_result);
485 }
486
487 #[simd_test(enable = "avx512vpopcntdq,avx512vl")]
488 unsafe fn test_mm256_popcnt_epi64() {
489 let test_data = _mm256_set_epi64x(0, 1, -1, -100);
490 let actual_result = _mm256_popcnt_epi64(test_data);
491 let reference_result = _mm256_set_epi64x(0, 1, 64, 60);
492 assert_eq_m256i(actual_result, reference_result);
493 }
494
495 #[simd_test(enable = "avx512vpopcntdq,avx512vl")]
496 unsafe fn test_mm256_mask_popcnt_epi64() {
497 let test_data = _mm256_set_epi64x(0, 1, -1, -100);
498 let mask = 0xE;
499 let actual_result = _mm256_mask_popcnt_epi64(test_data, mask, test_data);
500 let reference_result = _mm256_set_epi64x(0, 1, 64, -100);
501 assert_eq_m256i(actual_result, reference_result);
502 }
503
504 #[simd_test(enable = "avx512vpopcntdq,avx512vl")]
505 unsafe fn test_mm256_maskz_popcnt_epi64() {
506 let test_data = _mm256_set_epi64x(0, 1, -1, -100);
507 let mask = 0xE;
508 let actual_result = _mm256_maskz_popcnt_epi64(mask, test_data);
509 let reference_result = _mm256_set_epi64x(0, 1, 64, 0);
510 assert_eq_m256i(actual_result, reference_result);
511 }
512
513 #[simd_test(enable = "avx512vpopcntdq,avx512vl")]
514 unsafe fn test_mm_popcnt_epi64() {
515 let test_data = _mm_set_epi64x(0, 1);
516 let actual_result = _mm_popcnt_epi64(test_data);
517 let reference_result = _mm_set_epi64x(0, 1);
518 assert_eq_m128i(actual_result, reference_result);
519 let test_data = _mm_set_epi64x(-1, -100);
520 let actual_result = _mm_popcnt_epi64(test_data);
521 let reference_result = _mm_set_epi64x(64, 60);
522 assert_eq_m128i(actual_result, reference_result);
523 }
524
525 #[simd_test(enable = "avx512vpopcntdq,avx512vl")]
526 unsafe fn test_mm_mask_popcnt_epi64() {
527 let test_data = _mm_set_epi64x(0, -100);
528 let mask = 0x2;
529 let actual_result = _mm_mask_popcnt_epi64(test_data, mask, test_data);
530 let reference_result = _mm_set_epi64x(0, -100);
531 assert_eq_m128i(actual_result, reference_result);
532 let test_data = _mm_set_epi64x(-1, 1);
533 let mask = 0x2;
534 let actual_result = _mm_mask_popcnt_epi64(test_data, mask, test_data);
535 let reference_result = _mm_set_epi64x(64, 1);
536 assert_eq_m128i(actual_result, reference_result);
537 }
538
539 #[simd_test(enable = "avx512vpopcntdq,avx512vl")]
540 unsafe fn test_mm_maskz_popcnt_epi64() {
541 let test_data = _mm_set_epi64x(0, 1);
542 let mask = 0x2;
543 let actual_result = _mm_maskz_popcnt_epi64(mask, test_data);
544 let reference_result = _mm_set_epi64x(0, 0);
545 assert_eq_m128i(actual_result, reference_result);
546 let test_data = _mm_set_epi64x(-1, -100);
547 let mask = 0x2;
548 let actual_result = _mm_maskz_popcnt_epi64(mask, test_data);
549 let reference_result = _mm_set_epi64x(64, 0);
550 assert_eq_m128i(actual_result, reference_result);
551 }
552}