diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_cnProc.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_cnProc.h index 73252f341b..50e425753d 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_cnProc.h +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_cnProc.h @@ -30,6 +30,8 @@ #ifndef __NR_LDPC_CNPROC__H__ #define __NR_LDPC_CNPROC__H__ +//#include "/opt/rh/devtoolset-7/root/lib/gcc/x86_64-redhat-linux/7/include/avx2intrin.h" +#include "immintrin.h" /** \brief Performs CN processing for BG2 on the CN processing buffer and stores the results in the CN processing results buffer. diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_decoder.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_decoder.c index 40bcafe1a7..493888ebbe 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_decoder.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_decoder.c @@ -38,6 +38,8 @@ #include "nrLDPC_mPass.h" #include "nrLDPC_cnProc.h" #include "nrLDPC_bnProc.h" +#include "nrLDPC_tools/include/avx512fintrin.h" + /*********************************include files-> AVX2**********************/ /*----------------------------BG1-----------------------------------------*/ @@ -160,62 +162,64 @@ /**************************include files for -->AVX512 ******************/ +//test +#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z384_13_AVX512.c" -/#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z384_13_AVX512.c" -/*#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z352_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z320_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z288_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z256_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z240_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z224_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z208_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z192_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z176_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z160_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z144_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z128_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z120_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z112_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z104_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z96_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z88_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z80_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z72_13_AVX512.c" +/*#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z352_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z320_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z288_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z256_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z240_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z224_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z208_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z192_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z176_13_AVX2.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z64_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z60_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z56_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z52_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z48_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z44_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z40_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z36_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z32_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z30_13_AVX512.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z160_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z144_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z128_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z120_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z112_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z104_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z96_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z88_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z80_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z72_13_AVX2.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z28_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z26_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z24_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z22_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z20_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z18_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z16_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z15_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z14_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z13_13_AVX512.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z64_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z60_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z56_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z52_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z48_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z44_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z40_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z36_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z32_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z30_13_AVX2.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z12_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z11_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z10_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z9_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z8_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z7_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z6_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z5_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z4_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z3_13_AVX512.c" -#include "nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z2_13_AVX512.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z28_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z26_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z24_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z22_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z20_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z18_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z16_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z15_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z14_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z13_13_AVX2.c" + +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z12_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z11_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z10_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z9_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z8_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z7_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z6_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z5_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z4_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z3_13_AVX2.c" +#include "nrLDPC_tools/ldpc_gen_files/avx2/nrLDPC_cnProc_BG1_Z2_13_AVX2.c" */ #define NR_LDPC_ENABLE_PARITY_CHECK #define NR_LDPC_PROFILER_DETAIL @@ -328,11 +332,11 @@ static inline uint32_t nrLDPC_decoder_core(int8_t* p_llr, int8_t* p_out, t_nrLDP switch (Z) { case 384: - { //we test here - //nrLDPC_cnProc_BG1_Z384_13_AVX2(p_procBuf->cnProcBuf,p_procBuf->cnProcBufRes); - nrLDPC_cnProc_BG1_Z384_13_AVX512(p_procBuf->cnProcBuf,p_procBuf->cnProcBufRes); + { + // nrLDPC_cnProc_BG1_Z384_13_AVX2(p_procBuf->cnProcBuf,p_procBuf->cnProcBufRes); + nrLDPC_cnProc_BG1_Z384_13_AVX512(p_procBuf->cnProcBuf,p_procBuf->cnProcBufRes); //were test here break; - } + } case 352: { nrLDPC_cnProc_BG1_Z352_13_AVX2(p_procBuf->cnProcBuf,p_procBuf->cnProcBufRes); @@ -683,7 +687,7 @@ static inline uint32_t nrLDPC_decoder_core(int8_t* p_llr, int8_t* p_out, t_nrLDP case 384: { nrLDPC_cnProc_BG2_Z384_13_AVX2(p_procBuf->cnProcBuf,p_procBuf->cnProcBufRes); - // nrLDPC_cnProc_BG2_Z384_13_AVX512(p_procBuf->cnProcBuf,p_procBuf->cnProcBufRes); + // nrLDPC_cnProc_BG2_Z384_13_AVX512(p_procBuf->cnProcBuf,p_procBuf->cnProcBufRes); break; } case 352: @@ -1118,8 +1122,8 @@ static inline uint32_t nrLDPC_decoder_core(int8_t* p_llr, int8_t* p_out, t_nrLDP { case 384: { - nrLDPC_cnProc_BG1_Z384_13_AVX2(p_procBuf->cnProcBuf,p_procBuf->cnProcBufRes); - // nrLDPC_cnProc_BG1_Z384_13_AVX512(p_procBuf->cnProcBuf,p_procBuf->cnProcBufRes); + //nrLDPC_cnProc_BG1_Z384_13_AVX2(p_procBuf->cnProcBuf,p_procBuf->cnProcBufRes); + nrLDPC_cnProc_BG1_Z384_13_AVX512(p_procBuf->cnProcBuf,p_procBuf->cnProcBufRes); //we test here break; } case 352: @@ -1917,15 +1921,15 @@ static inline uint32_t nrLDPC_decoder_core(int8_t* p_llr, int8_t* p_out, t_nrLDP switch (Z) { case 384: - {//we test here - //nrLDPC_cnProc_BG1_Z384_13_AVX2(p_procBuf->cnProcBuf,p_procBuf->cnProcBufRes); - nrLDPC_cnProc_BG1_Z384_13_AVX512(p_procBuf->cnProcBuf,p_procBuf->cnProcBufRes); + { + // nrLDPC_cnProc_BG1_Z384_13_AVX2(p_procBuf->cnProcBuf,p_procBuf->cnProcBufRes); + nrLDPC_cnProc_BG1_Z384_13_AVX512(p_procBuf->cnProcBuf,p_procBuf->cnProcBufRes); //we test here break; } case 352: { - nrLDPC_cnProc_BG1_Z352_13_AVX2(p_procBuf->cnProcBuf,p_procBuf->cnProcBufRes); - // nrLDPC_cnProc_BG1_Z352_13_AVX512(p_procBuf->cnProcBuf,p_procBuf->cnProcBufRes); + //nrLDPC_cnProc_BG1_Z352_13_AVX2(p_procBuf->cnProcBuf,p_procBuf->cnProcBufRes); + nrLDPC_cnProc_BG1_Z352_13_AVX512(p_procBuf->cnProcBuf,p_procBuf->cnProcBufRes); break; } @@ -2267,9 +2271,9 @@ static inline uint32_t nrLDPC_decoder_core(int8_t* p_llr, int8_t* p_out, t_nrLDP switch (Z) { case 384: - { //we test here - //nrLDPC_cnProc_BG2_Z384_13_AVX2(p_procBuf->cnProcBuf,p_procBuf->cnProcBufRes); - nrLDPC_cnProc_BG2_Z384_13_AVX512(p_procBuf->cnProcBuf,p_procBuf->cnProcBufRes); + { + nrLDPC_cnProc_BG2_Z384_13_AVX2(p_procBuf->cnProcBuf,p_procBuf->cnProcBufRes); + // nrLDPC_cnProc_BG2_Z384_13_AVX512(p_procBuf->cnProcBuf,p_procBuf->cnProcBufRes); break; } case 352: diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/generator_avx2/cnProc_gen_BG1_avx2.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/generator_avx2/cnProc_gen_BG1_avx2.c index b9a8e19f21..562caa6f26 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/generator_avx2/cnProc_gen_BG1_avx2.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/generator_avx2/cnProc_gen_BG1_avx2.c @@ -21,6 +21,8 @@ void nrLDPC_cnProc_BG1_generator_AVX2(uint16_t Z,int R) fprintf(fd,"#include \n"); fprintf(fd,"#include \n"); + fprintf(fd,"#include "\"../include/avx512fintrin.h>\"\n"); + fprintf(fd,"void nrLDPC_cnProc_BG1_Z%d_%s_AVX2(int8_t* cnProcBuf,int8_t* cnProcBufRes) {\n",Z,ratestr[R]); const uint8_t* lut_numCnInCnGroups; diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/generator_avx512/cnProc_gen_BG1_avx512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/generator_avx512/cnProc_gen_BG1_avx512.c index a6ffe84223..1d6f2f14d8 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/generator_avx512/cnProc_gen_BG1_avx512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/generator_avx512/cnProc_gen_BG1_avx512.c @@ -20,7 +20,7 @@ void nrLDPC_cnProc_BG1_generator_AVX512(uint16_t Z,int R) fprintf(fd,"#include \n"); fprintf(fd,"#include \n"); - + fprintf(fd, "#include \"../include/avx512fintrin.h\"\n"); fprintf(fd, "__m512i _mm512_sign_epi16(__m512i a, __m512i b){ \n"); /* Emulate _mm512_sign_epi16() with instructions that exist in the AVX-512 instruction set */ fprintf(fd, "b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); \n" ); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/generator_avx512/cnProc_gen_BG2_avx512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/generator_avx512/cnProc_gen_BG2_avx512.c index 0b95e84336..282cede4d6 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/generator_avx512/cnProc_gen_BG2_avx512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/generator_avx512/cnProc_gen_BG2_avx512.c @@ -20,6 +20,7 @@ void nrLDPC_cnProc_BG2_generator_AVX512(uint16_t Z,int R) fprintf(fd,"#include \n"); fprintf(fd,"#include \n"); + fprintf(fd, "#include \"../include/avx512fintrin.h\"\n"); fprintf(fd, "__m512i _mm512_sign_epi16(__m512i a, __m512i b){ \n"); /* Emulate _mm512_sign_epi16() with instructions that exist in the AVX-512 instruction set */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/generator_avx512/cnProc_gen_avx512 b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/generator_avx512/cnProc_gen_avx512 deleted file mode 100644 index 776516a4d7..0000000000 Binary files a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/generator_avx512/cnProc_gen_avx512 and /dev/null differ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/adxintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/adxintrin.h new file mode 100644 index 0000000000..7acdaf4ab6 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/adxintrin.h @@ -0,0 +1,81 @@ +/* Copyright (C) 2012-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#if !defined _X86INTRIN_H_INCLUDED && !defined _IMMINTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _ADXINTRIN_H_INCLUDED +#define _ADXINTRIN_H_INCLUDED + +extern __inline unsigned char +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_subborrow_u32 (unsigned char __CF, unsigned int __X, + unsigned int __Y, unsigned int *__P) +{ + return __builtin_ia32_sbb_u32 (__CF, __X, __Y, __P); +} + +extern __inline unsigned char +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_addcarry_u32 (unsigned char __CF, unsigned int __X, + unsigned int __Y, unsigned int *__P) +{ + return __builtin_ia32_addcarryx_u32 (__CF, __X, __Y, __P); +} + +extern __inline unsigned char +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_addcarryx_u32 (unsigned char __CF, unsigned int __X, + unsigned int __Y, unsigned int *__P) +{ + return __builtin_ia32_addcarryx_u32 (__CF, __X, __Y, __P); +} + +#ifdef __x86_64__ +extern __inline unsigned char +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_subborrow_u64 (unsigned char __CF, unsigned long long __X, + unsigned long long __Y, unsigned long long *__P) +{ + return __builtin_ia32_sbb_u64 (__CF, __X, __Y, __P); +} + +extern __inline unsigned char +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_addcarry_u64 (unsigned char __CF, unsigned long long __X, + unsigned long long __Y, unsigned long long *__P) +{ + return __builtin_ia32_addcarryx_u64 (__CF, __X, __Y, __P); +} + +extern __inline unsigned char +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_addcarryx_u64 (unsigned char __CF, unsigned long long __X, + unsigned long long __Y, unsigned long long *__P) +{ + return __builtin_ia32_addcarryx_u64 (__CF, __X, __Y, __P); +} +#endif + +#endif /* _ADXINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/ammintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/ammintrin.h new file mode 100644 index 0000000000..0093092781 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/ammintrin.h @@ -0,0 +1,93 @@ +/* Copyright (C) 2007-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +/* Implemented from the specification included in the AMD Programmers + Manual Update, version 2.x */ + +#ifndef _AMMINTRIN_H_INCLUDED +#define _AMMINTRIN_H_INCLUDED + +/* We need definitions from the SSE3, SSE2 and SSE header files*/ +#include + +#ifndef __SSE4A__ +#pragma GCC push_options +#pragma GCC target("sse4a") +#define __DISABLE_SSE4A__ +#endif /* __SSE4A__ */ + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_stream_sd (double * __P, __m128d __Y) +{ + __builtin_ia32_movntsd (__P, (__v2df) __Y); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_stream_ss (float * __P, __m128 __Y) +{ + __builtin_ia32_movntss (__P, (__v4sf) __Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_extract_si64 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_extrq ((__v2di) __X, (__v16qi) __Y); +} + +#ifdef __OPTIMIZE__ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_extracti_si64 (__m128i __X, unsigned const int __I, unsigned const int __L) +{ + return (__m128i) __builtin_ia32_extrqi ((__v2di) __X, __I, __L); +} +#else +#define _mm_extracti_si64(X, I, L) \ + ((__m128i) __builtin_ia32_extrqi ((__v2di)(__m128i)(X), \ + (unsigned int)(I), (unsigned int)(L))) +#endif + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_insert_si64 (__m128i __X,__m128i __Y) +{ + return (__m128i) __builtin_ia32_insertq ((__v2di)__X, (__v2di)__Y); +} + +#ifdef __OPTIMIZE__ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_inserti_si64(__m128i __X, __m128i __Y, unsigned const int __I, unsigned const int __L) +{ + return (__m128i) __builtin_ia32_insertqi ((__v2di)__X, (__v2di)__Y, __I, __L); +} +#else +#define _mm_inserti_si64(X, Y, I, L) \ + ((__m128i) __builtin_ia32_insertqi ((__v2di)(__m128i)(X), \ + (__v2di)(__m128i)(Y), \ + (unsigned int)(I), (unsigned int)(L))) +#endif + +#ifdef __DISABLE_SSE4A__ +#undef __DISABLE_SSE4A__ +#pragma GCC pop_options +#endif /* __DISABLE_SSE4A__ */ + +#endif /* _AMMINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx2intrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx2intrin.h new file mode 100644 index 0000000000..82f170a3d6 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx2intrin.h @@ -0,0 +1,1908 @@ +/* Copyright (C) 2011-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _AVX2INTRIN_H_INCLUDED +#define _AVX2INTRIN_H_INCLUDED + +#ifndef __AVX2__ +#pragma GCC push_options +#pragma GCC target("avx2") +#define __DISABLE_AVX2__ +#endif /* __AVX2__ */ + +/* Sum absolute 8-bit integer difference of adjacent groups of 4 + byte integers in the first 2 operands. Starting offsets within + operands are determined by the 3rd mask operand. */ +#ifdef __OPTIMIZE__ +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mpsadbw_epu8 (__m256i __X, __m256i __Y, const int __M) +{ + return (__m256i) __builtin_ia32_mpsadbw256 ((__v32qi)__X, + (__v32qi)__Y, __M); +} +#else +#define _mm256_mpsadbw_epu8(X, Y, M) \ + ((__m256i) __builtin_ia32_mpsadbw256 ((__v32qi)(__m256i)(X), \ + (__v32qi)(__m256i)(Y), (int)(M))) +#endif + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_abs_epi8 (__m256i __A) +{ + return (__m256i)__builtin_ia32_pabsb256 ((__v32qi)__A); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_abs_epi16 (__m256i __A) +{ + return (__m256i)__builtin_ia32_pabsw256 ((__v16hi)__A); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_abs_epi32 (__m256i __A) +{ + return (__m256i)__builtin_ia32_pabsd256 ((__v8si)__A); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_packs_epi32 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_packssdw256 ((__v8si)__A, (__v8si)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_packs_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_packsswb256 ((__v16hi)__A, (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_packus_epi32 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_packusdw256 ((__v8si)__A, (__v8si)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_packus_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_packuswb256 ((__v16hi)__A, (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_add_epi8 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v32qu)__A + (__v32qu)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_add_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v16hu)__A + (__v16hu)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_add_epi32 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v8su)__A + (__v8su)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_add_epi64 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v4du)__A + (__v4du)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_adds_epi8 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_paddsb256 ((__v32qi)__A, (__v32qi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_adds_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_paddsw256 ((__v16hi)__A, (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_adds_epu8 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_paddusb256 ((__v32qi)__A, (__v32qi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_adds_epu16 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_paddusw256 ((__v16hi)__A, (__v16hi)__B); +} + +#ifdef __OPTIMIZE__ +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_alignr_epi8 (__m256i __A, __m256i __B, const int __N) +{ + return (__m256i) __builtin_ia32_palignr256 ((__v4di)__A, + (__v4di)__B, + __N * 8); +} +#else +/* In that case (__N*8) will be in vreg, and insn will not be matched. */ +/* Use define instead */ +#define _mm256_alignr_epi8(A, B, N) \ + ((__m256i) __builtin_ia32_palignr256 ((__v4di)(__m256i)(A), \ + (__v4di)(__m256i)(B), \ + (int)(N) * 8)) +#endif + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_and_si256 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v4du)__A & (__v4du)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_andnot_si256 (__m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_andnotsi256 ((__v4di)__A, (__v4di)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_avg_epu8 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pavgb256 ((__v32qi)__A, (__v32qi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_avg_epu16 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pavgw256 ((__v16hi)__A, (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_blendv_epi8 (__m256i __X, __m256i __Y, __m256i __M) +{ + return (__m256i) __builtin_ia32_pblendvb256 ((__v32qi)__X, + (__v32qi)__Y, + (__v32qi)__M); +} + +#ifdef __OPTIMIZE__ +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_blend_epi16 (__m256i __X, __m256i __Y, const int __M) +{ + return (__m256i) __builtin_ia32_pblendw256 ((__v16hi)__X, + (__v16hi)__Y, + __M); +} +#else +#define _mm256_blend_epi16(X, Y, M) \ + ((__m256i) __builtin_ia32_pblendw256 ((__v16hi)(__m256i)(X), \ + (__v16hi)(__m256i)(Y), (int)(M))) +#endif + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpeq_epi8 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v32qi)__A == (__v32qi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpeq_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v16hi)__A == (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpeq_epi32 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v8si)__A == (__v8si)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpeq_epi64 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v4di)__A == (__v4di)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpgt_epi8 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v32qi)__A > (__v32qi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpgt_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v16hi)__A > (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpgt_epi32 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v8si)__A > (__v8si)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpgt_epi64 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v4di)__A > (__v4di)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_hadd_epi16 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_phaddw256 ((__v16hi)__X, + (__v16hi)__Y); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_hadd_epi32 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_phaddd256 ((__v8si)__X, (__v8si)__Y); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_hadds_epi16 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_phaddsw256 ((__v16hi)__X, + (__v16hi)__Y); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_hsub_epi16 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_phsubw256 ((__v16hi)__X, + (__v16hi)__Y); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_hsub_epi32 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_phsubd256 ((__v8si)__X, (__v8si)__Y); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_hsubs_epi16 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_phsubsw256 ((__v16hi)__X, + (__v16hi)__Y); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maddubs_epi16 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_pmaddubsw256 ((__v32qi)__X, + (__v32qi)__Y); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_madd_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pmaddwd256 ((__v16hi)__A, + (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_max_epi8 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pmaxsb256 ((__v32qi)__A, (__v32qi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_max_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pmaxsw256 ((__v16hi)__A, (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_max_epi32 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pmaxsd256 ((__v8si)__A, (__v8si)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_max_epu8 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pmaxub256 ((__v32qi)__A, (__v32qi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_max_epu16 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pmaxuw256 ((__v16hi)__A, (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_max_epu32 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pmaxud256 ((__v8si)__A, (__v8si)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_min_epi8 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pminsb256 ((__v32qi)__A, (__v32qi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_min_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pminsw256 ((__v16hi)__A, (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_min_epi32 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pminsd256 ((__v8si)__A, (__v8si)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_min_epu8 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pminub256 ((__v32qi)__A, (__v32qi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_min_epu16 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pminuw256 ((__v16hi)__A, (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_min_epu32 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pminud256 ((__v8si)__A, (__v8si)__B); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_movemask_epi8 (__m256i __A) +{ + return __builtin_ia32_pmovmskb256 ((__v32qi)__A); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi8_epi16 (__m128i __X) +{ + return (__m256i) __builtin_ia32_pmovsxbw256 ((__v16qi)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi8_epi32 (__m128i __X) +{ + return (__m256i) __builtin_ia32_pmovsxbd256 ((__v16qi)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi8_epi64 (__m128i __X) +{ + return (__m256i) __builtin_ia32_pmovsxbq256 ((__v16qi)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi16_epi32 (__m128i __X) +{ + return (__m256i) __builtin_ia32_pmovsxwd256 ((__v8hi)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi16_epi64 (__m128i __X) +{ + return (__m256i) __builtin_ia32_pmovsxwq256 ((__v8hi)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi32_epi64 (__m128i __X) +{ + return (__m256i) __builtin_ia32_pmovsxdq256 ((__v4si)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepu8_epi16 (__m128i __X) +{ + return (__m256i) __builtin_ia32_pmovzxbw256 ((__v16qi)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepu8_epi32 (__m128i __X) +{ + return (__m256i) __builtin_ia32_pmovzxbd256 ((__v16qi)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepu8_epi64 (__m128i __X) +{ + return (__m256i) __builtin_ia32_pmovzxbq256 ((__v16qi)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepu16_epi32 (__m128i __X) +{ + return (__m256i) __builtin_ia32_pmovzxwd256 ((__v8hi)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepu16_epi64 (__m128i __X) +{ + return (__m256i) __builtin_ia32_pmovzxwq256 ((__v8hi)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepu32_epi64 (__m128i __X) +{ + return (__m256i) __builtin_ia32_pmovzxdq256 ((__v4si)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mul_epi32 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_pmuldq256 ((__v8si)__X, (__v8si)__Y); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mulhrs_epi16 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_pmulhrsw256 ((__v16hi)__X, + (__v16hi)__Y); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mulhi_epu16 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pmulhuw256 ((__v16hi)__A, (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mulhi_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pmulhw256 ((__v16hi)__A, (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mullo_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v16hu)__A * (__v16hu)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mullo_epi32 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v8su)__A * (__v8su)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mul_epu32 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pmuludq256 ((__v8si)__A, (__v8si)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_or_si256 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v4du)__A | (__v4du)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sad_epu8 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_psadbw256 ((__v32qi)__A, (__v32qi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_shuffle_epi8 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_pshufb256 ((__v32qi)__X, + (__v32qi)__Y); +} + +#ifdef __OPTIMIZE__ +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_shuffle_epi32 (__m256i __A, const int __mask) +{ + return (__m256i)__builtin_ia32_pshufd256 ((__v8si)__A, __mask); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_shufflehi_epi16 (__m256i __A, const int __mask) +{ + return (__m256i)__builtin_ia32_pshufhw256 ((__v16hi)__A, __mask); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_shufflelo_epi16 (__m256i __A, const int __mask) +{ + return (__m256i)__builtin_ia32_pshuflw256 ((__v16hi)__A, __mask); +} +#else +#define _mm256_shuffle_epi32(A, N) \ + ((__m256i)__builtin_ia32_pshufd256 ((__v8si)(__m256i)(A), (int)(N))) +#define _mm256_shufflehi_epi16(A, N) \ + ((__m256i)__builtin_ia32_pshufhw256 ((__v16hi)(__m256i)(A), (int)(N))) +#define _mm256_shufflelo_epi16(A, N) \ + ((__m256i)__builtin_ia32_pshuflw256 ((__v16hi)(__m256i)(A), (int)(N))) +#endif + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sign_epi8 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_psignb256 ((__v32qi)__X, (__v32qi)__Y); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sign_epi16 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_psignw256 ((__v16hi)__X, (__v16hi)__Y); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sign_epi32 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_psignd256 ((__v8si)__X, (__v8si)__Y); +} + +#ifdef __OPTIMIZE__ +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_bslli_epi128 (__m256i __A, const int __N) +{ + return (__m256i)__builtin_ia32_pslldqi256 (__A, __N * 8); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_slli_si256 (__m256i __A, const int __N) +{ + return (__m256i)__builtin_ia32_pslldqi256 (__A, __N * 8); +} +#else +#define _mm256_bslli_epi128(A, N) \ + ((__m256i)__builtin_ia32_pslldqi256 ((__m256i)(A), (int)(N) * 8)) +#define _mm256_slli_si256(A, N) \ + ((__m256i)__builtin_ia32_pslldqi256 ((__m256i)(A), (int)(N) * 8)) +#endif + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_slli_epi16 (__m256i __A, int __B) +{ + return (__m256i)__builtin_ia32_psllwi256 ((__v16hi)__A, __B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sll_epi16 (__m256i __A, __m128i __B) +{ + return (__m256i)__builtin_ia32_psllw256((__v16hi)__A, (__v8hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_slli_epi32 (__m256i __A, int __B) +{ + return (__m256i)__builtin_ia32_pslldi256 ((__v8si)__A, __B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sll_epi32 (__m256i __A, __m128i __B) +{ + return (__m256i)__builtin_ia32_pslld256((__v8si)__A, (__v4si)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_slli_epi64 (__m256i __A, int __B) +{ + return (__m256i)__builtin_ia32_psllqi256 ((__v4di)__A, __B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sll_epi64 (__m256i __A, __m128i __B) +{ + return (__m256i)__builtin_ia32_psllq256((__v4di)__A, (__v2di)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_srai_epi16 (__m256i __A, int __B) +{ + return (__m256i)__builtin_ia32_psrawi256 ((__v16hi)__A, __B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sra_epi16 (__m256i __A, __m128i __B) +{ + return (__m256i)__builtin_ia32_psraw256 ((__v16hi)__A, (__v8hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_srai_epi32 (__m256i __A, int __B) +{ + return (__m256i)__builtin_ia32_psradi256 ((__v8si)__A, __B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sra_epi32 (__m256i __A, __m128i __B) +{ + return (__m256i)__builtin_ia32_psrad256 ((__v8si)__A, (__v4si)__B); +} + +#ifdef __OPTIMIZE__ +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_bsrli_epi128 (__m256i __A, const int __N) +{ + return (__m256i)__builtin_ia32_psrldqi256 (__A, __N * 8); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_srli_si256 (__m256i __A, const int __N) +{ + return (__m256i)__builtin_ia32_psrldqi256 (__A, __N * 8); +} +#else +#define _mm256_bsrli_epi128(A, N) \ + ((__m256i)__builtin_ia32_psrldqi256 ((__m256i)(A), (int)(N) * 8)) +#define _mm256_srli_si256(A, N) \ + ((__m256i)__builtin_ia32_psrldqi256 ((__m256i)(A), (int)(N) * 8)) +#endif + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_srli_epi16 (__m256i __A, int __B) +{ + return (__m256i)__builtin_ia32_psrlwi256 ((__v16hi)__A, __B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_srl_epi16 (__m256i __A, __m128i __B) +{ + return (__m256i)__builtin_ia32_psrlw256((__v16hi)__A, (__v8hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_srli_epi32 (__m256i __A, int __B) +{ + return (__m256i)__builtin_ia32_psrldi256 ((__v8si)__A, __B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_srl_epi32 (__m256i __A, __m128i __B) +{ + return (__m256i)__builtin_ia32_psrld256((__v8si)__A, (__v4si)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_srli_epi64 (__m256i __A, int __B) +{ + return (__m256i)__builtin_ia32_psrlqi256 ((__v4di)__A, __B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_srl_epi64 (__m256i __A, __m128i __B) +{ + return (__m256i)__builtin_ia32_psrlq256((__v4di)__A, (__v2di)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sub_epi8 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v32qu)__A - (__v32qu)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sub_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v16hu)__A - (__v16hu)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sub_epi32 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v8su)__A - (__v8su)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sub_epi64 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v4du)__A - (__v4du)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_subs_epi8 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_psubsb256 ((__v32qi)__A, (__v32qi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_subs_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_psubsw256 ((__v16hi)__A, (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_subs_epu8 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_psubusb256 ((__v32qi)__A, (__v32qi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_subs_epu16 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_psubusw256 ((__v16hi)__A, (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_unpackhi_epi8 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_punpckhbw256 ((__v32qi)__A, (__v32qi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_unpackhi_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_punpckhwd256 ((__v16hi)__A, (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_unpackhi_epi32 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_punpckhdq256 ((__v8si)__A, (__v8si)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_unpackhi_epi64 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_punpckhqdq256 ((__v4di)__A, (__v4di)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_unpacklo_epi8 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_punpcklbw256 ((__v32qi)__A, (__v32qi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_unpacklo_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_punpcklwd256 ((__v16hi)__A, (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_unpacklo_epi32 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_punpckldq256 ((__v8si)__A, (__v8si)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_unpacklo_epi64 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_punpcklqdq256 ((__v4di)__A, (__v4di)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_xor_si256 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v4du)__A ^ (__v4du)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_stream_load_si256 (__m256i const *__X) +{ + return (__m256i) __builtin_ia32_movntdqa256 ((__v4di *) __X); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_broadcastss_ps (__m128 __X) +{ + return (__m128) __builtin_ia32_vbroadcastss_ps ((__v4sf)__X); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcastss_ps (__m128 __X) +{ + return (__m256) __builtin_ia32_vbroadcastss_ps256 ((__v4sf)__X); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcastsd_pd (__m128d __X) +{ + return (__m256d) __builtin_ia32_vbroadcastsd_pd256 ((__v2df)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcastsi128_si256 (__m128i __X) +{ + return (__m256i) __builtin_ia32_vbroadcastsi256 ((__v2di)__X); +} + +#ifdef __OPTIMIZE__ +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_blend_epi32 (__m128i __X, __m128i __Y, const int __M) +{ + return (__m128i) __builtin_ia32_pblendd128 ((__v4si)__X, + (__v4si)__Y, + __M); +} +#else +#define _mm_blend_epi32(X, Y, M) \ + ((__m128i) __builtin_ia32_pblendd128 ((__v4si)(__m128i)(X), \ + (__v4si)(__m128i)(Y), (int)(M))) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_blend_epi32 (__m256i __X, __m256i __Y, const int __M) +{ + return (__m256i) __builtin_ia32_pblendd256 ((__v8si)__X, + (__v8si)__Y, + __M); +} +#else +#define _mm256_blend_epi32(X, Y, M) \ + ((__m256i) __builtin_ia32_pblendd256 ((__v8si)(__m256i)(X), \ + (__v8si)(__m256i)(Y), (int)(M))) +#endif + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcastb_epi8 (__m128i __X) +{ + return (__m256i) __builtin_ia32_pbroadcastb256 ((__v16qi)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcastw_epi16 (__m128i __X) +{ + return (__m256i) __builtin_ia32_pbroadcastw256 ((__v8hi)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcastd_epi32 (__m128i __X) +{ + return (__m256i) __builtin_ia32_pbroadcastd256 ((__v4si)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcastq_epi64 (__m128i __X) +{ + return (__m256i) __builtin_ia32_pbroadcastq256 ((__v2di)__X); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_broadcastb_epi8 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pbroadcastb128 ((__v16qi)__X); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_broadcastw_epi16 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pbroadcastw128 ((__v8hi)__X); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_broadcastd_epi32 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pbroadcastd128 ((__v4si)__X); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_broadcastq_epi64 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pbroadcastq128 ((__v2di)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permutevar8x32_epi32 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_permvarsi256 ((__v8si)__X, (__v8si)__Y); +} + +#ifdef __OPTIMIZE__ +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permute4x64_pd (__m256d __X, const int __M) +{ + return (__m256d) __builtin_ia32_permdf256 ((__v4df)__X, __M); +} +#else +#define _mm256_permute4x64_pd(X, M) \ + ((__m256d) __builtin_ia32_permdf256 ((__v4df)(__m256d)(X), (int)(M))) +#endif + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permutevar8x32_ps (__m256 __X, __m256i __Y) +{ + return (__m256) __builtin_ia32_permvarsf256 ((__v8sf)__X, (__v8si)__Y); +} + +#ifdef __OPTIMIZE__ +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permute4x64_epi64 (__m256i __X, const int __M) +{ + return (__m256i) __builtin_ia32_permdi256 ((__v4di)__X, __M); +} +#else +#define _mm256_permute4x64_epi64(X, M) \ + ((__m256i) __builtin_ia32_permdi256 ((__v4di)(__m256i)(X), (int)(M))) +#endif + + +#ifdef __OPTIMIZE__ +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permute2x128_si256 (__m256i __X, __m256i __Y, const int __M) +{ + return (__m256i) __builtin_ia32_permti256 ((__v4di)__X, (__v4di)__Y, __M); +} +#else +#define _mm256_permute2x128_si256(X, Y, M) \ + ((__m256i) __builtin_ia32_permti256 ((__v4di)(__m256i)(X), (__v4di)(__m256i)(Y), (int)(M))) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_extracti128_si256 (__m256i __X, const int __M) +{ + return (__m128i) __builtin_ia32_extract128i256 ((__v4di)__X, __M); +} +#else +#define _mm256_extracti128_si256(X, M) \ + ((__m128i) __builtin_ia32_extract128i256 ((__v4di)(__m256i)(X), (int)(M))) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_inserti128_si256 (__m256i __X, __m128i __Y, const int __M) +{ + return (__m256i) __builtin_ia32_insert128i256 ((__v4di)__X, (__v2di)__Y, __M); +} +#else +#define _mm256_inserti128_si256(X, Y, M) \ + ((__m256i) __builtin_ia32_insert128i256 ((__v4di)(__m256i)(X), \ + (__v2di)(__m128i)(Y), \ + (int)(M))) +#endif + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskload_epi32 (int const *__X, __m256i __M ) +{ + return (__m256i) __builtin_ia32_maskloadd256 ((const __v8si *)__X, + (__v8si)__M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskload_epi64 (long long const *__X, __m256i __M ) +{ + return (__m256i) __builtin_ia32_maskloadq256 ((const __v4di *)__X, + (__v4di)__M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskload_epi32 (int const *__X, __m128i __M ) +{ + return (__m128i) __builtin_ia32_maskloadd ((const __v4si *)__X, + (__v4si)__M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskload_epi64 (long long const *__X, __m128i __M ) +{ + return (__m128i) __builtin_ia32_maskloadq ((const __v2di *)__X, + (__v2di)__M); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskstore_epi32 (int *__X, __m256i __M, __m256i __Y ) +{ + __builtin_ia32_maskstored256 ((__v8si *)__X, (__v8si)__M, (__v8si)__Y); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskstore_epi64 (long long *__X, __m256i __M, __m256i __Y ) +{ + __builtin_ia32_maskstoreq256 ((__v4di *)__X, (__v4di)__M, (__v4di)__Y); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskstore_epi32 (int *__X, __m128i __M, __m128i __Y ) +{ + __builtin_ia32_maskstored ((__v4si *)__X, (__v4si)__M, (__v4si)__Y); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskstore_epi64 (long long *__X, __m128i __M, __m128i __Y ) +{ + __builtin_ia32_maskstoreq (( __v2di *)__X, (__v2di)__M, (__v2di)__Y); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sllv_epi32 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_psllv8si ((__v8si)__X, (__v8si)__Y); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sllv_epi32 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_psllv4si ((__v4si)__X, (__v4si)__Y); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sllv_epi64 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_psllv4di ((__v4di)__X, (__v4di)__Y); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sllv_epi64 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_psllv2di ((__v2di)__X, (__v2di)__Y); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_srav_epi32 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_psrav8si ((__v8si)__X, (__v8si)__Y); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srav_epi32 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_psrav4si ((__v4si)__X, (__v4si)__Y); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_srlv_epi32 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_psrlv8si ((__v8si)__X, (__v8si)__Y); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srlv_epi32 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_psrlv4si ((__v4si)__X, (__v4si)__Y); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_srlv_epi64 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_psrlv4di ((__v4di)__X, (__v4di)__Y); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srlv_epi64 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_psrlv2di ((__v2di)__X, (__v2di)__Y); +} + +#ifdef __OPTIMIZE__ +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_i32gather_pd (double const *__base, __m128i __index, const int __scale) +{ + __v2df __zero = _mm_setzero_pd (); + __v2df __mask = _mm_cmpeq_pd (__zero, __zero); + + return (__m128d) __builtin_ia32_gathersiv2df (_mm_undefined_pd (), + __base, + (__v4si)__index, + __mask, + __scale); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_i32gather_pd (__m128d __src, double const *__base, __m128i __index, + __m128d __mask, const int __scale) +{ + return (__m128d) __builtin_ia32_gathersiv2df ((__v2df)__src, + __base, + (__v4si)__index, + (__v2df)__mask, + __scale); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_i32gather_pd (double const *__base, __m128i __index, const int __scale) +{ + __v4df __zero = _mm256_setzero_pd (); + __v4df __mask = _mm256_cmp_pd (__zero, __zero, _CMP_EQ_OQ); + + return (__m256d) __builtin_ia32_gathersiv4df (_mm256_undefined_pd (), + __base, + (__v4si)__index, + __mask, + __scale); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_i32gather_pd (__m256d __src, double const *__base, + __m128i __index, __m256d __mask, const int __scale) +{ + return (__m256d) __builtin_ia32_gathersiv4df ((__v4df)__src, + __base, + (__v4si)__index, + (__v4df)__mask, + __scale); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_i64gather_pd (double const *__base, __m128i __index, const int __scale) +{ + __v2df __src = _mm_setzero_pd (); + __v2df __mask = _mm_cmpeq_pd (__src, __src); + + return (__m128d) __builtin_ia32_gatherdiv2df (__src, + __base, + (__v2di)__index, + __mask, + __scale); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_i64gather_pd (__m128d __src, double const *__base, __m128i __index, + __m128d __mask, const int __scale) +{ + return (__m128d) __builtin_ia32_gatherdiv2df ((__v2df)__src, + __base, + (__v2di)__index, + (__v2df)__mask, + __scale); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_i64gather_pd (double const *__base, __m256i __index, const int __scale) +{ + __v4df __src = _mm256_setzero_pd (); + __v4df __mask = _mm256_cmp_pd (__src, __src, _CMP_EQ_OQ); + + return (__m256d) __builtin_ia32_gatherdiv4df (__src, + __base, + (__v4di)__index, + __mask, + __scale); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_i64gather_pd (__m256d __src, double const *__base, + __m256i __index, __m256d __mask, const int __scale) +{ + return (__m256d) __builtin_ia32_gatherdiv4df ((__v4df)__src, + __base, + (__v4di)__index, + (__v4df)__mask, + __scale); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_i32gather_ps (float const *__base, __m128i __index, const int __scale) +{ + __v4sf __src = _mm_setzero_ps (); + __v4sf __mask = _mm_cmpeq_ps (__src, __src); + + return (__m128) __builtin_ia32_gathersiv4sf (__src, + __base, + (__v4si)__index, + __mask, + __scale); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_i32gather_ps (__m128 __src, float const *__base, __m128i __index, + __m128 __mask, const int __scale) +{ + return (__m128) __builtin_ia32_gathersiv4sf ((__v4sf)__src, + __base, + (__v4si)__index, + (__v4sf)__mask, + __scale); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_i32gather_ps (float const *__base, __m256i __index, const int __scale) +{ + __v8sf __src = _mm256_setzero_ps (); + __v8sf __mask = _mm256_cmp_ps (__src, __src, _CMP_EQ_OQ); + + return (__m256) __builtin_ia32_gathersiv8sf (__src, + __base, + (__v8si)__index, + __mask, + __scale); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_i32gather_ps (__m256 __src, float const *__base, + __m256i __index, __m256 __mask, const int __scale) +{ + return (__m256) __builtin_ia32_gathersiv8sf ((__v8sf)__src, + __base, + (__v8si)__index, + (__v8sf)__mask, + __scale); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_i64gather_ps (float const *__base, __m128i __index, const int __scale) +{ + __v4sf __src = _mm_setzero_ps (); + __v4sf __mask = _mm_cmpeq_ps (__src, __src); + + return (__m128) __builtin_ia32_gatherdiv4sf (__src, + __base, + (__v2di)__index, + __mask, + __scale); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_i64gather_ps (__m128 __src, float const *__base, __m128i __index, + __m128 __mask, const int __scale) +{ + return (__m128) __builtin_ia32_gatherdiv4sf ((__v4sf)__src, + __base, + (__v2di)__index, + (__v4sf)__mask, + __scale); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_i64gather_ps (float const *__base, __m256i __index, const int __scale) +{ + __v4sf __src = _mm_setzero_ps (); + __v4sf __mask = _mm_cmpeq_ps (__src, __src); + + return (__m128) __builtin_ia32_gatherdiv4sf256 (__src, + __base, + (__v4di)__index, + __mask, + __scale); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_i64gather_ps (__m128 __src, float const *__base, + __m256i __index, __m128 __mask, const int __scale) +{ + return (__m128) __builtin_ia32_gatherdiv4sf256 ((__v4sf)__src, + __base, + (__v4di)__index, + (__v4sf)__mask, + __scale); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_i32gather_epi64 (long long int const *__base, + __m128i __index, const int __scale) +{ + __v2di __src = __extension__ (__v2di){ 0, 0 }; + __v2di __mask = __extension__ (__v2di){ ~0, ~0 }; + + return (__m128i) __builtin_ia32_gathersiv2di (__src, + __base, + (__v4si)__index, + __mask, + __scale); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_i32gather_epi64 (__m128i __src, long long int const *__base, + __m128i __index, __m128i __mask, const int __scale) +{ + return (__m128i) __builtin_ia32_gathersiv2di ((__v2di)__src, + __base, + (__v4si)__index, + (__v2di)__mask, + __scale); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_i32gather_epi64 (long long int const *__base, + __m128i __index, const int __scale) +{ + __v4di __src = __extension__ (__v4di){ 0, 0, 0, 0 }; + __v4di __mask = __extension__ (__v4di){ ~0, ~0, ~0, ~0 }; + + return (__m256i) __builtin_ia32_gathersiv4di (__src, + __base, + (__v4si)__index, + __mask, + __scale); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_i32gather_epi64 (__m256i __src, long long int const *__base, + __m128i __index, __m256i __mask, + const int __scale) +{ + return (__m256i) __builtin_ia32_gathersiv4di ((__v4di)__src, + __base, + (__v4si)__index, + (__v4di)__mask, + __scale); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_i64gather_epi64 (long long int const *__base, + __m128i __index, const int __scale) +{ + __v2di __src = __extension__ (__v2di){ 0, 0 }; + __v2di __mask = __extension__ (__v2di){ ~0, ~0 }; + + return (__m128i) __builtin_ia32_gatherdiv2di (__src, + __base, + (__v2di)__index, + __mask, + __scale); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_i64gather_epi64 (__m128i __src, long long int const *__base, + __m128i __index, __m128i __mask, const int __scale) +{ + return (__m128i) __builtin_ia32_gatherdiv2di ((__v2di)__src, + __base, + (__v2di)__index, + (__v2di)__mask, + __scale); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_i64gather_epi64 (long long int const *__base, + __m256i __index, const int __scale) +{ + __v4di __src = __extension__ (__v4di){ 0, 0, 0, 0 }; + __v4di __mask = __extension__ (__v4di){ ~0, ~0, ~0, ~0 }; + + return (__m256i) __builtin_ia32_gatherdiv4di (__src, + __base, + (__v4di)__index, + __mask, + __scale); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_i64gather_epi64 (__m256i __src, long long int const *__base, + __m256i __index, __m256i __mask, + const int __scale) +{ + return (__m256i) __builtin_ia32_gatherdiv4di ((__v4di)__src, + __base, + (__v4di)__index, + (__v4di)__mask, + __scale); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_i32gather_epi32 (int const *__base, __m128i __index, const int __scale) +{ + __v4si __src = __extension__ (__v4si){ 0, 0, 0, 0 }; + __v4si __mask = __extension__ (__v4si){ ~0, ~0, ~0, ~0 }; + + return (__m128i) __builtin_ia32_gathersiv4si (__src, + __base, + (__v4si)__index, + __mask, + __scale); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_i32gather_epi32 (__m128i __src, int const *__base, __m128i __index, + __m128i __mask, const int __scale) +{ + return (__m128i) __builtin_ia32_gathersiv4si ((__v4si)__src, + __base, + (__v4si)__index, + (__v4si)__mask, + __scale); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_i32gather_epi32 (int const *__base, __m256i __index, const int __scale) +{ + __v8si __src = __extension__ (__v8si){ 0, 0, 0, 0, 0, 0, 0, 0 }; + __v8si __mask = __extension__ (__v8si){ ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0 }; + + return (__m256i) __builtin_ia32_gathersiv8si (__src, + __base, + (__v8si)__index, + __mask, + __scale); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_i32gather_epi32 (__m256i __src, int const *__base, + __m256i __index, __m256i __mask, + const int __scale) +{ + return (__m256i) __builtin_ia32_gathersiv8si ((__v8si)__src, + __base, + (__v8si)__index, + (__v8si)__mask, + __scale); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_i64gather_epi32 (int const *__base, __m128i __index, const int __scale) +{ + __v4si __src = __extension__ (__v4si){ 0, 0, 0, 0 }; + __v4si __mask = __extension__ (__v4si){ ~0, ~0, ~0, ~0 }; + + return (__m128i) __builtin_ia32_gatherdiv4si (__src, + __base, + (__v2di)__index, + __mask, + __scale); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_i64gather_epi32 (__m128i __src, int const *__base, __m128i __index, + __m128i __mask, const int __scale) +{ + return (__m128i) __builtin_ia32_gatherdiv4si ((__v4si)__src, + __base, + (__v2di)__index, + (__v4si)__mask, + __scale); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_i64gather_epi32 (int const *__base, __m256i __index, const int __scale) +{ + __v4si __src = __extension__ (__v4si){ 0, 0, 0, 0 }; + __v4si __mask = __extension__ (__v4si){ ~0, ~0, ~0, ~0 }; + + return (__m128i) __builtin_ia32_gatherdiv4si256 (__src, + __base, + (__v4di)__index, + __mask, + __scale); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_i64gather_epi32 (__m128i __src, int const *__base, + __m256i __index, __m128i __mask, + const int __scale) +{ + return (__m128i) __builtin_ia32_gatherdiv4si256 ((__v4si)__src, + __base, + (__v4di)__index, + (__v4si)__mask, + __scale); +} +#else /* __OPTIMIZE__ */ +#define _mm_i32gather_pd(BASE, INDEX, SCALE) \ + (__m128d) __builtin_ia32_gathersiv2df ((__v2df) _mm_setzero_pd (), \ + (double const *)BASE, \ + (__v4si)(__m128i)INDEX, \ + (__v2df)_mm_set1_pd( \ + (double)(long long int) -1), \ + (int)SCALE) + +#define _mm_mask_i32gather_pd(SRC, BASE, INDEX, MASK, SCALE) \ + (__m128d) __builtin_ia32_gathersiv2df ((__v2df)(__m128d)SRC, \ + (double const *)BASE, \ + (__v4si)(__m128i)INDEX, \ + (__v2df)(__m128d)MASK, \ + (int)SCALE) + +#define _mm256_i32gather_pd(BASE, INDEX, SCALE) \ + (__m256d) __builtin_ia32_gathersiv4df ((__v4df) _mm256_setzero_pd (), \ + (double const *)BASE, \ + (__v4si)(__m128i)INDEX, \ + (__v4df)_mm256_set1_pd( \ + (double)(long long int) -1), \ + (int)SCALE) + +#define _mm256_mask_i32gather_pd(SRC, BASE, INDEX, MASK, SCALE) \ + (__m256d) __builtin_ia32_gathersiv4df ((__v4df)(__m256d)SRC, \ + (double const *)BASE, \ + (__v4si)(__m128i)INDEX, \ + (__v4df)(__m256d)MASK, \ + (int)SCALE) + +#define _mm_i64gather_pd(BASE, INDEX, SCALE) \ + (__m128d) __builtin_ia32_gatherdiv2df ((__v2df) _mm_setzero_pd (), \ + (double const *)BASE, \ + (__v2di)(__m128i)INDEX, \ + (__v2df)_mm_set1_pd( \ + (double)(long long int) -1), \ + (int)SCALE) + +#define _mm_mask_i64gather_pd(SRC, BASE, INDEX, MASK, SCALE) \ + (__m128d) __builtin_ia32_gatherdiv2df ((__v2df)(__m128d)SRC, \ + (double const *)BASE, \ + (__v2di)(__m128i)INDEX, \ + (__v2df)(__m128d)MASK, \ + (int)SCALE) + +#define _mm256_i64gather_pd(BASE, INDEX, SCALE) \ + (__m256d) __builtin_ia32_gatherdiv4df ((__v4df) _mm256_setzero_pd (), \ + (double const *)BASE, \ + (__v4di)(__m256i)INDEX, \ + (__v4df)_mm256_set1_pd( \ + (double)(long long int) -1), \ + (int)SCALE) + +#define _mm256_mask_i64gather_pd(SRC, BASE, INDEX, MASK, SCALE) \ + (__m256d) __builtin_ia32_gatherdiv4df ((__v4df)(__m256d)SRC, \ + (double const *)BASE, \ + (__v4di)(__m256i)INDEX, \ + (__v4df)(__m256d)MASK, \ + (int)SCALE) + +#define _mm_i32gather_ps(BASE, INDEX, SCALE) \ + (__m128) __builtin_ia32_gathersiv4sf ((__v4sf) _mm_setzero_ps (), \ + (float const *)BASE, \ + (__v4si)(__m128i)INDEX, \ + _mm_set1_ps ((float)(int) -1), \ + (int)SCALE) + +#define _mm_mask_i32gather_ps(SRC, BASE, INDEX, MASK, SCALE) \ + (__m128) __builtin_ia32_gathersiv4sf ((__v4sf)(__m128d)SRC, \ + (float const *)BASE, \ + (__v4si)(__m128i)INDEX, \ + (__v4sf)(__m128d)MASK, \ + (int)SCALE) + +#define _mm256_i32gather_ps(BASE, INDEX, SCALE) \ + (__m256) __builtin_ia32_gathersiv8sf ((__v8sf) _mm256_setzero_ps (), \ + (float const *)BASE, \ + (__v8si)(__m256i)INDEX, \ + (__v8sf)_mm256_set1_ps ( \ + (float)(int) -1), \ + (int)SCALE) + +#define _mm256_mask_i32gather_ps(SRC, BASE, INDEX, MASK, SCALE) \ + (__m256) __builtin_ia32_gathersiv8sf ((__v8sf)(__m256)SRC, \ + (float const *)BASE, \ + (__v8si)(__m256i)INDEX, \ + (__v8sf)(__m256d)MASK, \ + (int)SCALE) + +#define _mm_i64gather_ps(BASE, INDEX, SCALE) \ + (__m128) __builtin_ia32_gatherdiv4sf ((__v4sf) _mm_setzero_pd (), \ + (float const *)BASE, \ + (__v2di)(__m128i)INDEX, \ + (__v4sf)_mm_set1_ps ( \ + (float)(int) -1), \ + (int)SCALE) + +#define _mm_mask_i64gather_ps(SRC, BASE, INDEX, MASK, SCALE) \ + (__m128) __builtin_ia32_gatherdiv4sf ((__v4sf)(__m128)SRC, \ + (float const *)BASE, \ + (__v2di)(__m128i)INDEX, \ + (__v4sf)(__m128d)MASK, \ + (int)SCALE) + +#define _mm256_i64gather_ps(BASE, INDEX, SCALE) \ + (__m128) __builtin_ia32_gatherdiv4sf256 ((__v4sf) _mm_setzero_ps (), \ + (float const *)BASE, \ + (__v4di)(__m256i)INDEX, \ + (__v4sf)_mm_set1_ps( \ + (float)(int) -1), \ + (int)SCALE) + +#define _mm256_mask_i64gather_ps(SRC, BASE, INDEX, MASK, SCALE) \ + (__m128) __builtin_ia32_gatherdiv4sf256 ((__v4sf)(__m128)SRC, \ + (float const *)BASE, \ + (__v4di)(__m256i)INDEX, \ + (__v4sf)(__m128)MASK, \ + (int)SCALE) + +#define _mm_i32gather_epi64(BASE, INDEX, SCALE) \ + (__m128i) __builtin_ia32_gathersiv2di ((__v2di) _mm_setzero_si128 (), \ + (long long const *)BASE, \ + (__v4si)(__m128i)INDEX, \ + (__v2di)_mm_set1_epi64x (-1), \ + (int)SCALE) + +#define _mm_mask_i32gather_epi64(SRC, BASE, INDEX, MASK, SCALE) \ + (__m128i) __builtin_ia32_gathersiv2di ((__v2di)(__m128i)SRC, \ + (long long const *)BASE, \ + (__v4si)(__m128i)INDEX, \ + (__v2di)(__m128i)MASK, \ + (int)SCALE) + +#define _mm256_i32gather_epi64(BASE, INDEX, SCALE) \ + (__m256i) __builtin_ia32_gathersiv4di ((__v4di) _mm256_setzero_si256 (), \ + (long long const *)BASE, \ + (__v4si)(__m128i)INDEX, \ + (__v4di)_mm256_set1_epi64x (-1), \ + (int)SCALE) + +#define _mm256_mask_i32gather_epi64(SRC, BASE, INDEX, MASK, SCALE) \ + (__m256i) __builtin_ia32_gathersiv4di ((__v4di)(__m256i)SRC, \ + (long long const *)BASE, \ + (__v4si)(__m128i)INDEX, \ + (__v4di)(__m256i)MASK, \ + (int)SCALE) + +#define _mm_i64gather_epi64(BASE, INDEX, SCALE) \ + (__m128i) __builtin_ia32_gatherdiv2di ((__v2di) _mm_setzero_si128 (), \ + (long long const *)BASE, \ + (__v2di)(__m128i)INDEX, \ + (__v2di)_mm_set1_epi64x (-1), \ + (int)SCALE) + +#define _mm_mask_i64gather_epi64(SRC, BASE, INDEX, MASK, SCALE) \ + (__m128i) __builtin_ia32_gatherdiv2di ((__v2di)(__m128i)SRC, \ + (long long const *)BASE, \ + (__v2di)(__m128i)INDEX, \ + (__v2di)(__m128i)MASK, \ + (int)SCALE) + +#define _mm256_i64gather_epi64(BASE, INDEX, SCALE) \ + (__m256i) __builtin_ia32_gatherdiv4di ((__v4di) _mm256_setzero_si256 (), \ + (long long const *)BASE, \ + (__v4di)(__m256i)INDEX, \ + (__v4di)_mm256_set1_epi64x (-1), \ + (int)SCALE) + +#define _mm256_mask_i64gather_epi64(SRC, BASE, INDEX, MASK, SCALE) \ + (__m256i) __builtin_ia32_gatherdiv4di ((__v4di)(__m256i)SRC, \ + (long long const *)BASE, \ + (__v4di)(__m256i)INDEX, \ + (__v4di)(__m256i)MASK, \ + (int)SCALE) + +#define _mm_i32gather_epi32(BASE, INDEX, SCALE) \ + (__m128i) __builtin_ia32_gathersiv4si ((__v4si) _mm_setzero_si128 (), \ + (int const *)BASE, \ + (__v4si)(__m128i)INDEX, \ + (__v4si)_mm_set1_epi32 (-1), \ + (int)SCALE) + +#define _mm_mask_i32gather_epi32(SRC, BASE, INDEX, MASK, SCALE) \ + (__m128i) __builtin_ia32_gathersiv4si ((__v4si)(__m128i)SRC, \ + (int const *)BASE, \ + (__v4si)(__m128i)INDEX, \ + (__v4si)(__m128i)MASK, \ + (int)SCALE) + +#define _mm256_i32gather_epi32(BASE, INDEX, SCALE) \ + (__m256i) __builtin_ia32_gathersiv8si ((__v8si) _mm256_setzero_si256 (), \ + (int const *)BASE, \ + (__v8si)(__m256i)INDEX, \ + (__v8si)_mm256_set1_epi32 (-1), \ + (int)SCALE) + +#define _mm256_mask_i32gather_epi32(SRC, BASE, INDEX, MASK, SCALE) \ + (__m256i) __builtin_ia32_gathersiv8si ((__v8si)(__m256i)SRC, \ + (int const *)BASE, \ + (__v8si)(__m256i)INDEX, \ + (__v8si)(__m256i)MASK, \ + (int)SCALE) + +#define _mm_i64gather_epi32(BASE, INDEX, SCALE) \ + (__m128i) __builtin_ia32_gatherdiv4si ((__v4si) _mm_setzero_si128 (), \ + (int const *)BASE, \ + (__v2di)(__m128i)INDEX, \ + (__v4si)_mm_set1_epi32 (-1), \ + (int)SCALE) + +#define _mm_mask_i64gather_epi32(SRC, BASE, INDEX, MASK, SCALE) \ + (__m128i) __builtin_ia32_gatherdiv4si ((__v4si)(__m128i)SRC, \ + (int const *)BASE, \ + (__v2di)(__m128i)INDEX, \ + (__v4si)(__m128i)MASK, \ + (int)SCALE) + +#define _mm256_i64gather_epi32(BASE, INDEX, SCALE) \ + (__m128i) __builtin_ia32_gatherdiv4si256 ((__v4si) _mm_setzero_si128 (), \ + (int const *)BASE, \ + (__v4di)(__m256i)INDEX, \ + (__v4si)_mm_set1_epi32(-1), \ + (int)SCALE) + +#define _mm256_mask_i64gather_epi32(SRC, BASE, INDEX, MASK, SCALE) \ + (__m128i) __builtin_ia32_gatherdiv4si256 ((__v4si)(__m128i)SRC, \ + (int const *)BASE, \ + (__v4di)(__m256i)INDEX, \ + (__v4si)(__m128i)MASK, \ + (int)SCALE) +#endif /* __OPTIMIZE__ */ + +#ifdef __DISABLE_AVX2__ +#undef __DISABLE_AVX2__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX2__ */ + +#endif /* _AVX2INTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx5124fmapsintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx5124fmapsintrin.h new file mode 100644 index 0000000000..45ac0d4f5d --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx5124fmapsintrin.h @@ -0,0 +1,216 @@ +/* Copyright (C) 2015-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#if !defined _IMMINTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _AVX5124FMAPSINTRIN_H_INCLUDED +#define _AVX5124FMAPSINTRIN_H_INCLUDED + +#ifndef __AVX5124FMAPS__ +#pragma GCC push_options +//#pragma GCC target("avx5124fmaps") +#define __DISABLE_AVX5124FMAPS__ +#endif /* __AVX5124FMAPS__ */ + +extern __inline __m512 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_4fmadd_ps (__m512 __A, __m512 __B, __m512 __C, + __m512 __D, __m512 __E, __m128 *__F) +{ + return (__m512) __builtin_ia32_4fmaddps ((__v16sf) __B, + (__v16sf) __C, + (__v16sf) __D, + (__v16sf) __E, + (__v16sf) __A, + (const __v4sf *) __F); +} + +extern __inline __m512 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_4fmadd_ps (__m512 __A, __mmask16 __U, __m512 __B, + __m512 __C, __m512 __D, __m512 __E, __m128 *__F) +{ + return (__m512) __builtin_ia32_4fmaddps_mask ((__v16sf) __B, + (__v16sf) __C, + (__v16sf) __D, + (__v16sf) __E, + (__v16sf) __A, + (const __v4sf *) __F, + (__v16sf) __A, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_4fmadd_ps (__mmask16 __U, + __m512 __A, __m512 __B, __m512 __C, + __m512 __D, __m512 __E, __m128 *__F) +{ + return (__m512) __builtin_ia32_4fmaddps_mask ((__v16sf) __B, + (__v16sf) __C, + (__v16sf) __D, + (__v16sf) __E, + (__v16sf) __A, + (const __v4sf *) __F, + (__v16sf) _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m128 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_4fmadd_ss (__m128 __A, __m128 __B, __m128 __C, + __m128 __D, __m128 __E, __m128 *__F) +{ + return (__m128) __builtin_ia32_4fmaddss ((__v4sf) __B, + (__v4sf) __C, + (__v4sf) __D, + (__v4sf) __E, + (__v4sf) __A, + (const __v4sf *) __F); +} + +extern __inline __m128 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_4fmadd_ss (__m128 __A, __mmask8 __U, __m128 __B, __m128 __C, + __m128 __D, __m128 __E, __m128 *__F) +{ + return (__m128) __builtin_ia32_4fmaddss_mask ((__v4sf) __B, + (__v4sf) __C, + (__v4sf) __D, + (__v4sf) __E, + (__v4sf) __A, + (const __v4sf *) __F, + (__v4sf) __A, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_4fmadd_ss (__mmask8 __U, __m128 __A, __m128 __B, __m128 __C, + __m128 __D, __m128 __E, __m128 *__F) +{ + return (__m128) __builtin_ia32_4fmaddss_mask ((__v4sf) __B, + (__v4sf) __C, + (__v4sf) __D, + (__v4sf) __E, + (__v4sf) __A, + (const __v4sf *) __F, + (__v4sf) _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_4fnmadd_ps (__m512 __A, __m512 __B, __m512 __C, + __m512 __D, __m512 __E, __m128 *__F) +{ + return (__m512) __builtin_ia32_4fnmaddps ((__v16sf) __B, + (__v16sf) __C, + (__v16sf) __D, + (__v16sf) __E, + (__v16sf) __A, + (const __v4sf *) __F); +} + +extern __inline __m512 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_4fnmadd_ps (__m512 __A, __mmask16 __U, __m512 __B, + __m512 __C, __m512 __D, __m512 __E, __m128 *__F) +{ + return (__m512) __builtin_ia32_4fnmaddps_mask ((__v16sf) __B, + (__v16sf) __C, + (__v16sf) __D, + (__v16sf) __E, + (__v16sf) __A, + (const __v4sf *) __F, + (__v16sf) __A, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_4fnmadd_ps (__mmask16 __U, + __m512 __A, __m512 __B, __m512 __C, + __m512 __D, __m512 __E, __m128 *__F) +{ + return (__m512) __builtin_ia32_4fnmaddps_mask ((__v16sf) __B, + (__v16sf) __C, + (__v16sf) __D, + (__v16sf) __E, + (__v16sf) __A, + (const __v4sf *) __F, + (__v16sf) _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m128 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_4fnmadd_ss (__m128 __A, __m128 __B, __m128 __C, + __m128 __D, __m128 __E, __m128 *__F) +{ + return (__m128) __builtin_ia32_4fnmaddss ((__v4sf) __B, + (__v4sf) __C, + (__v4sf) __D, + (__v4sf) __E, + (__v4sf) __A, + (const __v4sf *) __F); +} + +extern __inline __m128 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_4fnmadd_ss (__m128 __A, __mmask8 __U, __m128 __B, __m128 __C, + __m128 __D, __m128 __E, __m128 *__F) +{ + return (__m128) __builtin_ia32_4fnmaddss_mask ((__v4sf) __B, + (__v4sf) __C, + (__v4sf) __D, + (__v4sf) __E, + (__v4sf) __A, + (const __v4sf *) __F, + (__v4sf) __A, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_4fnmadd_ss (__mmask8 __U, __m128 __A, __m128 __B, __m128 __C, + __m128 __D, __m128 __E, __m128 *__F) +{ + return (__m128) __builtin_ia32_4fnmaddss_mask ((__v4sf) __B, + (__v4sf) __C, + (__v4sf) __D, + (__v4sf) __E, + (__v4sf) __A, + (const __v4sf *) __F, + (__v4sf) _mm_setzero_ps (), + (__mmask8) __U); +} + +#ifdef __DISABLE_AVX5124FMAPS__ +#undef __DISABLE_AVX5124FMAPS__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX5124FMAPS__ */ + +#endif /* _AVX5124FMAPSINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx5124vnniwintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx5124vnniwintrin.h new file mode 100644 index 0000000000..483096ec26 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx5124vnniwintrin.h @@ -0,0 +1,132 @@ +/* Copyright (C) 2015-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#if !defined _IMMINTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _AVX5124VNNIWINTRIN_H_INCLUDED +#define _AVX5124VNNIWINTRIN_H_INCLUDED + +#ifndef __AVX5124VNNIW__ +#pragma GCC push_options +//#pragma GCC target("avx5124vnniw") +#define __DISABLE_AVX5124VNNIW__ +#endif /* __AVX5124VNNIW__ */ + +extern __inline __m512i +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_4dpwssd_epi32 (__m512i __A, __m512i __B, __m512i __C, + __m512i __D, __m512i __E, __m128i *__F) +{ + return (__m512i) __builtin_ia32_vp4dpwssd ((__v16si) __B, + (__v16si) __C, + (__v16si) __D, + (__v16si) __E, + (__v16si) __A, + (const __v4si *) __F); +} + +extern __inline __m512i +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_4dpwssd_epi32 (__m512i __A, __mmask16 __U, __m512i __B, + __m512i __C, __m512i __D, __m512i __E, + __m128i *__F) +{ + return (__m512i) __builtin_ia32_vp4dpwssd_mask ((__v16si) __B, + (__v16si) __C, + (__v16si) __D, + (__v16si) __E, + (__v16si) __A, + (const __v4si *) __F, + (__v16si) __A, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_4dpwssd_epi32 (__mmask16 __U, __m512i __A, __m512i __B, + __m512i __C, __m512i __D, __m512i __E, + __m128i *__F) +{ + return (__m512i) __builtin_ia32_vp4dpwssd_mask ((__v16si) __B, + (__v16si) __C, + (__v16si) __D, + (__v16si) __E, + (__v16si) __A, + (const __v4si *) __F, + (__v16si) _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_4dpwssds_epi32 (__m512i __A, __m512i __B, __m512i __C, + __m512i __D, __m512i __E, __m128i *__F) +{ + return (__m512i) __builtin_ia32_vp4dpwssds ((__v16si) __B, + (__v16si) __C, + (__v16si) __D, + (__v16si) __E, + (__v16si) __A, + (const __v4si *) __F); +} + +extern __inline __m512i +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_4dpwssds_epi32 (__m512i __A, __mmask16 __U, __m512i __B, + __m512i __C, __m512i __D, __m512i __E, + __m128i *__F) +{ + return (__m512i) __builtin_ia32_vp4dpwssds_mask ((__v16si) __B, + (__v16si) __C, + (__v16si) __D, + (__v16si) __E, + (__v16si) __A, + (const __v4si *) __F, + (__v16si) __A, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_4dpwssds_epi32 (__mmask16 __U, __m512i __A, __m512i __B, + __m512i __C, __m512i __D, __m512i __E, + __m128i *__F) +{ + return (__m512i) __builtin_ia32_vp4dpwssds_mask ((__v16si) __B, + (__v16si) __C, + (__v16si) __D, + (__v16si) __E, + (__v16si) __A, + (const __v4si *) __F, + (__v16si) _mm512_setzero_ps (), + (__mmask16) __U); +} + +#ifdef __DISABLE_AVX5124VNNIW__ +#undef __DISABLE_AVX5124VNNIW__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX5124VNNIW__ */ + +#endif /* _AVX5124VNNIWINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512bwintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512bwintrin.h new file mode 100644 index 0000000000..bf316cf48e --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512bwintrin.h @@ -0,0 +1,3280 @@ +/* Copyright (C) 2014-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +#error "Never use directly; include instead." +#endif + +#ifndef _AVX512BWINTRIN_H_INCLUDED +#define _AVX512BWINTRIN_H_INCLUDED + +#ifndef __AVX512BW__ +#pragma GCC push_options +//#pragma GCC target("avx512bw") +#define __DISABLE_AVX512BW__ +#endif /* __AVX512BW__ */ + +/* Internal data types for implementing the intrinsics. */ +typedef short __v32hi __attribute__ ((__vector_size__ (64))); +typedef char __v64qi __attribute__ ((__vector_size__ (64))); + +typedef unsigned long long __mmask64; + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_ktest_mask32_u8 (__mmask32 __A, __mmask32 __B, unsigned char *__CF) +{ + *__CF = (unsigned char) __builtin_ia32_ktestcsi (__A, __B); + return (unsigned char) __builtin_ia32_ktestzsi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_ktest_mask64_u8 (__mmask64 __A, __mmask64 __B, unsigned char *__CF) +{ + *__CF = (unsigned char) __builtin_ia32_ktestcdi (__A, __B); + return (unsigned char) __builtin_ia32_ktestzdi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_ktestz_mask32_u8 (__mmask32 __A, __mmask32 __B) +{ + return (unsigned char) __builtin_ia32_ktestzsi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_ktestz_mask64_u8 (__mmask64 __A, __mmask64 __B) +{ + return (unsigned char) __builtin_ia32_ktestzdi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_ktestc_mask32_u8 (__mmask32 __A, __mmask32 __B) +{ + return (unsigned char) __builtin_ia32_ktestcsi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_ktestc_mask64_u8 (__mmask64 __A, __mmask64 __B) +{ + return (unsigned char) __builtin_ia32_ktestcdi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kortest_mask32_u8 (__mmask32 __A, __mmask32 __B, unsigned char *__CF) +{ + *__CF = (unsigned char) __builtin_ia32_kortestcsi (__A, __B); + return (unsigned char) __builtin_ia32_kortestzsi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kortest_mask64_u8 (__mmask64 __A, __mmask64 __B, unsigned char *__CF) +{ + *__CF = (unsigned char) __builtin_ia32_kortestcdi (__A, __B); + return (unsigned char) __builtin_ia32_kortestzdi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kortestz_mask32_u8 (__mmask32 __A, __mmask32 __B) +{ + return (unsigned char) __builtin_ia32_kortestzsi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kortestz_mask64_u8 (__mmask64 __A, __mmask64 __B) +{ + return (unsigned char) __builtin_ia32_kortestzdi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kortestc_mask32_u8 (__mmask32 __A, __mmask32 __B) +{ + return (unsigned char) __builtin_ia32_kortestcsi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kortestc_mask64_u8 (__mmask64 __A, __mmask64 __B) +{ + return (unsigned char) __builtin_ia32_kortestcdi (__A, __B); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kadd_mask32 (__mmask32 __A, __mmask32 __B) +{ + return (__mmask32) __builtin_ia32_kaddsi ((__mmask32) __A, (__mmask32) __B); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kadd_mask64 (__mmask64 __A, __mmask64 __B) +{ + return (__mmask64) __builtin_ia32_kadddi ((__mmask64) __A, (__mmask64) __B); +} + +extern __inline unsigned int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_cvtmask32_u32 (__mmask32 __A) +{ + return (unsigned int) __builtin_ia32_kmovd ((__mmask32) __A); +} + +extern __inline unsigned long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_cvtmask64_u64 (__mmask64 __A) +{ + return (unsigned long long) __builtin_ia32_kmovq ((__mmask64) __A); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_cvtu32_mask32 (unsigned int __A) +{ + return (__mmask32) __builtin_ia32_kmovd ((__mmask32) __A); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_cvtu64_mask64 (unsigned long long __A) +{ + return (__mmask64) __builtin_ia32_kmovq ((__mmask64) __A); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_load_mask32 (__mmask32 *__A) +{ + return (__mmask32) __builtin_ia32_kmovd (*__A); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_load_mask64 (__mmask64 *__A) +{ + return (__mmask64) __builtin_ia32_kmovq (*(__mmask64 *) __A); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_store_mask32 (__mmask32 *__A, __mmask32 __B) +{ + *(__mmask32 *) __A = __builtin_ia32_kmovd (__B); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_store_mask64 (__mmask64 *__A, __mmask64 __B) +{ + *(__mmask64 *) __A = __builtin_ia32_kmovq (__B); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_knot_mask32 (__mmask32 __A) +{ + return (__mmask32) __builtin_ia32_knotsi ((__mmask32) __A); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_knot_mask64 (__mmask64 __A) +{ + return (__mmask64) __builtin_ia32_knotdi ((__mmask64) __A); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kor_mask32 (__mmask32 __A, __mmask32 __B) +{ + return (__mmask32) __builtin_ia32_korsi ((__mmask32) __A, (__mmask32) __B); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kor_mask64 (__mmask64 __A, __mmask64 __B) +{ + return (__mmask64) __builtin_ia32_kordi ((__mmask64) __A, (__mmask64) __B); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kxnor_mask32 (__mmask32 __A, __mmask32 __B) +{ + return (__mmask32) __builtin_ia32_kxnorsi ((__mmask32) __A, (__mmask32) __B); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kxnor_mask64 (__mmask64 __A, __mmask64 __B) +{ + return (__mmask64) __builtin_ia32_kxnordi ((__mmask64) __A, (__mmask64) __B); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kxor_mask32 (__mmask32 __A, __mmask32 __B) +{ + return (__mmask32) __builtin_ia32_kxorsi ((__mmask32) __A, (__mmask32) __B); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kxor_mask64 (__mmask64 __A, __mmask64 __B) +{ + return (__mmask64) __builtin_ia32_kxordi ((__mmask64) __A, (__mmask64) __B); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kand_mask32 (__mmask32 __A, __mmask32 __B) +{ + return (__mmask32) __builtin_ia32_kandsi ((__mmask32) __A, (__mmask32) __B); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kand_mask64 (__mmask64 __A, __mmask64 __B) +{ + return (__mmask64) __builtin_ia32_kanddi ((__mmask64) __A, (__mmask64) __B); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kandn_mask32 (__mmask32 __A, __mmask32 __B) +{ + return (__mmask32) __builtin_ia32_kandnsi ((__mmask32) __A, (__mmask32) __B); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kandn_mask64 (__mmask64 __A, __mmask64 __B) +{ + return (__mmask64) __builtin_ia32_kandndi ((__mmask64) __A, (__mmask64) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mov_epi16 (__m512i __W, __mmask32 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_movdquhi512_mask ((__v32hi) __A, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mov_epi16 (__mmask32 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_movdquhi512_mask ((__v32hi) __A, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_loadu_epi16 (__m512i __W, __mmask32 __U, void const *__P) +{ + return (__m512i) __builtin_ia32_loaddquhi512_mask ((const short *) __P, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_loadu_epi16 (__mmask32 __U, void const *__P) +{ + return (__m512i) __builtin_ia32_loaddquhi512_mask ((const short *) __P, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_storeu_epi16 (void *__P, __mmask32 __U, __m512i __A) +{ + __builtin_ia32_storedquhi512_mask ((short *) __P, + (__v32hi) __A, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mov_epi8 (__m512i __W, __mmask64 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_movdquqi512_mask ((__v64qi) __A, + (__v64qi) __W, + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mov_epi8 (__mmask64 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_movdquqi512_mask ((__v64qi) __A, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __U); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_kunpackw (__mmask32 __A, __mmask32 __B) +{ + return (__mmask32) __builtin_ia32_kunpcksi ((__mmask32) __A, + (__mmask32) __B); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kunpackw_mask32 (__mmask16 __A, __mmask16 __B) +{ + return (__mmask32) __builtin_ia32_kunpcksi ((__mmask32) __A, + (__mmask32) __B); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_kunpackd (__mmask64 __A, __mmask64 __B) +{ + return (__mmask64) __builtin_ia32_kunpckdi ((__mmask64) __A, + (__mmask64) __B); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kunpackd_mask64 (__mmask32 __A, __mmask32 __B) +{ + return (__mmask64) __builtin_ia32_kunpckdi ((__mmask64) __A, + (__mmask64) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_loadu_epi8 (__m512i __W, __mmask64 __U, void const *__P) +{ + return (__m512i) __builtin_ia32_loaddquqi512_mask ((const char *) __P, + (__v64qi) __W, + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_loadu_epi8 (__mmask64 __U, void const *__P) +{ + return (__m512i) __builtin_ia32_loaddquqi512_mask ((const char *) __P, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_storeu_epi8 (void *__P, __mmask64 __U, __m512i __A) +{ + __builtin_ia32_storedquqi512_mask ((char *) __P, + (__v64qi) __A, + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sad_epu8 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psadbw512 ((__v64qi) __A, + (__v64qi) __B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepi16_epi8 (__m512i __A) +{ + return (__m256i) __builtin_ia32_pmovwb512_mask ((__v32hi) __A, + (__v32qi) _mm256_undefined_si256(), + (__mmask32) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi16_epi8 (__m256i __O, __mmask32 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovwb512_mask ((__v32hi) __A, + (__v32qi) __O, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepi16_epi8 (__mmask32 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovwb512_mask ((__v32hi) __A, + (__v32qi) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtsepi16_epi8 (__m512i __A) +{ + return (__m256i) __builtin_ia32_pmovswb512_mask ((__v32hi) __A, + (__v32qi)_mm256_undefined_si256(), + (__mmask32) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtsepi16_epi8 (__m256i __O, __mmask32 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovswb512_mask ((__v32hi) __A, + (__v32qi)__O, + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtsepi16_epi8 (__mmask32 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovswb512_mask ((__v32hi) __A, + (__v32qi) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtusepi16_epi8 (__m512i __A) +{ + return (__m256i) __builtin_ia32_pmovuswb512_mask ((__v32hi) __A, + (__v32qi)_mm256_undefined_si256(), + (__mmask32) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtusepi16_epi8 (__m256i __O, __mmask32 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovuswb512_mask ((__v32hi) __A, + (__v32qi) __O, + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtusepi16_epi8 (__mmask32 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovuswb512_mask ((__v32hi) __A, + (__v32qi) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcastb_epi8 (__m128i __A) +{ + return (__m512i) __builtin_ia32_pbroadcastb512_mask ((__v16qi) __A, + (__v64qi)_mm512_undefined_epi32(), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_broadcastb_epi8 (__m512i __O, __mmask64 __M, __m128i __A) +{ + return (__m512i) __builtin_ia32_pbroadcastb512_mask ((__v16qi) __A, + (__v64qi) __O, + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_broadcastb_epi8 (__mmask64 __M, __m128i __A) +{ + return (__m512i) __builtin_ia32_pbroadcastb512_mask ((__v16qi) __A, + (__v64qi) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_set1_epi8 (__m512i __O, __mmask64 __M, char __A) +{ + return (__m512i) __builtin_ia32_pbroadcastb512_gpr_mask (__A, + (__v64qi) __O, + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_set1_epi8 (__mmask64 __M, char __A) +{ + return (__m512i) + __builtin_ia32_pbroadcastb512_gpr_mask (__A, + (__v64qi) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcastw_epi16 (__m128i __A) +{ + return (__m512i) __builtin_ia32_pbroadcastw512_mask ((__v8hi) __A, + (__v32hi)_mm512_undefined_epi32(), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_broadcastw_epi16 (__m512i __O, __mmask32 __M, __m128i __A) +{ + return (__m512i) __builtin_ia32_pbroadcastw512_mask ((__v8hi) __A, + (__v32hi) __O, + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_broadcastw_epi16 (__mmask32 __M, __m128i __A) +{ + return (__m512i) __builtin_ia32_pbroadcastw512_mask ((__v8hi) __A, + (__v32hi) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_set1_epi16 (__m512i __O, __mmask32 __M, short __A) +{ + return (__m512i) __builtin_ia32_pbroadcastw512_gpr_mask (__A, + (__v32hi) __O, + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_set1_epi16 (__mmask32 __M, short __A) +{ + return (__m512i) + __builtin_ia32_pbroadcastw512_gpr_mask (__A, + (__v32hi) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mulhrs_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmulhrsw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mulhrs_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pmulhrsw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mulhrs_epi16 (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmulhrsw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mulhi_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmulhw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mulhi_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pmulhw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mulhi_epi16 (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmulhw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mulhi_epu16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmulhuw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mulhi_epu16 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pmulhuw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mulhi_epu16 (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmulhuw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mullo_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v32hu) __A * (__v32hu) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mullo_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pmullw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mullo_epi16 (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmullw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepi8_epi16 (__m256i __A) +{ + return (__m512i) __builtin_ia32_pmovsxbw512_mask ((__v32qi) __A, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi8_epi16 (__m512i __W, __mmask32 __U, __m256i __A) +{ + return (__m512i) __builtin_ia32_pmovsxbw512_mask ((__v32qi) __A, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepi8_epi16 (__mmask32 __U, __m256i __A) +{ + return (__m512i) __builtin_ia32_pmovsxbw512_mask ((__v32qi) __A, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepu8_epi16 (__m256i __A) +{ + return (__m512i) __builtin_ia32_pmovzxbw512_mask ((__v32qi) __A, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepu8_epi16 (__m512i __W, __mmask32 __U, __m256i __A) +{ + return (__m512i) __builtin_ia32_pmovzxbw512_mask ((__v32qi) __A, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepu8_epi16 (__mmask32 __U, __m256i __A) +{ + return (__m512i) __builtin_ia32_pmovzxbw512_mask ((__v32qi) __A, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permutexvar_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_permvarhi512_mask ((__v32hi) __B, + (__v32hi) __A, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permutexvar_epi16 (__mmask32 __M, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_permvarhi512_mask ((__v32hi) __B, + (__v32hi) __A, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permutexvar_epi16 (__m512i __W, __mmask32 __M, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_permvarhi512_mask ((__v32hi) __B, + (__v32hi) __A, + (__v32hi) __W, + (__mmask32) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permutex2var_epi16 (__m512i __A, __m512i __I, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpermt2varhi512_mask ((__v32hi) __I + /* idx */ , + (__v32hi) __A, + (__v32hi) __B, + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permutex2var_epi16 (__m512i __A, __mmask32 __U, + __m512i __I, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpermt2varhi512_mask ((__v32hi) __I + /* idx */ , + (__v32hi) __A, + (__v32hi) __B, + (__mmask32) + __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask2_permutex2var_epi16 (__m512i __A, __m512i __I, + __mmask32 __U, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpermi2varhi512_mask ((__v32hi) __A, + (__v32hi) __I + /* idx */ , + (__v32hi) __B, + (__mmask32) + __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permutex2var_epi16 (__mmask32 __U, __m512i __A, + __m512i __I, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpermt2varhi512_maskz ((__v32hi) __I + /* idx */ , + (__v32hi) __A, + (__v32hi) __B, + (__mmask32) + __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_avg_epu8 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pavgb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_avg_epu8 (__m512i __W, __mmask64 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pavgb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) __W, + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_avg_epu8 (__mmask64 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pavgb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_add_epi8 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v64qu) __A + (__v64qu) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_add_epi8 (__m512i __W, __mmask64 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_paddb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) __W, + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_add_epi8 (__mmask64 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_paddb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sub_epi8 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v64qu) __A - (__v64qu) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sub_epi8 (__m512i __W, __mmask64 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_psubb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) __W, + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sub_epi8 (__mmask64 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psubb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_avg_epu16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pavgw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_avg_epu16 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pavgw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_avg_epu16 (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pavgw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_subs_epi8 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psubsb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_subs_epi8 (__m512i __W, __mmask64 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_psubsb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) __W, + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_subs_epi8 (__mmask64 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psubsb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_subs_epu8 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psubusb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_subs_epu8 (__m512i __W, __mmask64 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_psubusb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) __W, + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_subs_epu8 (__mmask64 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psubusb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_adds_epi8 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_paddsb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_adds_epi8 (__m512i __W, __mmask64 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_paddsb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) __W, + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_adds_epi8 (__mmask64 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_paddsb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_adds_epu8 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_paddusb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_adds_epu8 (__m512i __W, __mmask64 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_paddusb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) __W, + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_adds_epu8 (__mmask64 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_paddusb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sub_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v32hu) __A - (__v32hu) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sub_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_psubw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sub_epi16 (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psubw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_subs_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psubsw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_subs_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_psubsw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_subs_epi16 (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psubsw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_subs_epu16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psubusw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_subs_epu16 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_psubusw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_subs_epu16 (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psubusw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_add_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v32hu) __A + (__v32hu) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_add_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_paddw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_add_epi16 (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_paddw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_adds_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_paddsw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_adds_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_paddsw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_adds_epi16 (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_paddsw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_adds_epu16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_paddusw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_adds_epu16 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_paddusw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_adds_epu16 (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_paddusw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_srl_epi16 (__m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psrlw512_mask ((__v32hi) __A, + (__v8hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_srl_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + __m128i __B) +{ + return (__m512i) __builtin_ia32_psrlw512_mask ((__v32hi) __A, + (__v8hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_srl_epi16 (__mmask32 __U, __m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psrlw512_mask ((__v32hi) __A, + (__v8hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_packs_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_packsswb512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sll_epi16 (__m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psllw512_mask ((__v32hi) __A, + (__v8hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sll_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + __m128i __B) +{ + return (__m512i) __builtin_ia32_psllw512_mask ((__v32hi) __A, + (__v8hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sll_epi16 (__mmask32 __U, __m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psllw512_mask ((__v32hi) __A, + (__v8hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maddubs_epi16 (__m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_pmaddubsw512_mask ((__v64qi) __X, + (__v64qi) __Y, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_maddubs_epi16 (__m512i __W, __mmask32 __U, __m512i __X, + __m512i __Y) +{ + return (__m512i) __builtin_ia32_pmaddubsw512_mask ((__v64qi) __X, + (__v64qi) __Y, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_maddubs_epi16 (__mmask32 __U, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_pmaddubsw512_mask ((__v64qi) __X, + (__v64qi) __Y, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_madd_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaddwd512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_madd_epi16 (__m512i __W, __mmask16 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaddwd512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_madd_epi16 (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaddwd512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_unpackhi_epi8 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpckhbw512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_unpackhi_epi8 (__m512i __W, __mmask64 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_punpckhbw512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) __W, + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_unpackhi_epi8 (__mmask64 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpckhbw512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_unpackhi_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpckhwd512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_unpackhi_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_punpckhwd512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_unpackhi_epi16 (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpckhwd512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_unpacklo_epi8 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpcklbw512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_unpacklo_epi8 (__m512i __W, __mmask64 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_punpcklbw512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) __W, + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_unpacklo_epi8 (__mmask64 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpcklbw512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_unpacklo_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpcklwd512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_unpacklo_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_punpcklwd512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_unpacklo_epi16 (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpcklwd512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpeq_epu8_mask (__m512i __A, __m512i __B) +{ + return (__mmask64) __builtin_ia32_ucmpb512_mask ((__v64qi) __A, + (__v64qi) __B, 0, + (__mmask64) -1); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpeq_epi8_mask (__m512i __A, __m512i __B) +{ + return (__mmask64) __builtin_ia32_pcmpeqb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__mmask64) -1); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpeq_epu8_mask (__mmask64 __U, __m512i __A, __m512i __B) +{ + return (__mmask64) __builtin_ia32_ucmpb512_mask ((__v64qi) __A, + (__v64qi) __B, 0, + __U); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpeq_epi8_mask (__mmask64 __U, __m512i __A, __m512i __B) +{ + return (__mmask64) __builtin_ia32_pcmpeqb512_mask ((__v64qi) __A, + (__v64qi) __B, + __U); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpeq_epu16_mask (__m512i __A, __m512i __B) +{ + return (__mmask32) __builtin_ia32_ucmpw512_mask ((__v32hi) __A, + (__v32hi) __B, 0, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpeq_epi16_mask (__m512i __A, __m512i __B) +{ + return (__mmask32) __builtin_ia32_pcmpeqw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpeq_epu16_mask (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__mmask32) __builtin_ia32_ucmpw512_mask ((__v32hi) __A, + (__v32hi) __B, 0, + __U); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpeq_epi16_mask (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__mmask32) __builtin_ia32_pcmpeqw512_mask ((__v32hi) __A, + (__v32hi) __B, + __U); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpgt_epu8_mask (__m512i __A, __m512i __B) +{ + return (__mmask64) __builtin_ia32_ucmpb512_mask ((__v64qi) __A, + (__v64qi) __B, 6, + (__mmask64) -1); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpgt_epi8_mask (__m512i __A, __m512i __B) +{ + return (__mmask64) __builtin_ia32_pcmpgtb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__mmask64) -1); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpgt_epu8_mask (__mmask64 __U, __m512i __A, __m512i __B) +{ + return (__mmask64) __builtin_ia32_ucmpb512_mask ((__v64qi) __A, + (__v64qi) __B, 6, + __U); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpgt_epi8_mask (__mmask64 __U, __m512i __A, __m512i __B) +{ + return (__mmask64) __builtin_ia32_pcmpgtb512_mask ((__v64qi) __A, + (__v64qi) __B, + __U); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpgt_epu16_mask (__m512i __A, __m512i __B) +{ + return (__mmask32) __builtin_ia32_ucmpw512_mask ((__v32hi) __A, + (__v32hi) __B, 6, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpgt_epi16_mask (__m512i __A, __m512i __B) +{ + return (__mmask32) __builtin_ia32_pcmpgtw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpgt_epu16_mask (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__mmask32) __builtin_ia32_ucmpw512_mask ((__v32hi) __A, + (__v32hi) __B, 6, + __U); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpgt_epi16_mask (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__mmask32) __builtin_ia32_pcmpgtw512_mask ((__v32hi) __A, + (__v32hi) __B, + __U); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_movepi8_mask (__m512i __A) +{ + return (__mmask64) __builtin_ia32_cvtb2mask512 ((__v64qi) __A); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_movepi16_mask (__m512i __A) +{ + return (__mmask32) __builtin_ia32_cvtw2mask512 ((__v32hi) __A); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_movm_epi8 (__mmask64 __A) +{ + return (__m512i) __builtin_ia32_cvtmask2b512 (__A); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_movm_epi16 (__mmask32 __A) +{ + return (__m512i) __builtin_ia32_cvtmask2w512 (__A); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_test_epi8_mask (__m512i __A, __m512i __B) +{ + return (__mmask64) __builtin_ia32_ptestmb512 ((__v64qi) __A, + (__v64qi) __B, + (__mmask64) -1); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_test_epi8_mask (__mmask64 __U, __m512i __A, __m512i __B) +{ + return (__mmask64) __builtin_ia32_ptestmb512 ((__v64qi) __A, + (__v64qi) __B, __U); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_test_epi16_mask (__m512i __A, __m512i __B) +{ + return (__mmask32) __builtin_ia32_ptestmw512 ((__v32hi) __A, + (__v32hi) __B, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_test_epi16_mask (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__mmask32) __builtin_ia32_ptestmw512 ((__v32hi) __A, + (__v32hi) __B, __U); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_testn_epi8_mask (__m512i __A, __m512i __B) +{ + return (__mmask64) __builtin_ia32_ptestnmb512 ((__v64qi) __A, + (__v64qi) __B, + (__mmask64) -1); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_testn_epi8_mask (__mmask64 __U, __m512i __A, __m512i __B) +{ + return (__mmask64) __builtin_ia32_ptestnmb512 ((__v64qi) __A, + (__v64qi) __B, __U); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_testn_epi16_mask (__m512i __A, __m512i __B) +{ + return (__mmask32) __builtin_ia32_ptestnmw512 ((__v32hi) __A, + (__v32hi) __B, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_testn_epi16_mask (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__mmask32) __builtin_ia32_ptestnmw512 ((__v32hi) __A, + (__v32hi) __B, __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_shuffle_epi8 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pshufb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_shuffle_epi8 (__m512i __W, __mmask64 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pshufb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) __W, + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_shuffle_epi8 (__mmask64 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pshufb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_min_epu16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminuw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_min_epu16 (__mmask32 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminuw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_min_epu16 (__m512i __W, __mmask32 __M, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pminuw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_min_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminsw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_min_epi16 (__mmask32 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminsw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_min_epi16 (__m512i __W, __mmask32 __M, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pminsw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_max_epu8 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxub512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_max_epu8 (__mmask64 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxub512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_max_epu8 (__m512i __W, __mmask64 __M, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxub512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) __W, + (__mmask64) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_max_epi8 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxsb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_max_epi8 (__mmask64 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxsb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_max_epi8 (__m512i __W, __mmask64 __M, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxsb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) __W, + (__mmask64) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_min_epu8 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminub512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_min_epu8 (__mmask64 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminub512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_min_epu8 (__m512i __W, __mmask64 __M, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pminub512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) __W, + (__mmask64) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_min_epi8 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminsb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_min_epi8 (__mmask64 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminsb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_min_epi8 (__m512i __W, __mmask64 __M, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pminsb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) __W, + (__mmask64) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_max_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxsw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_max_epi16 (__mmask32 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxsw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_max_epi16 (__m512i __W, __mmask32 __M, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxsw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_max_epu16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxuw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_max_epu16 (__mmask32 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxuw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_max_epu16 (__m512i __W, __mmask32 __M, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxuw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sra_epi16 (__m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psraw512_mask ((__v32hi) __A, + (__v8hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sra_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + __m128i __B) +{ + return (__m512i) __builtin_ia32_psraw512_mask ((__v32hi) __A, + (__v8hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sra_epi16 (__mmask32 __U, __m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psraw512_mask ((__v32hi) __A, + (__v8hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_srav_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psrav32hi_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_srav_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_psrav32hi_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_srav_epi16 (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psrav32hi_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_srlv_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psrlv32hi_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_srlv_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_psrlv32hi_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_srlv_epi16 (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psrlv32hi_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sllv_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psllv32hi_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sllv_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_psllv32hi_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sllv_epi16 (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psllv32hi_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_packs_epi16 (__m512i __W, __mmask64 __M, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_packsswb512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v64qi) __W, + (__mmask64) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_packs_epi16 (__mmask64 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_packsswb512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v64qi) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_packus_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_packuswb512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_packus_epi16 (__m512i __W, __mmask64 __M, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_packuswb512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v64qi) __W, + (__mmask64) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_packus_epi16 (__mmask64 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_packuswb512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_abs_epi8 (__m512i __A) +{ + return (__m512i) __builtin_ia32_pabsb512_mask ((__v64qi) __A, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_abs_epi8 (__m512i __W, __mmask64 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_pabsb512_mask ((__v64qi) __A, + (__v64qi) __W, + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_abs_epi8 (__mmask64 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_pabsb512_mask ((__v64qi) __A, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_abs_epi16 (__m512i __A) +{ + return (__m512i) __builtin_ia32_pabsw512_mask ((__v32hi) __A, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_abs_epi16 (__m512i __W, __mmask32 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_pabsw512_mask ((__v32hi) __A, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_abs_epi16 (__mmask32 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_pabsw512_mask ((__v32hi) __A, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpneq_epu8_mask (__mmask64 __M, __m512i __X, __m512i __Y) +{ + return (__mmask64) __builtin_ia32_ucmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, 4, + (__mmask64) __M); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmplt_epu8_mask (__mmask64 __M, __m512i __X, __m512i __Y) +{ + return (__mmask64) __builtin_ia32_ucmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, 1, + (__mmask64) __M); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpge_epu8_mask (__mmask64 __M, __m512i __X, __m512i __Y) +{ + return (__mmask64) __builtin_ia32_ucmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, 5, + (__mmask64) __M); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmple_epu8_mask (__mmask64 __M, __m512i __X, __m512i __Y) +{ + return (__mmask64) __builtin_ia32_ucmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, 2, + (__mmask64) __M); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpneq_epu16_mask (__mmask32 __M, __m512i __X, __m512i __Y) +{ + return (__mmask32) __builtin_ia32_ucmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, 4, + (__mmask32) __M); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmplt_epu16_mask (__mmask32 __M, __m512i __X, __m512i __Y) +{ + return (__mmask32) __builtin_ia32_ucmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, 1, + (__mmask32) __M); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpge_epu16_mask (__mmask32 __M, __m512i __X, __m512i __Y) +{ + return (__mmask32) __builtin_ia32_ucmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, 5, + (__mmask32) __M); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmple_epu16_mask (__mmask32 __M, __m512i __X, __m512i __Y) +{ + return (__mmask32) __builtin_ia32_ucmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, 2, + (__mmask32) __M); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpneq_epi8_mask (__mmask64 __M, __m512i __X, __m512i __Y) +{ + return (__mmask64) __builtin_ia32_cmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, 4, + (__mmask64) __M); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmplt_epi8_mask (__mmask64 __M, __m512i __X, __m512i __Y) +{ + return (__mmask64) __builtin_ia32_cmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, 1, + (__mmask64) __M); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpge_epi8_mask (__mmask64 __M, __m512i __X, __m512i __Y) +{ + return (__mmask64) __builtin_ia32_cmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, 5, + (__mmask64) __M); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmple_epi8_mask (__mmask64 __M, __m512i __X, __m512i __Y) +{ + return (__mmask64) __builtin_ia32_cmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, 2, + (__mmask64) __M); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpneq_epi16_mask (__mmask32 __M, __m512i __X, __m512i __Y) +{ + return (__mmask32) __builtin_ia32_cmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, 4, + (__mmask32) __M); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmplt_epi16_mask (__mmask32 __M, __m512i __X, __m512i __Y) +{ + return (__mmask32) __builtin_ia32_cmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, 1, + (__mmask32) __M); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpge_epi16_mask (__mmask32 __M, __m512i __X, __m512i __Y) +{ + return (__mmask32) __builtin_ia32_cmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, 5, + (__mmask32) __M); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmple_epi16_mask (__mmask32 __M, __m512i __X, __m512i __Y) +{ + return (__mmask32) __builtin_ia32_cmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, 2, + (__mmask32) __M); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpneq_epu8_mask (__m512i __X, __m512i __Y) +{ + return (__mmask64) __builtin_ia32_ucmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, 4, + (__mmask64) -1); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmplt_epu8_mask (__m512i __X, __m512i __Y) +{ + return (__mmask64) __builtin_ia32_ucmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, 1, + (__mmask64) -1); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpge_epu8_mask (__m512i __X, __m512i __Y) +{ + return (__mmask64) __builtin_ia32_ucmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, 5, + (__mmask64) -1); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmple_epu8_mask (__m512i __X, __m512i __Y) +{ + return (__mmask64) __builtin_ia32_ucmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, 2, + (__mmask64) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpneq_epu16_mask (__m512i __X, __m512i __Y) +{ + return (__mmask32) __builtin_ia32_ucmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, 4, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmplt_epu16_mask (__m512i __X, __m512i __Y) +{ + return (__mmask32) __builtin_ia32_ucmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, 1, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpge_epu16_mask (__m512i __X, __m512i __Y) +{ + return (__mmask32) __builtin_ia32_ucmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, 5, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmple_epu16_mask (__m512i __X, __m512i __Y) +{ + return (__mmask32) __builtin_ia32_ucmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, 2, + (__mmask32) -1); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpneq_epi8_mask (__m512i __X, __m512i __Y) +{ + return (__mmask64) __builtin_ia32_cmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, 4, + (__mmask64) -1); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmplt_epi8_mask (__m512i __X, __m512i __Y) +{ + return (__mmask64) __builtin_ia32_cmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, 1, + (__mmask64) -1); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpge_epi8_mask (__m512i __X, __m512i __Y) +{ + return (__mmask64) __builtin_ia32_cmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, 5, + (__mmask64) -1); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmple_epi8_mask (__m512i __X, __m512i __Y) +{ + return (__mmask64) __builtin_ia32_cmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, 2, + (__mmask64) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpneq_epi16_mask (__m512i __X, __m512i __Y) +{ + return (__mmask32) __builtin_ia32_cmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, 4, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmplt_epi16_mask (__m512i __X, __m512i __Y) +{ + return (__mmask32) __builtin_ia32_cmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, 1, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpge_epi16_mask (__m512i __X, __m512i __Y) +{ + return (__mmask32) __builtin_ia32_cmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, 5, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmple_epi16_mask (__m512i __X, __m512i __Y) +{ + return (__mmask32) __builtin_ia32_cmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, 2, + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_packs_epi32 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_packssdw512_mask ((__v16si) __A, + (__v16si) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_packs_epi32 (__mmask32 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_packssdw512_mask ((__v16si) __A, + (__v16si) __B, + (__v32hi) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_packs_epi32 (__m512i __W, __mmask32 __M, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_packssdw512_mask ((__v16si) __A, + (__v16si) __B, + (__v32hi) __W, + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_packus_epi32 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_packusdw512_mask ((__v16si) __A, + (__v16si) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_packus_epi32 (__mmask32 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_packusdw512_mask ((__v16si) __A, + (__v16si) __B, + (__v32hi) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_packus_epi32 (__m512i __W, __mmask32 __M, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_packusdw512_mask ((__v16si) __A, + (__v16si) __B, + (__v32hi) __W, + __M); +} + +#ifdef __OPTIMIZE__ +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kshiftli_mask32 (__mmask32 __A, unsigned int __B) +{ + return (__mmask32) __builtin_ia32_kshiftlisi ((__mmask32) __A, + (__mmask8) __B); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kshiftli_mask64 (__mmask64 __A, unsigned int __B) +{ + return (__mmask64) __builtin_ia32_kshiftlidi ((__mmask64) __A, + (__mmask8) __B); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kshiftri_mask32 (__mmask32 __A, unsigned int __B) +{ + return (__mmask32) __builtin_ia32_kshiftrisi ((__mmask32) __A, + (__mmask8) __B); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kshiftri_mask64 (__mmask64 __A, unsigned int __B) +{ + return (__mmask64) __builtin_ia32_kshiftridi ((__mmask64) __A, + (__mmask8) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_alignr_epi8 (__m512i __A, __m512i __B, const int __N) +{ + return (__m512i) __builtin_ia32_palignr512 ((__v8di) __A, + (__v8di) __B, __N * 8); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_alignr_epi8 (__m512i __W, __mmask64 __U, __m512i __A, + __m512i __B, const int __N) +{ + return (__m512i) __builtin_ia32_palignr512_mask ((__v8di) __A, + (__v8di) __B, + __N * 8, + (__v8di) __W, + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_alignr_epi8 (__mmask64 __U, __m512i __A, __m512i __B, + const int __N) +{ + return (__m512i) __builtin_ia32_palignr512_mask ((__v8di) __A, + (__v8di) __B, + __N * 8, + (__v8di) + _mm512_setzero_si512 (), + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_dbsad_epu8 (__m512i __A, __m512i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_dbpsadbw512_mask ((__v64qi) __A, + (__v64qi) __B, + __imm, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_dbsad_epu8 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_dbpsadbw512_mask ((__v64qi) __A, + (__v64qi) __B, + __imm, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_dbsad_epu8 (__mmask32 __U, __m512i __A, __m512i __B, + const int __imm) +{ + return (__m512i) __builtin_ia32_dbpsadbw512_mask ((__v64qi) __A, + (__v64qi) __B, + __imm, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_srli_epi16 (__m512i __A, const int __imm) +{ + return (__m512i) __builtin_ia32_psrlwi512_mask ((__v32hi) __A, __imm, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_srli_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + const int __imm) +{ + return (__m512i) __builtin_ia32_psrlwi512_mask ((__v32hi) __A, __imm, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_srli_epi16 (__mmask32 __U, __m512i __A, const int __imm) +{ + return (__m512i) __builtin_ia32_psrlwi512_mask ((__v32hi) __A, __imm, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_slli_epi16 (__m512i __A, const int __B) +{ + return (__m512i) __builtin_ia32_psllwi512_mask ((__v32hi) __A, __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_slli_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + const int __B) +{ + return (__m512i) __builtin_ia32_psllwi512_mask ((__v32hi) __A, __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_slli_epi16 (__mmask32 __U, __m512i __A, const int __B) +{ + return (__m512i) __builtin_ia32_psllwi512_mask ((__v32hi) __A, __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_shufflehi_epi16 (__m512i __A, const int __imm) +{ + return (__m512i) __builtin_ia32_pshufhw512_mask ((__v32hi) __A, + __imm, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_shufflehi_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + const int __imm) +{ + return (__m512i) __builtin_ia32_pshufhw512_mask ((__v32hi) __A, + __imm, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_shufflehi_epi16 (__mmask32 __U, __m512i __A, + const int __imm) +{ + return (__m512i) __builtin_ia32_pshufhw512_mask ((__v32hi) __A, + __imm, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_shufflelo_epi16 (__m512i __A, const int __imm) +{ + return (__m512i) __builtin_ia32_pshuflw512_mask ((__v32hi) __A, + __imm, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_shufflelo_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + const int __imm) +{ + return (__m512i) __builtin_ia32_pshuflw512_mask ((__v32hi) __A, + __imm, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_shufflelo_epi16 (__mmask32 __U, __m512i __A, + const int __imm) +{ + return (__m512i) __builtin_ia32_pshuflw512_mask ((__v32hi) __A, + __imm, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_srai_epi16 (__m512i __A, const int __imm) +{ + return (__m512i) __builtin_ia32_psrawi512_mask ((__v32hi) __A, __imm, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_srai_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + const int __imm) +{ + return (__m512i) __builtin_ia32_psrawi512_mask ((__v32hi) __A, __imm, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_srai_epi16 (__mmask32 __U, __m512i __A, const int __imm) +{ + return (__m512i) __builtin_ia32_psrawi512_mask ((__v32hi) __A, __imm, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_blend_epi16 (__mmask32 __U, __m512i __A, __m512i __W) +{ + return (__m512i) __builtin_ia32_blendmw_512_mask ((__v32hi) __A, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_blend_epi8 (__mmask64 __U, __m512i __A, __m512i __W) +{ + return (__m512i) __builtin_ia32_blendmb_512_mask ((__v64qi) __A, + (__v64qi) __W, + (__mmask64) __U); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmp_epi16_mask (__mmask32 __U, __m512i __X, __m512i __Y, + const int __P) +{ + return (__mmask32) __builtin_ia32_cmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, __P, + (__mmask32) __U); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmp_epi16_mask (__m512i __X, __m512i __Y, const int __P) +{ + return (__mmask32) __builtin_ia32_cmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, __P, + (__mmask32) -1); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmp_epi8_mask (__mmask32 __U, __m512i __X, __m512i __Y, + const int __P) +{ + return (__mmask64) __builtin_ia32_cmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, __P, + (__mmask64) __U); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmp_epi8_mask (__m512i __X, __m512i __Y, const int __P) +{ + return (__mmask64) __builtin_ia32_cmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, __P, + (__mmask64) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmp_epu16_mask (__mmask32 __U, __m512i __X, __m512i __Y, + const int __P) +{ + return (__mmask32) __builtin_ia32_ucmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, __P, + (__mmask32) __U); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmp_epu16_mask (__m512i __X, __m512i __Y, const int __P) +{ + return (__mmask32) __builtin_ia32_ucmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, __P, + (__mmask32) -1); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmp_epu8_mask (__mmask32 __U, __m512i __X, __m512i __Y, + const int __P) +{ + return (__mmask64) __builtin_ia32_ucmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, __P, + (__mmask64) __U); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmp_epu8_mask (__m512i __X, __m512i __Y, const int __P) +{ + return (__mmask64) __builtin_ia32_ucmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, __P, + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_bslli_epi128 (__m512i __A, const int __N) +{ + return (__m512i) __builtin_ia32_pslldq512 (__A, __N * 8); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_bsrli_epi128 (__m512i __A, const int __N) +{ + return (__m512i) __builtin_ia32_psrldq512 (__A, __N * 8); +} + +#else +#define _kshiftli_mask32(X, Y) \ + ((__mmask32) __builtin_ia32_kshiftlisi ((__mmask32)(X), (__mmask8)(Y))) + +#define _kshiftli_mask64(X, Y) \ + ((__mmask64) __builtin_ia32_kshiftlidi ((__mmask64)(X), (__mmask8)(Y))) + +#define _kshiftri_mask32(X, Y) \ + ((__mmask32) __builtin_ia32_kshiftrisi ((__mmask32)(X), (__mmask8)(Y))) + +#define _kshiftri_mask64(X, Y) \ + ((__mmask64) __builtin_ia32_kshiftridi ((__mmask64)(X), (__mmask8)(Y))) + +#define _mm512_alignr_epi8(X, Y, N) \ + ((__m512i) __builtin_ia32_palignr512 ((__v8di)(__m512i)(X), \ + (__v8di)(__m512i)(Y), \ + (int)(N * 8))) + +#define _mm512_mask_alignr_epi8(W, U, X, Y, N) \ + ((__m512i) __builtin_ia32_palignr512_mask ((__v8di)(__m512i)(X), \ + (__v8di)(__m512i)(Y), (int)(N * 8), \ + (__v8di)(__m512i)(W), (__mmask64)(U))) + +#define _mm512_maskz_alignr_epi8(U, X, Y, N) \ + ((__m512i) __builtin_ia32_palignr512_mask ((__v8di)(__m512i)(X), \ + (__v8di)(__m512i)(Y), (int)(N * 8), \ + (__v8di)(__m512i) \ + _mm512_setzero_si512 (), \ + (__mmask64)(U))) + +#define _mm512_dbsad_epu8(X, Y, C) \ + ((__m512i) __builtin_ia32_dbpsadbw512_mask ((__v64qi)(__m512i) (X), \ + (__v64qi)(__m512i) (Y), (int) (C), \ + (__v32hi)(__m512i) \ + _mm512_setzero_si512 (), \ + (__mmask32)-1)) + +#define _mm512_mask_dbsad_epu8(W, U, X, Y, C) \ + ((__m512i) __builtin_ia32_dbpsadbw512_mask ((__v64qi)(__m512i) (X), \ + (__v64qi)(__m512i) (Y), (int) (C), \ + (__v32hi)(__m512i)(W), \ + (__mmask32)(U))) + +#define _mm512_maskz_dbsad_epu8(U, X, Y, C) \ + ((__m512i) __builtin_ia32_dbpsadbw512_mask ((__v64qi)(__m512i) (X), \ + (__v64qi)(__m512i) (Y), (int) (C), \ + (__v32hi)(__m512i) \ + _mm512_setzero_si512 (), \ + (__mmask32)(U))) + +#define _mm512_srli_epi16(A, B) \ + ((__m512i) __builtin_ia32_psrlwi512_mask ((__v32hi)(__m512i)(A), \ + (int)(B), (__v32hi)_mm512_setzero_si512 (), (__mmask32)-1)) + +#define _mm512_mask_srli_epi16(W, U, A, B) \ + ((__m512i) __builtin_ia32_psrlwi512_mask ((__v32hi)(__m512i)(A), \ + (int)(B), (__v32hi)(__m512i)(W), (__mmask32)(U))) + +#define _mm512_maskz_srli_epi16(U, A, B) \ + ((__m512i) __builtin_ia32_psrlwi512_mask ((__v32hi)(__m512i)(A), \ + (int)(B), (__v32hi)_mm512_setzero_si512 (), (__mmask32)(U))) + +#define _mm512_slli_epi16(X, C) \ + ((__m512i)__builtin_ia32_psllwi512_mask ((__v32hi)(__m512i)(X), (int)(C),\ + (__v32hi)(__m512i)_mm512_setzero_si512 (), \ + (__mmask32)-1)) + +#define _mm512_mask_slli_epi16(W, U, X, C) \ + ((__m512i)__builtin_ia32_psllwi512_mask ((__v32hi)(__m512i)(X), (int)(C),\ + (__v32hi)(__m512i)(W),\ + (__mmask32)(U))) + +#define _mm512_maskz_slli_epi16(U, X, C) \ + ((__m512i)__builtin_ia32_psllwi512_mask ((__v32hi)(__m512i)(X), (int)(C),\ + (__v32hi)(__m512i)_mm512_setzero_si512 (), \ + (__mmask32)(U))) + +#define _mm512_shufflehi_epi16(A, B) \ + ((__m512i) __builtin_ia32_pshufhw512_mask ((__v32hi)(__m512i)(A), (int)(B), \ + (__v32hi)(__m512i) \ + _mm512_setzero_si512 (), \ + (__mmask32)-1)) + +#define _mm512_mask_shufflehi_epi16(W, U, A, B) \ + ((__m512i) __builtin_ia32_pshufhw512_mask ((__v32hi)(__m512i)(A), (int)(B), \ + (__v32hi)(__m512i)(W), \ + (__mmask32)(U))) + +#define _mm512_maskz_shufflehi_epi16(U, A, B) \ + ((__m512i) __builtin_ia32_pshufhw512_mask ((__v32hi)(__m512i)(A), (int)(B), \ + (__v32hi)(__m512i) \ + _mm512_setzero_si512 (), \ + (__mmask32)(U))) + +#define _mm512_shufflelo_epi16(A, B) \ + ((__m512i) __builtin_ia32_pshuflw512_mask ((__v32hi)(__m512i)(A), (int)(B), \ + (__v32hi)(__m512i) \ + _mm512_setzero_si512 (), \ + (__mmask32)-1)) + +#define _mm512_mask_shufflelo_epi16(W, U, A, B) \ + ((__m512i) __builtin_ia32_pshuflw512_mask ((__v32hi)(__m512i)(A), (int)(B), \ + (__v32hi)(__m512i)(W), \ + (__mmask32)(U))) + +#define _mm512_maskz_shufflelo_epi16(U, A, B) \ + ((__m512i) __builtin_ia32_pshuflw512_mask ((__v32hi)(__m512i)(A), (int)(B), \ + (__v32hi)(__m512i) \ + _mm512_setzero_si512 (), \ + (__mmask32)(U))) + +#define _mm512_srai_epi16(A, B) \ + ((__m512i) __builtin_ia32_psrawi512_mask ((__v32hi)(__m512i)(A), \ + (int)(B), (__v32hi)_mm512_setzero_si512 (), (__mmask32)-1)) + +#define _mm512_mask_srai_epi16(W, U, A, B) \ + ((__m512i) __builtin_ia32_psrawi512_mask ((__v32hi)(__m512i)(A), \ + (int)(B), (__v32hi)(__m512i)(W), (__mmask32)(U))) + +#define _mm512_maskz_srai_epi16(U, A, B) \ + ((__m512i) __builtin_ia32_psrawi512_mask ((__v32hi)(__m512i)(A), \ + (int)(B), (__v32hi)_mm512_setzero_si512 (), (__mmask32)(U))) + +#define _mm512_mask_blend_epi16(__U, __A, __W) \ + ((__m512i) __builtin_ia32_blendmw_512_mask ((__v32hi) (__A), \ + (__v32hi) (__W), \ + (__mmask32) (__U))) + +#define _mm512_mask_blend_epi8(__U, __A, __W) \ + ((__m512i) __builtin_ia32_blendmb_512_mask ((__v64qi) (__A), \ + (__v64qi) (__W), \ + (__mmask64) (__U))) + +#define _mm512_cmp_epi16_mask(X, Y, P) \ + ((__mmask32) __builtin_ia32_cmpw512_mask ((__v32hi)(__m512i)(X), \ + (__v32hi)(__m512i)(Y), (int)(P),\ + (__mmask32)(-1))) + +#define _mm512_cmp_epi8_mask(X, Y, P) \ + ((__mmask64) __builtin_ia32_cmpb512_mask ((__v64qi)(__m512i)(X), \ + (__v64qi)(__m512i)(Y), (int)(P),\ + (__mmask64)(-1))) + +#define _mm512_cmp_epu16_mask(X, Y, P) \ + ((__mmask32) __builtin_ia32_ucmpw512_mask ((__v32hi)(__m512i)(X), \ + (__v32hi)(__m512i)(Y), (int)(P),\ + (__mmask32)(-1))) + +#define _mm512_cmp_epu8_mask(X, Y, P) \ + ((__mmask64) __builtin_ia32_ucmpb512_mask ((__v64qi)(__m512i)(X), \ + (__v64qi)(__m512i)(Y), (int)(P),\ + (__mmask64)(-1))) + +#define _mm512_mask_cmp_epi16_mask(M, X, Y, P) \ + ((__mmask32) __builtin_ia32_cmpw512_mask ((__v32hi)(__m512i)(X), \ + (__v32hi)(__m512i)(Y), (int)(P),\ + (__mmask32)(M))) + +#define _mm512_mask_cmp_epi8_mask(M, X, Y, P) \ + ((__mmask64) __builtin_ia32_cmpb512_mask ((__v64qi)(__m512i)(X), \ + (__v64qi)(__m512i)(Y), (int)(P),\ + (__mmask64)(M))) + +#define _mm512_mask_cmp_epu16_mask(M, X, Y, P) \ + ((__mmask32) __builtin_ia32_ucmpw512_mask ((__v32hi)(__m512i)(X), \ + (__v32hi)(__m512i)(Y), (int)(P),\ + (__mmask32)(M))) + +#define _mm512_mask_cmp_epu8_mask(M, X, Y, P) \ + ((__mmask64) __builtin_ia32_ucmpb512_mask ((__v64qi)(__m512i)(X), \ + (__v64qi)(__m512i)(Y), (int)(P),\ + (__mmask64)(M))) + +#define _mm512_bslli_epi128(A, N) \ + ((__m512i)__builtin_ia32_pslldq512 ((__m512i)(A), (int)(N) * 8)) + +#define _mm512_bsrli_epi128(A, N) \ + ((__m512i)__builtin_ia32_psrldq512 ((__m512i)(A), (int)(N) * 8)) + +#endif + +#ifdef __DISABLE_AVX512BW__ +#undef __DISABLE_AVX512BW__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX512BW__ */ + +#endif /* _AVX512BWINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512cdintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512cdintrin.h new file mode 100644 index 0000000000..7fc5851cf1 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512cdintrin.h @@ -0,0 +1,184 @@ +/* Copyright (C) 2013-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +#error "Never use directly; include instead." +#endif + +#ifndef _AVX512CDINTRIN_H_INCLUDED +#define _AVX512CDINTRIN_H_INCLUDED + +#ifndef __AVX512CD__ +#pragma GCC push_options +//#pragma GCC target("avx512cd") +#define __DISABLE_AVX512CD__ +#endif /* __AVX512CD__ */ + +/* Internal data types for implementing the intrinsics. */ +typedef long long __v8di __attribute__ ((__vector_size__ (64))); +typedef int __v16si __attribute__ ((__vector_size__ (64))); + +/* The Intel API is flexible enough that we must allow aliasing with other + vector types, and their scalar components. */ +typedef long long __m512i __attribute__ ((__vector_size__ (64), __may_alias__)); +typedef double __m512d __attribute__ ((__vector_size__ (64), __may_alias__)); + +typedef unsigned char __mmask8; +typedef unsigned short __mmask16; + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_conflict_epi32 (__m512i __A) +{ + return (__m512i) + __builtin_ia32_vpconflictsi_512_mask ((__v16si) __A, + (__v16si) _mm512_setzero_si512 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_conflict_epi32 (__m512i __W, __mmask16 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_vpconflictsi_512_mask ((__v16si) __A, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_conflict_epi32 (__mmask16 __U, __m512i __A) +{ + return (__m512i) + __builtin_ia32_vpconflictsi_512_mask ((__v16si) __A, + (__v16si) _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_conflict_epi64 (__m512i __A) +{ + return (__m512i) + __builtin_ia32_vpconflictdi_512_mask ((__v8di) __A, + (__v8di) _mm512_setzero_si512 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_conflict_epi64 (__m512i __W, __mmask8 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_vpconflictdi_512_mask ((__v8di) __A, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_conflict_epi64 (__mmask8 __U, __m512i __A) +{ + return (__m512i) + __builtin_ia32_vpconflictdi_512_mask ((__v8di) __A, + (__v8di) _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_lzcnt_epi64 (__m512i __A) +{ + return (__m512i) + __builtin_ia32_vplzcntq_512_mask ((__v8di) __A, + (__v8di) _mm512_setzero_si512 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_lzcnt_epi64 (__m512i __W, __mmask8 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_vplzcntq_512_mask ((__v8di) __A, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_lzcnt_epi64 (__mmask8 __U, __m512i __A) +{ + return (__m512i) + __builtin_ia32_vplzcntq_512_mask ((__v8di) __A, + (__v8di) _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_lzcnt_epi32 (__m512i __A) +{ + return (__m512i) + __builtin_ia32_vplzcntd_512_mask ((__v16si) __A, + (__v16si) _mm512_setzero_si512 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_lzcnt_epi32 (__m512i __W, __mmask16 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_vplzcntd_512_mask ((__v16si) __A, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_lzcnt_epi32 (__mmask16 __U, __m512i __A) +{ + return (__m512i) + __builtin_ia32_vplzcntd_512_mask ((__v16si) __A, + (__v16si) _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcastmb_epi64 (__mmask8 __A) +{ + return (__m512i) __builtin_ia32_broadcastmb512 (__A); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcastmw_epi32 (__mmask16 __A) +{ + return (__m512i) __builtin_ia32_broadcastmw512 (__A); +} + +#ifdef __DISABLE_AVX512CD__ +#undef __DISABLE_AVX512CD__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX512CD__ */ + +#endif /* _AVX512CDINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512dqintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512dqintrin.h new file mode 100644 index 0000000000..8da9478f5f --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512dqintrin.h @@ -0,0 +1,2466 @@ +/* Copyright (C) 2014-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +#error "Never use directly; include instead." +#endif + +#ifndef _AVX512DQINTRIN_H_INCLUDED +#define _AVX512DQINTRIN_H_INCLUDED + +#ifndef __AVX512DQ__ +#pragma GCC push_options +//#pragma GCC target("avx512dq") +#define __DISABLE_AVX512DQ__ +#endif /* __AVX512DQ__ */ + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_ktest_mask8_u8 (__mmask8 __A, __mmask8 __B, unsigned char *__CF) +{ + *__CF = (unsigned char) __builtin_ia32_ktestcqi (__A, __B); + return (unsigned char) __builtin_ia32_ktestzqi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_ktestz_mask8_u8 (__mmask8 __A, __mmask8 __B) +{ + return (unsigned char) __builtin_ia32_ktestzqi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_ktestc_mask8_u8 (__mmask8 __A, __mmask8 __B) +{ + return (unsigned char) __builtin_ia32_ktestcqi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_ktest_mask16_u8 (__mmask16 __A, __mmask16 __B, unsigned char *__CF) +{ + *__CF = (unsigned char) __builtin_ia32_ktestchi (__A, __B); + return (unsigned char) __builtin_ia32_ktestzhi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_ktestz_mask16_u8 (__mmask16 __A, __mmask16 __B) +{ + return (unsigned char) __builtin_ia32_ktestzhi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_ktestc_mask16_u8 (__mmask16 __A, __mmask16 __B) +{ + return (unsigned char) __builtin_ia32_ktestchi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kortest_mask8_u8 (__mmask8 __A, __mmask8 __B, unsigned char *__CF) +{ + *__CF = (unsigned char) __builtin_ia32_kortestcqi (__A, __B); + return (unsigned char) __builtin_ia32_kortestzqi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kortestz_mask8_u8 (__mmask8 __A, __mmask8 __B) +{ + return (unsigned char) __builtin_ia32_kortestzqi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kortestc_mask8_u8 (__mmask8 __A, __mmask8 __B) +{ + return (unsigned char) __builtin_ia32_kortestcqi (__A, __B); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kadd_mask8 (__mmask8 __A, __mmask8 __B) +{ + return (__mmask8) __builtin_ia32_kaddqi ((__mmask8) __A, (__mmask8) __B); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kadd_mask16 (__mmask16 __A, __mmask16 __B) +{ + return (__mmask16) __builtin_ia32_kaddhi ((__mmask16) __A, (__mmask16) __B); +} + +extern __inline unsigned int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_cvtmask8_u32 (__mmask8 __A) +{ + return (unsigned int) __builtin_ia32_kmovb ((__mmask8 ) __A); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_cvtu32_mask8 (unsigned int __A) +{ + return (__mmask8) __builtin_ia32_kmovb ((__mmask8) __A); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_load_mask8 (__mmask8 *__A) +{ + return (__mmask8) __builtin_ia32_kmovb (*(__mmask8 *) __A); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_store_mask8 (__mmask8 *__A, __mmask8 __B) +{ + *(__mmask8 *) __A = __builtin_ia32_kmovb (__B); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_knot_mask8 (__mmask8 __A) +{ + return (__mmask8) __builtin_ia32_knotqi ((__mmask8) __A); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kor_mask8 (__mmask8 __A, __mmask8 __B) +{ + return (__mmask8) __builtin_ia32_korqi ((__mmask8) __A, (__mmask8) __B); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kxnor_mask8 (__mmask8 __A, __mmask8 __B) +{ + return (__mmask8) __builtin_ia32_kxnorqi ((__mmask8) __A, (__mmask8) __B); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kxor_mask8 (__mmask8 __A, __mmask8 __B) +{ + return (__mmask8) __builtin_ia32_kxorqi ((__mmask8) __A, (__mmask8) __B); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kand_mask8 (__mmask8 __A, __mmask8 __B) +{ + return (__mmask8) __builtin_ia32_kandqi ((__mmask8) __A, (__mmask8) __B); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kandn_mask8 (__mmask8 __A, __mmask8 __B) +{ + return (__mmask8) __builtin_ia32_kandnqi ((__mmask8) __A, (__mmask8) __B); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcast_f64x2 (__m128d __A) +{ + return (__m512d) + __builtin_ia32_broadcastf64x2_512_mask ((__v2df) __A, + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_broadcast_f64x2 (__m512d __O, __mmask8 __M, __m128d __A) +{ + return (__m512d) __builtin_ia32_broadcastf64x2_512_mask ((__v2df) + __A, + (__v8df) + __O, __M); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_broadcast_f64x2 (__mmask8 __M, __m128d __A) +{ + return (__m512d) __builtin_ia32_broadcastf64x2_512_mask ((__v2df) + __A, + (__v8df) + _mm512_setzero_ps (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcast_i64x2 (__m128i __A) +{ + return (__m512i) + __builtin_ia32_broadcasti64x2_512_mask ((__v2di) __A, + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_broadcast_i64x2 (__m512i __O, __mmask8 __M, __m128i __A) +{ + return (__m512i) __builtin_ia32_broadcasti64x2_512_mask ((__v2di) + __A, + (__v8di) + __O, __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_broadcast_i64x2 (__mmask8 __M, __m128i __A) +{ + return (__m512i) __builtin_ia32_broadcasti64x2_512_mask ((__v2di) + __A, + (__v8di) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcast_f32x2 (__m128 __A) +{ + return (__m512) + __builtin_ia32_broadcastf32x2_512_mask ((__v4sf) __A, + (__v16sf)_mm512_undefined_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_broadcast_f32x2 (__m512 __O, __mmask16 __M, __m128 __A) +{ + return (__m512) __builtin_ia32_broadcastf32x2_512_mask ((__v4sf) __A, + (__v16sf) + __O, __M); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_broadcast_f32x2 (__mmask16 __M, __m128 __A) +{ + return (__m512) __builtin_ia32_broadcastf32x2_512_mask ((__v4sf) __A, + (__v16sf) + _mm512_setzero_ps (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcast_i32x2 (__m128i __A) +{ + return (__m512i) + __builtin_ia32_broadcasti32x2_512_mask ((__v4si) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_broadcast_i32x2 (__m512i __O, __mmask16 __M, __m128i __A) +{ + return (__m512i) __builtin_ia32_broadcasti32x2_512_mask ((__v4si) + __A, + (__v16si) + __O, __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_broadcast_i32x2 (__mmask16 __M, __m128i __A) +{ + return (__m512i) __builtin_ia32_broadcasti32x2_512_mask ((__v4si) + __A, + (__v16si) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcast_f32x8 (__m256 __A) +{ + return (__m512) + __builtin_ia32_broadcastf32x8_512_mask ((__v8sf) __A, + _mm512_undefined_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_broadcast_f32x8 (__m512 __O, __mmask16 __M, __m256 __A) +{ + return (__m512) __builtin_ia32_broadcastf32x8_512_mask ((__v8sf) __A, + (__v16sf)__O, + __M); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_broadcast_f32x8 (__mmask16 __M, __m256 __A) +{ + return (__m512) __builtin_ia32_broadcastf32x8_512_mask ((__v8sf) __A, + (__v16sf) + _mm512_setzero_ps (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcast_i32x8 (__m256i __A) +{ + return (__m512i) + __builtin_ia32_broadcasti32x8_512_mask ((__v8si) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_broadcast_i32x8 (__m512i __O, __mmask16 __M, __m256i __A) +{ + return (__m512i) __builtin_ia32_broadcasti32x8_512_mask ((__v8si) + __A, + (__v16si)__O, + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_broadcast_i32x8 (__mmask16 __M, __m256i __A) +{ + return (__m512i) __builtin_ia32_broadcasti32x8_512_mask ((__v8si) + __A, + (__v16si) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mullo_epi64 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v8du) __A * (__v8du) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mullo_epi64 (__m512i __W, __mmask8 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pmullq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mullo_epi64 (__mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmullq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_xor_pd (__m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_xorpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_xor_pd (__m512d __W, __mmask8 __U, __m512d __A, + __m512d __B) +{ + return (__m512d) __builtin_ia32_xorpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_xor_pd (__mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_xorpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_xor_ps (__m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_xorps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_xor_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_xorps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_xor_ps (__mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_xorps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_or_pd (__m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_orpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_or_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_orpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_or_pd (__mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_orpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_or_ps (__m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_orps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_or_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_orps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_or_ps (__mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_orps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_and_pd (__m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_andpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_and_pd (__m512d __W, __mmask8 __U, __m512d __A, + __m512d __B) +{ + return (__m512d) __builtin_ia32_andpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_and_pd (__mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_andpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_and_ps (__m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_andps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_and_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_andps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_and_ps (__mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_andps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_andnot_pd (__m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_andnpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_andnot_pd (__m512d __W, __mmask8 __U, __m512d __A, + __m512d __B) +{ + return (__m512d) __builtin_ia32_andnpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_andnot_pd (__mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_andnpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_andnot_ps (__m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_andnps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_andnot_ps (__m512 __W, __mmask16 __U, __m512 __A, + __m512 __B) +{ + return (__m512) __builtin_ia32_andnps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_andnot_ps (__mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_andnps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_movepi32_mask (__m512i __A) +{ + return (__mmask16) __builtin_ia32_cvtd2mask512 ((__v16si) __A); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_movepi64_mask (__m512i __A) +{ + return (__mmask8) __builtin_ia32_cvtq2mask512 ((__v8di) __A); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_movm_epi32 (__mmask16 __A) +{ + return (__m512i) __builtin_ia32_cvtmask2d512 (__A); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_movm_epi64 (__mmask8 __A) +{ + return (__m512i) __builtin_ia32_cvtmask2q512 (__A); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvttpd_epi64 (__m512d __A) +{ + return (__m512i) __builtin_ia32_cvttpd2qq512_mask ((__v8df) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvttpd_epi64 (__m512i __W, __mmask8 __U, __m512d __A) +{ + return (__m512i) __builtin_ia32_cvttpd2qq512_mask ((__v8df) __A, + (__v8di) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvttpd_epi64 (__mmask8 __U, __m512d __A) +{ + return (__m512i) __builtin_ia32_cvttpd2qq512_mask ((__v8df) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvttpd_epu64 (__m512d __A) +{ + return (__m512i) __builtin_ia32_cvttpd2uqq512_mask ((__v8df) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvttpd_epu64 (__m512i __W, __mmask8 __U, __m512d __A) +{ + return (__m512i) __builtin_ia32_cvttpd2uqq512_mask ((__v8df) __A, + (__v8di) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvttpd_epu64 (__mmask8 __U, __m512d __A) +{ + return (__m512i) __builtin_ia32_cvttpd2uqq512_mask ((__v8df) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvttps_epi64 (__m256 __A) +{ + return (__m512i) __builtin_ia32_cvttps2qq512_mask ((__v8sf) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvttps_epi64 (__m512i __W, __mmask8 __U, __m256 __A) +{ + return (__m512i) __builtin_ia32_cvttps2qq512_mask ((__v8sf) __A, + (__v8di) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvttps_epi64 (__mmask8 __U, __m256 __A) +{ + return (__m512i) __builtin_ia32_cvttps2qq512_mask ((__v8sf) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvttps_epu64 (__m256 __A) +{ + return (__m512i) __builtin_ia32_cvttps2uqq512_mask ((__v8sf) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvttps_epu64 (__m512i __W, __mmask8 __U, __m256 __A) +{ + return (__m512i) __builtin_ia32_cvttps2uqq512_mask ((__v8sf) __A, + (__v8di) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvttps_epu64 (__mmask8 __U, __m256 __A) +{ + return (__m512i) __builtin_ia32_cvttps2uqq512_mask ((__v8sf) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtpd_epi64 (__m512d __A) +{ + return (__m512i) __builtin_ia32_cvtpd2qq512_mask ((__v8df) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtpd_epi64 (__m512i __W, __mmask8 __U, __m512d __A) +{ + return (__m512i) __builtin_ia32_cvtpd2qq512_mask ((__v8df) __A, + (__v8di) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtpd_epi64 (__mmask8 __U, __m512d __A) +{ + return (__m512i) __builtin_ia32_cvtpd2qq512_mask ((__v8df) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtpd_epu64 (__m512d __A) +{ + return (__m512i) __builtin_ia32_cvtpd2uqq512_mask ((__v8df) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtpd_epu64 (__m512i __W, __mmask8 __U, __m512d __A) +{ + return (__m512i) __builtin_ia32_cvtpd2uqq512_mask ((__v8df) __A, + (__v8di) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtpd_epu64 (__mmask8 __U, __m512d __A) +{ + return (__m512i) __builtin_ia32_cvtpd2uqq512_mask ((__v8df) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtps_epi64 (__m256 __A) +{ + return (__m512i) __builtin_ia32_cvtps2qq512_mask ((__v8sf) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtps_epi64 (__m512i __W, __mmask8 __U, __m256 __A) +{ + return (__m512i) __builtin_ia32_cvtps2qq512_mask ((__v8sf) __A, + (__v8di) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtps_epi64 (__mmask8 __U, __m256 __A) +{ + return (__m512i) __builtin_ia32_cvtps2qq512_mask ((__v8sf) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtps_epu64 (__m256 __A) +{ + return (__m512i) __builtin_ia32_cvtps2uqq512_mask ((__v8sf) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtps_epu64 (__m512i __W, __mmask8 __U, __m256 __A) +{ + return (__m512i) __builtin_ia32_cvtps2uqq512_mask ((__v8sf) __A, + (__v8di) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtps_epu64 (__mmask8 __U, __m256 __A) +{ + return (__m512i) __builtin_ia32_cvtps2uqq512_mask ((__v8sf) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepi64_ps (__m512i __A) +{ + return (__m256) __builtin_ia32_cvtqq2ps512_mask ((__v8di) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi64_ps (__m256 __W, __mmask8 __U, __m512i __A) +{ + return (__m256) __builtin_ia32_cvtqq2ps512_mask ((__v8di) __A, + (__v8sf) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepi64_ps (__mmask8 __U, __m512i __A) +{ + return (__m256) __builtin_ia32_cvtqq2ps512_mask ((__v8di) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepu64_ps (__m512i __A) +{ + return (__m256) __builtin_ia32_cvtuqq2ps512_mask ((__v8di) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepu64_ps (__m256 __W, __mmask8 __U, __m512i __A) +{ + return (__m256) __builtin_ia32_cvtuqq2ps512_mask ((__v8di) __A, + (__v8sf) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepu64_ps (__mmask8 __U, __m512i __A) +{ + return (__m256) __builtin_ia32_cvtuqq2ps512_mask ((__v8di) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepi64_pd (__m512i __A) +{ + return (__m512d) __builtin_ia32_cvtqq2pd512_mask ((__v8di) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi64_pd (__m512d __W, __mmask8 __U, __m512i __A) +{ + return (__m512d) __builtin_ia32_cvtqq2pd512_mask ((__v8di) __A, + (__v8df) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepi64_pd (__mmask8 __U, __m512i __A) +{ + return (__m512d) __builtin_ia32_cvtqq2pd512_mask ((__v8di) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepu64_pd (__m512i __A) +{ + return (__m512d) __builtin_ia32_cvtuqq2pd512_mask ((__v8di) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepu64_pd (__m512d __W, __mmask8 __U, __m512i __A) +{ + return (__m512d) __builtin_ia32_cvtuqq2pd512_mask ((__v8di) __A, + (__v8df) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepu64_pd (__mmask8 __U, __m512i __A) +{ + return (__m512d) __builtin_ia32_cvtuqq2pd512_mask ((__v8di) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +#ifdef __OPTIMIZE__ +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kshiftli_mask8 (__mmask8 __A, unsigned int __B) +{ + return (__mmask8) __builtin_ia32_kshiftliqi ((__mmask8) __A, (__mmask8) __B); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kshiftri_mask8 (__mmask8 __A, unsigned int __B) +{ + return (__mmask8) __builtin_ia32_kshiftriqi ((__mmask8) __A, (__mmask8) __B); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_range_pd (__m512d __A, __m512d __B, int __C) +{ + return (__m512d) __builtin_ia32_rangepd512_mask ((__v8df) __A, + (__v8df) __B, __C, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_range_pd (__m512d __W, __mmask8 __U, + __m512d __A, __m512d __B, int __C) +{ + return (__m512d) __builtin_ia32_rangepd512_mask ((__v8df) __A, + (__v8df) __B, __C, + (__v8df) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_range_pd (__mmask8 __U, __m512d __A, __m512d __B, int __C) +{ + return (__m512d) __builtin_ia32_rangepd512_mask ((__v8df) __A, + (__v8df) __B, __C, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_range_ps (__m512 __A, __m512 __B, int __C) +{ + return (__m512) __builtin_ia32_rangeps512_mask ((__v16sf) __A, + (__v16sf) __B, __C, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_range_ps (__m512 __W, __mmask16 __U, + __m512 __A, __m512 __B, int __C) +{ + return (__m512) __builtin_ia32_rangeps512_mask ((__v16sf) __A, + (__v16sf) __B, __C, + (__v16sf) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_range_ps (__mmask16 __U, __m512 __A, __m512 __B, int __C) +{ + return (__m512) __builtin_ia32_rangeps512_mask ((__v16sf) __A, + (__v16sf) __B, __C, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_reduce_sd (__m128d __A, __m128d __B, int __C) +{ + return (__m128d) __builtin_ia32_reducesd ((__v2df) __A, + (__v2df) __B, __C); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_reduce_ss (__m128 __A, __m128 __B, int __C) +{ + return (__m128) __builtin_ia32_reducess ((__v4sf) __A, + (__v4sf) __B, __C); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_range_sd (__m128d __A, __m128d __B, int __C) +{ + return (__m128d) __builtin_ia32_rangesd128_round ((__v2df) __A, + (__v2df) __B, __C, + _MM_FROUND_CUR_DIRECTION); +} + + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_range_ss (__m128 __A, __m128 __B, int __C) +{ + return (__m128) __builtin_ia32_rangess128_round ((__v4sf) __A, + (__v4sf) __B, __C, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_range_round_sd (__m128d __A, __m128d __B, int __C, const int __R) +{ + return (__m128d) __builtin_ia32_rangesd128_round ((__v2df) __A, + (__v2df) __B, __C, + __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_range_round_ss (__m128 __A, __m128 __B, int __C, const int __R) +{ + return (__m128) __builtin_ia32_rangess128_round ((__v4sf) __A, + (__v4sf) __B, __C, + __R); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fpclass_ss_mask (__m128 __A, const int __imm) +{ + return (__mmask8) __builtin_ia32_fpclassss ((__v4sf) __A, __imm); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fpclass_sd_mask (__m128d __A, const int __imm) +{ + return (__mmask8) __builtin_ia32_fpclasssd ((__v2df) __A, __imm); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtt_roundpd_epi64 (__m512d __A, const int __R) +{ + return (__m512i) __builtin_ia32_cvttpd2qq512_mask ((__v8df) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtt_roundpd_epi64 (__m512i __W, __mmask8 __U, __m512d __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvttpd2qq512_mask ((__v8df) __A, + (__v8di) __W, + (__mmask8) __U, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtt_roundpd_epi64 (__mmask8 __U, __m512d __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvttpd2qq512_mask ((__v8df) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtt_roundpd_epu64 (__m512d __A, const int __R) +{ + return (__m512i) __builtin_ia32_cvttpd2uqq512_mask ((__v8df) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtt_roundpd_epu64 (__m512i __W, __mmask8 __U, __m512d __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvttpd2uqq512_mask ((__v8df) __A, + (__v8di) __W, + (__mmask8) __U, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtt_roundpd_epu64 (__mmask8 __U, __m512d __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvttpd2uqq512_mask ((__v8df) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtt_roundps_epi64 (__m256 __A, const int __R) +{ + return (__m512i) __builtin_ia32_cvttps2qq512_mask ((__v8sf) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtt_roundps_epi64 (__m512i __W, __mmask8 __U, __m256 __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvttps2qq512_mask ((__v8sf) __A, + (__v8di) __W, + (__mmask8) __U, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtt_roundps_epi64 (__mmask8 __U, __m256 __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvttps2qq512_mask ((__v8sf) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtt_roundps_epu64 (__m256 __A, const int __R) +{ + return (__m512i) __builtin_ia32_cvttps2uqq512_mask ((__v8sf) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtt_roundps_epu64 (__m512i __W, __mmask8 __U, __m256 __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvttps2uqq512_mask ((__v8sf) __A, + (__v8di) __W, + (__mmask8) __U, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtt_roundps_epu64 (__mmask8 __U, __m256 __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvttps2uqq512_mask ((__v8sf) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundpd_epi64 (__m512d __A, const int __R) +{ + return (__m512i) __builtin_ia32_cvtpd2qq512_mask ((__v8df) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundpd_epi64 (__m512i __W, __mmask8 __U, __m512d __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvtpd2qq512_mask ((__v8df) __A, + (__v8di) __W, + (__mmask8) __U, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundpd_epi64 (__mmask8 __U, __m512d __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvtpd2qq512_mask ((__v8df) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundpd_epu64 (__m512d __A, const int __R) +{ + return (__m512i) __builtin_ia32_cvtpd2uqq512_mask ((__v8df) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundpd_epu64 (__m512i __W, __mmask8 __U, __m512d __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvtpd2uqq512_mask ((__v8df) __A, + (__v8di) __W, + (__mmask8) __U, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundpd_epu64 (__mmask8 __U, __m512d __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvtpd2uqq512_mask ((__v8df) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundps_epi64 (__m256 __A, const int __R) +{ + return (__m512i) __builtin_ia32_cvtps2qq512_mask ((__v8sf) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundps_epi64 (__m512i __W, __mmask8 __U, __m256 __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvtps2qq512_mask ((__v8sf) __A, + (__v8di) __W, + (__mmask8) __U, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundps_epi64 (__mmask8 __U, __m256 __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvtps2qq512_mask ((__v8sf) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundps_epu64 (__m256 __A, const int __R) +{ + return (__m512i) __builtin_ia32_cvtps2uqq512_mask ((__v8sf) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundps_epu64 (__m512i __W, __mmask8 __U, __m256 __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvtps2uqq512_mask ((__v8sf) __A, + (__v8di) __W, + (__mmask8) __U, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundps_epu64 (__mmask8 __U, __m256 __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvtps2uqq512_mask ((__v8sf) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U, + __R); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundepi64_ps (__m512i __A, const int __R) +{ + return (__m256) __builtin_ia32_cvtqq2ps512_mask ((__v8di) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) -1, + __R); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundepi64_ps (__m256 __W, __mmask8 __U, __m512i __A, + const int __R) +{ + return (__m256) __builtin_ia32_cvtqq2ps512_mask ((__v8di) __A, + (__v8sf) __W, + (__mmask8) __U, + __R); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundepi64_ps (__mmask8 __U, __m512i __A, + const int __R) +{ + return (__m256) __builtin_ia32_cvtqq2ps512_mask ((__v8di) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U, + __R); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundepu64_ps (__m512i __A, const int __R) +{ + return (__m256) __builtin_ia32_cvtuqq2ps512_mask ((__v8di) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) -1, + __R); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundepu64_ps (__m256 __W, __mmask8 __U, __m512i __A, + const int __R) +{ + return (__m256) __builtin_ia32_cvtuqq2ps512_mask ((__v8di) __A, + (__v8sf) __W, + (__mmask8) __U, + __R); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundepu64_ps (__mmask8 __U, __m512i __A, + const int __R) +{ + return (__m256) __builtin_ia32_cvtuqq2ps512_mask ((__v8di) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U, + __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundepi64_pd (__m512i __A, const int __R) +{ + return (__m512d) __builtin_ia32_cvtqq2pd512_mask ((__v8di) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) -1, + __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundepi64_pd (__m512d __W, __mmask8 __U, __m512i __A, + const int __R) +{ + return (__m512d) __builtin_ia32_cvtqq2pd512_mask ((__v8di) __A, + (__v8df) __W, + (__mmask8) __U, + __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundepi64_pd (__mmask8 __U, __m512i __A, + const int __R) +{ + return (__m512d) __builtin_ia32_cvtqq2pd512_mask ((__v8di) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, + __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundepu64_pd (__m512i __A, const int __R) +{ + return (__m512d) __builtin_ia32_cvtuqq2pd512_mask ((__v8di) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) -1, + __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundepu64_pd (__m512d __W, __mmask8 __U, __m512i __A, + const int __R) +{ + return (__m512d) __builtin_ia32_cvtuqq2pd512_mask ((__v8di) __A, + (__v8df) __W, + (__mmask8) __U, + __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundepu64_pd (__mmask8 __U, __m512i __A, + const int __R) +{ + return (__m512d) __builtin_ia32_cvtuqq2pd512_mask ((__v8di) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, + __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_pd (__m512d __A, int __B) +{ + return (__m512d) __builtin_ia32_reducepd512_mask ((__v8df) __A, __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_pd (__m512d __W, __mmask8 __U, __m512d __A, int __B) +{ + return (__m512d) __builtin_ia32_reducepd512_mask ((__v8df) __A, __B, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_reduce_pd (__mmask8 __U, __m512d __A, int __B) +{ + return (__m512d) __builtin_ia32_reducepd512_mask ((__v8df) __A, __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_ps (__m512 __A, int __B) +{ + return (__m512) __builtin_ia32_reduceps512_mask ((__v16sf) __A, __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_ps (__m512 __W, __mmask16 __U, __m512 __A, int __B) +{ + return (__m512) __builtin_ia32_reduceps512_mask ((__v16sf) __A, __B, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_reduce_ps (__mmask16 __U, __m512 __A, int __B) +{ + return (__m512) __builtin_ia32_reduceps512_mask ((__v16sf) __A, __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_extractf32x8_ps (__m512 __A, const int __imm) +{ + return (__m256) __builtin_ia32_extractf32x8_mask ((__v16sf) __A, + __imm, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_extractf32x8_ps (__m256 __W, __mmask8 __U, __m512 __A, + const int __imm) +{ + return (__m256) __builtin_ia32_extractf32x8_mask ((__v16sf) __A, + __imm, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_extractf32x8_ps (__mmask8 __U, __m512 __A, + const int __imm) +{ + return (__m256) __builtin_ia32_extractf32x8_mask ((__v16sf) __A, + __imm, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_extractf64x2_pd (__m512d __A, const int __imm) +{ + return (__m128d) __builtin_ia32_extractf64x2_512_mask ((__v8df) __A, + __imm, + (__v2df) + _mm_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_extractf64x2_pd (__m128d __W, __mmask8 __U, __m512d __A, + const int __imm) +{ + return (__m128d) __builtin_ia32_extractf64x2_512_mask ((__v8df) __A, + __imm, + (__v2df) __W, + (__mmask8) + __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_extractf64x2_pd (__mmask8 __U, __m512d __A, + const int __imm) +{ + return (__m128d) __builtin_ia32_extractf64x2_512_mask ((__v8df) __A, + __imm, + (__v2df) + _mm_setzero_pd (), + (__mmask8) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_extracti32x8_epi32 (__m512i __A, const int __imm) +{ + return (__m256i) __builtin_ia32_extracti32x8_mask ((__v16si) __A, + __imm, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_extracti32x8_epi32 (__m256i __W, __mmask8 __U, __m512i __A, + const int __imm) +{ + return (__m256i) __builtin_ia32_extracti32x8_mask ((__v16si) __A, + __imm, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_extracti32x8_epi32 (__mmask8 __U, __m512i __A, + const int __imm) +{ + return (__m256i) __builtin_ia32_extracti32x8_mask ((__v16si) __A, + __imm, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_extracti64x2_epi64 (__m512i __A, const int __imm) +{ + return (__m128i) __builtin_ia32_extracti64x2_512_mask ((__v8di) __A, + __imm, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_extracti64x2_epi64 (__m128i __W, __mmask8 __U, __m512i __A, + const int __imm) +{ + return (__m128i) __builtin_ia32_extracti64x2_512_mask ((__v8di) __A, + __imm, + (__v2di) __W, + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_extracti64x2_epi64 (__mmask8 __U, __m512i __A, + const int __imm) +{ + return (__m128i) __builtin_ia32_extracti64x2_512_mask ((__v8di) __A, + __imm, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) + __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_range_round_pd (__m512d __A, __m512d __B, int __C, + const int __R) +{ + return (__m512d) __builtin_ia32_rangepd512_mask ((__v8df) __A, + (__v8df) __B, __C, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) -1, + __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_range_round_pd (__m512d __W, __mmask8 __U, + __m512d __A, __m512d __B, int __C, + const int __R) +{ + return (__m512d) __builtin_ia32_rangepd512_mask ((__v8df) __A, + (__v8df) __B, __C, + (__v8df) __W, + (__mmask8) __U, + __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_range_round_pd (__mmask8 __U, __m512d __A, __m512d __B, + int __C, const int __R) +{ + return (__m512d) __builtin_ia32_rangepd512_mask ((__v8df) __A, + (__v8df) __B, __C, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, + __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_range_round_ps (__m512 __A, __m512 __B, int __C, const int __R) +{ + return (__m512) __builtin_ia32_rangeps512_mask ((__v16sf) __A, + (__v16sf) __B, __C, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) -1, + __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_range_round_ps (__m512 __W, __mmask16 __U, + __m512 __A, __m512 __B, int __C, + const int __R) +{ + return (__m512) __builtin_ia32_rangeps512_mask ((__v16sf) __A, + (__v16sf) __B, __C, + (__v16sf) __W, + (__mmask16) __U, + __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_range_round_ps (__mmask16 __U, __m512 __A, __m512 __B, + int __C, const int __R) +{ + return (__m512) __builtin_ia32_rangeps512_mask ((__v16sf) __A, + (__v16sf) __B, __C, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_inserti32x8 (__m512i __A, __m256i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_inserti32x8_mask ((__v16si) __A, + (__v8si) __B, + __imm, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_inserti32x8 (__m512i __W, __mmask16 __U, __m512i __A, + __m256i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_inserti32x8_mask ((__v16si) __A, + (__v8si) __B, + __imm, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_inserti32x8 (__mmask16 __U, __m512i __A, __m256i __B, + const int __imm) +{ + return (__m512i) __builtin_ia32_inserti32x8_mask ((__v16si) __A, + (__v8si) __B, + __imm, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_insertf32x8 (__m512 __A, __m256 __B, const int __imm) +{ + return (__m512) __builtin_ia32_insertf32x8_mask ((__v16sf) __A, + (__v8sf) __B, + __imm, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_insertf32x8 (__m512 __W, __mmask16 __U, __m512 __A, + __m256 __B, const int __imm) +{ + return (__m512) __builtin_ia32_insertf32x8_mask ((__v16sf) __A, + (__v8sf) __B, + __imm, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_insertf32x8 (__mmask16 __U, __m512 __A, __m256 __B, + const int __imm) +{ + return (__m512) __builtin_ia32_insertf32x8_mask ((__v16sf) __A, + (__v8sf) __B, + __imm, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_inserti64x2 (__m512i __A, __m128i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_inserti64x2_512_mask ((__v8di) __A, + (__v2di) __B, + __imm, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_inserti64x2 (__m512i __W, __mmask8 __U, __m512i __A, + __m128i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_inserti64x2_512_mask ((__v8di) __A, + (__v2di) __B, + __imm, + (__v8di) __W, + (__mmask8) + __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_inserti64x2 (__mmask8 __U, __m512i __A, __m128i __B, + const int __imm) +{ + return (__m512i) __builtin_ia32_inserti64x2_512_mask ((__v8di) __A, + (__v2di) __B, + __imm, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) + __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_insertf64x2 (__m512d __A, __m128d __B, const int __imm) +{ + return (__m512d) __builtin_ia32_insertf64x2_512_mask ((__v8df) __A, + (__v2df) __B, + __imm, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_insertf64x2 (__m512d __W, __mmask8 __U, __m512d __A, + __m128d __B, const int __imm) +{ + return (__m512d) __builtin_ia32_insertf64x2_512_mask ((__v8df) __A, + (__v2df) __B, + __imm, + (__v8df) __W, + (__mmask8) + __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_insertf64x2 (__mmask8 __U, __m512d __A, __m128d __B, + const int __imm) +{ + return (__m512d) __builtin_ia32_insertf64x2_512_mask ((__v8df) __A, + (__v2df) __B, + __imm, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) + __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fpclass_pd_mask (__mmask8 __U, __m512d __A, + const int __imm) +{ + return (__mmask8) __builtin_ia32_fpclasspd512_mask ((__v8df) __A, + __imm, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fpclass_pd_mask (__m512d __A, const int __imm) +{ + return (__mmask8) __builtin_ia32_fpclasspd512_mask ((__v8df) __A, + __imm, + (__mmask8) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fpclass_ps_mask (__mmask16 __U, __m512 __A, + const int __imm) +{ + return (__mmask16) __builtin_ia32_fpclassps512_mask ((__v16sf) __A, + __imm, __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fpclass_ps_mask (__m512 __A, const int __imm) +{ + return (__mmask16) __builtin_ia32_fpclassps512_mask ((__v16sf) __A, + __imm, + (__mmask16) -1); +} + +#else +#define _kshiftli_mask8(X, Y) \ + ((__mmask8) __builtin_ia32_kshiftliqi ((__mmask8)(X), (__mmask8)(Y))) + +#define _kshiftri_mask8(X, Y) \ + ((__mmask8) __builtin_ia32_kshiftriqi ((__mmask8)(X), (__mmask8)(Y))) + +#define _mm_range_sd(A, B, C) \ + ((__m128d) __builtin_ia32_rangesd128_round ((__v2df)(__m128d)(A), \ + (__v2df)(__m128d)(B), (int)(C), \ + _MM_FROUND_CUR_DIRECTION)) + +#define _mm_range_ss(A, B, C) \ + ((__m128) __builtin_ia32_rangess128_round ((__v4sf)(__m128)(A), \ + (__v4sf)(__m128)(B), (int)(C), \ + _MM_FROUND_CUR_DIRECTION)) + +#define _mm_range_round_sd(A, B, C, R) \ + ((__m128d) __builtin_ia32_rangesd128_round ((__v2df)(__m128d)(A), \ + (__v2df)(__m128d)(B), (int)(C), (R))) + +#define _mm_range_round_ss(A, B, C, R) \ + ((__m128) __builtin_ia32_rangess128_round ((__v4sf)(__m128)(A), \ + (__v4sf)(__m128)(B), (int)(C), (R))) + +#define _mm512_cvtt_roundpd_epi64(A, B) \ + ((__m512i)__builtin_ia32_cvttpd2qq512_mask ((A), (__v8di) \ + _mm512_setzero_si512 (), \ + -1, (B))) + +#define _mm512_mask_cvtt_roundpd_epi64(W, U, A, B) \ + ((__m512i)__builtin_ia32_cvttpd2qq512_mask ((A), (__v8di)(W), (U), (B))) + +#define _mm512_maskz_cvtt_roundpd_epi64(U, A, B) \ + ((__m512i)__builtin_ia32_cvttpd2qq512_mask ((A), (__v8di)_mm512_setzero_si512 (), (U), (B))) + +#define _mm512_cvtt_roundpd_epu64(A, B) \ + ((__m512i)__builtin_ia32_cvttpd2uqq512_mask ((A), (__v8di)_mm512_setzero_si512 (), -1, (B))) + +#define _mm512_mask_cvtt_roundpd_epu64(W, U, A, B) \ + ((__m512i)__builtin_ia32_cvttpd2uqq512_mask ((A), (__v8di)(W), (U), (B))) + +#define _mm512_maskz_cvtt_roundpd_epu64(U, A, B) \ + ((__m512i)__builtin_ia32_cvttpd2uqq512_mask ((A), (__v8di)_mm512_setzero_si512 (), (U), (B))) + +#define _mm512_cvtt_roundps_epi64(A, B) \ + ((__m512i)__builtin_ia32_cvttps2qq512_mask ((A), (__v8di)_mm512_setzero_si512 (), -1, (B))) + +#define _mm512_mask_cvtt_roundps_epi64(W, U, A, B) \ + ((__m512i)__builtin_ia32_cvttps2qq512_mask ((A), (__v8di)(W), (U), (B))) + +#define _mm512_maskz_cvtt_roundps_epi64(U, A, B) \ + ((__m512i)__builtin_ia32_cvttps2qq512_mask ((A), (__v8di)_mm512_setzero_si512 (), (U), (B))) + +#define _mm512_cvtt_roundps_epu64(A, B) \ + ((__m512i)__builtin_ia32_cvttps2uqq512_mask ((A), (__v8di)_mm512_setzero_si512 (), -1, (B))) + +#define _mm512_mask_cvtt_roundps_epu64(W, U, A, B) \ + ((__m512i)__builtin_ia32_cvttps2uqq512_mask ((A), (__v8di)(W), (U), (B))) + +#define _mm512_maskz_cvtt_roundps_epu64(U, A, B) \ + ((__m512i)__builtin_ia32_cvttps2uqq512_mask ((A), (__v8di)_mm512_setzero_si512 (), (U), (B))) + +#define _mm512_cvt_roundpd_epi64(A, B) \ + ((__m512i)__builtin_ia32_cvtpd2qq512_mask ((A), (__v8di)_mm512_setzero_si512 (), -1, (B))) + +#define _mm512_mask_cvt_roundpd_epi64(W, U, A, B) \ + ((__m512i)__builtin_ia32_cvtpd2qq512_mask ((A), (__v8di)(W), (U), (B))) + +#define _mm512_maskz_cvt_roundpd_epi64(U, A, B) \ + ((__m512i)__builtin_ia32_cvtpd2qq512_mask ((A), (__v8di)_mm512_setzero_si512 (), (U), (B))) + +#define _mm512_cvt_roundpd_epu64(A, B) \ + ((__m512i)__builtin_ia32_cvtpd2uqq512_mask ((A), (__v8di)_mm512_setzero_si512 (), -1, (B))) + +#define _mm512_mask_cvt_roundpd_epu64(W, U, A, B) \ + ((__m512i)__builtin_ia32_cvtpd2uqq512_mask ((A), (__v8di)(W), (U), (B))) + +#define _mm512_maskz_cvt_roundpd_epu64(U, A, B) \ + ((__m512i)__builtin_ia32_cvtpd2uqq512_mask ((A), (__v8di)_mm512_setzero_si512 (), (U), (B))) + +#define _mm512_cvt_roundps_epi64(A, B) \ + ((__m512i)__builtin_ia32_cvtps2qq512_mask ((A), (__v8di)_mm512_setzero_si512 (), -1, (B))) + +#define _mm512_mask_cvt_roundps_epi64(W, U, A, B) \ + ((__m512i)__builtin_ia32_cvtps2qq512_mask ((A), (__v8di)(W), (U), (B))) + +#define _mm512_maskz_cvt_roundps_epi64(U, A, B) \ + ((__m512i)__builtin_ia32_cvtps2qq512_mask ((A), (__v8di)_mm512_setzero_si512 (), (U), (B))) + +#define _mm512_cvt_roundps_epu64(A, B) \ + ((__m512i)__builtin_ia32_cvtps2uqq512_mask ((A), (__v8di)_mm512_setzero_si512 (), -1, (B))) + +#define _mm512_mask_cvt_roundps_epu64(W, U, A, B) \ + ((__m512i)__builtin_ia32_cvtps2uqq512_mask ((A), (__v8di)(W), (U), (B))) + +#define _mm512_maskz_cvt_roundps_epu64(U, A, B) \ + ((__m512i)__builtin_ia32_cvtps2uqq512_mask ((A), (__v8di)_mm512_setzero_si512 (), (U), (B))) + +#define _mm512_cvt_roundepi64_ps(A, B) \ + ((__m256)__builtin_ia32_cvtqq2ps512_mask ((__v8di)(A), (__v8sf)_mm256_setzero_ps (), -1, (B))) + +#define _mm512_mask_cvt_roundepi64_ps(W, U, A, B) \ + ((__m256)__builtin_ia32_cvtqq2ps512_mask ((__v8di)(A), (W), (U), (B))) + +#define _mm512_maskz_cvt_roundepi64_ps(U, A, B) \ + ((__m256)__builtin_ia32_cvtqq2ps512_mask ((__v8di)(A), (__v8sf)_mm256_setzero_ps (), (U), (B))) + +#define _mm512_cvt_roundepu64_ps(A, B) \ + ((__m256)__builtin_ia32_cvtuqq2ps512_mask ((__v8di)(A), (__v8sf)_mm256_setzero_ps (), -1, (B))) + +#define _mm512_mask_cvt_roundepu64_ps(W, U, A, B) \ + ((__m256)__builtin_ia32_cvtuqq2ps512_mask ((__v8di)(A), (W), (U), (B))) + +#define _mm512_maskz_cvt_roundepu64_ps(U, A, B) \ + ((__m256)__builtin_ia32_cvtuqq2ps512_mask ((__v8di)(A), (__v8sf)_mm256_setzero_ps (), (U), (B))) + +#define _mm512_cvt_roundepi64_pd(A, B) \ + ((__m512d)__builtin_ia32_cvtqq2pd512_mask ((__v8di)(A), (__v8df)_mm512_setzero_pd (), -1, (B))) + +#define _mm512_mask_cvt_roundepi64_pd(W, U, A, B) \ + ((__m512d)__builtin_ia32_cvtqq2pd512_mask ((__v8di)(A), (W), (U), (B))) + +#define _mm512_maskz_cvt_roundepi64_pd(U, A, B) \ + ((__m512d)__builtin_ia32_cvtqq2pd512_mask ((__v8di)(A), (__v8df)_mm512_setzero_pd (), (U), (B))) + +#define _mm512_cvt_roundepu64_pd(A, B) \ + ((__m512d)__builtin_ia32_cvtuqq2pd512_mask ((__v8di)(A), (__v8df)_mm512_setzero_pd (), -1, (B))) + +#define _mm512_mask_cvt_roundepu64_pd(W, U, A, B) \ + ((__m512d)__builtin_ia32_cvtuqq2pd512_mask ((__v8di)(A), (W), (U), (B))) + +#define _mm512_maskz_cvt_roundepu64_pd(U, A, B) \ + ((__m512d)__builtin_ia32_cvtuqq2pd512_mask ((__v8di)(A), (__v8df)_mm512_setzero_pd (), (U), (B))) + +#define _mm512_reduce_pd(A, B) \ + ((__m512d) __builtin_ia32_reducepd512_mask ((__v8df)(__m512d)(A), \ + (int)(B), (__v8df)_mm512_setzero_pd (), (__mmask8)-1)) + +#define _mm512_mask_reduce_pd(W, U, A, B) \ + ((__m512d) __builtin_ia32_reducepd512_mask ((__v8df)(__m512d)(A), \ + (int)(B), (__v8df)(__m512d)(W), (__mmask8)(U))) + +#define _mm512_maskz_reduce_pd(U, A, B) \ + ((__m512d) __builtin_ia32_reducepd512_mask ((__v8df)(__m512d)(A), \ + (int)(B), (__v8df)_mm512_setzero_pd (), (__mmask8)(U))) + +#define _mm512_reduce_ps(A, B) \ + ((__m512) __builtin_ia32_reduceps512_mask ((__v16sf)(__m512)(A), \ + (int)(B), (__v16sf)_mm512_setzero_ps (), (__mmask16)-1)) + +#define _mm512_mask_reduce_ps(W, U, A, B) \ + ((__m512) __builtin_ia32_reduceps512_mask ((__v16sf)(__m512)(A), \ + (int)(B), (__v16sf)(__m512)(W), (__mmask16)(U))) + +#define _mm512_maskz_reduce_ps(U, A, B) \ + ((__m512) __builtin_ia32_reduceps512_mask ((__v16sf)(__m512)(A), \ + (int)(B), (__v16sf)_mm512_setzero_ps (), (__mmask16)(U))) + +#define _mm512_extractf32x8_ps(X, C) \ + ((__m256) __builtin_ia32_extractf32x8_mask ((__v16sf)(__m512) (X), \ + (int) (C), (__v8sf)(__m256) _mm256_setzero_ps (), (__mmask8)-1)) + +#define _mm512_mask_extractf32x8_ps(W, U, X, C) \ + ((__m256) __builtin_ia32_extractf32x8_mask ((__v16sf)(__m512) (X), \ + (int) (C), (__v8sf)(__m256) (W), (__mmask8) (U))) + +#define _mm512_maskz_extractf32x8_ps(U, X, C) \ + ((__m256) __builtin_ia32_extractf32x8_mask ((__v16sf)(__m512) (X), \ + (int) (C), (__v8sf)(__m256) _mm256_setzero_ps (), (__mmask8) (U))) + +#define _mm512_extractf64x2_pd(X, C) \ + ((__m128d) __builtin_ia32_extractf64x2_512_mask ((__v8df)(__m512d) (X),\ + (int) (C), (__v2df)(__m128d) _mm_setzero_pd (), (__mmask8)-1)) + +#define _mm512_mask_extractf64x2_pd(W, U, X, C) \ + ((__m128d) __builtin_ia32_extractf64x2_512_mask ((__v8df)(__m512d) (X),\ + (int) (C), (__v2df)(__m128d) (W), (__mmask8) (U))) + +#define _mm512_maskz_extractf64x2_pd(U, X, C) \ + ((__m128d) __builtin_ia32_extractf64x2_512_mask ((__v8df)(__m512d) (X),\ + (int) (C), (__v2df)(__m128d) _mm_setzero_pd (), (__mmask8) (U))) + +#define _mm512_extracti32x8_epi32(X, C) \ + ((__m256i) __builtin_ia32_extracti32x8_mask ((__v16si)(__m512i) (X), \ + (int) (C), (__v8si)(__m256i) _mm256_setzero_si256 (), (__mmask8)-1)) + +#define _mm512_mask_extracti32x8_epi32(W, U, X, C) \ + ((__m256i) __builtin_ia32_extracti32x8_mask ((__v16si)(__m512i) (X), \ + (int) (C), (__v8si)(__m256i) (W), (__mmask8) (U))) + +#define _mm512_maskz_extracti32x8_epi32(U, X, C) \ + ((__m256i) __builtin_ia32_extracti32x8_mask ((__v16si)(__m512i) (X), \ + (int) (C), (__v8si)(__m256i) _mm256_setzero_si256 (), (__mmask8) (U))) + +#define _mm512_extracti64x2_epi64(X, C) \ + ((__m128i) __builtin_ia32_extracti64x2_512_mask ((__v8di)(__m512i) (X),\ + (int) (C), (__v2di)(__m128i) _mm_setzero_si128 (), (__mmask8)-1)) + +#define _mm512_mask_extracti64x2_epi64(W, U, X, C) \ + ((__m128i) __builtin_ia32_extracti64x2_512_mask ((__v8di)(__m512i) (X),\ + (int) (C), (__v2di)(__m128i) (W), (__mmask8) (U))) + +#define _mm512_maskz_extracti64x2_epi64(U, X, C) \ + ((__m128i) __builtin_ia32_extracti64x2_512_mask ((__v8di)(__m512i) (X),\ + (int) (C), (__v2di)(__m128i) _mm_setzero_si128 (), (__mmask8) (U))) + +#define _mm512_range_pd(A, B, C) \ + ((__m512d) __builtin_ia32_rangepd512_mask ((__v8df)(__m512d)(A), \ + (__v8df)(__m512d)(B), (int)(C), \ + (__v8df)_mm512_setzero_pd (), (__mmask8)-1, _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_mask_range_pd(W, U, A, B, C) \ + ((__m512d) __builtin_ia32_rangepd512_mask ((__v8df)(__m512d)(A), \ + (__v8df)(__m512d)(B), (int)(C), \ + (__v8df)(__m512d)(W), (__mmask8)(U), _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_maskz_range_pd(U, A, B, C) \ + ((__m512d) __builtin_ia32_rangepd512_mask ((__v8df)(__m512d)(A), \ + (__v8df)(__m512d)(B), (int)(C), \ + (__v8df)_mm512_setzero_pd (), (__mmask8)(U), _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_range_ps(A, B, C) \ + ((__m512) __builtin_ia32_rangeps512_mask ((__v16sf)(__m512)(A), \ + (__v16sf)(__m512)(B), (int)(C), \ + (__v16sf)_mm512_setzero_ps (), (__mmask16)-1, _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_mask_range_ps(W, U, A, B, C) \ + ((__m512) __builtin_ia32_rangeps512_mask ((__v16sf)(__m512)(A), \ + (__v16sf)(__m512)(B), (int)(C), \ + (__v16sf)(__m512)(W), (__mmask16)(U), _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_maskz_range_ps(U, A, B, C) \ + ((__m512) __builtin_ia32_rangeps512_mask ((__v16sf)(__m512)(A), \ + (__v16sf)(__m512)(B), (int)(C), \ + (__v16sf)_mm512_setzero_ps (), (__mmask16)(U), _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_range_round_pd(A, B, C, R) \ + ((__m512d) __builtin_ia32_rangepd512_mask ((__v8df)(__m512d)(A), \ + (__v8df)(__m512d)(B), (int)(C), \ + (__v8df)_mm512_setzero_pd (), (__mmask8)-1, (R))) + +#define _mm512_mask_range_round_pd(W, U, A, B, C, R) \ + ((__m512d) __builtin_ia32_rangepd512_mask ((__v8df)(__m512d)(A), \ + (__v8df)(__m512d)(B), (int)(C), \ + (__v8df)(__m512d)(W), (__mmask8)(U), (R))) + +#define _mm512_maskz_range_round_pd(U, A, B, C, R) \ + ((__m512d) __builtin_ia32_rangepd512_mask ((__v8df)(__m512d)(A), \ + (__v8df)(__m512d)(B), (int)(C), \ + (__v8df)_mm512_setzero_pd (), (__mmask8)(U), (R))) + +#define _mm512_range_round_ps(A, B, C, R) \ + ((__m512) __builtin_ia32_rangeps512_mask ((__v16sf)(__m512)(A), \ + (__v16sf)(__m512)(B), (int)(C), \ + (__v16sf)_mm512_setzero_ps (), (__mmask16)-1, (R))) + +#define _mm512_mask_range_round_ps(W, U, A, B, C, R) \ + ((__m512) __builtin_ia32_rangeps512_mask ((__v16sf)(__m512)(A), \ + (__v16sf)(__m512)(B), (int)(C), \ + (__v16sf)(__m512)(W), (__mmask16)(U), (R))) + +#define _mm512_maskz_range_round_ps(U, A, B, C, R) \ + ((__m512) __builtin_ia32_rangeps512_mask ((__v16sf)(__m512)(A), \ + (__v16sf)(__m512)(B), (int)(C), \ + (__v16sf)_mm512_setzero_ps (), (__mmask16)(U), (R))) + +#define _mm512_insertf64x2(X, Y, C) \ + ((__m512d) __builtin_ia32_insertf64x2_512_mask ((__v8df)(__m512d) (X),\ + (__v2df)(__m128d) (Y), (int) (C), (__v8df)(__m512d) (X), \ + (__mmask8)-1)) + +#define _mm512_mask_insertf64x2(W, U, X, Y, C) \ + ((__m512d) __builtin_ia32_insertf64x2_512_mask ((__v8df)(__m512d) (X),\ + (__v2df)(__m128d) (Y), (int) (C), (__v8df)(__m512d) (W), \ + (__mmask8) (U))) + +#define _mm512_maskz_insertf64x2(U, X, Y, C) \ + ((__m512d) __builtin_ia32_insertf64x2_512_mask ((__v8df)(__m512d) (X),\ + (__v2df)(__m128d) (Y), (int) (C), \ + (__v8df)(__m512d) _mm512_setzero_pd (), (__mmask8) (U))) + +#define _mm512_inserti64x2(X, Y, C) \ + ((__m512i) __builtin_ia32_inserti64x2_512_mask ((__v8di)(__m512i) (X),\ + (__v2di)(__m128i) (Y), (int) (C), (__v8di)(__m512i) (X), (__mmask8)-1)) + +#define _mm512_mask_inserti64x2(W, U, X, Y, C) \ + ((__m512i) __builtin_ia32_inserti64x2_512_mask ((__v8di)(__m512i) (X),\ + (__v2di)(__m128i) (Y), (int) (C), (__v8di)(__m512i) (W), \ + (__mmask8) (U))) + +#define _mm512_maskz_inserti64x2(U, X, Y, C) \ + ((__m512i) __builtin_ia32_inserti64x2_512_mask ((__v8di)(__m512i) (X),\ + (__v2di)(__m128i) (Y), (int) (C), \ + (__v8di)(__m512i) _mm512_setzero_si512 (), (__mmask8) (U))) + +#define _mm512_insertf32x8(X, Y, C) \ + ((__m512) __builtin_ia32_insertf32x8_mask ((__v16sf)(__m512) (X), \ + (__v8sf)(__m256) (Y), (int) (C),\ + (__v16sf)(__m512)_mm512_setzero_ps (),\ + (__mmask16)-1)) + +#define _mm512_mask_insertf32x8(W, U, X, Y, C) \ + ((__m512) __builtin_ia32_insertf32x8_mask ((__v16sf)(__m512) (X), \ + (__v8sf)(__m256) (Y), (int) (C),\ + (__v16sf)(__m512)(W),\ + (__mmask16)(U))) + +#define _mm512_maskz_insertf32x8(U, X, Y, C) \ + ((__m512) __builtin_ia32_insertf32x8_mask ((__v16sf)(__m512) (X), \ + (__v8sf)(__m256) (Y), (int) (C),\ + (__v16sf)(__m512)_mm512_setzero_ps (),\ + (__mmask16)(U))) + +#define _mm512_inserti32x8(X, Y, C) \ + ((__m512i) __builtin_ia32_inserti32x8_mask ((__v16si)(__m512i) (X), \ + (__v8si)(__m256i) (Y), (int) (C),\ + (__v16si)(__m512i)_mm512_setzero_si512 (),\ + (__mmask16)-1)) + +#define _mm512_mask_inserti32x8(W, U, X, Y, C) \ + ((__m512i) __builtin_ia32_inserti32x8_mask ((__v16si)(__m512i) (X), \ + (__v8si)(__m256i) (Y), (int) (C),\ + (__v16si)(__m512i)(W),\ + (__mmask16)(U))) + +#define _mm512_maskz_inserti32x8(U, X, Y, C) \ + ((__m512i) __builtin_ia32_inserti32x8_mask ((__v16si)(__m512i) (X), \ + (__v8si)(__m256i) (Y), (int) (C),\ + (__v16si)(__m512i)_mm512_setzero_si512 (),\ + (__mmask16)(U))) + +#define _mm_fpclass_ss_mask(X, C) \ + ((__mmask8) __builtin_ia32_fpclassss ((__v4sf) (__m128) (X), (int) (C))) \ + +#define _mm_fpclass_sd_mask(X, C) \ + ((__mmask8) __builtin_ia32_fpclasssd ((__v2df) (__m128d) (X), (int) (C))) \ + +#define _mm512_mask_fpclass_pd_mask(u, X, C) \ + ((__mmask8) __builtin_ia32_fpclasspd512_mask ((__v8df) (__m512d) (X), \ + (int) (C), (__mmask8)(u))) + +#define _mm512_mask_fpclass_ps_mask(u, x, c) \ + ((__mmask16) __builtin_ia32_fpclassps512_mask ((__v16sf) (__m512) (x),\ + (int) (c),(__mmask8)(u))) + +#define _mm512_fpclass_pd_mask(X, C) \ + ((__mmask8) __builtin_ia32_fpclasspd512_mask ((__v8df) (__m512d) (X), \ + (int) (C), (__mmask8)-1)) + +#define _mm512_fpclass_ps_mask(x, c) \ + ((__mmask16) __builtin_ia32_fpclassps512_mask ((__v16sf) (__m512) (x),\ + (int) (c),(__mmask8)-1)) + +#define _mm_reduce_sd(A, B, C) \ + ((__m128d) __builtin_ia32_reducesd ((__v2df)(__m128d)(A), \ + (__v2df)(__m128d)(B), (int)(C))) \ + +#define _mm_reduce_ss(A, B, C) \ + ((__m128) __builtin_ia32_reducess ((__v4sf)(__m128)(A), \ + (__v4sf)(__m128)(A), (int)(C))) \ + +#endif + +#ifdef __DISABLE_AVX512DQ__ +#undef __DISABLE_AVX512DQ__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX512DQ__ */ + +#endif /* _AVX512DQINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512erintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512erintrin.h new file mode 100644 index 0000000000..b46ec62aa4 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512erintrin.h @@ -0,0 +1,394 @@ +/* Copyright (C) 2013-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +#error "Never use directly; include instead." +#endif + +#ifndef _AVX512ERINTRIN_H_INCLUDED +#define _AVX512ERINTRIN_H_INCLUDED + +#ifndef __AVX512ER__ +//#pragma GCC push_options +#pragma GCC target("avx512er") +#define __DISABLE_AVX512ER__ +#endif /* __AVX512ER__ */ + +/* Internal data types for implementing the intrinsics. */ +typedef double __v8df __attribute__ ((__vector_size__ (64))); +typedef float __v16sf __attribute__ ((__vector_size__ (64))); + +/* The Intel API is flexible enough that we must allow aliasing with other + vector types, and their scalar components. */ +typedef float __m512 __attribute__ ((__vector_size__ (64), __may_alias__)); +typedef double __m512d __attribute__ ((__vector_size__ (64), __may_alias__)); + +typedef unsigned char __mmask8; +typedef unsigned short __mmask16; + +#ifdef __OPTIMIZE__ +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_exp2a23_round_pd (__m512d __A, int __R) +{ + __m512d __W; + return (__m512d) __builtin_ia32_exp2pd_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_exp2a23_round_pd (__m512d __W, __mmask8 __U, __m512d __A, int __R) +{ + return (__m512d) __builtin_ia32_exp2pd_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_exp2a23_round_pd (__mmask8 __U, __m512d __A, int __R) +{ + return (__m512d) __builtin_ia32_exp2pd_mask ((__v8df) __A, + (__v8df) _mm512_setzero_pd (), + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_exp2a23_round_ps (__m512 __A, int __R) +{ + __m512 __W; + return (__m512) __builtin_ia32_exp2ps_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_exp2a23_round_ps (__m512 __W, __mmask16 __U, __m512 __A, int __R) +{ + return (__m512) __builtin_ia32_exp2ps_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_exp2a23_round_ps (__mmask16 __U, __m512 __A, int __R) +{ + return (__m512) __builtin_ia32_exp2ps_mask ((__v16sf) __A, + (__v16sf) _mm512_setzero_ps (), + (__mmask16) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_rcp28_round_pd (__m512d __A, int __R) +{ + __m512d __W; + return (__m512d) __builtin_ia32_rcp28pd_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_rcp28_round_pd (__m512d __W, __mmask8 __U, __m512d __A, int __R) +{ + return (__m512d) __builtin_ia32_rcp28pd_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_rcp28_round_pd (__mmask8 __U, __m512d __A, int __R) +{ + return (__m512d) __builtin_ia32_rcp28pd_mask ((__v8df) __A, + (__v8df) _mm512_setzero_pd (), + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_rcp28_round_ps (__m512 __A, int __R) +{ + __m512 __W; + return (__m512) __builtin_ia32_rcp28ps_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_rcp28_round_ps (__m512 __W, __mmask16 __U, __m512 __A, int __R) +{ + return (__m512) __builtin_ia32_rcp28ps_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_rcp28_round_ps (__mmask16 __U, __m512 __A, int __R) +{ + return (__m512) __builtin_ia32_rcp28ps_mask ((__v16sf) __A, + (__v16sf) _mm512_setzero_ps (), + (__mmask16) __U, __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rcp28_round_sd (__m128d __A, __m128d __B, int __R) +{ + return (__m128d) __builtin_ia32_rcp28sd_round ((__v2df) __B, + (__v2df) __A, + __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rcp28_round_ss (__m128 __A, __m128 __B, int __R) +{ + return (__m128) __builtin_ia32_rcp28ss_round ((__v4sf) __B, + (__v4sf) __A, + __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_rsqrt28_round_pd (__m512d __A, int __R) +{ + __m512d __W; + return (__m512d) __builtin_ia32_rsqrt28pd_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_rsqrt28_round_pd (__m512d __W, __mmask8 __U, __m512d __A, int __R) +{ + return (__m512d) __builtin_ia32_rsqrt28pd_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_rsqrt28_round_pd (__mmask8 __U, __m512d __A, int __R) +{ + return (__m512d) __builtin_ia32_rsqrt28pd_mask ((__v8df) __A, + (__v8df) _mm512_setzero_pd (), + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_rsqrt28_round_ps (__m512 __A, int __R) +{ + __m512 __W; + return (__m512) __builtin_ia32_rsqrt28ps_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_rsqrt28_round_ps (__m512 __W, __mmask16 __U, __m512 __A, int __R) +{ + return (__m512) __builtin_ia32_rsqrt28ps_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_rsqrt28_round_ps (__mmask16 __U, __m512 __A, int __R) +{ + return (__m512) __builtin_ia32_rsqrt28ps_mask ((__v16sf) __A, + (__v16sf) _mm512_setzero_ps (), + (__mmask16) __U, __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rsqrt28_round_sd (__m128d __A, __m128d __B, int __R) +{ + return (__m128d) __builtin_ia32_rsqrt28sd_round ((__v2df) __B, + (__v2df) __A, + __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rsqrt28_round_ss (__m128 __A, __m128 __B, int __R) +{ + return (__m128) __builtin_ia32_rsqrt28ss_round ((__v4sf) __B, + (__v4sf) __A, + __R); +} + +#else +#define _mm512_exp2a23_round_pd(A, C) \ + __builtin_ia32_exp2pd_mask(A, (__v8df)_mm512_setzero_pd(), -1, C) + +#define _mm512_mask_exp2a23_round_pd(W, U, A, C) \ + __builtin_ia32_exp2pd_mask(A, W, U, C) + +#define _mm512_maskz_exp2a23_round_pd(U, A, C) \ + __builtin_ia32_exp2pd_mask(A, (__v8df)_mm512_setzero_pd(), U, C) + +#define _mm512_exp2a23_round_ps(A, C) \ + __builtin_ia32_exp2ps_mask(A, (__v16sf)_mm512_setzero_ps(), -1, C) + +#define _mm512_mask_exp2a23_round_ps(W, U, A, C) \ + __builtin_ia32_exp2ps_mask(A, W, U, C) + +#define _mm512_maskz_exp2a23_round_ps(U, A, C) \ + __builtin_ia32_exp2ps_mask(A, (__v16sf)_mm512_setzero_ps(), U, C) + +#define _mm512_rcp28_round_pd(A, C) \ + __builtin_ia32_rcp28pd_mask(A, (__v8df)_mm512_setzero_pd(), -1, C) + +#define _mm512_mask_rcp28_round_pd(W, U, A, C) \ + __builtin_ia32_rcp28pd_mask(A, W, U, C) + +#define _mm512_maskz_rcp28_round_pd(U, A, C) \ + __builtin_ia32_rcp28pd_mask(A, (__v8df)_mm512_setzero_pd(), U, C) + +#define _mm512_rcp28_round_ps(A, C) \ + __builtin_ia32_rcp28ps_mask(A, (__v16sf)_mm512_setzero_ps(), -1, C) + +#define _mm512_mask_rcp28_round_ps(W, U, A, C) \ + __builtin_ia32_rcp28ps_mask(A, W, U, C) + +#define _mm512_maskz_rcp28_round_ps(U, A, C) \ + __builtin_ia32_rcp28ps_mask(A, (__v16sf)_mm512_setzero_ps(), U, C) + +#define _mm512_rsqrt28_round_pd(A, C) \ + __builtin_ia32_rsqrt28pd_mask(A, (__v8df)_mm512_setzero_pd(), -1, C) + +#define _mm512_mask_rsqrt28_round_pd(W, U, A, C) \ + __builtin_ia32_rsqrt28pd_mask(A, W, U, C) + +#define _mm512_maskz_rsqrt28_round_pd(U, A, C) \ + __builtin_ia32_rsqrt28pd_mask(A, (__v8df)_mm512_setzero_pd(), U, C) + +#define _mm512_rsqrt28_round_ps(A, C) \ + __builtin_ia32_rsqrt28ps_mask(A, (__v16sf)_mm512_setzero_ps(), -1, C) + +#define _mm512_mask_rsqrt28_round_ps(W, U, A, C) \ + __builtin_ia32_rsqrt28ps_mask(A, W, U, C) + +#define _mm512_maskz_rsqrt28_round_ps(U, A, C) \ + __builtin_ia32_rsqrt28ps_mask(A, (__v16sf)_mm512_setzero_ps(), U, C) + +#define _mm_rcp28_round_sd(A, B, R) \ + __builtin_ia32_rcp28sd_round(A, B, R) + +#define _mm_rcp28_round_ss(A, B, R) \ + __builtin_ia32_rcp28ss_round(A, B, R) + +#define _mm_rsqrt28_round_sd(A, B, R) \ + __builtin_ia32_rsqrt28sd_round(A, B, R) + +#define _mm_rsqrt28_round_ss(A, B, R) \ + __builtin_ia32_rsqrt28ss_round(A, B, R) + +#endif + +#define _mm512_exp2a23_pd(A) \ + _mm512_exp2a23_round_pd(A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_mask_exp2a23_pd(W, U, A) \ + _mm512_mask_exp2a23_round_pd(W, U, A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_maskz_exp2a23_pd(U, A) \ + _mm512_maskz_exp2a23_round_pd(U, A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_exp2a23_ps(A) \ + _mm512_exp2a23_round_ps(A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_mask_exp2a23_ps(W, U, A) \ + _mm512_mask_exp2a23_round_ps(W, U, A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_maskz_exp2a23_ps(U, A) \ + _mm512_maskz_exp2a23_round_ps(U, A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_rcp28_pd(A) \ + _mm512_rcp28_round_pd(A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_mask_rcp28_pd(W, U, A) \ + _mm512_mask_rcp28_round_pd(W, U, A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_maskz_rcp28_pd(U, A) \ + _mm512_maskz_rcp28_round_pd(U, A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_rcp28_ps(A) \ + _mm512_rcp28_round_ps(A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_mask_rcp28_ps(W, U, A) \ + _mm512_mask_rcp28_round_ps(W, U, A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_maskz_rcp28_ps(U, A) \ + _mm512_maskz_rcp28_round_ps(U, A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_rsqrt28_pd(A) \ + _mm512_rsqrt28_round_pd(A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_mask_rsqrt28_pd(W, U, A) \ + _mm512_mask_rsqrt28_round_pd(W, U, A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_maskz_rsqrt28_pd(U, A) \ + _mm512_maskz_rsqrt28_round_pd(U, A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_rsqrt28_ps(A) \ + _mm512_rsqrt28_round_ps(A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_mask_rsqrt28_ps(W, U, A) \ + _mm512_mask_rsqrt28_round_ps(W, U, A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_maskz_rsqrt28_ps(U, A) \ + _mm512_maskz_rsqrt28_round_ps(U, A, _MM_FROUND_CUR_DIRECTION) + +#define _mm_rcp28_sd(A, B) \ + __builtin_ia32_rcp28sd_round(B, A, _MM_FROUND_CUR_DIRECTION) + +#define _mm_rcp28_ss(A, B) \ + __builtin_ia32_rcp28ss_round(B, A, _MM_FROUND_CUR_DIRECTION) + +#define _mm_rsqrt28_sd(A, B) \ + __builtin_ia32_rsqrt28sd_round(B, A, _MM_FROUND_CUR_DIRECTION) + +#define _mm_rsqrt28_ss(A, B) \ + __builtin_ia32_rsqrt28ss_round(B, A, _MM_FROUND_CUR_DIRECTION) + +#ifdef __DISABLE_AVX512ER__ +#undef __DISABLE_AVX512ER__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX512ER__ */ + +#endif /* _AVX512ERINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512fintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512fintrin.h new file mode 100644 index 0000000000..15e3b0ba94 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512fintrin.h @@ -0,0 +1,13754 @@ +/* Copyright (C) 2013-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +#error "Never use directly; include instead." +#endif + +#ifndef _AVX512FINTRIN_H_INCLUDED +#define _AVX512FINTRIN_H_INCLUDED + +#ifndef __AVX512F__ +#pragma GCC push_options +//#pragma GCC target("avx512f") +#define __DISABLE_AVX512F__ +#endif /* __AVX512F__ */ + +/* Internal data types for implementing the intrinsics. */ +typedef double __v8df __attribute__ ((__vector_size__ (64))); +typedef float __v16sf __attribute__ ((__vector_size__ (64))); +typedef long long __v8di __attribute__ ((__vector_size__ (64))); +typedef unsigned long long __v8du __attribute__ ((__vector_size__ (64))); +typedef int __v16si __attribute__ ((__vector_size__ (64))); +typedef unsigned int __v16su __attribute__ ((__vector_size__ (64))); +typedef short __v32hi __attribute__ ((__vector_size__ (64))); +typedef unsigned short __v32hu __attribute__ ((__vector_size__ (64))); +typedef char __v64qi __attribute__ ((__vector_size__ (64))); +typedef unsigned char __v64qu __attribute__ ((__vector_size__ (64))); + +/* The Intel API is flexible enough that we must allow aliasing with other + vector types, and their scalar components. */ +typedef float __m512 __attribute__ ((__vector_size__ (64), __may_alias__)); +typedef long long __m512i __attribute__ ((__vector_size__ (64), __may_alias__)); +typedef double __m512d __attribute__ ((__vector_size__ (64), __may_alias__)); + +/* Unaligned version of the same type. */ +typedef float __m512_u __attribute__ ((__vector_size__ (64), __may_alias__, __aligned__ (1))); +typedef long long __m512i_u __attribute__ ((__vector_size__ (64), __may_alias__, __aligned__ (1))); +typedef double __m512d_u __attribute__ ((__vector_size__ (64), __may_alias__, __aligned__ (1))); + +typedef unsigned char __mmask8; +typedef unsigned short __mmask16; + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_int2mask (int __M) +{ + return (__mmask16) __M; +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask2int (__mmask16 __M) +{ + return (int) __M; +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_set_epi64 (long long __A, long long __B, long long __C, + long long __D, long long __E, long long __F, + long long __G, long long __H) +{ + return __extension__ (__m512i) (__v8di) + { __H, __G, __F, __E, __D, __C, __B, __A }; +} + +/* Create the vector [A B C D E F G H I J K L M N O P]. */ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_set_epi32 (int __A, int __B, int __C, int __D, + int __E, int __F, int __G, int __H, + int __I, int __J, int __K, int __L, + int __M, int __N, int __O, int __P) +{ + return __extension__ (__m512i)(__v16si) + { __P, __O, __N, __M, __L, __K, __J, __I, + __H, __G, __F, __E, __D, __C, __B, __A }; +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_set_pd (double __A, double __B, double __C, double __D, + double __E, double __F, double __G, double __H) +{ + return __extension__ (__m512d) + { __H, __G, __F, __E, __D, __C, __B, __A }; +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_set_ps (float __A, float __B, float __C, float __D, + float __E, float __F, float __G, float __H, + float __I, float __J, float __K, float __L, + float __M, float __N, float __O, float __P) +{ + return __extension__ (__m512) + { __P, __O, __N, __M, __L, __K, __J, __I, + __H, __G, __F, __E, __D, __C, __B, __A }; +} + +#define _mm512_setr_epi64(e0,e1,e2,e3,e4,e5,e6,e7) \ + _mm512_set_epi64(e7,e6,e5,e4,e3,e2,e1,e0) + +#define _mm512_setr_epi32(e0,e1,e2,e3,e4,e5,e6,e7, \ + e8,e9,e10,e11,e12,e13,e14,e15) \ + _mm512_set_epi32(e15,e14,e13,e12,e11,e10,e9,e8,e7,e6,e5,e4,e3,e2,e1,e0) + +#define _mm512_setr_pd(e0,e1,e2,e3,e4,e5,e6,e7) \ + _mm512_set_pd(e7,e6,e5,e4,e3,e2,e1,e0) + +#define _mm512_setr_ps(e0,e1,e2,e3,e4,e5,e6,e7,e8,e9,e10,e11,e12,e13,e14,e15) \ + _mm512_set_ps(e15,e14,e13,e12,e11,e10,e9,e8,e7,e6,e5,e4,e3,e2,e1,e0) + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_undefined_ps (void) +{ + __m512 __Y = __Y; + return __Y; +} + +#define _mm512_undefined _mm512_undefined_ps + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_undefined_pd (void) +{ + __m512d __Y = __Y; + return __Y; +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_undefined_epi32 (void) +{ + __m512i __Y = __Y; + return __Y; +} + +#define _mm512_undefined_si512 _mm512_undefined_epi32 + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_set1_epi8 (char __A) +{ + return __extension__ (__m512i)(__v64qi) + { __A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A }; +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_set1_epi16 (short __A) +{ + return __extension__ (__m512i)(__v32hi) + { __A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A }; +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_set1_pd (double __A) +{ + return (__m512d) __builtin_ia32_broadcastsd512 (__extension__ + (__v2df) { __A, }, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_set1_ps (float __A) +{ + return (__m512) __builtin_ia32_broadcastss512 (__extension__ + (__v4sf) { __A, }, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1); +} + +/* Create the vector [A B C D A B C D A B C D A B C D]. */ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_set4_epi32 (int __A, int __B, int __C, int __D) +{ + return __extension__ (__m512i)(__v16si) + { __D, __C, __B, __A, __D, __C, __B, __A, + __D, __C, __B, __A, __D, __C, __B, __A }; +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_set4_epi64 (long long __A, long long __B, long long __C, + long long __D) +{ + return __extension__ (__m512i) (__v8di) + { __D, __C, __B, __A, __D, __C, __B, __A }; +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_set4_pd (double __A, double __B, double __C, double __D) +{ + return __extension__ (__m512d) + { __D, __C, __B, __A, __D, __C, __B, __A }; +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_set4_ps (float __A, float __B, float __C, float __D) +{ + return __extension__ (__m512) + { __D, __C, __B, __A, __D, __C, __B, __A, + __D, __C, __B, __A, __D, __C, __B, __A }; +} + +#define _mm512_setr4_epi64(e0,e1,e2,e3) \ + _mm512_set4_epi64(e3,e2,e1,e0) + +#define _mm512_setr4_epi32(e0,e1,e2,e3) \ + _mm512_set4_epi32(e3,e2,e1,e0) + +#define _mm512_setr4_pd(e0,e1,e2,e3) \ + _mm512_set4_pd(e3,e2,e1,e0) + +#define _mm512_setr4_ps(e0,e1,e2,e3) \ + _mm512_set4_ps(e3,e2,e1,e0) + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_setzero_ps (void) +{ + return __extension__ (__m512){ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 }; +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_setzero_pd (void) +{ + return __extension__ (__m512d) { 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 }; +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_setzero_epi32 (void) +{ + return __extension__ (__m512i)(__v8di){ 0, 0, 0, 0, 0, 0, 0, 0 }; +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_setzero_si512 (void) +{ + return __extension__ (__m512i)(__v8di){ 0, 0, 0, 0, 0, 0, 0, 0 }; +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mov_pd (__m512d __W, __mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_movapd512_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mov_pd (__mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_movapd512_mask ((__v8df) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mov_ps (__m512 __W, __mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_movaps512_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mov_ps (__mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_movaps512_mask ((__v16sf) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_load_pd (void const *__P) +{ + return *(__m512d *) __P; +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_load_pd (__m512d __W, __mmask8 __U, void const *__P) +{ + return (__m512d) __builtin_ia32_loadapd512_mask ((const __v8df *) __P, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_load_pd (__mmask8 __U, void const *__P) +{ + return (__m512d) __builtin_ia32_loadapd512_mask ((const __v8df *) __P, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_store_pd (void *__P, __m512d __A) +{ + *(__m512d *) __P = __A; +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_store_pd (void *__P, __mmask8 __U, __m512d __A) +{ + __builtin_ia32_storeapd512_mask ((__v8df *) __P, (__v8df) __A, + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_load_ps (void const *__P) +{ + return *(__m512 *) __P; +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_load_ps (__m512 __W, __mmask16 __U, void const *__P) +{ + return (__m512) __builtin_ia32_loadaps512_mask ((const __v16sf *) __P, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_load_ps (__mmask16 __U, void const *__P) +{ + return (__m512) __builtin_ia32_loadaps512_mask ((const __v16sf *) __P, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_store_ps (void *__P, __m512 __A) +{ + *(__m512 *) __P = __A; +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_store_ps (void *__P, __mmask16 __U, __m512 __A) +{ + __builtin_ia32_storeaps512_mask ((__v16sf *) __P, (__v16sf) __A, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mov_epi64 (__m512i __W, __mmask8 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_movdqa64_512_mask ((__v8di) __A, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mov_epi64 (__mmask8 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_movdqa64_512_mask ((__v8di) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_load_epi64 (void const *__P) +{ + return *(__m512i *) __P; +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_load_epi64 (__m512i __W, __mmask8 __U, void const *__P) +{ + return (__m512i) __builtin_ia32_movdqa64load512_mask ((const __v8di *) __P, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_load_epi64 (__mmask8 __U, void const *__P) +{ + return (__m512i) __builtin_ia32_movdqa64load512_mask ((const __v8di *) __P, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_store_epi64 (void *__P, __m512i __A) +{ + *(__m512i *) __P = __A; +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_store_epi64 (void *__P, __mmask8 __U, __m512i __A) +{ + __builtin_ia32_movdqa64store512_mask ((__v8di *) __P, (__v8di) __A, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mov_epi32 (__m512i __W, __mmask16 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_movdqa32_512_mask ((__v16si) __A, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mov_epi32 (__mmask16 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_movdqa32_512_mask ((__v16si) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_load_si512 (void const *__P) +{ + return *(__m512i *) __P; +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_load_epi32 (void const *__P) +{ + return *(__m512i *) __P; +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_load_epi32 (__m512i __W, __mmask16 __U, void const *__P) +{ + return (__m512i) __builtin_ia32_movdqa32load512_mask ((const __v16si *) __P, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_load_epi32 (__mmask16 __U, void const *__P) +{ + return (__m512i) __builtin_ia32_movdqa32load512_mask ((const __v16si *) __P, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_store_si512 (void *__P, __m512i __A) +{ + *(__m512i *) __P = __A; +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_store_epi32 (void *__P, __m512i __A) +{ + *(__m512i *) __P = __A; +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_store_epi32 (void *__P, __mmask16 __U, __m512i __A) +{ + __builtin_ia32_movdqa32store512_mask ((__v16si *) __P, (__v16si) __A, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mullo_epi32 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v16su) __A * (__v16su) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mullo_epi32 (__mmask16 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmulld512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mullo_epi32 (__m512i __W, __mmask16 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmulld512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __W, __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sllv_epi32 (__m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psllv16si_mask ((__v16si) __X, + (__v16si) __Y, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sllv_epi32 (__m512i __W, __mmask16 __U, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psllv16si_mask ((__v16si) __X, + (__v16si) __Y, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sllv_epi32 (__mmask16 __U, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psllv16si_mask ((__v16si) __X, + (__v16si) __Y, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_srav_epi32 (__m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psrav16si_mask ((__v16si) __X, + (__v16si) __Y, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_srav_epi32 (__m512i __W, __mmask16 __U, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psrav16si_mask ((__v16si) __X, + (__v16si) __Y, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_srav_epi32 (__mmask16 __U, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psrav16si_mask ((__v16si) __X, + (__v16si) __Y, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_srlv_epi32 (__m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psrlv16si_mask ((__v16si) __X, + (__v16si) __Y, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_srlv_epi32 (__m512i __W, __mmask16 __U, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psrlv16si_mask ((__v16si) __X, + (__v16si) __Y, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_srlv_epi32 (__mmask16 __U, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psrlv16si_mask ((__v16si) __X, + (__v16si) __Y, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_add_epi64 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v8du) __A + (__v8du) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_add_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_paddq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_add_epi64 (__mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_paddq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sub_epi64 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v8du) __A - (__v8du) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sub_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psubq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sub_epi64 (__mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psubq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sllv_epi64 (__m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psllv8di_mask ((__v8di) __X, + (__v8di) __Y, + (__v8di) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sllv_epi64 (__m512i __W, __mmask8 __U, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psllv8di_mask ((__v8di) __X, + (__v8di) __Y, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sllv_epi64 (__mmask8 __U, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psllv8di_mask ((__v8di) __X, + (__v8di) __Y, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_srav_epi64 (__m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psrav8di_mask ((__v8di) __X, + (__v8di) __Y, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_srav_epi64 (__m512i __W, __mmask8 __U, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psrav8di_mask ((__v8di) __X, + (__v8di) __Y, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_srav_epi64 (__mmask8 __U, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psrav8di_mask ((__v8di) __X, + (__v8di) __Y, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_srlv_epi64 (__m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psrlv8di_mask ((__v8di) __X, + (__v8di) __Y, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_srlv_epi64 (__m512i __W, __mmask8 __U, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psrlv8di_mask ((__v8di) __X, + (__v8di) __Y, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_srlv_epi64 (__mmask8 __U, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psrlv8di_mask ((__v8di) __X, + (__v8di) __Y, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_add_epi32 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v16su) __A + (__v16su) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_add_epi32 (__m512i __W, __mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_paddd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_add_epi32 (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_paddd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mul_epi32 (__m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_pmuldq512_mask ((__v16si) __X, + (__v16si) __Y, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mul_epi32 (__m512i __W, __mmask8 __M, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_pmuldq512_mask ((__v16si) __X, + (__v16si) __Y, + (__v8di) __W, __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mul_epi32 (__mmask8 __M, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_pmuldq512_mask ((__v16si) __X, + (__v16si) __Y, + (__v8di) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sub_epi32 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v16su) __A - (__v16su) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sub_epi32 (__m512i __W, __mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psubd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sub_epi32 (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psubd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mul_epu32 (__m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_pmuludq512_mask ((__v16si) __X, + (__v16si) __Y, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mul_epu32 (__m512i __W, __mmask8 __M, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_pmuludq512_mask ((__v16si) __X, + (__v16si) __Y, + (__v8di) __W, __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mul_epu32 (__mmask8 __M, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_pmuludq512_mask ((__v16si) __X, + (__v16si) __Y, + (__v8di) + _mm512_setzero_si512 (), + __M); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_slli_epi64 (__m512i __A, unsigned int __B) +{ + return (__m512i) __builtin_ia32_psllqi512_mask ((__v8di) __A, __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_slli_epi64 (__m512i __W, __mmask8 __U, __m512i __A, + unsigned int __B) +{ + return (__m512i) __builtin_ia32_psllqi512_mask ((__v8di) __A, __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_slli_epi64 (__mmask8 __U, __m512i __A, unsigned int __B) +{ + return (__m512i) __builtin_ia32_psllqi512_mask ((__v8di) __A, __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} +#else +#define _mm512_slli_epi64(X, C) \ + ((__m512i) __builtin_ia32_psllqi512_mask ((__v8di)(__m512i)(X), (int)(C),\ + (__v8di)(__m512i)_mm512_undefined_epi32 (),\ + (__mmask8)-1)) + +#define _mm512_mask_slli_epi64(W, U, X, C) \ + ((__m512i) __builtin_ia32_psllqi512_mask ((__v8di)(__m512i)(X), (int)(C),\ + (__v8di)(__m512i)(W),\ + (__mmask8)(U))) + +#define _mm512_maskz_slli_epi64(U, X, C) \ + ((__m512i) __builtin_ia32_psllqi512_mask ((__v8di)(__m512i)(X), (int)(C),\ + (__v8di)(__m512i)_mm512_setzero_si512 (),\ + (__mmask8)(U))) +#endif + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sll_epi64 (__m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psllq512_mask ((__v8di) __A, + (__v2di) __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sll_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psllq512_mask ((__v8di) __A, + (__v2di) __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sll_epi64 (__mmask8 __U, __m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psllq512_mask ((__v8di) __A, + (__v2di) __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_srli_epi64 (__m512i __A, unsigned int __B) +{ + return (__m512i) __builtin_ia32_psrlqi512_mask ((__v8di) __A, __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_srli_epi64 (__m512i __W, __mmask8 __U, + __m512i __A, unsigned int __B) +{ + return (__m512i) __builtin_ia32_psrlqi512_mask ((__v8di) __A, __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_srli_epi64 (__mmask8 __U, __m512i __A, unsigned int __B) +{ + return (__m512i) __builtin_ia32_psrlqi512_mask ((__v8di) __A, __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} +#else +#define _mm512_srli_epi64(X, C) \ + ((__m512i) __builtin_ia32_psrlqi512_mask ((__v8di)(__m512i)(X), (int)(C),\ + (__v8di)(__m512i)_mm512_undefined_epi32 (),\ + (__mmask8)-1)) + +#define _mm512_mask_srli_epi64(W, U, X, C) \ + ((__m512i) __builtin_ia32_psrlqi512_mask ((__v8di)(__m512i)(X), (int)(C),\ + (__v8di)(__m512i)(W),\ + (__mmask8)(U))) + +#define _mm512_maskz_srli_epi64(U, X, C) \ + ((__m512i) __builtin_ia32_psrlqi512_mask ((__v8di)(__m512i)(X), (int)(C),\ + (__v8di)(__m512i)_mm512_setzero_si512 (),\ + (__mmask8)(U))) +#endif + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_srl_epi64 (__m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psrlq512_mask ((__v8di) __A, + (__v2di) __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_srl_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psrlq512_mask ((__v8di) __A, + (__v2di) __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_srl_epi64 (__mmask8 __U, __m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psrlq512_mask ((__v8di) __A, + (__v2di) __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_srai_epi64 (__m512i __A, unsigned int __B) +{ + return (__m512i) __builtin_ia32_psraqi512_mask ((__v8di) __A, __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_srai_epi64 (__m512i __W, __mmask8 __U, __m512i __A, + unsigned int __B) +{ + return (__m512i) __builtin_ia32_psraqi512_mask ((__v8di) __A, __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_srai_epi64 (__mmask8 __U, __m512i __A, unsigned int __B) +{ + return (__m512i) __builtin_ia32_psraqi512_mask ((__v8di) __A, __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} +#else +#define _mm512_srai_epi64(X, C) \ + ((__m512i) __builtin_ia32_psraqi512_mask ((__v8di)(__m512i)(X), (int)(C),\ + (__v8di)(__m512i)_mm512_undefined_epi32 (),\ + (__mmask8)-1)) + +#define _mm512_mask_srai_epi64(W, U, X, C) \ + ((__m512i) __builtin_ia32_psraqi512_mask ((__v8di)(__m512i)(X), (int)(C),\ + (__v8di)(__m512i)(W),\ + (__mmask8)(U))) + +#define _mm512_maskz_srai_epi64(U, X, C) \ + ((__m512i) __builtin_ia32_psraqi512_mask ((__v8di)(__m512i)(X), (int)(C),\ + (__v8di)(__m512i)_mm512_setzero_si512 (),\ + (__mmask8)(U))) +#endif + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sra_epi64 (__m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psraq512_mask ((__v8di) __A, + (__v2di) __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sra_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psraq512_mask ((__v8di) __A, + (__v2di) __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sra_epi64 (__mmask8 __U, __m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psraq512_mask ((__v8di) __A, + (__v2di) __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_slli_epi32 (__m512i __A, unsigned int __B) +{ + return (__m512i) __builtin_ia32_pslldi512_mask ((__v16si) __A, __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_slli_epi32 (__m512i __W, __mmask16 __U, __m512i __A, + unsigned int __B) +{ + return (__m512i) __builtin_ia32_pslldi512_mask ((__v16si) __A, __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_slli_epi32 (__mmask16 __U, __m512i __A, unsigned int __B) +{ + return (__m512i) __builtin_ia32_pslldi512_mask ((__v16si) __A, __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} +#else +#define _mm512_slli_epi32(X, C) \ + ((__m512i) __builtin_ia32_pslldi512_mask ((__v16si)(__m512i)(X), (int)(C),\ + (__v16si)(__m512i)_mm512_undefined_epi32 (),\ + (__mmask16)-1)) + +#define _mm512_mask_slli_epi32(W, U, X, C) \ + ((__m512i) __builtin_ia32_pslldi512_mask ((__v16si)(__m512i)(X), (int)(C),\ + (__v16si)(__m512i)(W),\ + (__mmask16)(U))) + +#define _mm512_maskz_slli_epi32(U, X, C) \ + ((__m512i) __builtin_ia32_pslldi512_mask ((__v16si)(__m512i)(X), (int)(C),\ + (__v16si)(__m512i)_mm512_setzero_si512 (),\ + (__mmask16)(U))) +#endif + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sll_epi32 (__m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_pslld512_mask ((__v16si) __A, + (__v4si) __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sll_epi32 (__m512i __W, __mmask16 __U, __m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_pslld512_mask ((__v16si) __A, + (__v4si) __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sll_epi32 (__mmask16 __U, __m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_pslld512_mask ((__v16si) __A, + (__v4si) __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_srli_epi32 (__m512i __A, unsigned int __B) +{ + return (__m512i) __builtin_ia32_psrldi512_mask ((__v16si) __A, __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_srli_epi32 (__m512i __W, __mmask16 __U, + __m512i __A, unsigned int __B) +{ + return (__m512i) __builtin_ia32_psrldi512_mask ((__v16si) __A, __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_srli_epi32 (__mmask16 __U, __m512i __A, unsigned int __B) +{ + return (__m512i) __builtin_ia32_psrldi512_mask ((__v16si) __A, __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} +#else +#define _mm512_srli_epi32(X, C) \ + ((__m512i) __builtin_ia32_psrldi512_mask ((__v16si)(__m512i)(X), (int)(C),\ + (__v16si)(__m512i)_mm512_undefined_epi32 (),\ + (__mmask16)-1)) + +#define _mm512_mask_srli_epi32(W, U, X, C) \ + ((__m512i) __builtin_ia32_psrldi512_mask ((__v16si)(__m512i)(X), (int)(C),\ + (__v16si)(__m512i)(W),\ + (__mmask16)(U))) + +#define _mm512_maskz_srli_epi32(U, X, C) \ + ((__m512i) __builtin_ia32_psrldi512_mask ((__v16si)(__m512i)(X), (int)(C),\ + (__v16si)(__m512i)_mm512_setzero_si512 (),\ + (__mmask16)(U))) +#endif + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_srl_epi32 (__m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psrld512_mask ((__v16si) __A, + (__v4si) __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_srl_epi32 (__m512i __W, __mmask16 __U, __m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psrld512_mask ((__v16si) __A, + (__v4si) __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_srl_epi32 (__mmask16 __U, __m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psrld512_mask ((__v16si) __A, + (__v4si) __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_srai_epi32 (__m512i __A, unsigned int __B) +{ + return (__m512i) __builtin_ia32_psradi512_mask ((__v16si) __A, __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_srai_epi32 (__m512i __W, __mmask16 __U, __m512i __A, + unsigned int __B) +{ + return (__m512i) __builtin_ia32_psradi512_mask ((__v16si) __A, __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_srai_epi32 (__mmask16 __U, __m512i __A, unsigned int __B) +{ + return (__m512i) __builtin_ia32_psradi512_mask ((__v16si) __A, __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} +#else +#define _mm512_srai_epi32(X, C) \ + ((__m512i) __builtin_ia32_psradi512_mask ((__v16si)(__m512i)(X), (int)(C),\ + (__v16si)(__m512i)_mm512_undefined_epi32 (),\ + (__mmask16)-1)) + +#define _mm512_mask_srai_epi32(W, U, X, C) \ + ((__m512i) __builtin_ia32_psradi512_mask ((__v16si)(__m512i)(X), (int)(C),\ + (__v16si)(__m512i)(W),\ + (__mmask16)(U))) + +#define _mm512_maskz_srai_epi32(U, X, C) \ + ((__m512i) __builtin_ia32_psradi512_mask ((__v16si)(__m512i)(X), (int)(C),\ + (__v16si)(__m512i)_mm512_setzero_si512 (),\ + (__mmask16)(U))) +#endif + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sra_epi32 (__m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psrad512_mask ((__v16si) __A, + (__v4si) __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sra_epi32 (__m512i __W, __mmask16 __U, __m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psrad512_mask ((__v16si) __A, + (__v4si) __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sra_epi32 (__mmask16 __U, __m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psrad512_mask ((__v16si) __A, + (__v4si) __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +#ifdef __OPTIMIZE__ +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_add_round_sd (__m128d __A, __m128d __B, const int __R) +{ + return (__m128d) __builtin_ia32_addsd_round ((__v2df) __A, + (__v2df) __B, + __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_add_round_ss (__m128 __A, __m128 __B, const int __R) +{ + return (__m128) __builtin_ia32_addss_round ((__v4sf) __A, + (__v4sf) __B, + __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sub_round_sd (__m128d __A, __m128d __B, const int __R) +{ + return (__m128d) __builtin_ia32_subsd_round ((__v2df) __A, + (__v2df) __B, + __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sub_round_ss (__m128 __A, __m128 __B, const int __R) +{ + return (__m128) __builtin_ia32_subss_round ((__v4sf) __A, + (__v4sf) __B, + __R); +} + +#else +#define _mm_add_round_sd(A, B, C) \ + (__m128d)__builtin_ia32_addsd_round(A, B, C) + +#define _mm_add_round_ss(A, B, C) \ + (__m128)__builtin_ia32_addss_round(A, B, C) + +#define _mm_sub_round_sd(A, B, C) \ + (__m128d)__builtin_ia32_subsd_round(A, B, C) + +#define _mm_sub_round_ss(A, B, C) \ + (__m128)__builtin_ia32_subss_round(A, B, C) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_ternarylogic_epi64 (__m512i __A, __m512i __B, __m512i __C, + const int __imm) +{ + return (__m512i) __builtin_ia32_pternlogq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __C, __imm, + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_ternarylogic_epi64 (__m512i __A, __mmask8 __U, __m512i __B, + __m512i __C, const int __imm) +{ + return (__m512i) __builtin_ia32_pternlogq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __C, __imm, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_ternarylogic_epi64 (__mmask8 __U, __m512i __A, __m512i __B, + __m512i __C, const int __imm) +{ + return (__m512i) __builtin_ia32_pternlogq512_maskz ((__v8di) __A, + (__v8di) __B, + (__v8di) __C, + __imm, (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_ternarylogic_epi32 (__m512i __A, __m512i __B, __m512i __C, + const int __imm) +{ + return (__m512i) __builtin_ia32_pternlogd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __C, + __imm, (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_ternarylogic_epi32 (__m512i __A, __mmask16 __U, __m512i __B, + __m512i __C, const int __imm) +{ + return (__m512i) __builtin_ia32_pternlogd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __C, + __imm, (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_ternarylogic_epi32 (__mmask16 __U, __m512i __A, __m512i __B, + __m512i __C, const int __imm) +{ + return (__m512i) __builtin_ia32_pternlogd512_maskz ((__v16si) __A, + (__v16si) __B, + (__v16si) __C, + __imm, (__mmask16) __U); +} +#else +#define _mm512_ternarylogic_epi64(A, B, C, I) \ + ((__m512i) __builtin_ia32_pternlogq512_mask ((__v8di)(__m512i)(A), \ + (__v8di)(__m512i)(B), (__v8di)(__m512i)(C), (int)(I), (__mmask8)-1)) +#define _mm512_mask_ternarylogic_epi64(A, U, B, C, I) \ + ((__m512i) __builtin_ia32_pternlogq512_mask ((__v8di)(__m512i)(A), \ + (__v8di)(__m512i)(B), (__v8di)(__m512i)(C), (int)(I), (__mmask8)(U))) +#define _mm512_maskz_ternarylogic_epi64(U, A, B, C, I) \ + ((__m512i) __builtin_ia32_pternlogq512_maskz ((__v8di)(__m512i)(A), \ + (__v8di)(__m512i)(B), (__v8di)(__m512i)(C), (int)(I), (__mmask8)(U))) +#define _mm512_ternarylogic_epi32(A, B, C, I) \ + ((__m512i) __builtin_ia32_pternlogd512_mask ((__v16si)(__m512i)(A), \ + (__v16si)(__m512i)(B), (__v16si)(__m512i)(C), (int)(I), \ + (__mmask16)-1)) +#define _mm512_mask_ternarylogic_epi32(A, U, B, C, I) \ + ((__m512i) __builtin_ia32_pternlogd512_mask ((__v16si)(__m512i)(A), \ + (__v16si)(__m512i)(B), (__v16si)(__m512i)(C), (int)(I), \ + (__mmask16)(U))) +#define _mm512_maskz_ternarylogic_epi32(U, A, B, C, I) \ + ((__m512i) __builtin_ia32_pternlogd512_maskz ((__v16si)(__m512i)(A), \ + (__v16si)(__m512i)(B), (__v16si)(__m512i)(C), (int)(I), \ + (__mmask16)(U))) +#endif + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_rcp14_pd (__m512d __A) +{ + return (__m512d) __builtin_ia32_rcp14pd512_mask ((__v8df) __A, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_rcp14_pd (__m512d __W, __mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_rcp14pd512_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_rcp14_pd (__mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_rcp14pd512_mask ((__v8df) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_rcp14_ps (__m512 __A) +{ + return (__m512) __builtin_ia32_rcp14ps512_mask ((__v16sf) __A, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_rcp14_ps (__m512 __W, __mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_rcp14ps512_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_rcp14_ps (__mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_rcp14ps512_mask ((__v16sf) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rcp14_sd (__m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_rcp14sd ((__v2df) __B, + (__v2df) __A); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rcp14_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_rcp14ss ((__v4sf) __B, + (__v4sf) __A); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_rsqrt14_pd (__m512d __A) +{ + return (__m512d) __builtin_ia32_rsqrt14pd512_mask ((__v8df) __A, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_rsqrt14_pd (__m512d __W, __mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_rsqrt14pd512_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_rsqrt14_pd (__mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_rsqrt14pd512_mask ((__v8df) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_rsqrt14_ps (__m512 __A) +{ + return (__m512) __builtin_ia32_rsqrt14ps512_mask ((__v16sf) __A, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_rsqrt14_ps (__m512 __W, __mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_rsqrt14ps512_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_rsqrt14_ps (__mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_rsqrt14ps512_mask ((__v16sf) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rsqrt14_sd (__m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_rsqrt14sd ((__v2df) __B, + (__v2df) __A); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rsqrt14_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_rsqrt14ss ((__v4sf) __B, + (__v4sf) __A); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sqrt_round_pd (__m512d __A, const int __R) +{ + return (__m512d) __builtin_ia32_sqrtpd512_mask ((__v8df) __A, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sqrt_round_pd (__m512d __W, __mmask8 __U, __m512d __A, + const int __R) +{ + return (__m512d) __builtin_ia32_sqrtpd512_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sqrt_round_pd (__mmask8 __U, __m512d __A, const int __R) +{ + return (__m512d) __builtin_ia32_sqrtpd512_mask ((__v8df) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sqrt_round_ps (__m512 __A, const int __R) +{ + return (__m512) __builtin_ia32_sqrtps512_mask ((__v16sf) __A, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sqrt_round_ps (__m512 __W, __mmask16 __U, __m512 __A, const int __R) +{ + return (__m512) __builtin_ia32_sqrtps512_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sqrt_round_ps (__mmask16 __U, __m512 __A, const int __R) +{ + return (__m512) __builtin_ia32_sqrtps512_mask ((__v16sf) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sqrt_round_sd (__m128d __A, __m128d __B, const int __R) +{ + return (__m128d) __builtin_ia32_sqrtsd_round ((__v2df) __B, + (__v2df) __A, + __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sqrt_round_ss (__m128 __A, __m128 __B, const int __R) +{ + return (__m128) __builtin_ia32_sqrtss_round ((__v4sf) __B, + (__v4sf) __A, + __R); +} +#else +#define _mm512_sqrt_round_pd(A, C) \ + (__m512d)__builtin_ia32_sqrtpd512_mask(A, (__v8df)_mm512_undefined_pd(), -1, C) + +#define _mm512_mask_sqrt_round_pd(W, U, A, C) \ + (__m512d)__builtin_ia32_sqrtpd512_mask(A, W, U, C) + +#define _mm512_maskz_sqrt_round_pd(U, A, C) \ + (__m512d)__builtin_ia32_sqrtpd512_mask(A, (__v8df)_mm512_setzero_pd(), U, C) + +#define _mm512_sqrt_round_ps(A, C) \ + (__m512)__builtin_ia32_sqrtps512_mask(A, (__v16sf)_mm512_undefined_ps(), -1, C) + +#define _mm512_mask_sqrt_round_ps(W, U, A, C) \ + (__m512)__builtin_ia32_sqrtps512_mask(A, W, U, C) + +#define _mm512_maskz_sqrt_round_ps(U, A, C) \ + (__m512)__builtin_ia32_sqrtps512_mask(A, (__v16sf)_mm512_setzero_ps(), U, C) + +#define _mm_sqrt_round_sd(A, B, C) \ + (__m128d)__builtin_ia32_sqrtsd_round(A, B, C) + +#define _mm_sqrt_round_ss(A, B, C) \ + (__m128)__builtin_ia32_sqrtss_round(A, B, C) +#endif + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepi8_epi32 (__m128i __A) +{ + return (__m512i) __builtin_ia32_pmovsxbd512_mask ((__v16qi) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi8_epi32 (__m512i __W, __mmask16 __U, __m128i __A) +{ + return (__m512i) __builtin_ia32_pmovsxbd512_mask ((__v16qi) __A, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepi8_epi32 (__mmask16 __U, __m128i __A) +{ + return (__m512i) __builtin_ia32_pmovsxbd512_mask ((__v16qi) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepi8_epi64 (__m128i __A) +{ + return (__m512i) __builtin_ia32_pmovsxbq512_mask ((__v16qi) __A, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi8_epi64 (__m512i __W, __mmask8 __U, __m128i __A) +{ + return (__m512i) __builtin_ia32_pmovsxbq512_mask ((__v16qi) __A, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepi8_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m512i) __builtin_ia32_pmovsxbq512_mask ((__v16qi) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepi16_epi32 (__m256i __A) +{ + return (__m512i) __builtin_ia32_pmovsxwd512_mask ((__v16hi) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi16_epi32 (__m512i __W, __mmask16 __U, __m256i __A) +{ + return (__m512i) __builtin_ia32_pmovsxwd512_mask ((__v16hi) __A, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepi16_epi32 (__mmask16 __U, __m256i __A) +{ + return (__m512i) __builtin_ia32_pmovsxwd512_mask ((__v16hi) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepi16_epi64 (__m128i __A) +{ + return (__m512i) __builtin_ia32_pmovsxwq512_mask ((__v8hi) __A, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi16_epi64 (__m512i __W, __mmask8 __U, __m128i __A) +{ + return (__m512i) __builtin_ia32_pmovsxwq512_mask ((__v8hi) __A, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepi16_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m512i) __builtin_ia32_pmovsxwq512_mask ((__v8hi) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepi32_epi64 (__m256i __X) +{ + return (__m512i) __builtin_ia32_pmovsxdq512_mask ((__v8si) __X, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi32_epi64 (__m512i __W, __mmask8 __U, __m256i __X) +{ + return (__m512i) __builtin_ia32_pmovsxdq512_mask ((__v8si) __X, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepi32_epi64 (__mmask8 __U, __m256i __X) +{ + return (__m512i) __builtin_ia32_pmovsxdq512_mask ((__v8si) __X, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepu8_epi32 (__m128i __A) +{ + return (__m512i) __builtin_ia32_pmovzxbd512_mask ((__v16qi) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepu8_epi32 (__m512i __W, __mmask16 __U, __m128i __A) +{ + return (__m512i) __builtin_ia32_pmovzxbd512_mask ((__v16qi) __A, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepu8_epi32 (__mmask16 __U, __m128i __A) +{ + return (__m512i) __builtin_ia32_pmovzxbd512_mask ((__v16qi) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepu8_epi64 (__m128i __A) +{ + return (__m512i) __builtin_ia32_pmovzxbq512_mask ((__v16qi) __A, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepu8_epi64 (__m512i __W, __mmask8 __U, __m128i __A) +{ + return (__m512i) __builtin_ia32_pmovzxbq512_mask ((__v16qi) __A, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepu8_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m512i) __builtin_ia32_pmovzxbq512_mask ((__v16qi) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepu16_epi32 (__m256i __A) +{ + return (__m512i) __builtin_ia32_pmovzxwd512_mask ((__v16hi) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepu16_epi32 (__m512i __W, __mmask16 __U, __m256i __A) +{ + return (__m512i) __builtin_ia32_pmovzxwd512_mask ((__v16hi) __A, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepu16_epi32 (__mmask16 __U, __m256i __A) +{ + return (__m512i) __builtin_ia32_pmovzxwd512_mask ((__v16hi) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepu16_epi64 (__m128i __A) +{ + return (__m512i) __builtin_ia32_pmovzxwq512_mask ((__v8hi) __A, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepu16_epi64 (__m512i __W, __mmask8 __U, __m128i __A) +{ + return (__m512i) __builtin_ia32_pmovzxwq512_mask ((__v8hi) __A, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepu16_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m512i) __builtin_ia32_pmovzxwq512_mask ((__v8hi) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepu32_epi64 (__m256i __X) +{ + return (__m512i) __builtin_ia32_pmovzxdq512_mask ((__v8si) __X, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepu32_epi64 (__m512i __W, __mmask8 __U, __m256i __X) +{ + return (__m512i) __builtin_ia32_pmovzxdq512_mask ((__v8si) __X, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepu32_epi64 (__mmask8 __U, __m256i __X) +{ + return (__m512i) __builtin_ia32_pmovzxdq512_mask ((__v8si) __X, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_add_round_pd (__m512d __A, __m512d __B, const int __R) +{ + return (__m512d) __builtin_ia32_addpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_add_round_pd (__m512d __W, __mmask8 __U, __m512d __A, + __m512d __B, const int __R) +{ + return (__m512d) __builtin_ia32_addpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_add_round_pd (__mmask8 __U, __m512d __A, __m512d __B, + const int __R) +{ + return (__m512d) __builtin_ia32_addpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_add_round_ps (__m512 __A, __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_addps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_add_round_ps (__m512 __W, __mmask16 __U, __m512 __A, + __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_addps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_add_round_ps (__mmask16 __U, __m512 __A, __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_addps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sub_round_pd (__m512d __A, __m512d __B, const int __R) +{ + return (__m512d) __builtin_ia32_subpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sub_round_pd (__m512d __W, __mmask8 __U, __m512d __A, + __m512d __B, const int __R) +{ + return (__m512d) __builtin_ia32_subpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sub_round_pd (__mmask8 __U, __m512d __A, __m512d __B, + const int __R) +{ + return (__m512d) __builtin_ia32_subpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sub_round_ps (__m512 __A, __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_subps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sub_round_ps (__m512 __W, __mmask16 __U, __m512 __A, + __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_subps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sub_round_ps (__mmask16 __U, __m512 __A, __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_subps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, __R); +} +#else +#define _mm512_add_round_pd(A, B, C) \ + (__m512d)__builtin_ia32_addpd512_mask(A, B, (__v8df)_mm512_undefined_pd(), -1, C) + +#define _mm512_mask_add_round_pd(W, U, A, B, C) \ + (__m512d)__builtin_ia32_addpd512_mask(A, B, W, U, C) + +#define _mm512_maskz_add_round_pd(U, A, B, C) \ + (__m512d)__builtin_ia32_addpd512_mask(A, B, (__v8df)_mm512_setzero_pd(), U, C) + +#define _mm512_add_round_ps(A, B, C) \ + (__m512)__builtin_ia32_addps512_mask(A, B, (__v16sf)_mm512_undefined_ps(), -1, C) + +#define _mm512_mask_add_round_ps(W, U, A, B, C) \ + (__m512)__builtin_ia32_addps512_mask(A, B, W, U, C) + +#define _mm512_maskz_add_round_ps(U, A, B, C) \ + (__m512)__builtin_ia32_addps512_mask(A, B, (__v16sf)_mm512_setzero_ps(), U, C) + +#define _mm512_sub_round_pd(A, B, C) \ + (__m512d)__builtin_ia32_subpd512_mask(A, B, (__v8df)_mm512_undefined_pd(), -1, C) + +#define _mm512_mask_sub_round_pd(W, U, A, B, C) \ + (__m512d)__builtin_ia32_subpd512_mask(A, B, W, U, C) + +#define _mm512_maskz_sub_round_pd(U, A, B, C) \ + (__m512d)__builtin_ia32_subpd512_mask(A, B, (__v8df)_mm512_setzero_pd(), U, C) + +#define _mm512_sub_round_ps(A, B, C) \ + (__m512)__builtin_ia32_subps512_mask(A, B, (__v16sf)_mm512_undefined_ps(), -1, C) + +#define _mm512_mask_sub_round_ps(W, U, A, B, C) \ + (__m512)__builtin_ia32_subps512_mask(A, B, W, U, C) + +#define _mm512_maskz_sub_round_ps(U, A, B, C) \ + (__m512)__builtin_ia32_subps512_mask(A, B, (__v16sf)_mm512_setzero_ps(), U, C) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mul_round_pd (__m512d __A, __m512d __B, const int __R) +{ + return (__m512d) __builtin_ia32_mulpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mul_round_pd (__m512d __W, __mmask8 __U, __m512d __A, + __m512d __B, const int __R) +{ + return (__m512d) __builtin_ia32_mulpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mul_round_pd (__mmask8 __U, __m512d __A, __m512d __B, + const int __R) +{ + return (__m512d) __builtin_ia32_mulpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mul_round_ps (__m512 __A, __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_mulps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mul_round_ps (__m512 __W, __mmask16 __U, __m512 __A, + __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_mulps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mul_round_ps (__mmask16 __U, __m512 __A, __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_mulps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_div_round_pd (__m512d __M, __m512d __V, const int __R) +{ + return (__m512d) __builtin_ia32_divpd512_mask ((__v8df) __M, + (__v8df) __V, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_div_round_pd (__m512d __W, __mmask8 __U, __m512d __M, + __m512d __V, const int __R) +{ + return (__m512d) __builtin_ia32_divpd512_mask ((__v8df) __M, + (__v8df) __V, + (__v8df) __W, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_div_round_pd (__mmask8 __U, __m512d __M, __m512d __V, + const int __R) +{ + return (__m512d) __builtin_ia32_divpd512_mask ((__v8df) __M, + (__v8df) __V, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_div_round_ps (__m512 __A, __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_divps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_div_round_ps (__m512 __W, __mmask16 __U, __m512 __A, + __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_divps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_div_round_ps (__mmask16 __U, __m512 __A, __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_divps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mul_round_sd (__m128d __A, __m128d __B, const int __R) +{ + return (__m128d) __builtin_ia32_mulsd_round ((__v2df) __A, + (__v2df) __B, + __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mul_round_ss (__m128 __A, __m128 __B, const int __R) +{ + return (__m128) __builtin_ia32_mulss_round ((__v4sf) __A, + (__v4sf) __B, + __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_div_round_sd (__m128d __A, __m128d __B, const int __R) +{ + return (__m128d) __builtin_ia32_divsd_round ((__v2df) __A, + (__v2df) __B, + __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_div_round_ss (__m128 __A, __m128 __B, const int __R) +{ + return (__m128) __builtin_ia32_divss_round ((__v4sf) __A, + (__v4sf) __B, + __R); +} + +#else +#define _mm512_mul_round_pd(A, B, C) \ + (__m512d)__builtin_ia32_mulpd512_mask(A, B, (__v8df)_mm512_undefined_pd(), -1, C) + +#define _mm512_mask_mul_round_pd(W, U, A, B, C) \ + (__m512d)__builtin_ia32_mulpd512_mask(A, B, W, U, C) + +#define _mm512_maskz_mul_round_pd(U, A, B, C) \ + (__m512d)__builtin_ia32_mulpd512_mask(A, B, (__v8df)_mm512_setzero_pd(), U, C) + +#define _mm512_mul_round_ps(A, B, C) \ + (__m512)__builtin_ia32_mulps512_mask(A, B, (__v16sf)_mm512_undefined_ps(), -1, C) + +#define _mm512_mask_mul_round_ps(W, U, A, B, C) \ + (__m512)__builtin_ia32_mulps512_mask(A, B, W, U, C) + +#define _mm512_maskz_mul_round_ps(U, A, B, C) \ + (__m512)__builtin_ia32_mulps512_mask(A, B, (__v16sf)_mm512_setzero_ps(), U, C) + +#define _mm512_div_round_pd(A, B, C) \ + (__m512d)__builtin_ia32_divpd512_mask(A, B, (__v8df)_mm512_undefined_pd(), -1, C) + +#define _mm512_mask_div_round_pd(W, U, A, B, C) \ + (__m512d)__builtin_ia32_divpd512_mask(A, B, W, U, C) + +#define _mm512_maskz_div_round_pd(U, A, B, C) \ + (__m512d)__builtin_ia32_divpd512_mask(A, B, (__v8df)_mm512_setzero_pd(), U, C) + +#define _mm512_div_round_ps(A, B, C) \ + (__m512)__builtin_ia32_divps512_mask(A, B, (__v16sf)_mm512_undefined_ps(), -1, C) + +#define _mm512_mask_div_round_ps(W, U, A, B, C) \ + (__m512)__builtin_ia32_divps512_mask(A, B, W, U, C) + +#define _mm512_maskz_div_round_ps(U, A, B, C) \ + (__m512)__builtin_ia32_divps512_mask(A, B, (__v16sf)_mm512_setzero_ps(), U, C) + +#define _mm_mul_round_sd(A, B, C) \ + (__m128d)__builtin_ia32_mulsd_round(A, B, C) + +#define _mm_mul_round_ss(A, B, C) \ + (__m128)__builtin_ia32_mulss_round(A, B, C) + +#define _mm_div_round_sd(A, B, C) \ + (__m128d)__builtin_ia32_divsd_round(A, B, C) + +#define _mm_div_round_ss(A, B, C) \ + (__m128)__builtin_ia32_divss_round(A, B, C) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_max_round_pd (__m512d __A, __m512d __B, const int __R) +{ + return (__m512d) __builtin_ia32_maxpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_max_round_pd (__m512d __W, __mmask8 __U, __m512d __A, + __m512d __B, const int __R) +{ + return (__m512d) __builtin_ia32_maxpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_max_round_pd (__mmask8 __U, __m512d __A, __m512d __B, + const int __R) +{ + return (__m512d) __builtin_ia32_maxpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_max_round_ps (__m512 __A, __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_maxps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_max_round_ps (__m512 __W, __mmask16 __U, __m512 __A, + __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_maxps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_max_round_ps (__mmask16 __U, __m512 __A, __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_maxps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_min_round_pd (__m512d __A, __m512d __B, const int __R) +{ + return (__m512d) __builtin_ia32_minpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_min_round_pd (__m512d __W, __mmask8 __U, __m512d __A, + __m512d __B, const int __R) +{ + return (__m512d) __builtin_ia32_minpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_min_round_pd (__mmask8 __U, __m512d __A, __m512d __B, + const int __R) +{ + return (__m512d) __builtin_ia32_minpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_min_round_ps (__m512 __A, __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_minps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_min_round_ps (__m512 __W, __mmask16 __U, __m512 __A, + __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_minps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_min_round_ps (__mmask16 __U, __m512 __A, __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_minps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, __R); +} +#else +#define _mm512_max_round_pd(A, B, R) \ + (__m512d)__builtin_ia32_maxpd512_mask(A, B, (__v8df)_mm512_undefined_pd(), -1, R) + +#define _mm512_mask_max_round_pd(W, U, A, B, R) \ + (__m512d)__builtin_ia32_maxpd512_mask(A, B, W, U, R) + +#define _mm512_maskz_max_round_pd(U, A, B, R) \ + (__m512d)__builtin_ia32_maxpd512_mask(A, B, (__v8df)_mm512_setzero_pd(), U, R) + +#define _mm512_max_round_ps(A, B, R) \ + (__m512)__builtin_ia32_maxps512_mask(A, B, (__v16sf)_mm512_undefined_pd(), -1, R) + +#define _mm512_mask_max_round_ps(W, U, A, B, R) \ + (__m512)__builtin_ia32_maxps512_mask(A, B, W, U, R) + +#define _mm512_maskz_max_round_ps(U, A, B, R) \ + (__m512)__builtin_ia32_maxps512_mask(A, B, (__v16sf)_mm512_setzero_ps(), U, R) + +#define _mm512_min_round_pd(A, B, R) \ + (__m512d)__builtin_ia32_minpd512_mask(A, B, (__v8df)_mm512_undefined_pd(), -1, R) + +#define _mm512_mask_min_round_pd(W, U, A, B, R) \ + (__m512d)__builtin_ia32_minpd512_mask(A, B, W, U, R) + +#define _mm512_maskz_min_round_pd(U, A, B, R) \ + (__m512d)__builtin_ia32_minpd512_mask(A, B, (__v8df)_mm512_setzero_pd(), U, R) + +#define _mm512_min_round_ps(A, B, R) \ + (__m512)__builtin_ia32_minps512_mask(A, B, (__v16sf)_mm512_undefined_ps(), -1, R) + +#define _mm512_mask_min_round_ps(W, U, A, B, R) \ + (__m512)__builtin_ia32_minps512_mask(A, B, W, U, R) + +#define _mm512_maskz_min_round_ps(U, A, B, R) \ + (__m512)__builtin_ia32_minps512_mask(A, B, (__v16sf)_mm512_setzero_ps(), U, R) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_scalef_round_pd (__m512d __A, __m512d __B, const int __R) +{ + return (__m512d) __builtin_ia32_scalefpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_scalef_round_pd (__m512d __W, __mmask8 __U, __m512d __A, + __m512d __B, const int __R) +{ + return (__m512d) __builtin_ia32_scalefpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_scalef_round_pd (__mmask8 __U, __m512d __A, __m512d __B, + const int __R) +{ + return (__m512d) __builtin_ia32_scalefpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_scalef_round_ps (__m512 __A, __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_scalefps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_scalef_round_ps (__m512 __W, __mmask16 __U, __m512 __A, + __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_scalefps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_scalef_round_ps (__mmask16 __U, __m512 __A, __m512 __B, + const int __R) +{ + return (__m512) __builtin_ia32_scalefps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_scalef_round_sd (__m128d __A, __m128d __B, const int __R) +{ + return (__m128d) __builtin_ia32_scalefsd_round ((__v2df) __A, + (__v2df) __B, + __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_scalef_round_ss (__m128 __A, __m128 __B, const int __R) +{ + return (__m128) __builtin_ia32_scalefss_round ((__v4sf) __A, + (__v4sf) __B, + __R); +} +#else +#define _mm512_scalef_round_pd(A, B, C) \ + (__m512d)__builtin_ia32_scalefpd512_mask(A, B, (__v8df)_mm512_undefined_pd(), -1, C) + +#define _mm512_mask_scalef_round_pd(W, U, A, B, C) \ + (__m512d)__builtin_ia32_scalefpd512_mask(A, B, W, U, C) + +#define _mm512_maskz_scalef_round_pd(U, A, B, C) \ + (__m512d)__builtin_ia32_scalefpd512_mask(A, B, (__v8df)_mm512_setzero_pd(), U, C) + +#define _mm512_scalef_round_ps(A, B, C) \ + (__m512)__builtin_ia32_scalefps512_mask(A, B, (__v16sf)_mm512_undefined_ps(), -1, C) + +#define _mm512_mask_scalef_round_ps(W, U, A, B, C) \ + (__m512)__builtin_ia32_scalefps512_mask(A, B, W, U, C) + +#define _mm512_maskz_scalef_round_ps(U, A, B, C) \ + (__m512)__builtin_ia32_scalefps512_mask(A, B, (__v16sf)_mm512_setzero_ps(), U, C) + +#define _mm_scalef_round_sd(A, B, C) \ + (__m128d)__builtin_ia32_scalefsd_round(A, B, C) + +#define _mm_scalef_round_ss(A, B, C) \ + (__m128)__builtin_ia32_scalefss_round(A, B, C) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fmadd_round_pd (__m512d __A, __m512d __B, __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fmadd_round_pd (__m512d __A, __mmask8 __U, __m512d __B, + __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fmadd_round_pd (__m512d __A, __m512d __B, __m512d __C, + __mmask8 __U, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_mask3 ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fmadd_round_pd (__mmask8 __U, __m512d __A, __m512d __B, + __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_maskz ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fmadd_round_ps (__m512 __A, __m512 __B, __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fmadd_round_ps (__m512 __A, __mmask16 __U, __m512 __B, + __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fmadd_round_ps (__m512 __A, __m512 __B, __m512 __C, + __mmask16 __U, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddps512_mask3 ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fmadd_round_ps (__mmask16 __U, __m512 __A, __m512 __B, + __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddps512_maskz ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fmsub_round_pd (__m512d __A, __m512d __B, __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A, + (__v8df) __B, + -(__v8df) __C, + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fmsub_round_pd (__m512d __A, __mmask8 __U, __m512d __B, + __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A, + (__v8df) __B, + -(__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fmsub_round_pd (__m512d __A, __m512d __B, __m512d __C, + __mmask8 __U, const int __R) +{ + return (__m512d) __builtin_ia32_vfmsubpd512_mask3 ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fmsub_round_pd (__mmask8 __U, __m512d __A, __m512d __B, + __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_maskz ((__v8df) __A, + (__v8df) __B, + -(__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fmsub_round_ps (__m512 __A, __m512 __B, __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A, + (__v16sf) __B, + -(__v16sf) __C, + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fmsub_round_ps (__m512 __A, __mmask16 __U, __m512 __B, + __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A, + (__v16sf) __B, + -(__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fmsub_round_ps (__m512 __A, __m512 __B, __m512 __C, + __mmask16 __U, const int __R) +{ + return (__m512) __builtin_ia32_vfmsubps512_mask3 ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fmsub_round_ps (__mmask16 __U, __m512 __A, __m512 __B, + __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddps512_maskz ((__v16sf) __A, + (__v16sf) __B, + -(__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fmaddsub_round_pd (__m512d __A, __m512d __B, __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddsubpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fmaddsub_round_pd (__m512d __A, __mmask8 __U, __m512d __B, + __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddsubpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fmaddsub_round_pd (__m512d __A, __m512d __B, __m512d __C, + __mmask8 __U, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddsubpd512_mask3 ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fmaddsub_round_pd (__mmask8 __U, __m512d __A, __m512d __B, + __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddsubpd512_maskz ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fmaddsub_round_ps (__m512 __A, __m512 __B, __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddsubps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fmaddsub_round_ps (__m512 __A, __mmask16 __U, __m512 __B, + __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddsubps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fmaddsub_round_ps (__m512 __A, __m512 __B, __m512 __C, + __mmask16 __U, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddsubps512_mask3 ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fmaddsub_round_ps (__mmask16 __U, __m512 __A, __m512 __B, + __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddsubps512_maskz ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fmsubadd_round_pd (__m512d __A, __m512d __B, __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddsubpd512_mask ((__v8df) __A, + (__v8df) __B, + -(__v8df) __C, + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fmsubadd_round_pd (__m512d __A, __mmask8 __U, __m512d __B, + __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddsubpd512_mask ((__v8df) __A, + (__v8df) __B, + -(__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fmsubadd_round_pd (__m512d __A, __m512d __B, __m512d __C, + __mmask8 __U, const int __R) +{ + return (__m512d) __builtin_ia32_vfmsubaddpd512_mask3 ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fmsubadd_round_pd (__mmask8 __U, __m512d __A, __m512d __B, + __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddsubpd512_maskz ((__v8df) __A, + (__v8df) __B, + -(__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fmsubadd_round_ps (__m512 __A, __m512 __B, __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddsubps512_mask ((__v16sf) __A, + (__v16sf) __B, + -(__v16sf) __C, + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fmsubadd_round_ps (__m512 __A, __mmask16 __U, __m512 __B, + __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddsubps512_mask ((__v16sf) __A, + (__v16sf) __B, + -(__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fmsubadd_round_ps (__m512 __A, __m512 __B, __m512 __C, + __mmask16 __U, const int __R) +{ + return (__m512) __builtin_ia32_vfmsubaddps512_mask3 ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fmsubadd_round_ps (__mmask16 __U, __m512 __A, __m512 __B, + __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddsubps512_maskz ((__v16sf) __A, + (__v16sf) __B, + -(__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fnmadd_round_pd (__m512d __A, __m512d __B, __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_mask (-(__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fnmadd_round_pd (__m512d __A, __mmask8 __U, __m512d __B, + __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfnmaddpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fnmadd_round_pd (__m512d __A, __m512d __B, __m512d __C, + __mmask8 __U, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_mask3 (-(__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fnmadd_round_pd (__mmask8 __U, __m512d __A, __m512d __B, + __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_maskz (-(__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fnmadd_round_ps (__m512 __A, __m512 __B, __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddps512_mask (-(__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fnmadd_round_ps (__m512 __A, __mmask16 __U, __m512 __B, + __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfnmaddps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fnmadd_round_ps (__m512 __A, __m512 __B, __m512 __C, + __mmask16 __U, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddps512_mask3 (-(__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fnmadd_round_ps (__mmask16 __U, __m512 __A, __m512 __B, + __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddps512_maskz (-(__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fnmsub_round_pd (__m512d __A, __m512d __B, __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_mask (-(__v8df) __A, + (__v8df) __B, + -(__v8df) __C, + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fnmsub_round_pd (__m512d __A, __mmask8 __U, __m512d __B, + __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfnmsubpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fnmsub_round_pd (__m512d __A, __m512d __B, __m512d __C, + __mmask8 __U, const int __R) +{ + return (__m512d) __builtin_ia32_vfnmsubpd512_mask3 ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fnmsub_round_pd (__mmask8 __U, __m512d __A, __m512d __B, + __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_maskz (-(__v8df) __A, + (__v8df) __B, + -(__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fnmsub_round_ps (__m512 __A, __m512 __B, __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddps512_mask (-(__v16sf) __A, + (__v16sf) __B, + -(__v16sf) __C, + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fnmsub_round_ps (__m512 __A, __mmask16 __U, __m512 __B, + __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfnmsubps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fnmsub_round_ps (__m512 __A, __m512 __B, __m512 __C, + __mmask16 __U, const int __R) +{ + return (__m512) __builtin_ia32_vfnmsubps512_mask3 ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fnmsub_round_ps (__mmask16 __U, __m512 __A, __m512 __B, + __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddps512_maskz (-(__v16sf) __A, + (__v16sf) __B, + -(__v16sf) __C, + (__mmask16) __U, __R); +} +#else +#define _mm512_fmadd_round_pd(A, B, C, R) \ + (__m512d)__builtin_ia32_vfmaddpd512_mask(A, B, C, -1, R) + +#define _mm512_mask_fmadd_round_pd(A, U, B, C, R) \ + (__m512d)__builtin_ia32_vfmaddpd512_mask(A, B, C, U, R) + +#define _mm512_mask3_fmadd_round_pd(A, B, C, U, R) \ + (__m512d)__builtin_ia32_vfmaddpd512_mask3(A, B, C, U, R) + +#define _mm512_maskz_fmadd_round_pd(U, A, B, C, R) \ + (__m512d)__builtin_ia32_vfmaddpd512_maskz(A, B, C, U, R) + +#define _mm512_fmadd_round_ps(A, B, C, R) \ + (__m512)__builtin_ia32_vfmaddps512_mask(A, B, C, -1, R) + +#define _mm512_mask_fmadd_round_ps(A, U, B, C, R) \ + (__m512)__builtin_ia32_vfmaddps512_mask(A, B, C, U, R) + +#define _mm512_mask3_fmadd_round_ps(A, B, C, U, R) \ + (__m512)__builtin_ia32_vfmaddps512_mask3(A, B, C, U, R) + +#define _mm512_maskz_fmadd_round_ps(U, A, B, C, R) \ + (__m512)__builtin_ia32_vfmaddps512_maskz(A, B, C, U, R) + +#define _mm512_fmsub_round_pd(A, B, C, R) \ + (__m512d)__builtin_ia32_vfmaddpd512_mask(A, B, -(C), -1, R) + +#define _mm512_mask_fmsub_round_pd(A, U, B, C, R) \ + (__m512d)__builtin_ia32_vfmaddpd512_mask(A, B, -(C), U, R) + +#define _mm512_mask3_fmsub_round_pd(A, B, C, U, R) \ + (__m512d)__builtin_ia32_vfmsubpd512_mask3(A, B, C, U, R) + +#define _mm512_maskz_fmsub_round_pd(U, A, B, C, R) \ + (__m512d)__builtin_ia32_vfmaddpd512_maskz(A, B, -(C), U, R) + +#define _mm512_fmsub_round_ps(A, B, C, R) \ + (__m512)__builtin_ia32_vfmaddps512_mask(A, B, -(C), -1, R) + +#define _mm512_mask_fmsub_round_ps(A, U, B, C, R) \ + (__m512)__builtin_ia32_vfmaddps512_mask(A, B, -(C), U, R) + +#define _mm512_mask3_fmsub_round_ps(A, B, C, U, R) \ + (__m512)__builtin_ia32_vfmsubps512_mask3(A, B, C, U, R) + +#define _mm512_maskz_fmsub_round_ps(U, A, B, C, R) \ + (__m512)__builtin_ia32_vfmaddps512_maskz(A, B, -(C), U, R) + +#define _mm512_fmaddsub_round_pd(A, B, C, R) \ + (__m512d)__builtin_ia32_vfmaddsubpd512_mask(A, B, C, -1, R) + +#define _mm512_mask_fmaddsub_round_pd(A, U, B, C, R) \ + (__m512d)__builtin_ia32_vfmaddpd512_mask(A, B, C, U, R) + +#define _mm512_mask3_fmaddsub_round_pd(A, B, C, U, R) \ + (__m512d)__builtin_ia32_vfmaddsubpd512_mask3(A, B, C, U, R) + +#define _mm512_maskz_fmaddsub_round_pd(U, A, B, C, R) \ + (__m512d)__builtin_ia32_vfmaddsubpd512_maskz(A, B, C, U, R) + +#define _mm512_fmaddsub_round_ps(A, B, C, R) \ + (__m512)__builtin_ia32_vfmaddsubps512_mask(A, B, C, -1, R) + +#define _mm512_mask_fmaddsub_round_ps(A, U, B, C, R) \ + (__m512)__builtin_ia32_vfmaddsubps512_mask(A, B, C, U, R) + +#define _mm512_mask3_fmaddsub_round_ps(A, B, C, U, R) \ + (__m512)__builtin_ia32_vfmaddsubps512_mask3(A, B, C, U, R) + +#define _mm512_maskz_fmaddsub_round_ps(U, A, B, C, R) \ + (__m512)__builtin_ia32_vfmaddsubps512_maskz(A, B, C, U, R) + +#define _mm512_fmsubadd_round_pd(A, B, C, R) \ + (__m512d)__builtin_ia32_vfmaddsubpd512_mask(A, B, -(C), -1, R) + +#define _mm512_mask_fmsubadd_round_pd(A, U, B, C, R) \ + (__m512d)__builtin_ia32_vfmaddsubpd512_mask(A, B, -(C), U, R) + +#define _mm512_mask3_fmsubadd_round_pd(A, B, C, U, R) \ + (__m512d)__builtin_ia32_vfmsubaddpd512_mask3(A, B, C, U, R) + +#define _mm512_maskz_fmsubadd_round_pd(U, A, B, C, R) \ + (__m512d)__builtin_ia32_vfmaddsubpd512_maskz(A, B, -(C), U, R) + +#define _mm512_fmsubadd_round_ps(A, B, C, R) \ + (__m512)__builtin_ia32_vfmaddsubps512_mask(A, B, -(C), -1, R) + +#define _mm512_mask_fmsubadd_round_ps(A, U, B, C, R) \ + (__m512)__builtin_ia32_vfmaddsubps512_mask(A, B, -(C), U, R) + +#define _mm512_mask3_fmsubadd_round_ps(A, B, C, U, R) \ + (__m512)__builtin_ia32_vfmsubaddps512_mask3(A, B, C, U, R) + +#define _mm512_maskz_fmsubadd_round_ps(U, A, B, C, R) \ + (__m512)__builtin_ia32_vfmaddsubps512_maskz(A, B, -(C), U, R) + +#define _mm512_fnmadd_round_pd(A, B, C, R) \ + (__m512d)__builtin_ia32_vfmaddpd512_mask(-(A), B, C, -1, R) + +#define _mm512_mask_fnmadd_round_pd(A, U, B, C, R) \ + (__m512d)__builtin_ia32_vfnmaddpd512_mask(-(A), B, C, U, R) + +#define _mm512_mask3_fnmadd_round_pd(A, B, C, U, R) \ + (__m512d)__builtin_ia32_vfmaddpd512_mask3(-(A), B, C, U, R) + +#define _mm512_maskz_fnmadd_round_pd(U, A, B, C, R) \ + (__m512d)__builtin_ia32_vfmaddpd512_maskz(-(A), B, C, U, R) + +#define _mm512_fnmadd_round_ps(A, B, C, R) \ + (__m512)__builtin_ia32_vfmaddps512_mask(-(A), B, C, -1, R) + +#define _mm512_mask_fnmadd_round_ps(A, U, B, C, R) \ + (__m512)__builtin_ia32_vfnmaddps512_mask(-(A), B, C, U, R) + +#define _mm512_mask3_fnmadd_round_ps(A, B, C, U, R) \ + (__m512)__builtin_ia32_vfmaddps512_mask3(-(A), B, C, U, R) + +#define _mm512_maskz_fnmadd_round_ps(U, A, B, C, R) \ + (__m512)__builtin_ia32_vfmaddps512_maskz(-(A), B, C, U, R) + +#define _mm512_fnmsub_round_pd(A, B, C, R) \ + (__m512d)__builtin_ia32_vfmaddpd512_mask(-(A), B, -(C), -1, R) + +#define _mm512_mask_fnmsub_round_pd(A, U, B, C, R) \ + (__m512d)__builtin_ia32_vfnmsubpd512_mask(A, B, C, U, R) + +#define _mm512_mask3_fnmsub_round_pd(A, B, C, U, R) \ + (__m512d)__builtin_ia32_vfnmsubpd512_mask3(A, B, C, U, R) + +#define _mm512_maskz_fnmsub_round_pd(U, A, B, C, R) \ + (__m512d)__builtin_ia32_vfmaddpd512_maskz(-(A), B, -(C), U, R) + +#define _mm512_fnmsub_round_ps(A, B, C, R) \ + (__m512)__builtin_ia32_vfmaddps512_mask(-(A), B, -(C), -1, R) + +#define _mm512_mask_fnmsub_round_ps(A, U, B, C, R) \ + (__m512)__builtin_ia32_vfnmsubps512_mask(A, B, C, U, R) + +#define _mm512_mask3_fnmsub_round_ps(A, B, C, U, R) \ + (__m512)__builtin_ia32_vfnmsubps512_mask3(A, B, C, U, R) + +#define _mm512_maskz_fnmsub_round_ps(U, A, B, C, R) \ + (__m512)__builtin_ia32_vfmaddps512_maskz(-(A), B, -(C), U, R) +#endif + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_abs_epi64 (__m512i __A) +{ + return (__m512i) __builtin_ia32_pabsq512_mask ((__v8di) __A, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_abs_epi64 (__m512i __W, __mmask8 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_pabsq512_mask ((__v8di) __A, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_abs_epi64 (__mmask8 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_pabsq512_mask ((__v8di) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_abs_epi32 (__m512i __A) +{ + return (__m512i) __builtin_ia32_pabsd512_mask ((__v16si) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_abs_epi32 (__m512i __W, __mmask16 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_pabsd512_mask ((__v16si) __A, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_abs_epi32 (__mmask16 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_pabsd512_mask ((__v16si) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcastss_ps (__m128 __A) +{ + return (__m512) __builtin_ia32_broadcastss512 ((__v4sf) __A, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_broadcastss_ps (__m512 __O, __mmask16 __M, __m128 __A) +{ + return (__m512) __builtin_ia32_broadcastss512 ((__v4sf) __A, + (__v16sf) __O, __M); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_broadcastss_ps (__mmask16 __M, __m128 __A) +{ + return (__m512) __builtin_ia32_broadcastss512 ((__v4sf) __A, + (__v16sf) + _mm512_setzero_ps (), + __M); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcastsd_pd (__m128d __A) +{ + return (__m512d) __builtin_ia32_broadcastsd512 ((__v2df) __A, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_broadcastsd_pd (__m512d __O, __mmask8 __M, __m128d __A) +{ + return (__m512d) __builtin_ia32_broadcastsd512 ((__v2df) __A, + (__v8df) __O, __M); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_broadcastsd_pd (__mmask8 __M, __m128d __A) +{ + return (__m512d) __builtin_ia32_broadcastsd512 ((__v2df) __A, + (__v8df) + _mm512_setzero_pd (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcastd_epi32 (__m128i __A) +{ + return (__m512i) __builtin_ia32_pbroadcastd512 ((__v4si) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_broadcastd_epi32 (__m512i __O, __mmask16 __M, __m128i __A) +{ + return (__m512i) __builtin_ia32_pbroadcastd512 ((__v4si) __A, + (__v16si) __O, __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_broadcastd_epi32 (__mmask16 __M, __m128i __A) +{ + return (__m512i) __builtin_ia32_pbroadcastd512 ((__v4si) __A, + (__v16si) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_set1_epi32 (int __A) +{ + return (__m512i) __builtin_ia32_pbroadcastd512_gpr_mask (__A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16)(-1)); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_set1_epi32 (__m512i __O, __mmask16 __M, int __A) +{ + return (__m512i) __builtin_ia32_pbroadcastd512_gpr_mask (__A, (__v16si) __O, + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_set1_epi32 (__mmask16 __M, int __A) +{ + return (__m512i) + __builtin_ia32_pbroadcastd512_gpr_mask (__A, + (__v16si) _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcastq_epi64 (__m128i __A) +{ + return (__m512i) __builtin_ia32_pbroadcastq512 ((__v2di) __A, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_broadcastq_epi64 (__m512i __O, __mmask8 __M, __m128i __A) +{ + return (__m512i) __builtin_ia32_pbroadcastq512 ((__v2di) __A, + (__v8di) __O, __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_broadcastq_epi64 (__mmask8 __M, __m128i __A) +{ + return (__m512i) __builtin_ia32_pbroadcastq512 ((__v2di) __A, + (__v8di) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_set1_epi64 (long long __A) +{ + return (__m512i) __builtin_ia32_pbroadcastq512_gpr_mask (__A, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8)(-1)); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_set1_epi64 (__m512i __O, __mmask8 __M, long long __A) +{ + return (__m512i) __builtin_ia32_pbroadcastq512_gpr_mask (__A, (__v8di) __O, + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_set1_epi64 (__mmask8 __M, long long __A) +{ + return (__m512i) + __builtin_ia32_pbroadcastq512_gpr_mask (__A, + (__v8di) _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcast_f32x4 (__m128 __A) +{ + return (__m512) __builtin_ia32_broadcastf32x4_512 ((__v4sf) __A, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_broadcast_f32x4 (__m512 __O, __mmask16 __M, __m128 __A) +{ + return (__m512) __builtin_ia32_broadcastf32x4_512 ((__v4sf) __A, + (__v16sf) __O, + __M); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_broadcast_f32x4 (__mmask16 __M, __m128 __A) +{ + return (__m512) __builtin_ia32_broadcastf32x4_512 ((__v4sf) __A, + (__v16sf) + _mm512_setzero_ps (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcast_i32x4 (__m128i __A) +{ + return (__m512i) __builtin_ia32_broadcasti32x4_512 ((__v4si) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_broadcast_i32x4 (__m512i __O, __mmask16 __M, __m128i __A) +{ + return (__m512i) __builtin_ia32_broadcasti32x4_512 ((__v4si) __A, + (__v16si) __O, + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_broadcast_i32x4 (__mmask16 __M, __m128i __A) +{ + return (__m512i) __builtin_ia32_broadcasti32x4_512 ((__v4si) __A, + (__v16si) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcast_f64x4 (__m256d __A) +{ + return (__m512d) __builtin_ia32_broadcastf64x4_512 ((__v4df) __A, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_broadcast_f64x4 (__m512d __O, __mmask8 __M, __m256d __A) +{ + return (__m512d) __builtin_ia32_broadcastf64x4_512 ((__v4df) __A, + (__v8df) __O, + __M); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_broadcast_f64x4 (__mmask8 __M, __m256d __A) +{ + return (__m512d) __builtin_ia32_broadcastf64x4_512 ((__v4df) __A, + (__v8df) + _mm512_setzero_pd (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcast_i64x4 (__m256i __A) +{ + return (__m512i) __builtin_ia32_broadcasti64x4_512 ((__v4di) __A, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_broadcast_i64x4 (__m512i __O, __mmask8 __M, __m256i __A) +{ + return (__m512i) __builtin_ia32_broadcasti64x4_512 ((__v4di) __A, + (__v8di) __O, + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_broadcast_i64x4 (__mmask8 __M, __m256i __A) +{ + return (__m512i) __builtin_ia32_broadcasti64x4_512 ((__v4di) __A, + (__v8di) + _mm512_setzero_si512 (), + __M); +} + +typedef enum +{ + _MM_PERM_AAAA = 0x00, _MM_PERM_AAAB = 0x01, _MM_PERM_AAAC = 0x02, + _MM_PERM_AAAD = 0x03, _MM_PERM_AABA = 0x04, _MM_PERM_AABB = 0x05, + _MM_PERM_AABC = 0x06, _MM_PERM_AABD = 0x07, _MM_PERM_AACA = 0x08, + _MM_PERM_AACB = 0x09, _MM_PERM_AACC = 0x0A, _MM_PERM_AACD = 0x0B, + _MM_PERM_AADA = 0x0C, _MM_PERM_AADB = 0x0D, _MM_PERM_AADC = 0x0E, + _MM_PERM_AADD = 0x0F, _MM_PERM_ABAA = 0x10, _MM_PERM_ABAB = 0x11, + _MM_PERM_ABAC = 0x12, _MM_PERM_ABAD = 0x13, _MM_PERM_ABBA = 0x14, + _MM_PERM_ABBB = 0x15, _MM_PERM_ABBC = 0x16, _MM_PERM_ABBD = 0x17, + _MM_PERM_ABCA = 0x18, _MM_PERM_ABCB = 0x19, _MM_PERM_ABCC = 0x1A, + _MM_PERM_ABCD = 0x1B, _MM_PERM_ABDA = 0x1C, _MM_PERM_ABDB = 0x1D, + _MM_PERM_ABDC = 0x1E, _MM_PERM_ABDD = 0x1F, _MM_PERM_ACAA = 0x20, + _MM_PERM_ACAB = 0x21, _MM_PERM_ACAC = 0x22, _MM_PERM_ACAD = 0x23, + _MM_PERM_ACBA = 0x24, _MM_PERM_ACBB = 0x25, _MM_PERM_ACBC = 0x26, + _MM_PERM_ACBD = 0x27, _MM_PERM_ACCA = 0x28, _MM_PERM_ACCB = 0x29, + _MM_PERM_ACCC = 0x2A, _MM_PERM_ACCD = 0x2B, _MM_PERM_ACDA = 0x2C, + _MM_PERM_ACDB = 0x2D, _MM_PERM_ACDC = 0x2E, _MM_PERM_ACDD = 0x2F, + _MM_PERM_ADAA = 0x30, _MM_PERM_ADAB = 0x31, _MM_PERM_ADAC = 0x32, + _MM_PERM_ADAD = 0x33, _MM_PERM_ADBA = 0x34, _MM_PERM_ADBB = 0x35, + _MM_PERM_ADBC = 0x36, _MM_PERM_ADBD = 0x37, _MM_PERM_ADCA = 0x38, + _MM_PERM_ADCB = 0x39, _MM_PERM_ADCC = 0x3A, _MM_PERM_ADCD = 0x3B, + _MM_PERM_ADDA = 0x3C, _MM_PERM_ADDB = 0x3D, _MM_PERM_ADDC = 0x3E, + _MM_PERM_ADDD = 0x3F, _MM_PERM_BAAA = 0x40, _MM_PERM_BAAB = 0x41, + _MM_PERM_BAAC = 0x42, _MM_PERM_BAAD = 0x43, _MM_PERM_BABA = 0x44, + _MM_PERM_BABB = 0x45, _MM_PERM_BABC = 0x46, _MM_PERM_BABD = 0x47, + _MM_PERM_BACA = 0x48, _MM_PERM_BACB = 0x49, _MM_PERM_BACC = 0x4A, + _MM_PERM_BACD = 0x4B, _MM_PERM_BADA = 0x4C, _MM_PERM_BADB = 0x4D, + _MM_PERM_BADC = 0x4E, _MM_PERM_BADD = 0x4F, _MM_PERM_BBAA = 0x50, + _MM_PERM_BBAB = 0x51, _MM_PERM_BBAC = 0x52, _MM_PERM_BBAD = 0x53, + _MM_PERM_BBBA = 0x54, _MM_PERM_BBBB = 0x55, _MM_PERM_BBBC = 0x56, + _MM_PERM_BBBD = 0x57, _MM_PERM_BBCA = 0x58, _MM_PERM_BBCB = 0x59, + _MM_PERM_BBCC = 0x5A, _MM_PERM_BBCD = 0x5B, _MM_PERM_BBDA = 0x5C, + _MM_PERM_BBDB = 0x5D, _MM_PERM_BBDC = 0x5E, _MM_PERM_BBDD = 0x5F, + _MM_PERM_BCAA = 0x60, _MM_PERM_BCAB = 0x61, _MM_PERM_BCAC = 0x62, + _MM_PERM_BCAD = 0x63, _MM_PERM_BCBA = 0x64, _MM_PERM_BCBB = 0x65, + _MM_PERM_BCBC = 0x66, _MM_PERM_BCBD = 0x67, _MM_PERM_BCCA = 0x68, + _MM_PERM_BCCB = 0x69, _MM_PERM_BCCC = 0x6A, _MM_PERM_BCCD = 0x6B, + _MM_PERM_BCDA = 0x6C, _MM_PERM_BCDB = 0x6D, _MM_PERM_BCDC = 0x6E, + _MM_PERM_BCDD = 0x6F, _MM_PERM_BDAA = 0x70, _MM_PERM_BDAB = 0x71, + _MM_PERM_BDAC = 0x72, _MM_PERM_BDAD = 0x73, _MM_PERM_BDBA = 0x74, + _MM_PERM_BDBB = 0x75, _MM_PERM_BDBC = 0x76, _MM_PERM_BDBD = 0x77, + _MM_PERM_BDCA = 0x78, _MM_PERM_BDCB = 0x79, _MM_PERM_BDCC = 0x7A, + _MM_PERM_BDCD = 0x7B, _MM_PERM_BDDA = 0x7C, _MM_PERM_BDDB = 0x7D, + _MM_PERM_BDDC = 0x7E, _MM_PERM_BDDD = 0x7F, _MM_PERM_CAAA = 0x80, + _MM_PERM_CAAB = 0x81, _MM_PERM_CAAC = 0x82, _MM_PERM_CAAD = 0x83, + _MM_PERM_CABA = 0x84, _MM_PERM_CABB = 0x85, _MM_PERM_CABC = 0x86, + _MM_PERM_CABD = 0x87, _MM_PERM_CACA = 0x88, _MM_PERM_CACB = 0x89, + _MM_PERM_CACC = 0x8A, _MM_PERM_CACD = 0x8B, _MM_PERM_CADA = 0x8C, + _MM_PERM_CADB = 0x8D, _MM_PERM_CADC = 0x8E, _MM_PERM_CADD = 0x8F, + _MM_PERM_CBAA = 0x90, _MM_PERM_CBAB = 0x91, _MM_PERM_CBAC = 0x92, + _MM_PERM_CBAD = 0x93, _MM_PERM_CBBA = 0x94, _MM_PERM_CBBB = 0x95, + _MM_PERM_CBBC = 0x96, _MM_PERM_CBBD = 0x97, _MM_PERM_CBCA = 0x98, + _MM_PERM_CBCB = 0x99, _MM_PERM_CBCC = 0x9A, _MM_PERM_CBCD = 0x9B, + _MM_PERM_CBDA = 0x9C, _MM_PERM_CBDB = 0x9D, _MM_PERM_CBDC = 0x9E, + _MM_PERM_CBDD = 0x9F, _MM_PERM_CCAA = 0xA0, _MM_PERM_CCAB = 0xA1, + _MM_PERM_CCAC = 0xA2, _MM_PERM_CCAD = 0xA3, _MM_PERM_CCBA = 0xA4, + _MM_PERM_CCBB = 0xA5, _MM_PERM_CCBC = 0xA6, _MM_PERM_CCBD = 0xA7, + _MM_PERM_CCCA = 0xA8, _MM_PERM_CCCB = 0xA9, _MM_PERM_CCCC = 0xAA, + _MM_PERM_CCCD = 0xAB, _MM_PERM_CCDA = 0xAC, _MM_PERM_CCDB = 0xAD, + _MM_PERM_CCDC = 0xAE, _MM_PERM_CCDD = 0xAF, _MM_PERM_CDAA = 0xB0, + _MM_PERM_CDAB = 0xB1, _MM_PERM_CDAC = 0xB2, _MM_PERM_CDAD = 0xB3, + _MM_PERM_CDBA = 0xB4, _MM_PERM_CDBB = 0xB5, _MM_PERM_CDBC = 0xB6, + _MM_PERM_CDBD = 0xB7, _MM_PERM_CDCA = 0xB8, _MM_PERM_CDCB = 0xB9, + _MM_PERM_CDCC = 0xBA, _MM_PERM_CDCD = 0xBB, _MM_PERM_CDDA = 0xBC, + _MM_PERM_CDDB = 0xBD, _MM_PERM_CDDC = 0xBE, _MM_PERM_CDDD = 0xBF, + _MM_PERM_DAAA = 0xC0, _MM_PERM_DAAB = 0xC1, _MM_PERM_DAAC = 0xC2, + _MM_PERM_DAAD = 0xC3, _MM_PERM_DABA = 0xC4, _MM_PERM_DABB = 0xC5, + _MM_PERM_DABC = 0xC6, _MM_PERM_DABD = 0xC7, _MM_PERM_DACA = 0xC8, + _MM_PERM_DACB = 0xC9, _MM_PERM_DACC = 0xCA, _MM_PERM_DACD = 0xCB, + _MM_PERM_DADA = 0xCC, _MM_PERM_DADB = 0xCD, _MM_PERM_DADC = 0xCE, + _MM_PERM_DADD = 0xCF, _MM_PERM_DBAA = 0xD0, _MM_PERM_DBAB = 0xD1, + _MM_PERM_DBAC = 0xD2, _MM_PERM_DBAD = 0xD3, _MM_PERM_DBBA = 0xD4, + _MM_PERM_DBBB = 0xD5, _MM_PERM_DBBC = 0xD6, _MM_PERM_DBBD = 0xD7, + _MM_PERM_DBCA = 0xD8, _MM_PERM_DBCB = 0xD9, _MM_PERM_DBCC = 0xDA, + _MM_PERM_DBCD = 0xDB, _MM_PERM_DBDA = 0xDC, _MM_PERM_DBDB = 0xDD, + _MM_PERM_DBDC = 0xDE, _MM_PERM_DBDD = 0xDF, _MM_PERM_DCAA = 0xE0, + _MM_PERM_DCAB = 0xE1, _MM_PERM_DCAC = 0xE2, _MM_PERM_DCAD = 0xE3, + _MM_PERM_DCBA = 0xE4, _MM_PERM_DCBB = 0xE5, _MM_PERM_DCBC = 0xE6, + _MM_PERM_DCBD = 0xE7, _MM_PERM_DCCA = 0xE8, _MM_PERM_DCCB = 0xE9, + _MM_PERM_DCCC = 0xEA, _MM_PERM_DCCD = 0xEB, _MM_PERM_DCDA = 0xEC, + _MM_PERM_DCDB = 0xED, _MM_PERM_DCDC = 0xEE, _MM_PERM_DCDD = 0xEF, + _MM_PERM_DDAA = 0xF0, _MM_PERM_DDAB = 0xF1, _MM_PERM_DDAC = 0xF2, + _MM_PERM_DDAD = 0xF3, _MM_PERM_DDBA = 0xF4, _MM_PERM_DDBB = 0xF5, + _MM_PERM_DDBC = 0xF6, _MM_PERM_DDBD = 0xF7, _MM_PERM_DDCA = 0xF8, + _MM_PERM_DDCB = 0xF9, _MM_PERM_DDCC = 0xFA, _MM_PERM_DDCD = 0xFB, + _MM_PERM_DDDA = 0xFC, _MM_PERM_DDDB = 0xFD, _MM_PERM_DDDC = 0xFE, + _MM_PERM_DDDD = 0xFF +} _MM_PERM_ENUM; + +#ifdef __OPTIMIZE__ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_shuffle_epi32 (__m512i __A, _MM_PERM_ENUM __mask) +{ + return (__m512i) __builtin_ia32_pshufd512_mask ((__v16si) __A, + __mask, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_shuffle_epi32 (__m512i __W, __mmask16 __U, __m512i __A, + _MM_PERM_ENUM __mask) +{ + return (__m512i) __builtin_ia32_pshufd512_mask ((__v16si) __A, + __mask, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_shuffle_epi32 (__mmask16 __U, __m512i __A, _MM_PERM_ENUM __mask) +{ + return (__m512i) __builtin_ia32_pshufd512_mask ((__v16si) __A, + __mask, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_shuffle_i64x2 (__m512i __A, __m512i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_shuf_i64x2_mask ((__v8di) __A, + (__v8di) __B, __imm, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_shuffle_i64x2 (__m512i __W, __mmask8 __U, __m512i __A, + __m512i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_shuf_i64x2_mask ((__v8di) __A, + (__v8di) __B, __imm, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_shuffle_i64x2 (__mmask8 __U, __m512i __A, __m512i __B, + const int __imm) +{ + return (__m512i) __builtin_ia32_shuf_i64x2_mask ((__v8di) __A, + (__v8di) __B, __imm, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_shuffle_i32x4 (__m512i __A, __m512i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_shuf_i32x4_mask ((__v16si) __A, + (__v16si) __B, + __imm, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_shuffle_i32x4 (__m512i __W, __mmask16 __U, __m512i __A, + __m512i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_shuf_i32x4_mask ((__v16si) __A, + (__v16si) __B, + __imm, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_shuffle_i32x4 (__mmask16 __U, __m512i __A, __m512i __B, + const int __imm) +{ + return (__m512i) __builtin_ia32_shuf_i32x4_mask ((__v16si) __A, + (__v16si) __B, + __imm, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_shuffle_f64x2 (__m512d __A, __m512d __B, const int __imm) +{ + return (__m512d) __builtin_ia32_shuf_f64x2_mask ((__v8df) __A, + (__v8df) __B, __imm, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_shuffle_f64x2 (__m512d __W, __mmask8 __U, __m512d __A, + __m512d __B, const int __imm) +{ + return (__m512d) __builtin_ia32_shuf_f64x2_mask ((__v8df) __A, + (__v8df) __B, __imm, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_shuffle_f64x2 (__mmask8 __U, __m512d __A, __m512d __B, + const int __imm) +{ + return (__m512d) __builtin_ia32_shuf_f64x2_mask ((__v8df) __A, + (__v8df) __B, __imm, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_shuffle_f32x4 (__m512 __A, __m512 __B, const int __imm) +{ + return (__m512) __builtin_ia32_shuf_f32x4_mask ((__v16sf) __A, + (__v16sf) __B, __imm, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_shuffle_f32x4 (__m512 __W, __mmask16 __U, __m512 __A, + __m512 __B, const int __imm) +{ + return (__m512) __builtin_ia32_shuf_f32x4_mask ((__v16sf) __A, + (__v16sf) __B, __imm, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_shuffle_f32x4 (__mmask16 __U, __m512 __A, __m512 __B, + const int __imm) +{ + return (__m512) __builtin_ia32_shuf_f32x4_mask ((__v16sf) __A, + (__v16sf) __B, __imm, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +#else +#define _mm512_shuffle_epi32(X, C) \ + ((__m512i) __builtin_ia32_pshufd512_mask ((__v16si)(__m512i)(X), (int)(C),\ + (__v16si)(__m512i)_mm512_undefined_epi32 (),\ + (__mmask16)-1)) + +#define _mm512_mask_shuffle_epi32(W, U, X, C) \ + ((__m512i) __builtin_ia32_pshufd512_mask ((__v16si)(__m512i)(X), (int)(C),\ + (__v16si)(__m512i)(W),\ + (__mmask16)(U))) + +#define _mm512_maskz_shuffle_epi32(U, X, C) \ + ((__m512i) __builtin_ia32_pshufd512_mask ((__v16si)(__m512i)(X), (int)(C),\ + (__v16si)(__m512i)_mm512_setzero_si512 (),\ + (__mmask16)(U))) + +#define _mm512_shuffle_i64x2(X, Y, C) \ + ((__m512i) __builtin_ia32_shuf_i64x2_mask ((__v8di)(__m512i)(X), \ + (__v8di)(__m512i)(Y), (int)(C),\ + (__v8di)(__m512i)_mm512_undefined_epi32 (),\ + (__mmask8)-1)) + +#define _mm512_mask_shuffle_i64x2(W, U, X, Y, C) \ + ((__m512i) __builtin_ia32_shuf_i64x2_mask ((__v8di)(__m512i)(X), \ + (__v8di)(__m512i)(Y), (int)(C),\ + (__v8di)(__m512i)(W),\ + (__mmask8)(U))) + +#define _mm512_maskz_shuffle_i64x2(U, X, Y, C) \ + ((__m512i) __builtin_ia32_shuf_i64x2_mask ((__v8di)(__m512i)(X), \ + (__v8di)(__m512i)(Y), (int)(C),\ + (__v8di)(__m512i)_mm512_setzero_si512 (),\ + (__mmask8)(U))) + +#define _mm512_shuffle_i32x4(X, Y, C) \ + ((__m512i) __builtin_ia32_shuf_i32x4_mask ((__v16si)(__m512i)(X), \ + (__v16si)(__m512i)(Y), (int)(C),\ + (__v16si)(__m512i)_mm512_undefined_epi32 (),\ + (__mmask16)-1)) + +#define _mm512_mask_shuffle_i32x4(W, U, X, Y, C) \ + ((__m512i) __builtin_ia32_shuf_i32x4_mask ((__v16si)(__m512i)(X), \ + (__v16si)(__m512i)(Y), (int)(C),\ + (__v16si)(__m512i)(W),\ + (__mmask16)(U))) + +#define _mm512_maskz_shuffle_i32x4(U, X, Y, C) \ + ((__m512i) __builtin_ia32_shuf_i32x4_mask ((__v16si)(__m512i)(X), \ + (__v16si)(__m512i)(Y), (int)(C),\ + (__v16si)(__m512i)_mm512_setzero_si512 (),\ + (__mmask16)(U))) + +#define _mm512_shuffle_f64x2(X, Y, C) \ + ((__m512d) __builtin_ia32_shuf_f64x2_mask ((__v8df)(__m512d)(X), \ + (__v8df)(__m512d)(Y), (int)(C),\ + (__v8df)(__m512d)_mm512_undefined_pd(),\ + (__mmask8)-1)) + +#define _mm512_mask_shuffle_f64x2(W, U, X, Y, C) \ + ((__m512d) __builtin_ia32_shuf_f64x2_mask ((__v8df)(__m512d)(X), \ + (__v8df)(__m512d)(Y), (int)(C),\ + (__v8df)(__m512d)(W),\ + (__mmask8)(U))) + +#define _mm512_maskz_shuffle_f64x2(U, X, Y, C) \ + ((__m512d) __builtin_ia32_shuf_f64x2_mask ((__v8df)(__m512d)(X), \ + (__v8df)(__m512d)(Y), (int)(C),\ + (__v8df)(__m512d)_mm512_setzero_pd(),\ + (__mmask8)(U))) + +#define _mm512_shuffle_f32x4(X, Y, C) \ + ((__m512) __builtin_ia32_shuf_f32x4_mask ((__v16sf)(__m512)(X), \ + (__v16sf)(__m512)(Y), (int)(C),\ + (__v16sf)(__m512)_mm512_undefined_ps(),\ + (__mmask16)-1)) + +#define _mm512_mask_shuffle_f32x4(W, U, X, Y, C) \ + ((__m512) __builtin_ia32_shuf_f32x4_mask ((__v16sf)(__m512)(X), \ + (__v16sf)(__m512)(Y), (int)(C),\ + (__v16sf)(__m512)(W),\ + (__mmask16)(U))) + +#define _mm512_maskz_shuffle_f32x4(U, X, Y, C) \ + ((__m512) __builtin_ia32_shuf_f32x4_mask ((__v16sf)(__m512)(X), \ + (__v16sf)(__m512)(Y), (int)(C),\ + (__v16sf)(__m512)_mm512_setzero_ps(),\ + (__mmask16)(U))) +#endif + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_rolv_epi32 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_prolvd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_rolv_epi32 (__m512i __W, __mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_prolvd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_rolv_epi32 (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_prolvd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_rorv_epi32 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_prorvd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_rorv_epi32 (__m512i __W, __mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_prorvd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_rorv_epi32 (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_prorvd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_rolv_epi64 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_prolvq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_rolv_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_prolvq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_rolv_epi64 (__mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_prolvq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_rorv_epi64 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_prorvq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_rorv_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_prorvq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_rorv_epi64 (__mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_prorvq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +#ifdef __OPTIMIZE__ +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtt_roundpd_epi32 (__m512d __A, const int __R) +{ + return (__m256i) __builtin_ia32_cvttpd2dq512_mask ((__v8df) __A, + (__v8si) + _mm256_undefined_si256 (), + (__mmask8) -1, __R); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtt_roundpd_epi32 (__m256i __W, __mmask8 __U, __m512d __A, + const int __R) +{ + return (__m256i) __builtin_ia32_cvttpd2dq512_mask ((__v8df) __A, + (__v8si) __W, + (__mmask8) __U, __R); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtt_roundpd_epi32 (__mmask8 __U, __m512d __A, const int __R) +{ + return (__m256i) __builtin_ia32_cvttpd2dq512_mask ((__v8df) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U, __R); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtt_roundpd_epu32 (__m512d __A, const int __R) +{ + return (__m256i) __builtin_ia32_cvttpd2udq512_mask ((__v8df) __A, + (__v8si) + _mm256_undefined_si256 (), + (__mmask8) -1, __R); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtt_roundpd_epu32 (__m256i __W, __mmask8 __U, __m512d __A, + const int __R) +{ + return (__m256i) __builtin_ia32_cvttpd2udq512_mask ((__v8df) __A, + (__v8si) __W, + (__mmask8) __U, __R); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtt_roundpd_epu32 (__mmask8 __U, __m512d __A, const int __R) +{ + return (__m256i) __builtin_ia32_cvttpd2udq512_mask ((__v8df) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U, __R); +} +#else +#define _mm512_cvtt_roundpd_epi32(A, B) \ + ((__m256i)__builtin_ia32_cvttpd2dq512_mask(A, (__v8si)_mm256_undefined_si256(), -1, B)) + +#define _mm512_mask_cvtt_roundpd_epi32(W, U, A, B) \ + ((__m256i)__builtin_ia32_cvttpd2dq512_mask(A, (__v8si)(W), U, B)) + +#define _mm512_maskz_cvtt_roundpd_epi32(U, A, B) \ + ((__m256i)__builtin_ia32_cvttpd2dq512_mask(A, (__v8si)_mm256_setzero_si256(), U, B)) + +#define _mm512_cvtt_roundpd_epu32(A, B) \ + ((__m256i)__builtin_ia32_cvttpd2udq512_mask(A, (__v8si)_mm256_undefined_si256(), -1, B)) + +#define _mm512_mask_cvtt_roundpd_epu32(W, U, A, B) \ + ((__m256i)__builtin_ia32_cvttpd2udq512_mask(A, (__v8si)(W), U, B)) + +#define _mm512_maskz_cvtt_roundpd_epu32(U, A, B) \ + ((__m256i)__builtin_ia32_cvttpd2udq512_mask(A, (__v8si)_mm256_setzero_si256(), U, B)) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundpd_epi32 (__m512d __A, const int __R) +{ + return (__m256i) __builtin_ia32_cvtpd2dq512_mask ((__v8df) __A, + (__v8si) + _mm256_undefined_si256 (), + (__mmask8) -1, __R); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundpd_epi32 (__m256i __W, __mmask8 __U, __m512d __A, + const int __R) +{ + return (__m256i) __builtin_ia32_cvtpd2dq512_mask ((__v8df) __A, + (__v8si) __W, + (__mmask8) __U, __R); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundpd_epi32 (__mmask8 __U, __m512d __A, const int __R) +{ + return (__m256i) __builtin_ia32_cvtpd2dq512_mask ((__v8df) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U, __R); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundpd_epu32 (__m512d __A, const int __R) +{ + return (__m256i) __builtin_ia32_cvtpd2udq512_mask ((__v8df) __A, + (__v8si) + _mm256_undefined_si256 (), + (__mmask8) -1, __R); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundpd_epu32 (__m256i __W, __mmask8 __U, __m512d __A, + const int __R) +{ + return (__m256i) __builtin_ia32_cvtpd2udq512_mask ((__v8df) __A, + (__v8si) __W, + (__mmask8) __U, __R); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundpd_epu32 (__mmask8 __U, __m512d __A, const int __R) +{ + return (__m256i) __builtin_ia32_cvtpd2udq512_mask ((__v8df) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U, __R); +} +#else +#define _mm512_cvt_roundpd_epi32(A, B) \ + ((__m256i)__builtin_ia32_cvtpd2dq512_mask(A, (__v8si)_mm256_undefined_si256(), -1, B)) + +#define _mm512_mask_cvt_roundpd_epi32(W, U, A, B) \ + ((__m256i)__builtin_ia32_cvtpd2dq512_mask(A, (__v8si)(W), U, B)) + +#define _mm512_maskz_cvt_roundpd_epi32(U, A, B) \ + ((__m256i)__builtin_ia32_cvtpd2dq512_mask(A, (__v8si)_mm256_setzero_si256(), U, B)) + +#define _mm512_cvt_roundpd_epu32(A, B) \ + ((__m256i)__builtin_ia32_cvtpd2udq512_mask(A, (__v8si)_mm256_undefined_si256(), -1, B)) + +#define _mm512_mask_cvt_roundpd_epu32(W, U, A, B) \ + ((__m256i)__builtin_ia32_cvtpd2udq512_mask(A, (__v8si)(W), U, B)) + +#define _mm512_maskz_cvt_roundpd_epu32(U, A, B) \ + ((__m256i)__builtin_ia32_cvtpd2udq512_mask(A, (__v8si)_mm256_setzero_si256(), U, B)) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtt_roundps_epi32 (__m512 __A, const int __R) +{ + return (__m512i) __builtin_ia32_cvttps2dq512_mask ((__v16sf) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1, __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtt_roundps_epi32 (__m512i __W, __mmask16 __U, __m512 __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvttps2dq512_mask ((__v16sf) __A, + (__v16si) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtt_roundps_epi32 (__mmask16 __U, __m512 __A, const int __R) +{ + return (__m512i) __builtin_ia32_cvttps2dq512_mask ((__v16sf) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U, __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtt_roundps_epu32 (__m512 __A, const int __R) +{ + return (__m512i) __builtin_ia32_cvttps2udq512_mask ((__v16sf) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1, __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtt_roundps_epu32 (__m512i __W, __mmask16 __U, __m512 __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvttps2udq512_mask ((__v16sf) __A, + (__v16si) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtt_roundps_epu32 (__mmask16 __U, __m512 __A, const int __R) +{ + return (__m512i) __builtin_ia32_cvttps2udq512_mask ((__v16sf) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U, __R); +} +#else +#define _mm512_cvtt_roundps_epi32(A, B) \ + ((__m512i)__builtin_ia32_cvttps2dq512_mask(A, (__v16si)_mm512_undefined_epi32 (), -1, B)) + +#define _mm512_mask_cvtt_roundps_epi32(W, U, A, B) \ + ((__m512i)__builtin_ia32_cvttps2dq512_mask(A, (__v16si)(W), U, B)) + +#define _mm512_maskz_cvtt_roundps_epi32(U, A, B) \ + ((__m512i)__builtin_ia32_cvttps2dq512_mask(A, (__v16si)_mm512_setzero_si512 (), U, B)) + +#define _mm512_cvtt_roundps_epu32(A, B) \ + ((__m512i)__builtin_ia32_cvttps2udq512_mask(A, (__v16si)_mm512_undefined_epi32 (), -1, B)) + +#define _mm512_mask_cvtt_roundps_epu32(W, U, A, B) \ + ((__m512i)__builtin_ia32_cvttps2udq512_mask(A, (__v16si)(W), U, B)) + +#define _mm512_maskz_cvtt_roundps_epu32(U, A, B) \ + ((__m512i)__builtin_ia32_cvttps2udq512_mask(A, (__v16si)_mm512_setzero_si512 (), U, B)) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundps_epi32 (__m512 __A, const int __R) +{ + return (__m512i) __builtin_ia32_cvtps2dq512_mask ((__v16sf) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1, __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundps_epi32 (__m512i __W, __mmask16 __U, __m512 __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvtps2dq512_mask ((__v16sf) __A, + (__v16si) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundps_epi32 (__mmask16 __U, __m512 __A, const int __R) +{ + return (__m512i) __builtin_ia32_cvtps2dq512_mask ((__v16sf) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U, __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundps_epu32 (__m512 __A, const int __R) +{ + return (__m512i) __builtin_ia32_cvtps2udq512_mask ((__v16sf) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1, __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundps_epu32 (__m512i __W, __mmask16 __U, __m512 __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvtps2udq512_mask ((__v16sf) __A, + (__v16si) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundps_epu32 (__mmask16 __U, __m512 __A, const int __R) +{ + return (__m512i) __builtin_ia32_cvtps2udq512_mask ((__v16sf) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U, __R); +} +#else +#define _mm512_cvt_roundps_epi32(A, B) \ + ((__m512i)__builtin_ia32_cvtps2dq512_mask(A, (__v16si)_mm512_undefined_epi32 (), -1, B)) + +#define _mm512_mask_cvt_roundps_epi32(W, U, A, B) \ + ((__m512i)__builtin_ia32_cvtps2dq512_mask(A, (__v16si)(W), U, B)) + +#define _mm512_maskz_cvt_roundps_epi32(U, A, B) \ + ((__m512i)__builtin_ia32_cvtps2dq512_mask(A, (__v16si)_mm512_setzero_si512 (), U, B)) + +#define _mm512_cvt_roundps_epu32(A, B) \ + ((__m512i)__builtin_ia32_cvtps2udq512_mask(A, (__v16si)_mm512_undefined_epi32 (), -1, B)) + +#define _mm512_mask_cvt_roundps_epu32(W, U, A, B) \ + ((__m512i)__builtin_ia32_cvtps2udq512_mask(A, (__v16si)(W), U, B)) + +#define _mm512_maskz_cvt_roundps_epu32(U, A, B) \ + ((__m512i)__builtin_ia32_cvtps2udq512_mask(A, (__v16si)_mm512_setzero_si512 (), U, B)) +#endif + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtu32_sd (__m128d __A, unsigned __B) +{ + return (__m128d) __builtin_ia32_cvtusi2sd32 ((__v2df) __A, __B); +} + +#ifdef __x86_64__ +#ifdef __OPTIMIZE__ +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundu64_sd (__m128d __A, unsigned long long __B, const int __R) +{ + return (__m128d) __builtin_ia32_cvtusi2sd64 ((__v2df) __A, __B, __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundi64_sd (__m128d __A, long long __B, const int __R) +{ + return (__m128d) __builtin_ia32_cvtsi2sd64 ((__v2df) __A, __B, __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundsi64_sd (__m128d __A, long long __B, const int __R) +{ + return (__m128d) __builtin_ia32_cvtsi2sd64 ((__v2df) __A, __B, __R); +} +#else +#define _mm_cvt_roundu64_sd(A, B, C) \ + (__m128d)__builtin_ia32_cvtusi2sd64(A, B, C) + +#define _mm_cvt_roundi64_sd(A, B, C) \ + (__m128d)__builtin_ia32_cvtsi2sd64(A, B, C) + +#define _mm_cvt_roundsi64_sd(A, B, C) \ + (__m128d)__builtin_ia32_cvtsi2sd64(A, B, C) +#endif + +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundu32_ss (__m128 __A, unsigned __B, const int __R) +{ + return (__m128) __builtin_ia32_cvtusi2ss32 ((__v4sf) __A, __B, __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundsi32_ss (__m128 __A, int __B, const int __R) +{ + return (__m128) __builtin_ia32_cvtsi2ss32 ((__v4sf) __A, __B, __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundi32_ss (__m128 __A, int __B, const int __R) +{ + return (__m128) __builtin_ia32_cvtsi2ss32 ((__v4sf) __A, __B, __R); +} +#else +#define _mm_cvt_roundu32_ss(A, B, C) \ + (__m128)__builtin_ia32_cvtusi2ss32(A, B, C) + +#define _mm_cvt_roundi32_ss(A, B, C) \ + (__m128)__builtin_ia32_cvtsi2ss32(A, B, C) + +#define _mm_cvt_roundsi32_ss(A, B, C) \ + (__m128)__builtin_ia32_cvtsi2ss32(A, B, C) +#endif + +#ifdef __x86_64__ +#ifdef __OPTIMIZE__ +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundu64_ss (__m128 __A, unsigned long long __B, const int __R) +{ + return (__m128) __builtin_ia32_cvtusi2ss64 ((__v4sf) __A, __B, __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundsi64_ss (__m128 __A, long long __B, const int __R) +{ + return (__m128) __builtin_ia32_cvtsi2ss64 ((__v4sf) __A, __B, __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundi64_ss (__m128 __A, long long __B, const int __R) +{ + return (__m128) __builtin_ia32_cvtsi2ss64 ((__v4sf) __A, __B, __R); +} +#else +#define _mm_cvt_roundu64_ss(A, B, C) \ + (__m128)__builtin_ia32_cvtusi2ss64(A, B, C) + +#define _mm_cvt_roundi64_ss(A, B, C) \ + (__m128)__builtin_ia32_cvtsi2ss64(A, B, C) + +#define _mm_cvt_roundsi64_ss(A, B, C) \ + (__m128)__builtin_ia32_cvtsi2ss64(A, B, C) +#endif + +#endif + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepi32_epi8 (__m512i __A) +{ + return (__m128i) __builtin_ia32_pmovdb512_mask ((__v16si) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask16) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi32_storeu_epi8 (void * __P, __mmask16 __M, __m512i __A) +{ + __builtin_ia32_pmovdb512mem_mask ((__v16qi *) __P, (__v16si) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi32_epi8 (__m128i __O, __mmask16 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovdb512_mask ((__v16si) __A, + (__v16qi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepi32_epi8 (__mmask16 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovdb512_mask ((__v16si) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtsepi32_epi8 (__m512i __A) +{ + return (__m128i) __builtin_ia32_pmovsdb512_mask ((__v16si) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask16) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtsepi32_storeu_epi8 (void * __P, __mmask16 __M, __m512i __A) +{ + __builtin_ia32_pmovsdb512mem_mask ((__v16qi *) __P, (__v16si) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtsepi32_epi8 (__m128i __O, __mmask16 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovsdb512_mask ((__v16si) __A, + (__v16qi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtsepi32_epi8 (__mmask16 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovsdb512_mask ((__v16si) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtusepi32_epi8 (__m512i __A) +{ + return (__m128i) __builtin_ia32_pmovusdb512_mask ((__v16si) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask16) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtusepi32_storeu_epi8 (void * __P, __mmask16 __M, __m512i __A) +{ + __builtin_ia32_pmovusdb512mem_mask ((__v16qi *) __P, (__v16si) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtusepi32_epi8 (__m128i __O, __mmask16 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovusdb512_mask ((__v16si) __A, + (__v16qi) __O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtusepi32_epi8 (__mmask16 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovusdb512_mask ((__v16si) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepi32_epi16 (__m512i __A) +{ + return (__m256i) __builtin_ia32_pmovdw512_mask ((__v16si) __A, + (__v16hi) + _mm256_undefined_si256 (), + (__mmask16) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi32_storeu_epi16 (void * __P, __mmask16 __M, __m512i __A) +{ + __builtin_ia32_pmovdw512mem_mask ((__v16hi *) __P, (__v16si) __A, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi32_epi16 (__m256i __O, __mmask16 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovdw512_mask ((__v16si) __A, + (__v16hi) __O, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepi32_epi16 (__mmask16 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovdw512_mask ((__v16si) __A, + (__v16hi) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtsepi32_epi16 (__m512i __A) +{ + return (__m256i) __builtin_ia32_pmovsdw512_mask ((__v16si) __A, + (__v16hi) + _mm256_undefined_si256 (), + (__mmask16) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtsepi32_storeu_epi16 (void *__P, __mmask16 __M, __m512i __A) +{ + __builtin_ia32_pmovsdw512mem_mask ((__v16hi*) __P, (__v16si) __A, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtsepi32_epi16 (__m256i __O, __mmask16 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovsdw512_mask ((__v16si) __A, + (__v16hi) __O, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtsepi32_epi16 (__mmask16 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovsdw512_mask ((__v16si) __A, + (__v16hi) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtusepi32_epi16 (__m512i __A) +{ + return (__m256i) __builtin_ia32_pmovusdw512_mask ((__v16si) __A, + (__v16hi) + _mm256_undefined_si256 (), + (__mmask16) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtusepi32_storeu_epi16 (void *__P, __mmask16 __M, __m512i __A) +{ + __builtin_ia32_pmovusdw512mem_mask ((__v16hi*) __P, (__v16si) __A, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtusepi32_epi16 (__m256i __O, __mmask16 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovusdw512_mask ((__v16si) __A, + (__v16hi) __O, + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtusepi32_epi16 (__mmask16 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovusdw512_mask ((__v16si) __A, + (__v16hi) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepi64_epi32 (__m512i __A) +{ + return (__m256i) __builtin_ia32_pmovqd512_mask ((__v8di) __A, + (__v8si) + _mm256_undefined_si256 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi64_storeu_epi32 (void* __P, __mmask8 __M, __m512i __A) +{ + __builtin_ia32_pmovqd512mem_mask ((__v8si *) __P, (__v8di) __A, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi64_epi32 (__m256i __O, __mmask8 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovqd512_mask ((__v8di) __A, + (__v8si) __O, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepi64_epi32 (__mmask8 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovqd512_mask ((__v8di) __A, + (__v8si) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtsepi64_epi32 (__m512i __A) +{ + return (__m256i) __builtin_ia32_pmovsqd512_mask ((__v8di) __A, + (__v8si) + _mm256_undefined_si256 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtsepi64_storeu_epi32 (void *__P, __mmask8 __M, __m512i __A) +{ + __builtin_ia32_pmovsqd512mem_mask ((__v8si *) __P, (__v8di) __A, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtsepi64_epi32 (__m256i __O, __mmask8 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovsqd512_mask ((__v8di) __A, + (__v8si) __O, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtsepi64_epi32 (__mmask8 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovsqd512_mask ((__v8di) __A, + (__v8si) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtusepi64_epi32 (__m512i __A) +{ + return (__m256i) __builtin_ia32_pmovusqd512_mask ((__v8di) __A, + (__v8si) + _mm256_undefined_si256 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtusepi64_storeu_epi32 (void* __P, __mmask8 __M, __m512i __A) +{ + __builtin_ia32_pmovusqd512mem_mask ((__v8si*) __P, (__v8di) __A, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtusepi64_epi32 (__m256i __O, __mmask8 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovusqd512_mask ((__v8di) __A, + (__v8si) __O, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtusepi64_epi32 (__mmask8 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovusqd512_mask ((__v8di) __A, + (__v8si) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepi64_epi16 (__m512i __A) +{ + return (__m128i) __builtin_ia32_pmovqw512_mask ((__v8di) __A, + (__v8hi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi64_storeu_epi16 (void *__P, __mmask8 __M, __m512i __A) +{ + __builtin_ia32_pmovqw512mem_mask ((__v8hi *) __P, (__v8di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi64_epi16 (__m128i __O, __mmask8 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovqw512_mask ((__v8di) __A, + (__v8hi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepi64_epi16 (__mmask8 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovqw512_mask ((__v8di) __A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtsepi64_epi16 (__m512i __A) +{ + return (__m128i) __builtin_ia32_pmovsqw512_mask ((__v8di) __A, + (__v8hi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtsepi64_storeu_epi16 (void * __P, __mmask8 __M, __m512i __A) +{ + __builtin_ia32_pmovsqw512mem_mask ((__v8hi *) __P, (__v8di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtsepi64_epi16 (__m128i __O, __mmask8 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovsqw512_mask ((__v8di) __A, + (__v8hi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtsepi64_epi16 (__mmask8 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovsqw512_mask ((__v8di) __A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtusepi64_epi16 (__m512i __A) +{ + return (__m128i) __builtin_ia32_pmovusqw512_mask ((__v8di) __A, + (__v8hi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtusepi64_storeu_epi16 (void *__P, __mmask8 __M, __m512i __A) +{ + __builtin_ia32_pmovusqw512mem_mask ((__v8hi*) __P, (__v8di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtusepi64_epi16 (__m128i __O, __mmask8 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovusqw512_mask ((__v8di) __A, + (__v8hi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtusepi64_epi16 (__mmask8 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovusqw512_mask ((__v8di) __A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepi64_epi8 (__m512i __A) +{ + return (__m128i) __builtin_ia32_pmovqb512_mask ((__v8di) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi64_storeu_epi8 (void * __P, __mmask8 __M, __m512i __A) +{ + __builtin_ia32_pmovqb512mem_mask ((__v16qi *) __P, (__v8di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi64_epi8 (__m128i __O, __mmask8 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovqb512_mask ((__v8di) __A, + (__v16qi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepi64_epi8 (__mmask8 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovqb512_mask ((__v8di) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtsepi64_epi8 (__m512i __A) +{ + return (__m128i) __builtin_ia32_pmovsqb512_mask ((__v8di) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtsepi64_storeu_epi8 (void * __P, __mmask8 __M, __m512i __A) +{ + __builtin_ia32_pmovsqb512mem_mask ((__v16qi *) __P, (__v8di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtsepi64_epi8 (__m128i __O, __mmask8 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovsqb512_mask ((__v8di) __A, + (__v16qi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtsepi64_epi8 (__mmask8 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovsqb512_mask ((__v8di) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtusepi64_epi8 (__m512i __A) +{ + return (__m128i) __builtin_ia32_pmovusqb512_mask ((__v8di) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtusepi64_storeu_epi8 (void * __P, __mmask8 __M, __m512i __A) +{ + __builtin_ia32_pmovusqb512mem_mask ((__v16qi *) __P, (__v8di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtusepi64_epi8 (__m128i __O, __mmask8 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovusqb512_mask ((__v8di) __A, + (__v16qi) __O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtusepi64_epi8 (__mmask8 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovusqb512_mask ((__v8di) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepi32_pd (__m256i __A) +{ + return (__m512d) __builtin_ia32_cvtdq2pd512_mask ((__v8si) __A, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi32_pd (__m512d __W, __mmask8 __U, __m256i __A) +{ + return (__m512d) __builtin_ia32_cvtdq2pd512_mask ((__v8si) __A, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepi32_pd (__mmask8 __U, __m256i __A) +{ + return (__m512d) __builtin_ia32_cvtdq2pd512_mask ((__v8si) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepu32_pd (__m256i __A) +{ + return (__m512d) __builtin_ia32_cvtudq2pd512_mask ((__v8si) __A, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepu32_pd (__m512d __W, __mmask8 __U, __m256i __A) +{ + return (__m512d) __builtin_ia32_cvtudq2pd512_mask ((__v8si) __A, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepu32_pd (__mmask8 __U, __m256i __A) +{ + return (__m512d) __builtin_ia32_cvtudq2pd512_mask ((__v8si) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundepi32_ps (__m512i __A, const int __R) +{ + return (__m512) __builtin_ia32_cvtdq2ps512_mask ((__v16si) __A, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundepi32_ps (__m512 __W, __mmask16 __U, __m512i __A, + const int __R) +{ + return (__m512) __builtin_ia32_cvtdq2ps512_mask ((__v16si) __A, + (__v16sf) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundepi32_ps (__mmask16 __U, __m512i __A, const int __R) +{ + return (__m512) __builtin_ia32_cvtdq2ps512_mask ((__v16si) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundepu32_ps (__m512i __A, const int __R) +{ + return (__m512) __builtin_ia32_cvtudq2ps512_mask ((__v16si) __A, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundepu32_ps (__m512 __W, __mmask16 __U, __m512i __A, + const int __R) +{ + return (__m512) __builtin_ia32_cvtudq2ps512_mask ((__v16si) __A, + (__v16sf) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundepu32_ps (__mmask16 __U, __m512i __A, const int __R) +{ + return (__m512) __builtin_ia32_cvtudq2ps512_mask ((__v16si) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, __R); +} + +#else +#define _mm512_cvt_roundepi32_ps(A, B) \ + (__m512)__builtin_ia32_cvtdq2ps512_mask((__v16si)(A), (__v16sf)_mm512_undefined_ps(), -1, B) + +#define _mm512_mask_cvt_roundepi32_ps(W, U, A, B) \ + (__m512)__builtin_ia32_cvtdq2ps512_mask((__v16si)(A), W, U, B) + +#define _mm512_maskz_cvt_roundepi32_ps(U, A, B) \ + (__m512)__builtin_ia32_cvtdq2ps512_mask((__v16si)(A), (__v16sf)_mm512_setzero_ps(), U, B) + +#define _mm512_cvt_roundepu32_ps(A, B) \ + (__m512)__builtin_ia32_cvtudq2ps512_mask((__v16si)(A), (__v16sf)_mm512_undefined_ps(), -1, B) + +#define _mm512_mask_cvt_roundepu32_ps(W, U, A, B) \ + (__m512)__builtin_ia32_cvtudq2ps512_mask((__v16si)(A), W, U, B) + +#define _mm512_maskz_cvt_roundepu32_ps(U, A, B) \ + (__m512)__builtin_ia32_cvtudq2ps512_mask((__v16si)(A), (__v16sf)_mm512_setzero_ps(), U, B) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_extractf64x4_pd (__m512d __A, const int __imm) +{ + return (__m256d) __builtin_ia32_extractf64x4_mask ((__v8df) __A, + __imm, + (__v4df) + _mm256_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_extractf64x4_pd (__m256d __W, __mmask8 __U, __m512d __A, + const int __imm) +{ + return (__m256d) __builtin_ia32_extractf64x4_mask ((__v8df) __A, + __imm, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_extractf64x4_pd (__mmask8 __U, __m512d __A, const int __imm) +{ + return (__m256d) __builtin_ia32_extractf64x4_mask ((__v8df) __A, + __imm, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_extractf32x4_ps (__m512 __A, const int __imm) +{ + return (__m128) __builtin_ia32_extractf32x4_mask ((__v16sf) __A, + __imm, + (__v4sf) + _mm_undefined_ps (), + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_extractf32x4_ps (__m128 __W, __mmask8 __U, __m512 __A, + const int __imm) +{ + return (__m128) __builtin_ia32_extractf32x4_mask ((__v16sf) __A, + __imm, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_extractf32x4_ps (__mmask8 __U, __m512 __A, const int __imm) +{ + return (__m128) __builtin_ia32_extractf32x4_mask ((__v16sf) __A, + __imm, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_extracti64x4_epi64 (__m512i __A, const int __imm) +{ + return (__m256i) __builtin_ia32_extracti64x4_mask ((__v8di) __A, + __imm, + (__v4di) + _mm256_undefined_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_extracti64x4_epi64 (__m256i __W, __mmask8 __U, __m512i __A, + const int __imm) +{ + return (__m256i) __builtin_ia32_extracti64x4_mask ((__v8di) __A, + __imm, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_extracti64x4_epi64 (__mmask8 __U, __m512i __A, const int __imm) +{ + return (__m256i) __builtin_ia32_extracti64x4_mask ((__v8di) __A, + __imm, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_extracti32x4_epi32 (__m512i __A, const int __imm) +{ + return (__m128i) __builtin_ia32_extracti32x4_mask ((__v16si) __A, + __imm, + (__v4si) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_extracti32x4_epi32 (__m128i __W, __mmask8 __U, __m512i __A, + const int __imm) +{ + return (__m128i) __builtin_ia32_extracti32x4_mask ((__v16si) __A, + __imm, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_extracti32x4_epi32 (__mmask8 __U, __m512i __A, const int __imm) +{ + return (__m128i) __builtin_ia32_extracti32x4_mask ((__v16si) __A, + __imm, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} +#else + +#define _mm512_extractf64x4_pd(X, C) \ + ((__m256d) __builtin_ia32_extractf64x4_mask ((__v8df)(__m512d) (X), \ + (int) (C),\ + (__v4df)(__m256d)_mm256_undefined_pd(),\ + (__mmask8)-1)) + +#define _mm512_mask_extractf64x4_pd(W, U, X, C) \ + ((__m256d) __builtin_ia32_extractf64x4_mask ((__v8df)(__m512d) (X), \ + (int) (C),\ + (__v4df)(__m256d)(W),\ + (__mmask8)(U))) + +#define _mm512_maskz_extractf64x4_pd(U, X, C) \ + ((__m256d) __builtin_ia32_extractf64x4_mask ((__v8df)(__m512d) (X), \ + (int) (C),\ + (__v4df)(__m256d)_mm256_setzero_pd(),\ + (__mmask8)(U))) + +#define _mm512_extractf32x4_ps(X, C) \ + ((__m128) __builtin_ia32_extractf32x4_mask ((__v16sf)(__m512) (X), \ + (int) (C),\ + (__v4sf)(__m128)_mm_undefined_ps(),\ + (__mmask8)-1)) + +#define _mm512_mask_extractf32x4_ps(W, U, X, C) \ + ((__m128) __builtin_ia32_extractf32x4_mask ((__v16sf)(__m512) (X), \ + (int) (C),\ + (__v4sf)(__m128)(W),\ + (__mmask8)(U))) + +#define _mm512_maskz_extractf32x4_ps(U, X, C) \ + ((__m128) __builtin_ia32_extractf32x4_mask ((__v16sf)(__m512) (X), \ + (int) (C),\ + (__v4sf)(__m128)_mm_setzero_ps(),\ + (__mmask8)(U))) + +#define _mm512_extracti64x4_epi64(X, C) \ + ((__m256i) __builtin_ia32_extracti64x4_mask ((__v8di)(__m512i) (X), \ + (int) (C),\ + (__v4di)(__m256i)_mm256_undefined_si256 (),\ + (__mmask8)-1)) + +#define _mm512_mask_extracti64x4_epi64(W, U, X, C) \ + ((__m256i) __builtin_ia32_extracti64x4_mask ((__v8di)(__m512i) (X), \ + (int) (C),\ + (__v4di)(__m256i)(W),\ + (__mmask8)(U))) + +#define _mm512_maskz_extracti64x4_epi64(U, X, C) \ + ((__m256i) __builtin_ia32_extracti64x4_mask ((__v8di)(__m512i) (X), \ + (int) (C),\ + (__v4di)(__m256i)_mm256_setzero_si256 (),\ + (__mmask8)(U))) + +#define _mm512_extracti32x4_epi32(X, C) \ + ((__m128i) __builtin_ia32_extracti32x4_mask ((__v16si)(__m512i) (X), \ + (int) (C),\ + (__v4si)(__m128i)_mm_undefined_si128 (),\ + (__mmask8)-1)) + +#define _mm512_mask_extracti32x4_epi32(W, U, X, C) \ + ((__m128i) __builtin_ia32_extracti32x4_mask ((__v16si)(__m512i) (X), \ + (int) (C),\ + (__v4si)(__m128i)(W),\ + (__mmask8)(U))) + +#define _mm512_maskz_extracti32x4_epi32(U, X, C) \ + ((__m128i) __builtin_ia32_extracti32x4_mask ((__v16si)(__m512i) (X), \ + (int) (C),\ + (__v4si)(__m128i)_mm_setzero_si128 (),\ + (__mmask8)(U))) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_inserti32x4 (__m512i __A, __m128i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_inserti32x4_mask ((__v16si) __A, + (__v4si) __B, + __imm, + (__v16si) __A, -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_insertf32x4 (__m512 __A, __m128 __B, const int __imm) +{ + return (__m512) __builtin_ia32_insertf32x4_mask ((__v16sf) __A, + (__v4sf) __B, + __imm, + (__v16sf) __A, -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_inserti64x4 (__m512i __A, __m256i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_inserti64x4_mask ((__v8di) __A, + (__v4di) __B, + __imm, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_inserti64x4 (__m512i __W, __mmask8 __U, __m512i __A, + __m256i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_inserti64x4_mask ((__v8di) __A, + (__v4di) __B, + __imm, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_inserti64x4 (__mmask8 __U, __m512i __A, __m256i __B, + const int __imm) +{ + return (__m512i) __builtin_ia32_inserti64x4_mask ((__v8di) __A, + (__v4di) __B, + __imm, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_insertf64x4 (__m512d __A, __m256d __B, const int __imm) +{ + return (__m512d) __builtin_ia32_insertf64x4_mask ((__v8df) __A, + (__v4df) __B, + __imm, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_insertf64x4 (__m512d __W, __mmask8 __U, __m512d __A, + __m256d __B, const int __imm) +{ + return (__m512d) __builtin_ia32_insertf64x4_mask ((__v8df) __A, + (__v4df) __B, + __imm, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_insertf64x4 (__mmask8 __U, __m512d __A, __m256d __B, + const int __imm) +{ + return (__m512d) __builtin_ia32_insertf64x4_mask ((__v8df) __A, + (__v4df) __B, + __imm, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} +#else +#define _mm512_insertf32x4(X, Y, C) \ + ((__m512) __builtin_ia32_insertf32x4_mask ((__v16sf)(__m512) (X), \ + (__v4sf)(__m128) (Y), (int) (C), (__v16sf)(__m512) (X), (__mmask16)(-1))) + +#define _mm512_inserti32x4(X, Y, C) \ + ((__m512i) __builtin_ia32_inserti32x4_mask ((__v16si)(__m512i) (X), \ + (__v4si)(__m128i) (Y), (int) (C), (__v16si)(__m512i) (X), (__mmask16)(-1))) + +#define _mm512_insertf64x4(X, Y, C) \ + ((__m512d) __builtin_ia32_insertf64x4_mask ((__v8df)(__m512d) (X), \ + (__v4df)(__m256d) (Y), (int) (C), \ + (__v8df)(__m512d)_mm512_undefined_pd(), \ + (__mmask8)-1)) + +#define _mm512_mask_insertf64x4(W, U, X, Y, C) \ + ((__m512d) __builtin_ia32_insertf64x4_mask ((__v8df)(__m512d) (X), \ + (__v4df)(__m256d) (Y), (int) (C), \ + (__v8df)(__m512d)(W), \ + (__mmask8)(U))) + +#define _mm512_maskz_insertf64x4(U, X, Y, C) \ + ((__m512d) __builtin_ia32_insertf64x4_mask ((__v8df)(__m512d) (X), \ + (__v4df)(__m256d) (Y), (int) (C), \ + (__v8df)(__m512d)_mm512_setzero_pd(), \ + (__mmask8)(U))) + +#define _mm512_inserti64x4(X, Y, C) \ + ((__m512i) __builtin_ia32_inserti64x4_mask ((__v8di)(__m512i) (X), \ + (__v4di)(__m256i) (Y), (int) (C), \ + (__v8di)(__m512i)_mm512_undefined_epi32 (), \ + (__mmask8)-1)) + +#define _mm512_mask_inserti64x4(W, U, X, Y, C) \ + ((__m512i) __builtin_ia32_inserti64x4_mask ((__v8di)(__m512i) (X), \ + (__v4di)(__m256i) (Y), (int) (C),\ + (__v8di)(__m512i)(W),\ + (__mmask8)(U))) + +#define _mm512_maskz_inserti64x4(U, X, Y, C) \ + ((__m512i) __builtin_ia32_inserti64x4_mask ((__v8di)(__m512i) (X), \ + (__v4di)(__m256i) (Y), (int) (C), \ + (__v8di)(__m512i)_mm512_setzero_si512 (), \ + (__mmask8)(U))) +#endif + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_loadu_pd (void const *__P) +{ + return *(__m512d_u *)__P; +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_loadu_pd (__m512d __W, __mmask8 __U, void const *__P) +{ + return (__m512d) __builtin_ia32_loadupd512_mask ((const double *) __P, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_loadu_pd (__mmask8 __U, void const *__P) +{ + return (__m512d) __builtin_ia32_loadupd512_mask ((const double *) __P, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_storeu_pd (void *__P, __m512d __A) +{ + *(__m512d_u *)__P = __A; +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_storeu_pd (void *__P, __mmask8 __U, __m512d __A) +{ + __builtin_ia32_storeupd512_mask ((double *) __P, (__v8df) __A, + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_loadu_ps (void const *__P) +{ + return *(__m512_u *)__P; +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_loadu_ps (__m512 __W, __mmask16 __U, void const *__P) +{ + return (__m512) __builtin_ia32_loadups512_mask ((const float *) __P, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_loadu_ps (__mmask16 __U, void const *__P) +{ + return (__m512) __builtin_ia32_loadups512_mask ((const float *) __P, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_storeu_ps (void *__P, __m512 __A) +{ + *(__m512_u *)__P = __A; +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_storeu_ps (void *__P, __mmask16 __U, __m512 __A) +{ + __builtin_ia32_storeups512_mask ((float *) __P, (__v16sf) __A, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_loadu_epi64 (__m512i __W, __mmask8 __U, void const *__P) +{ + return (__m512i) __builtin_ia32_loaddqudi512_mask ((const long long *) __P, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_loadu_epi64 (__mmask8 __U, void const *__P) +{ + return (__m512i) __builtin_ia32_loaddqudi512_mask ((const long long *) __P, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_storeu_epi64 (void *__P, __mmask8 __U, __m512i __A) +{ + __builtin_ia32_storedqudi512_mask ((long long *) __P, (__v8di) __A, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_loadu_si512 (void const *__P) +{ + return *(__m512i_u *)__P; +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_loadu_epi32 (__m512i __W, __mmask16 __U, void const *__P) +{ + return (__m512i) __builtin_ia32_loaddqusi512_mask ((const int *) __P, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_loadu_epi32 (__mmask16 __U, void const *__P) +{ + return (__m512i) __builtin_ia32_loaddqusi512_mask ((const int *) __P, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_storeu_si512 (void *__P, __m512i __A) +{ + *(__m512i_u *)__P = __A; +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_storeu_epi32 (void *__P, __mmask16 __U, __m512i __A) +{ + __builtin_ia32_storedqusi512_mask ((int *) __P, (__v16si) __A, + (__mmask16) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permutevar_pd (__m512d __A, __m512i __C) +{ + return (__m512d) __builtin_ia32_vpermilvarpd512_mask ((__v8df) __A, + (__v8di) __C, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permutevar_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512i __C) +{ + return (__m512d) __builtin_ia32_vpermilvarpd512_mask ((__v8df) __A, + (__v8di) __C, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permutevar_pd (__mmask8 __U, __m512d __A, __m512i __C) +{ + return (__m512d) __builtin_ia32_vpermilvarpd512_mask ((__v8df) __A, + (__v8di) __C, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permutevar_ps (__m512 __A, __m512i __C) +{ + return (__m512) __builtin_ia32_vpermilvarps512_mask ((__v16sf) __A, + (__v16si) __C, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permutevar_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512i __C) +{ + return (__m512) __builtin_ia32_vpermilvarps512_mask ((__v16sf) __A, + (__v16si) __C, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permutevar_ps (__mmask16 __U, __m512 __A, __m512i __C) +{ + return (__m512) __builtin_ia32_vpermilvarps512_mask ((__v16sf) __A, + (__v16si) __C, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permutex2var_epi64 (__m512i __A, __m512i __I, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpermt2varq512_mask ((__v8di) __I + /* idx */ , + (__v8di) __A, + (__v8di) __B, + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permutex2var_epi64 (__m512i __A, __mmask8 __U, __m512i __I, + __m512i __B) +{ + return (__m512i) __builtin_ia32_vpermt2varq512_mask ((__v8di) __I + /* idx */ , + (__v8di) __A, + (__v8di) __B, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask2_permutex2var_epi64 (__m512i __A, __m512i __I, + __mmask8 __U, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpermi2varq512_mask ((__v8di) __A, + (__v8di) __I + /* idx */ , + (__v8di) __B, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permutex2var_epi64 (__mmask8 __U, __m512i __A, + __m512i __I, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpermt2varq512_maskz ((__v8di) __I + /* idx */ , + (__v8di) __A, + (__v8di) __B, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permutex2var_epi32 (__m512i __A, __m512i __I, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpermt2vard512_mask ((__v16si) __I + /* idx */ , + (__v16si) __A, + (__v16si) __B, + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permutex2var_epi32 (__m512i __A, __mmask16 __U, + __m512i __I, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpermt2vard512_mask ((__v16si) __I + /* idx */ , + (__v16si) __A, + (__v16si) __B, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask2_permutex2var_epi32 (__m512i __A, __m512i __I, + __mmask16 __U, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpermi2vard512_mask ((__v16si) __A, + (__v16si) __I + /* idx */ , + (__v16si) __B, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permutex2var_epi32 (__mmask16 __U, __m512i __A, + __m512i __I, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpermt2vard512_maskz ((__v16si) __I + /* idx */ , + (__v16si) __A, + (__v16si) __B, + (__mmask16) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permutex2var_pd (__m512d __A, __m512i __I, __m512d __B) +{ + return (__m512d) __builtin_ia32_vpermt2varpd512_mask ((__v8di) __I + /* idx */ , + (__v8df) __A, + (__v8df) __B, + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permutex2var_pd (__m512d __A, __mmask8 __U, __m512i __I, + __m512d __B) +{ + return (__m512d) __builtin_ia32_vpermt2varpd512_mask ((__v8di) __I + /* idx */ , + (__v8df) __A, + (__v8df) __B, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask2_permutex2var_pd (__m512d __A, __m512i __I, __mmask8 __U, + __m512d __B) +{ + return (__m512d) __builtin_ia32_vpermi2varpd512_mask ((__v8df) __A, + (__v8di) __I + /* idx */ , + (__v8df) __B, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permutex2var_pd (__mmask8 __U, __m512d __A, __m512i __I, + __m512d __B) +{ + return (__m512d) __builtin_ia32_vpermt2varpd512_maskz ((__v8di) __I + /* idx */ , + (__v8df) __A, + (__v8df) __B, + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permutex2var_ps (__m512 __A, __m512i __I, __m512 __B) +{ + return (__m512) __builtin_ia32_vpermt2varps512_mask ((__v16si) __I + /* idx */ , + (__v16sf) __A, + (__v16sf) __B, + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permutex2var_ps (__m512 __A, __mmask16 __U, __m512i __I, __m512 __B) +{ + return (__m512) __builtin_ia32_vpermt2varps512_mask ((__v16si) __I + /* idx */ , + (__v16sf) __A, + (__v16sf) __B, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask2_permutex2var_ps (__m512 __A, __m512i __I, __mmask16 __U, + __m512 __B) +{ + return (__m512) __builtin_ia32_vpermi2varps512_mask ((__v16sf) __A, + (__v16si) __I + /* idx */ , + (__v16sf) __B, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permutex2var_ps (__mmask16 __U, __m512 __A, __m512i __I, + __m512 __B) +{ + return (__m512) __builtin_ia32_vpermt2varps512_maskz ((__v16si) __I + /* idx */ , + (__v16sf) __A, + (__v16sf) __B, + (__mmask16) __U); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permute_pd (__m512d __X, const int __C) +{ + return (__m512d) __builtin_ia32_vpermilpd512_mask ((__v8df) __X, __C, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permute_pd (__m512d __W, __mmask8 __U, __m512d __X, const int __C) +{ + return (__m512d) __builtin_ia32_vpermilpd512_mask ((__v8df) __X, __C, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permute_pd (__mmask8 __U, __m512d __X, const int __C) +{ + return (__m512d) __builtin_ia32_vpermilpd512_mask ((__v8df) __X, __C, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permute_ps (__m512 __X, const int __C) +{ + return (__m512) __builtin_ia32_vpermilps512_mask ((__v16sf) __X, __C, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permute_ps (__m512 __W, __mmask16 __U, __m512 __X, const int __C) +{ + return (__m512) __builtin_ia32_vpermilps512_mask ((__v16sf) __X, __C, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permute_ps (__mmask16 __U, __m512 __X, const int __C) +{ + return (__m512) __builtin_ia32_vpermilps512_mask ((__v16sf) __X, __C, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} +#else +#define _mm512_permute_pd(X, C) \ + ((__m512d) __builtin_ia32_vpermilpd512_mask ((__v8df)(__m512d)(X), (int)(C), \ + (__v8df)(__m512d)_mm512_undefined_pd(),\ + (__mmask8)(-1))) + +#define _mm512_mask_permute_pd(W, U, X, C) \ + ((__m512d) __builtin_ia32_vpermilpd512_mask ((__v8df)(__m512d)(X), (int)(C), \ + (__v8df)(__m512d)(W), \ + (__mmask8)(U))) + +#define _mm512_maskz_permute_pd(U, X, C) \ + ((__m512d) __builtin_ia32_vpermilpd512_mask ((__v8df)(__m512d)(X), (int)(C), \ + (__v8df)(__m512d)_mm512_setzero_pd(), \ + (__mmask8)(U))) + +#define _mm512_permute_ps(X, C) \ + ((__m512) __builtin_ia32_vpermilps512_mask ((__v16sf)(__m512)(X), (int)(C), \ + (__v16sf)(__m512)_mm512_undefined_ps(),\ + (__mmask16)(-1))) + +#define _mm512_mask_permute_ps(W, U, X, C) \ + ((__m512) __builtin_ia32_vpermilps512_mask ((__v16sf)(__m512)(X), (int)(C), \ + (__v16sf)(__m512)(W), \ + (__mmask16)(U))) + +#define _mm512_maskz_permute_ps(U, X, C) \ + ((__m512) __builtin_ia32_vpermilps512_mask ((__v16sf)(__m512)(X), (int)(C), \ + (__v16sf)(__m512)_mm512_setzero_ps(), \ + (__mmask16)(U))) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permutex_epi64 (__m512i __X, const int __I) +{ + return (__m512i) __builtin_ia32_permdi512_mask ((__v8di) __X, __I, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) (-1)); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permutex_epi64 (__m512i __W, __mmask8 __M, + __m512i __X, const int __I) +{ + return (__m512i) __builtin_ia32_permdi512_mask ((__v8di) __X, __I, + (__v8di) __W, + (__mmask8) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permutex_epi64 (__mmask8 __M, __m512i __X, const int __I) +{ + return (__m512i) __builtin_ia32_permdi512_mask ((__v8di) __X, __I, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __M); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permutex_pd (__m512d __X, const int __M) +{ + return (__m512d) __builtin_ia32_permdf512_mask ((__v8df) __X, __M, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permutex_pd (__m512d __W, __mmask8 __U, __m512d __X, const int __M) +{ + return (__m512d) __builtin_ia32_permdf512_mask ((__v8df) __X, __M, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permutex_pd (__mmask8 __U, __m512d __X, const int __M) +{ + return (__m512d) __builtin_ia32_permdf512_mask ((__v8df) __X, __M, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} +#else +#define _mm512_permutex_pd(X, M) \ + ((__m512d) __builtin_ia32_permdf512_mask ((__v8df)(__m512d)(X), (int)(M), \ + (__v8df)(__m512d)_mm512_undefined_pd(),\ + (__mmask8)-1)) + +#define _mm512_mask_permutex_pd(W, U, X, M) \ + ((__m512d) __builtin_ia32_permdf512_mask ((__v8df)(__m512d)(X), (int)(M), \ + (__v8df)(__m512d)(W), (__mmask8)(U))) + +#define _mm512_maskz_permutex_pd(U, X, M) \ + ((__m512d) __builtin_ia32_permdf512_mask ((__v8df)(__m512d)(X), (int)(M), \ + (__v8df)(__m512d)_mm512_setzero_pd(),\ + (__mmask8)(U))) + +#define _mm512_permutex_epi64(X, I) \ + ((__m512i) __builtin_ia32_permdi512_mask ((__v8di)(__m512i)(X), \ + (int)(I), \ + (__v8di)(__m512i) \ + (_mm512_undefined_epi32 ()),\ + (__mmask8)(-1))) + +#define _mm512_maskz_permutex_epi64(M, X, I) \ + ((__m512i) __builtin_ia32_permdi512_mask ((__v8di)(__m512i)(X), \ + (int)(I), \ + (__v8di)(__m512i) \ + (_mm512_setzero_si512 ()),\ + (__mmask8)(M))) + +#define _mm512_mask_permutex_epi64(W, M, X, I) \ + ((__m512i) __builtin_ia32_permdi512_mask ((__v8di)(__m512i)(X), \ + (int)(I), \ + (__v8di)(__m512i)(W), \ + (__mmask8)(M))) +#endif + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permutexvar_epi64 (__mmask8 __M, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_permvardi512_mask ((__v8di) __Y, + (__v8di) __X, + (__v8di) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permutexvar_epi64 (__m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_permvardi512_mask ((__v8di) __Y, + (__v8di) __X, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permutexvar_epi64 (__m512i __W, __mmask8 __M, __m512i __X, + __m512i __Y) +{ + return (__m512i) __builtin_ia32_permvardi512_mask ((__v8di) __Y, + (__v8di) __X, + (__v8di) __W, + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permutexvar_epi32 (__mmask16 __M, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_permvarsi512_mask ((__v16si) __Y, + (__v16si) __X, + (__v16si) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permutexvar_epi32 (__m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_permvarsi512_mask ((__v16si) __Y, + (__v16si) __X, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permutexvar_epi32 (__m512i __W, __mmask16 __M, __m512i __X, + __m512i __Y) +{ + return (__m512i) __builtin_ia32_permvarsi512_mask ((__v16si) __Y, + (__v16si) __X, + (__v16si) __W, + __M); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permutexvar_pd (__m512i __X, __m512d __Y) +{ + return (__m512d) __builtin_ia32_permvardf512_mask ((__v8df) __Y, + (__v8di) __X, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permutexvar_pd (__m512d __W, __mmask8 __U, __m512i __X, __m512d __Y) +{ + return (__m512d) __builtin_ia32_permvardf512_mask ((__v8df) __Y, + (__v8di) __X, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permutexvar_pd (__mmask8 __U, __m512i __X, __m512d __Y) +{ + return (__m512d) __builtin_ia32_permvardf512_mask ((__v8df) __Y, + (__v8di) __X, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permutexvar_ps (__m512i __X, __m512 __Y) +{ + return (__m512) __builtin_ia32_permvarsf512_mask ((__v16sf) __Y, + (__v16si) __X, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permutexvar_ps (__m512 __W, __mmask16 __U, __m512i __X, __m512 __Y) +{ + return (__m512) __builtin_ia32_permvarsf512_mask ((__v16sf) __Y, + (__v16si) __X, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permutexvar_ps (__mmask16 __U, __m512i __X, __m512 __Y) +{ + return (__m512) __builtin_ia32_permvarsf512_mask ((__v16sf) __Y, + (__v16si) __X, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_shuffle_ps (__m512 __M, __m512 __V, const int __imm) +{ + return (__m512) __builtin_ia32_shufps512_mask ((__v16sf) __M, + (__v16sf) __V, __imm, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_shuffle_ps (__m512 __W, __mmask16 __U, __m512 __M, + __m512 __V, const int __imm) +{ + return (__m512) __builtin_ia32_shufps512_mask ((__v16sf) __M, + (__v16sf) __V, __imm, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_shuffle_ps (__mmask16 __U, __m512 __M, __m512 __V, const int __imm) +{ + return (__m512) __builtin_ia32_shufps512_mask ((__v16sf) __M, + (__v16sf) __V, __imm, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_shuffle_pd (__m512d __M, __m512d __V, const int __imm) +{ + return (__m512d) __builtin_ia32_shufpd512_mask ((__v8df) __M, + (__v8df) __V, __imm, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_shuffle_pd (__m512d __W, __mmask8 __U, __m512d __M, + __m512d __V, const int __imm) +{ + return (__m512d) __builtin_ia32_shufpd512_mask ((__v8df) __M, + (__v8df) __V, __imm, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_shuffle_pd (__mmask8 __U, __m512d __M, __m512d __V, + const int __imm) +{ + return (__m512d) __builtin_ia32_shufpd512_mask ((__v8df) __M, + (__v8df) __V, __imm, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fixupimm_round_pd (__m512d __A, __m512d __B, __m512i __C, + const int __imm, const int __R) +{ + return (__m512d) __builtin_ia32_fixupimmpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8di) __C, + __imm, + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fixupimm_round_pd (__m512d __A, __mmask8 __U, __m512d __B, + __m512i __C, const int __imm, const int __R) +{ + return (__m512d) __builtin_ia32_fixupimmpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8di) __C, + __imm, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fixupimm_round_pd (__mmask8 __U, __m512d __A, __m512d __B, + __m512i __C, const int __imm, const int __R) +{ + return (__m512d) __builtin_ia32_fixupimmpd512_maskz ((__v8df) __A, + (__v8df) __B, + (__v8di) __C, + __imm, + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fixupimm_round_ps (__m512 __A, __m512 __B, __m512i __C, + const int __imm, const int __R) +{ + return (__m512) __builtin_ia32_fixupimmps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16si) __C, + __imm, + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fixupimm_round_ps (__m512 __A, __mmask16 __U, __m512 __B, + __m512i __C, const int __imm, const int __R) +{ + return (__m512) __builtin_ia32_fixupimmps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16si) __C, + __imm, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fixupimm_round_ps (__mmask16 __U, __m512 __A, __m512 __B, + __m512i __C, const int __imm, const int __R) +{ + return (__m512) __builtin_ia32_fixupimmps512_maskz ((__v16sf) __A, + (__v16sf) __B, + (__v16si) __C, + __imm, + (__mmask16) __U, __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fixupimm_round_sd (__m128d __A, __m128d __B, __m128i __C, + const int __imm, const int __R) +{ + return (__m128d) __builtin_ia32_fixupimmsd_mask ((__v2df) __A, + (__v2df) __B, + (__v2di) __C, __imm, + (__mmask8) -1, __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fixupimm_round_sd (__m128d __A, __mmask8 __U, __m128d __B, + __m128i __C, const int __imm, const int __R) +{ + return (__m128d) __builtin_ia32_fixupimmsd_mask ((__v2df) __A, + (__v2df) __B, + (__v2di) __C, __imm, + (__mmask8) __U, __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fixupimm_round_sd (__mmask8 __U, __m128d __A, __m128d __B, + __m128i __C, const int __imm, const int __R) +{ + return (__m128d) __builtin_ia32_fixupimmsd_maskz ((__v2df) __A, + (__v2df) __B, + (__v2di) __C, + __imm, + (__mmask8) __U, __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fixupimm_round_ss (__m128 __A, __m128 __B, __m128i __C, + const int __imm, const int __R) +{ + return (__m128) __builtin_ia32_fixupimmss_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4si) __C, __imm, + (__mmask8) -1, __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fixupimm_round_ss (__m128 __A, __mmask8 __U, __m128 __B, + __m128i __C, const int __imm, const int __R) +{ + return (__m128) __builtin_ia32_fixupimmss_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4si) __C, __imm, + (__mmask8) __U, __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fixupimm_round_ss (__mmask8 __U, __m128 __A, __m128 __B, + __m128i __C, const int __imm, const int __R) +{ + return (__m128) __builtin_ia32_fixupimmss_maskz ((__v4sf) __A, + (__v4sf) __B, + (__v4si) __C, __imm, + (__mmask8) __U, __R); +} + +#else +#define _mm512_shuffle_pd(X, Y, C) \ + ((__m512d)__builtin_ia32_shufpd512_mask ((__v8df)(__m512d)(X), \ + (__v8df)(__m512d)(Y), (int)(C),\ + (__v8df)(__m512d)_mm512_undefined_pd(),\ + (__mmask8)-1)) + +#define _mm512_mask_shuffle_pd(W, U, X, Y, C) \ + ((__m512d)__builtin_ia32_shufpd512_mask ((__v8df)(__m512d)(X), \ + (__v8df)(__m512d)(Y), (int)(C),\ + (__v8df)(__m512d)(W),\ + (__mmask8)(U))) + +#define _mm512_maskz_shuffle_pd(U, X, Y, C) \ + ((__m512d)__builtin_ia32_shufpd512_mask ((__v8df)(__m512d)(X), \ + (__v8df)(__m512d)(Y), (int)(C),\ + (__v8df)(__m512d)_mm512_setzero_pd(),\ + (__mmask8)(U))) + +#define _mm512_shuffle_ps(X, Y, C) \ + ((__m512)__builtin_ia32_shufps512_mask ((__v16sf)(__m512)(X), \ + (__v16sf)(__m512)(Y), (int)(C),\ + (__v16sf)(__m512)_mm512_undefined_ps(),\ + (__mmask16)-1)) + +#define _mm512_mask_shuffle_ps(W, U, X, Y, C) \ + ((__m512)__builtin_ia32_shufps512_mask ((__v16sf)(__m512)(X), \ + (__v16sf)(__m512)(Y), (int)(C),\ + (__v16sf)(__m512)(W),\ + (__mmask16)(U))) + +#define _mm512_maskz_shuffle_ps(U, X, Y, C) \ + ((__m512)__builtin_ia32_shufps512_mask ((__v16sf)(__m512)(X), \ + (__v16sf)(__m512)(Y), (int)(C),\ + (__v16sf)(__m512)_mm512_setzero_ps(),\ + (__mmask16)(U))) + +#define _mm512_fixupimm_round_pd(X, Y, Z, C, R) \ + ((__m512d)__builtin_ia32_fixupimmpd512_mask ((__v8df)(__m512d)(X), \ + (__v8df)(__m512d)(Y), (__v8di)(__m512i)(Z), (int)(C), \ + (__mmask8)(-1), (R))) + +#define _mm512_mask_fixupimm_round_pd(X, U, Y, Z, C, R) \ + ((__m512d)__builtin_ia32_fixupimmpd512_mask ((__v8df)(__m512d)(X), \ + (__v8df)(__m512d)(Y), (__v8di)(__m512i)(Z), (int)(C), \ + (__mmask8)(U), (R))) + +#define _mm512_maskz_fixupimm_round_pd(U, X, Y, Z, C, R) \ + ((__m512d)__builtin_ia32_fixupimmpd512_maskz ((__v8df)(__m512d)(X), \ + (__v8df)(__m512d)(Y), (__v8di)(__m512i)(Z), (int)(C), \ + (__mmask8)(U), (R))) + +#define _mm512_fixupimm_round_ps(X, Y, Z, C, R) \ + ((__m512)__builtin_ia32_fixupimmps512_mask ((__v16sf)(__m512)(X), \ + (__v16sf)(__m512)(Y), (__v16si)(__m512i)(Z), (int)(C), \ + (__mmask16)(-1), (R))) + +#define _mm512_mask_fixupimm_round_ps(X, U, Y, Z, C, R) \ + ((__m512)__builtin_ia32_fixupimmps512_mask ((__v16sf)(__m512)(X), \ + (__v16sf)(__m512)(Y), (__v16si)(__m512i)(Z), (int)(C), \ + (__mmask16)(U), (R))) + +#define _mm512_maskz_fixupimm_round_ps(U, X, Y, Z, C, R) \ + ((__m512)__builtin_ia32_fixupimmps512_maskz ((__v16sf)(__m512)(X), \ + (__v16sf)(__m512)(Y), (__v16si)(__m512i)(Z), (int)(C), \ + (__mmask16)(U), (R))) + +#define _mm_fixupimm_round_sd(X, Y, Z, C, R) \ + ((__m128d)__builtin_ia32_fixupimmsd_mask ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (__v2di)(__m128i)(Z), (int)(C), \ + (__mmask8)(-1), (R))) + +#define _mm_mask_fixupimm_round_sd(X, U, Y, Z, C, R) \ + ((__m128d)__builtin_ia32_fixupimmsd_mask ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (__v2di)(__m128i)(Z), (int)(C), \ + (__mmask8)(U), (R))) + +#define _mm_maskz_fixupimm_round_sd(U, X, Y, Z, C, R) \ + ((__m128d)__builtin_ia32_fixupimmsd_maskz ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (__v2di)(__m128i)(Z), (int)(C), \ + (__mmask8)(U), (R))) + +#define _mm_fixupimm_round_ss(X, Y, Z, C, R) \ + ((__m128)__builtin_ia32_fixupimmss_mask ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (__v4si)(__m128i)(Z), (int)(C), \ + (__mmask8)(-1), (R))) + +#define _mm_mask_fixupimm_round_ss(X, U, Y, Z, C, R) \ + ((__m128)__builtin_ia32_fixupimmss_mask ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (__v4si)(__m128i)(Z), (int)(C), \ + (__mmask8)(U), (R))) + +#define _mm_maskz_fixupimm_round_ss(U, X, Y, Z, C, R) \ + ((__m128)__builtin_ia32_fixupimmss_maskz ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (__v4si)(__m128i)(Z), (int)(C), \ + (__mmask8)(U), (R))) +#endif + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_movehdup_ps (__m512 __A) +{ + return (__m512) __builtin_ia32_movshdup512_mask ((__v16sf) __A, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_movehdup_ps (__m512 __W, __mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_movshdup512_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_movehdup_ps (__mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_movshdup512_mask ((__v16sf) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_moveldup_ps (__m512 __A) +{ + return (__m512) __builtin_ia32_movsldup512_mask ((__v16sf) __A, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_moveldup_ps (__m512 __W, __mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_movsldup512_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_moveldup_ps (__mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_movsldup512_mask ((__v16sf) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_or_si512 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v16su) __A | (__v16su) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_or_epi32 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v16su) __A | (__v16su) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_or_epi32 (__m512i __W, __mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pord512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_or_epi32 (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pord512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_or_epi64 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v8du) __A | (__v8du) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_or_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_porq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_or_epi64 (__mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_porq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_xor_si512 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v16su) __A ^ (__v16su) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_xor_epi32 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v16su) __A ^ (__v16su) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_xor_epi32 (__m512i __W, __mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pxord512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_xor_epi32 (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pxord512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_xor_epi64 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v8du) __A ^ (__v8du) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_xor_epi64 (__m512i __W, __mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pxorq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_xor_epi64 (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pxorq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_rol_epi32 (__m512i __A, const int __B) +{ + return (__m512i) __builtin_ia32_prold512_mask ((__v16si) __A, __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_rol_epi32 (__m512i __W, __mmask16 __U, __m512i __A, const int __B) +{ + return (__m512i) __builtin_ia32_prold512_mask ((__v16si) __A, __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_rol_epi32 (__mmask16 __U, __m512i __A, const int __B) +{ + return (__m512i) __builtin_ia32_prold512_mask ((__v16si) __A, __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_ror_epi32 (__m512i __A, int __B) +{ + return (__m512i) __builtin_ia32_prord512_mask ((__v16si) __A, __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_ror_epi32 (__m512i __W, __mmask16 __U, __m512i __A, int __B) +{ + return (__m512i) __builtin_ia32_prord512_mask ((__v16si) __A, __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_ror_epi32 (__mmask16 __U, __m512i __A, int __B) +{ + return (__m512i) __builtin_ia32_prord512_mask ((__v16si) __A, __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_rol_epi64 (__m512i __A, const int __B) +{ + return (__m512i) __builtin_ia32_prolq512_mask ((__v8di) __A, __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_rol_epi64 (__m512i __W, __mmask8 __U, __m512i __A, const int __B) +{ + return (__m512i) __builtin_ia32_prolq512_mask ((__v8di) __A, __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_rol_epi64 (__mmask8 __U, __m512i __A, const int __B) +{ + return (__m512i) __builtin_ia32_prolq512_mask ((__v8di) __A, __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_ror_epi64 (__m512i __A, int __B) +{ + return (__m512i) __builtin_ia32_prorq512_mask ((__v8di) __A, __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_ror_epi64 (__m512i __W, __mmask8 __U, __m512i __A, int __B) +{ + return (__m512i) __builtin_ia32_prorq512_mask ((__v8di) __A, __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_ror_epi64 (__mmask8 __U, __m512i __A, int __B) +{ + return (__m512i) __builtin_ia32_prorq512_mask ((__v8di) __A, __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +#else +#define _mm512_rol_epi32(A, B) \ + ((__m512i)__builtin_ia32_prold512_mask ((__v16si)(__m512i)(A), \ + (int)(B), \ + (__v16si)_mm512_undefined_epi32 (), \ + (__mmask16)(-1))) +#define _mm512_mask_rol_epi32(W, U, A, B) \ + ((__m512i)__builtin_ia32_prold512_mask ((__v16si)(__m512i)(A), \ + (int)(B), \ + (__v16si)(__m512i)(W), \ + (__mmask16)(U))) +#define _mm512_maskz_rol_epi32(U, A, B) \ + ((__m512i)__builtin_ia32_prold512_mask ((__v16si)(__m512i)(A), \ + (int)(B), \ + (__v16si)_mm512_setzero_si512 (), \ + (__mmask16)(U))) +#define _mm512_ror_epi32(A, B) \ + ((__m512i)__builtin_ia32_prord512_mask ((__v16si)(__m512i)(A), \ + (int)(B), \ + (__v16si)_mm512_undefined_epi32 (), \ + (__mmask16)(-1))) +#define _mm512_mask_ror_epi32(W, U, A, B) \ + ((__m512i)__builtin_ia32_prord512_mask ((__v16si)(__m512i)(A), \ + (int)(B), \ + (__v16si)(__m512i)(W), \ + (__mmask16)(U))) +#define _mm512_maskz_ror_epi32(U, A, B) \ + ((__m512i)__builtin_ia32_prord512_mask ((__v16si)(__m512i)(A), \ + (int)(B), \ + (__v16si)_mm512_setzero_si512 (), \ + (__mmask16)(U))) +#define _mm512_rol_epi64(A, B) \ + ((__m512i)__builtin_ia32_prolq512_mask ((__v8di)(__m512i)(A), \ + (int)(B), \ + (__v8di)_mm512_undefined_epi32 (), \ + (__mmask8)(-1))) +#define _mm512_mask_rol_epi64(W, U, A, B) \ + ((__m512i)__builtin_ia32_prolq512_mask ((__v8di)(__m512i)(A), \ + (int)(B), \ + (__v8di)(__m512i)(W), \ + (__mmask8)(U))) +#define _mm512_maskz_rol_epi64(U, A, B) \ + ((__m512i)__builtin_ia32_prolq512_mask ((__v8di)(__m512i)(A), \ + (int)(B), \ + (__v8di)_mm512_setzero_si512 (), \ + (__mmask8)(U))) + +#define _mm512_ror_epi64(A, B) \ + ((__m512i)__builtin_ia32_prorq512_mask ((__v8di)(__m512i)(A), \ + (int)(B), \ + (__v8di)_mm512_undefined_epi32 (), \ + (__mmask8)(-1))) +#define _mm512_mask_ror_epi64(W, U, A, B) \ + ((__m512i)__builtin_ia32_prorq512_mask ((__v8di)(__m512i)(A), \ + (int)(B), \ + (__v8di)(__m512i)(W), \ + (__mmask8)(U))) +#define _mm512_maskz_ror_epi64(U, A, B) \ + ((__m512i)__builtin_ia32_prorq512_mask ((__v8di)(__m512i)(A), \ + (int)(B), \ + (__v8di)_mm512_setzero_si512 (), \ + (__mmask8)(U))) +#endif + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_and_si512 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v16su) __A & (__v16su) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_and_epi32 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v16su) __A & (__v16su) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_and_epi32 (__m512i __W, __mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pandd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_and_epi32 (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pandd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_and_epi64 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v8du) __A & (__v8du) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_and_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pandq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __W, __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_and_epi64 (__mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pandq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_setzero_pd (), + __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_andnot_si512 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pandnd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_andnot_epi32 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pandnd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_andnot_epi32 (__m512i __W, __mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pandnd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_andnot_epi32 (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pandnd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_andnot_epi64 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pandnq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_andnot_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pandnq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __W, __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_andnot_epi64 (__mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pandnq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_setzero_pd (), + __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_test_epi32_mask (__m512i __A, __m512i __B) +{ + return (__mmask16) __builtin_ia32_ptestmd512 ((__v16si) __A, + (__v16si) __B, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_test_epi32_mask (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__mmask16) __builtin_ia32_ptestmd512 ((__v16si) __A, + (__v16si) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_test_epi64_mask (__m512i __A, __m512i __B) +{ + return (__mmask8) __builtin_ia32_ptestmq512 ((__v8di) __A, + (__v8di) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_test_epi64_mask (__mmask8 __U, __m512i __A, __m512i __B) +{ + return (__mmask8) __builtin_ia32_ptestmq512 ((__v8di) __A, (__v8di) __B, __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_testn_epi32_mask (__m512i __A, __m512i __B) +{ + return (__mmask16) __builtin_ia32_ptestnmd512 ((__v16si) __A, + (__v16si) __B, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_testn_epi32_mask (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__mmask16) __builtin_ia32_ptestnmd512 ((__v16si) __A, + (__v16si) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_testn_epi64_mask (__m512i __A, __m512i __B) +{ + return (__mmask8) __builtin_ia32_ptestnmq512 ((__v8di) __A, + (__v8di) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_testn_epi64_mask (__mmask8 __U, __m512i __A, __m512i __B) +{ + return (__mmask8) __builtin_ia32_ptestnmq512 ((__v8di) __A, + (__v8di) __B, __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_abs_ps (__m512 __A) +{ + return (__m512) _mm512_and_epi32 ((__m512i) __A, + _mm512_set1_epi32 (0x7fffffff)); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_abs_ps (__m512 __W, __mmask16 __U, __m512 __A) +{ + return (__m512) _mm512_mask_and_epi32 ((__m512i) __W, __U, (__m512i) __A, + _mm512_set1_epi32 (0x7fffffff)); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_abs_pd (__m512 __A) +{ + return (__m512d) _mm512_and_epi64 ((__m512i) __A, + _mm512_set1_epi64 (0x7fffffffffffffffLL)); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_abs_pd (__m512d __W, __mmask8 __U, __m512 __A) +{ + return (__m512d) + _mm512_mask_and_epi64 ((__m512i) __W, __U, (__m512i) __A, + _mm512_set1_epi64 (0x7fffffffffffffffLL)); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_unpackhi_epi32 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpckhdq512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_unpackhi_epi32 (__m512i __W, __mmask16 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_punpckhdq512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_unpackhi_epi32 (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpckhdq512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_unpackhi_epi64 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpckhqdq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_unpackhi_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpckhqdq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_unpackhi_epi64 (__mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpckhqdq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_unpacklo_epi32 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpckldq512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_unpacklo_epi32 (__m512i __W, __mmask16 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_punpckldq512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_unpacklo_epi32 (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpckldq512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_unpacklo_epi64 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpcklqdq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_unpacklo_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpcklqdq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_unpacklo_epi64 (__mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpcklqdq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +#ifdef __x86_64__ +#ifdef __OPTIMIZE__ +extern __inline unsigned long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundss_u64 (__m128 __A, const int __R) +{ + return (unsigned long long) __builtin_ia32_vcvtss2usi64 ((__v4sf) __A, __R); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundss_si64 (__m128 __A, const int __R) +{ + return (long long) __builtin_ia32_vcvtss2si64 ((__v4sf) __A, __R); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundss_i64 (__m128 __A, const int __R) +{ + return (long long) __builtin_ia32_vcvtss2si64 ((__v4sf) __A, __R); +} + +extern __inline unsigned long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtt_roundss_u64 (__m128 __A, const int __R) +{ + return (unsigned long long) __builtin_ia32_vcvttss2usi64 ((__v4sf) __A, __R); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtt_roundss_i64 (__m128 __A, const int __R) +{ + return (long long) __builtin_ia32_vcvttss2si64 ((__v4sf) __A, __R); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtt_roundss_si64 (__m128 __A, const int __R) +{ + return (long long) __builtin_ia32_vcvttss2si64 ((__v4sf) __A, __R); +} +#else +#define _mm_cvt_roundss_u64(A, B) \ + ((unsigned long long)__builtin_ia32_vcvtss2usi64(A, B)) + +#define _mm_cvt_roundss_si64(A, B) \ + ((long long)__builtin_ia32_vcvtss2si64(A, B)) + +#define _mm_cvt_roundss_i64(A, B) \ + ((long long)__builtin_ia32_vcvtss2si64(A, B)) + +#define _mm_cvtt_roundss_u64(A, B) \ + ((unsigned long long)__builtin_ia32_vcvttss2usi64(A, B)) + +#define _mm_cvtt_roundss_i64(A, B) \ + ((long long)__builtin_ia32_vcvttss2si64(A, B)) + +#define _mm_cvtt_roundss_si64(A, B) \ + ((long long)__builtin_ia32_vcvttss2si64(A, B)) +#endif +#endif + +#ifdef __OPTIMIZE__ +extern __inline unsigned +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundss_u32 (__m128 __A, const int __R) +{ + return (unsigned) __builtin_ia32_vcvtss2usi32 ((__v4sf) __A, __R); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundss_si32 (__m128 __A, const int __R) +{ + return (int) __builtin_ia32_vcvtss2si32 ((__v4sf) __A, __R); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundss_i32 (__m128 __A, const int __R) +{ + return (int) __builtin_ia32_vcvtss2si32 ((__v4sf) __A, __R); +} + +extern __inline unsigned +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtt_roundss_u32 (__m128 __A, const int __R) +{ + return (unsigned) __builtin_ia32_vcvttss2usi32 ((__v4sf) __A, __R); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtt_roundss_i32 (__m128 __A, const int __R) +{ + return (int) __builtin_ia32_vcvttss2si32 ((__v4sf) __A, __R); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtt_roundss_si32 (__m128 __A, const int __R) +{ + return (int) __builtin_ia32_vcvttss2si32 ((__v4sf) __A, __R); +} +#else +#define _mm_cvt_roundss_u32(A, B) \ + ((unsigned)__builtin_ia32_vcvtss2usi32(A, B)) + +#define _mm_cvt_roundss_si32(A, B) \ + ((int)__builtin_ia32_vcvtss2si32(A, B)) + +#define _mm_cvt_roundss_i32(A, B) \ + ((int)__builtin_ia32_vcvtss2si32(A, B)) + +#define _mm_cvtt_roundss_u32(A, B) \ + ((unsigned)__builtin_ia32_vcvttss2usi32(A, B)) + +#define _mm_cvtt_roundss_si32(A, B) \ + ((int)__builtin_ia32_vcvttss2si32(A, B)) + +#define _mm_cvtt_roundss_i32(A, B) \ + ((int)__builtin_ia32_vcvttss2si32(A, B)) +#endif + +#ifdef __x86_64__ +#ifdef __OPTIMIZE__ +extern __inline unsigned long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundsd_u64 (__m128d __A, const int __R) +{ + return (unsigned long long) __builtin_ia32_vcvtsd2usi64 ((__v2df) __A, __R); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundsd_si64 (__m128d __A, const int __R) +{ + return (long long) __builtin_ia32_vcvtsd2si64 ((__v2df) __A, __R); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundsd_i64 (__m128d __A, const int __R) +{ + return (long long) __builtin_ia32_vcvtsd2si64 ((__v2df) __A, __R); +} + +extern __inline unsigned long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtt_roundsd_u64 (__m128d __A, const int __R) +{ + return (unsigned long long) __builtin_ia32_vcvttsd2usi64 ((__v2df) __A, __R); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtt_roundsd_si64 (__m128d __A, const int __R) +{ + return (long long) __builtin_ia32_vcvttsd2si64 ((__v2df) __A, __R); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtt_roundsd_i64 (__m128d __A, const int __R) +{ + return (long long) __builtin_ia32_vcvttsd2si64 ((__v2df) __A, __R); +} +#else +#define _mm_cvt_roundsd_u64(A, B) \ + ((unsigned long long)__builtin_ia32_vcvtsd2usi64(A, B)) + +#define _mm_cvt_roundsd_si64(A, B) \ + ((long long)__builtin_ia32_vcvtsd2si64(A, B)) + +#define _mm_cvt_roundsd_i64(A, B) \ + ((long long)__builtin_ia32_vcvtsd2si64(A, B)) + +#define _mm_cvtt_roundsd_u64(A, B) \ + ((unsigned long long)__builtin_ia32_vcvttsd2usi64(A, B)) + +#define _mm_cvtt_roundsd_si64(A, B) \ + ((long long)__builtin_ia32_vcvttsd2si64(A, B)) + +#define _mm_cvtt_roundsd_i64(A, B) \ + ((long long)__builtin_ia32_vcvttsd2si64(A, B)) +#endif +#endif + +#ifdef __OPTIMIZE__ +extern __inline unsigned +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundsd_u32 (__m128d __A, const int __R) +{ + return (unsigned) __builtin_ia32_vcvtsd2usi32 ((__v2df) __A, __R); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundsd_si32 (__m128d __A, const int __R) +{ + return (int) __builtin_ia32_vcvtsd2si32 ((__v2df) __A, __R); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundsd_i32 (__m128d __A, const int __R) +{ + return (int) __builtin_ia32_vcvtsd2si32 ((__v2df) __A, __R); +} + +extern __inline unsigned +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtt_roundsd_u32 (__m128d __A, const int __R) +{ + return (unsigned) __builtin_ia32_vcvttsd2usi32 ((__v2df) __A, __R); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtt_roundsd_i32 (__m128d __A, const int __R) +{ + return (int) __builtin_ia32_vcvttsd2si32 ((__v2df) __A, __R); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtt_roundsd_si32 (__m128d __A, const int __R) +{ + return (int) __builtin_ia32_vcvttsd2si32 ((__v2df) __A, __R); +} +#else +#define _mm_cvt_roundsd_u32(A, B) \ + ((unsigned)__builtin_ia32_vcvtsd2usi32(A, B)) + +#define _mm_cvt_roundsd_si32(A, B) \ + ((int)__builtin_ia32_vcvtsd2si32(A, B)) + +#define _mm_cvt_roundsd_i32(A, B) \ + ((int)__builtin_ia32_vcvtsd2si32(A, B)) + +#define _mm_cvtt_roundsd_u32(A, B) \ + ((unsigned)__builtin_ia32_vcvttsd2usi32(A, B)) + +#define _mm_cvtt_roundsd_si32(A, B) \ + ((int)__builtin_ia32_vcvttsd2si32(A, B)) + +#define _mm_cvtt_roundsd_i32(A, B) \ + ((int)__builtin_ia32_vcvttsd2si32(A, B)) +#endif + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_movedup_pd (__m512d __A) +{ + return (__m512d) __builtin_ia32_movddup512_mask ((__v8df) __A, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_movedup_pd (__m512d __W, __mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_movddup512_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_movedup_pd (__mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_movddup512_mask ((__v8df) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_unpacklo_pd (__m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_unpcklpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_unpacklo_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_unpcklpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_unpacklo_pd (__mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_unpcklpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_unpackhi_pd (__m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_unpckhpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_unpackhi_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_unpckhpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_unpackhi_pd (__mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_unpckhpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_unpackhi_ps (__m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_unpckhps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_unpackhi_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_unpckhps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_unpackhi_ps (__mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_unpckhps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundps_pd (__m256 __A, const int __R) +{ + return (__m512d) __builtin_ia32_cvtps2pd512_mask ((__v8sf) __A, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundps_pd (__m512d __W, __mmask8 __U, __m256 __A, + const int __R) +{ + return (__m512d) __builtin_ia32_cvtps2pd512_mask ((__v8sf) __A, + (__v8df) __W, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundps_pd (__mmask8 __U, __m256 __A, const int __R) +{ + return (__m512d) __builtin_ia32_cvtps2pd512_mask ((__v8sf) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundph_ps (__m256i __A, const int __R) +{ + return (__m512) __builtin_ia32_vcvtph2ps512_mask ((__v16hi) __A, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundph_ps (__m512 __W, __mmask16 __U, __m256i __A, + const int __R) +{ + return (__m512) __builtin_ia32_vcvtph2ps512_mask ((__v16hi) __A, + (__v16sf) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundph_ps (__mmask16 __U, __m256i __A, const int __R) +{ + return (__m512) __builtin_ia32_vcvtph2ps512_mask ((__v16hi) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, __R); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundps_ph (__m512 __A, const int __I) +{ + return (__m256i) __builtin_ia32_vcvtps2ph512_mask ((__v16sf) __A, + __I, + (__v16hi) + _mm256_undefined_si256 (), + -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtps_ph (__m512 __A, const int __I) +{ + return (__m256i) __builtin_ia32_vcvtps2ph512_mask ((__v16sf) __A, + __I, + (__v16hi) + _mm256_undefined_si256 (), + -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundps_ph (__m256i __U, __mmask16 __W, __m512 __A, + const int __I) +{ + return (__m256i) __builtin_ia32_vcvtps2ph512_mask ((__v16sf) __A, + __I, + (__v16hi) __U, + (__mmask16) __W); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtps_ph (__m256i __U, __mmask16 __W, __m512 __A, const int __I) +{ + return (__m256i) __builtin_ia32_vcvtps2ph512_mask ((__v16sf) __A, + __I, + (__v16hi) __U, + (__mmask16) __W); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundps_ph (__mmask16 __W, __m512 __A, const int __I) +{ + return (__m256i) __builtin_ia32_vcvtps2ph512_mask ((__v16sf) __A, + __I, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __W); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtps_ph (__mmask16 __W, __m512 __A, const int __I) +{ + return (__m256i) __builtin_ia32_vcvtps2ph512_mask ((__v16sf) __A, + __I, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __W); +} +#else +#define _mm512_cvt_roundps_pd(A, B) \ + (__m512d)__builtin_ia32_cvtps2pd512_mask(A, (__v8df)_mm512_undefined_pd(), -1, B) + +#define _mm512_mask_cvt_roundps_pd(W, U, A, B) \ + (__m512d)__builtin_ia32_cvtps2pd512_mask(A, (__v8df)(W), U, B) + +#define _mm512_maskz_cvt_roundps_pd(U, A, B) \ + (__m512d)__builtin_ia32_cvtps2pd512_mask(A, (__v8df)_mm512_setzero_pd(), U, B) + +#define _mm512_cvt_roundph_ps(A, B) \ + (__m512)__builtin_ia32_vcvtph2ps512_mask((__v16hi)(A), (__v16sf)_mm512_undefined_ps(), -1, B) + +#define _mm512_mask_cvt_roundph_ps(W, U, A, B) \ + (__m512)__builtin_ia32_vcvtph2ps512_mask((__v16hi)(A), (__v16sf)(W), U, B) + +#define _mm512_maskz_cvt_roundph_ps(U, A, B) \ + (__m512)__builtin_ia32_vcvtph2ps512_mask((__v16hi)(A), (__v16sf)_mm512_setzero_ps(), U, B) + +#define _mm512_cvt_roundps_ph(A, I) \ + ((__m256i) __builtin_ia32_vcvtps2ph512_mask ((__v16sf)(__m512) A, (int) (I),\ + (__v16hi)_mm256_undefined_si256 (), -1)) +#define _mm512_cvtps_ph(A, I) \ + ((__m256i) __builtin_ia32_vcvtps2ph512_mask ((__v16sf)(__m512) A, (int) (I),\ + (__v16hi)_mm256_undefined_si256 (), -1)) +#define _mm512_mask_cvt_roundps_ph(U, W, A, I) \ + ((__m256i) __builtin_ia32_vcvtps2ph512_mask ((__v16sf)(__m512) A, (int) (I),\ + (__v16hi)(__m256i)(U), (__mmask16) (W))) +#define _mm512_mask_cvtps_ph(U, W, A, I) \ + ((__m256i) __builtin_ia32_vcvtps2ph512_mask ((__v16sf)(__m512) A, (int) (I),\ + (__v16hi)(__m256i)(U), (__mmask16) (W))) +#define _mm512_maskz_cvt_roundps_ph(W, A, I) \ + ((__m256i) __builtin_ia32_vcvtps2ph512_mask ((__v16sf)(__m512) A, (int) (I),\ + (__v16hi)_mm256_setzero_si256 (), (__mmask16) (W))) +#define _mm512_maskz_cvtps_ph(W, A, I) \ + ((__m256i) __builtin_ia32_vcvtps2ph512_mask ((__v16sf)(__m512) A, (int) (I),\ + (__v16hi)_mm256_setzero_si256 (), (__mmask16) (W))) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundpd_ps (__m512d __A, const int __R) +{ + return (__m256) __builtin_ia32_cvtpd2ps512_mask ((__v8df) __A, + (__v8sf) + _mm256_undefined_ps (), + (__mmask8) -1, __R); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundpd_ps (__m256 __W, __mmask8 __U, __m512d __A, + const int __R) +{ + return (__m256) __builtin_ia32_cvtpd2ps512_mask ((__v8df) __A, + (__v8sf) __W, + (__mmask8) __U, __R); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundpd_ps (__mmask8 __U, __m512d __A, const int __R) +{ + return (__m256) __builtin_ia32_cvtpd2ps512_mask ((__v8df) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U, __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundsd_ss (__m128 __A, __m128d __B, const int __R) +{ + return (__m128) __builtin_ia32_cvtsd2ss_round ((__v4sf) __A, + (__v2df) __B, + __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundss_sd (__m128d __A, __m128 __B, const int __R) +{ + return (__m128d) __builtin_ia32_cvtss2sd_round ((__v2df) __A, + (__v4sf) __B, + __R); +} +#else +#define _mm512_cvt_roundpd_ps(A, B) \ + (__m256)__builtin_ia32_cvtpd2ps512_mask(A, (__v8sf)_mm256_undefined_ps(), -1, B) + +#define _mm512_mask_cvt_roundpd_ps(W, U, A, B) \ + (__m256)__builtin_ia32_cvtpd2ps512_mask(A, (__v8sf)(W), U, B) + +#define _mm512_maskz_cvt_roundpd_ps(U, A, B) \ + (__m256)__builtin_ia32_cvtpd2ps512_mask(A, (__v8sf)_mm256_setzero_ps(), U, B) + +#define _mm_cvt_roundsd_ss(A, B, C) \ + (__m128)__builtin_ia32_cvtsd2ss_round(A, B, C) + +#define _mm_cvt_roundss_sd(A, B, C) \ + (__m128d)__builtin_ia32_cvtss2sd_round(A, B, C) +#endif + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_stream_si512 (__m512i * __P, __m512i __A) +{ + __builtin_ia32_movntdq512 ((__v8di *) __P, (__v8di) __A); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_stream_ps (float *__P, __m512 __A) +{ + __builtin_ia32_movntps512 (__P, (__v16sf) __A); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_stream_pd (double *__P, __m512d __A) +{ + __builtin_ia32_movntpd512 (__P, (__v8df) __A); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_stream_load_si512 (void *__P) +{ + return __builtin_ia32_movntdqa512 ((__v8di *)__P); +} + +/* Constants for mantissa extraction */ +typedef enum +{ + _MM_MANT_NORM_1_2, /* interval [1, 2) */ + _MM_MANT_NORM_p5_2, /* interval [0.5, 2) */ + _MM_MANT_NORM_p5_1, /* interval [0.5, 1) */ + _MM_MANT_NORM_p75_1p5 /* interval [0.75, 1.5) */ +} _MM_MANTISSA_NORM_ENUM; + +typedef enum +{ + _MM_MANT_SIGN_src, /* sign = sign(SRC) */ + _MM_MANT_SIGN_zero, /* sign = 0 */ + _MM_MANT_SIGN_nan /* DEST = NaN if sign(SRC) = 1 */ +} _MM_MANTISSA_SIGN_ENUM; + +#ifdef __OPTIMIZE__ +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_getexp_round_ss (__m128 __A, __m128 __B, const int __R) +{ + return (__m128) __builtin_ia32_getexpss128_round ((__v4sf) __A, + (__v4sf) __B, + __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_getexp_round_sd (__m128d __A, __m128d __B, const int __R) +{ + return (__m128d) __builtin_ia32_getexpsd128_round ((__v2df) __A, + (__v2df) __B, + __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_getexp_round_ps (__m512 __A, const int __R) +{ + return (__m512) __builtin_ia32_getexpps512_mask ((__v16sf) __A, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_getexp_round_ps (__m512 __W, __mmask16 __U, __m512 __A, + const int __R) +{ + return (__m512) __builtin_ia32_getexpps512_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_getexp_round_ps (__mmask16 __U, __m512 __A, const int __R) +{ + return (__m512) __builtin_ia32_getexpps512_mask ((__v16sf) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_getexp_round_pd (__m512d __A, const int __R) +{ + return (__m512d) __builtin_ia32_getexppd512_mask ((__v8df) __A, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_getexp_round_pd (__m512d __W, __mmask8 __U, __m512d __A, + const int __R) +{ + return (__m512d) __builtin_ia32_getexppd512_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_getexp_round_pd (__mmask8 __U, __m512d __A, const int __R) +{ + return (__m512d) __builtin_ia32_getexppd512_mask ((__v8df) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_getmant_round_pd (__m512d __A, _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C, const int __R) +{ + return (__m512d) __builtin_ia32_getmantpd512_mask ((__v8df) __A, + (__C << 2) | __B, + _mm512_undefined_pd (), + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_getmant_round_pd (__m512d __W, __mmask8 __U, __m512d __A, + _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C, const int __R) +{ + return (__m512d) __builtin_ia32_getmantpd512_mask ((__v8df) __A, + (__C << 2) | __B, + (__v8df) __W, __U, + __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_getmant_round_pd (__mmask8 __U, __m512d __A, + _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C, const int __R) +{ + return (__m512d) __builtin_ia32_getmantpd512_mask ((__v8df) __A, + (__C << 2) | __B, + (__v8df) + _mm512_setzero_pd (), + __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_getmant_round_ps (__m512 __A, _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C, const int __R) +{ + return (__m512) __builtin_ia32_getmantps512_mask ((__v16sf) __A, + (__C << 2) | __B, + _mm512_undefined_ps (), + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_getmant_round_ps (__m512 __W, __mmask16 __U, __m512 __A, + _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C, const int __R) +{ + return (__m512) __builtin_ia32_getmantps512_mask ((__v16sf) __A, + (__C << 2) | __B, + (__v16sf) __W, __U, + __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_getmant_round_ps (__mmask16 __U, __m512 __A, + _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C, const int __R) +{ + return (__m512) __builtin_ia32_getmantps512_mask ((__v16sf) __A, + (__C << 2) | __B, + (__v16sf) + _mm512_setzero_ps (), + __U, __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_getmant_round_sd (__m128d __A, __m128d __B, + _MM_MANTISSA_NORM_ENUM __C, + _MM_MANTISSA_SIGN_ENUM __D, const int __R) +{ + return (__m128d) __builtin_ia32_getmantsd_round ((__v2df) __A, + (__v2df) __B, + (__D << 2) | __C, + __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_getmant_round_ss (__m128 __A, __m128 __B, + _MM_MANTISSA_NORM_ENUM __C, + _MM_MANTISSA_SIGN_ENUM __D, const int __R) +{ + return (__m128) __builtin_ia32_getmantss_round ((__v4sf) __A, + (__v4sf) __B, + (__D << 2) | __C, + __R); +} + +#else +#define _mm512_getmant_round_pd(X, B, C, R) \ + ((__m512d)__builtin_ia32_getmantpd512_mask ((__v8df)(__m512d)(X), \ + (int)(((C)<<2) | (B)), \ + (__v8df)(__m512d)_mm512_undefined_pd(), \ + (__mmask8)-1,\ + (R))) + +#define _mm512_mask_getmant_round_pd(W, U, X, B, C, R) \ + ((__m512d)__builtin_ia32_getmantpd512_mask ((__v8df)(__m512d)(X), \ + (int)(((C)<<2) | (B)), \ + (__v8df)(__m512d)(W), \ + (__mmask8)(U),\ + (R))) + +#define _mm512_maskz_getmant_round_pd(U, X, B, C, R) \ + ((__m512d)__builtin_ia32_getmantpd512_mask ((__v8df)(__m512d)(X), \ + (int)(((C)<<2) | (B)), \ + (__v8df)(__m512d)_mm512_setzero_pd(), \ + (__mmask8)(U),\ + (R))) +#define _mm512_getmant_round_ps(X, B, C, R) \ + ((__m512)__builtin_ia32_getmantps512_mask ((__v16sf)(__m512)(X), \ + (int)(((C)<<2) | (B)), \ + (__v16sf)(__m512)_mm512_undefined_ps(), \ + (__mmask16)-1,\ + (R))) + +#define _mm512_mask_getmant_round_ps(W, U, X, B, C, R) \ + ((__m512)__builtin_ia32_getmantps512_mask ((__v16sf)(__m512)(X), \ + (int)(((C)<<2) | (B)), \ + (__v16sf)(__m512)(W), \ + (__mmask16)(U),\ + (R))) + +#define _mm512_maskz_getmant_round_ps(U, X, B, C, R) \ + ((__m512)__builtin_ia32_getmantps512_mask ((__v16sf)(__m512)(X), \ + (int)(((C)<<2) | (B)), \ + (__v16sf)(__m512)_mm512_setzero_ps(), \ + (__mmask16)(U),\ + (R))) +#define _mm_getmant_round_sd(X, Y, C, D, R) \ + ((__m128d)__builtin_ia32_getmantsd_round ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), \ + (int)(((D)<<2) | (C)), \ + (R))) + +#define _mm_getmant_round_ss(X, Y, C, D, R) \ + ((__m128)__builtin_ia32_getmantss_round ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), \ + (int)(((D)<<2) | (C)), \ + (R))) + +#define _mm_getexp_round_ss(A, B, R) \ + ((__m128)__builtin_ia32_getexpss128_round((__v4sf)(__m128)(A), (__v4sf)(__m128)(B), R)) + +#define _mm_getexp_round_sd(A, B, R) \ + ((__m128d)__builtin_ia32_getexpsd128_round((__v2df)(__m128d)(A), (__v2df)(__m128d)(B), R)) + +#define _mm512_getexp_round_ps(A, R) \ + ((__m512)__builtin_ia32_getexpps512_mask((__v16sf)(__m512)(A), \ + (__v16sf)_mm512_undefined_ps(), (__mmask16)-1, R)) + +#define _mm512_mask_getexp_round_ps(W, U, A, R) \ + ((__m512)__builtin_ia32_getexpps512_mask((__v16sf)(__m512)(A), \ + (__v16sf)(__m512)(W), (__mmask16)(U), R)) + +#define _mm512_maskz_getexp_round_ps(U, A, R) \ + ((__m512)__builtin_ia32_getexpps512_mask((__v16sf)(__m512)(A), \ + (__v16sf)_mm512_setzero_ps(), (__mmask16)(U), R)) + +#define _mm512_getexp_round_pd(A, R) \ + ((__m512d)__builtin_ia32_getexppd512_mask((__v8df)(__m512d)(A), \ + (__v8df)_mm512_undefined_pd(), (__mmask8)-1, R)) + +#define _mm512_mask_getexp_round_pd(W, U, A, R) \ + ((__m512d)__builtin_ia32_getexppd512_mask((__v8df)(__m512d)(A), \ + (__v8df)(__m512d)(W), (__mmask8)(U), R)) + +#define _mm512_maskz_getexp_round_pd(U, A, R) \ + ((__m512d)__builtin_ia32_getexppd512_mask((__v8df)(__m512d)(A), \ + (__v8df)_mm512_setzero_pd(), (__mmask8)(U), R)) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_roundscale_round_ps (__m512 __A, const int __imm, const int __R) +{ + return (__m512) __builtin_ia32_rndscaleps_mask ((__v16sf) __A, __imm, + (__v16sf) + _mm512_undefined_ps (), + -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_roundscale_round_ps (__m512 __A, __mmask16 __B, __m512 __C, + const int __imm, const int __R) +{ + return (__m512) __builtin_ia32_rndscaleps_mask ((__v16sf) __C, __imm, + (__v16sf) __A, + (__mmask16) __B, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_roundscale_round_ps (__mmask16 __A, __m512 __B, + const int __imm, const int __R) +{ + return (__m512) __builtin_ia32_rndscaleps_mask ((__v16sf) __B, + __imm, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __A, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_roundscale_round_pd (__m512d __A, const int __imm, const int __R) +{ + return (__m512d) __builtin_ia32_rndscalepd_mask ((__v8df) __A, __imm, + (__v8df) + _mm512_undefined_pd (), + -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_roundscale_round_pd (__m512d __A, __mmask8 __B, + __m512d __C, const int __imm, const int __R) +{ + return (__m512d) __builtin_ia32_rndscalepd_mask ((__v8df) __C, __imm, + (__v8df) __A, + (__mmask8) __B, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_roundscale_round_pd (__mmask8 __A, __m512d __B, + const int __imm, const int __R) +{ + return (__m512d) __builtin_ia32_rndscalepd_mask ((__v8df) __B, + __imm, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __A, __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_roundscale_round_ss (__m128 __A, __m128 __B, const int __imm, const int __R) +{ + return (__m128) __builtin_ia32_rndscaless_round ((__v4sf) __A, + (__v4sf) __B, __imm, __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_roundscale_round_sd (__m128d __A, __m128d __B, const int __imm, + const int __R) +{ + return (__m128d) __builtin_ia32_rndscalesd_round ((__v2df) __A, + (__v2df) __B, __imm, __R); +} + +#else +#define _mm512_roundscale_round_ps(A, B, R) \ + ((__m512) __builtin_ia32_rndscaleps_mask ((__v16sf)(__m512)(A), (int)(B),\ + (__v16sf)_mm512_undefined_ps(), (__mmask16)(-1), R)) +#define _mm512_mask_roundscale_round_ps(A, B, C, D, R) \ + ((__m512) __builtin_ia32_rndscaleps_mask ((__v16sf)(__m512)(C), \ + (int)(D), \ + (__v16sf)(__m512)(A), \ + (__mmask16)(B), R)) +#define _mm512_maskz_roundscale_round_ps(A, B, C, R) \ + ((__m512) __builtin_ia32_rndscaleps_mask ((__v16sf)(__m512)(B), \ + (int)(C), \ + (__v16sf)_mm512_setzero_ps(),\ + (__mmask16)(A), R)) +#define _mm512_roundscale_round_pd(A, B, R) \ + ((__m512d) __builtin_ia32_rndscalepd_mask ((__v8df)(__m512d)(A), (int)(B),\ + (__v8df)_mm512_undefined_pd(), (__mmask8)(-1), R)) +#define _mm512_mask_roundscale_round_pd(A, B, C, D, R) \ + ((__m512d) __builtin_ia32_rndscalepd_mask ((__v8df)(__m512d)(C), \ + (int)(D), \ + (__v8df)(__m512d)(A), \ + (__mmask8)(B), R)) +#define _mm512_maskz_roundscale_round_pd(A, B, C, R) \ + ((__m512d) __builtin_ia32_rndscalepd_mask ((__v8df)(__m512d)(B), \ + (int)(C), \ + (__v8df)_mm512_setzero_pd(),\ + (__mmask8)(A), R)) +#define _mm_roundscale_round_ss(A, B, C, R) \ + ((__m128) __builtin_ia32_rndscaless_round ((__v4sf)(__m128)(A), \ + (__v4sf)(__m128)(B), (int)(C), R)) +#define _mm_roundscale_round_sd(A, B, C, R) \ + ((__m128d) __builtin_ia32_rndscalesd_round ((__v2df)(__m128d)(A), \ + (__v2df)(__m128d)(B), (int)(C), R)) +#endif + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_floor_ps (__m512 __A) +{ + return (__m512) __builtin_ia32_rndscaleps_mask ((__v16sf) __A, + _MM_FROUND_FLOOR, + (__v16sf) __A, -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_floor_pd (__m512d __A) +{ + return (__m512d) __builtin_ia32_rndscalepd_mask ((__v8df) __A, + _MM_FROUND_FLOOR, + (__v8df) __A, -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_ceil_ps (__m512 __A) +{ + return (__m512) __builtin_ia32_rndscaleps_mask ((__v16sf) __A, + _MM_FROUND_CEIL, + (__v16sf) __A, -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_ceil_pd (__m512d __A) +{ + return (__m512d) __builtin_ia32_rndscalepd_mask ((__v8df) __A, + _MM_FROUND_CEIL, + (__v8df) __A, -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_floor_ps (__m512 __W, __mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_rndscaleps_mask ((__v16sf) __A, + _MM_FROUND_FLOOR, + (__v16sf) __W, __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_floor_pd (__m512d __W, __mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_rndscalepd_mask ((__v8df) __A, + _MM_FROUND_FLOOR, + (__v8df) __W, __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_ceil_ps (__m512 __W, __mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_rndscaleps_mask ((__v16sf) __A, + _MM_FROUND_CEIL, + (__v16sf) __W, __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_ceil_pd (__m512d __W, __mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_rndscalepd_mask ((__v8df) __A, + _MM_FROUND_CEIL, + (__v8df) __W, __U, + _MM_FROUND_CUR_DIRECTION); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_alignr_epi32 (__m512i __A, __m512i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_alignd512_mask ((__v16si) __A, + (__v16si) __B, __imm, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_alignr_epi32 (__m512i __W, __mmask16 __U, __m512i __A, + __m512i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_alignd512_mask ((__v16si) __A, + (__v16si) __B, __imm, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_alignr_epi32 (__mmask16 __U, __m512i __A, __m512i __B, + const int __imm) +{ + return (__m512i) __builtin_ia32_alignd512_mask ((__v16si) __A, + (__v16si) __B, __imm, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_alignr_epi64 (__m512i __A, __m512i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_alignq512_mask ((__v8di) __A, + (__v8di) __B, __imm, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_alignr_epi64 (__m512i __W, __mmask8 __U, __m512i __A, + __m512i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_alignq512_mask ((__v8di) __A, + (__v8di) __B, __imm, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_alignr_epi64 (__mmask8 __U, __m512i __A, __m512i __B, + const int __imm) +{ + return (__m512i) __builtin_ia32_alignq512_mask ((__v8di) __A, + (__v8di) __B, __imm, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} +#else +#define _mm512_alignr_epi32(X, Y, C) \ + ((__m512i)__builtin_ia32_alignd512_mask ((__v16si)(__m512i)(X), \ + (__v16si)(__m512i)(Y), (int)(C), (__v16si)_mm512_undefined_epi32 (),\ + (__mmask16)-1)) + +#define _mm512_mask_alignr_epi32(W, U, X, Y, C) \ + ((__m512i)__builtin_ia32_alignd512_mask ((__v16si)(__m512i)(X), \ + (__v16si)(__m512i)(Y), (int)(C), (__v16si)(__m512i)(W), \ + (__mmask16)(U))) + +#define _mm512_maskz_alignr_epi32(U, X, Y, C) \ + ((__m512i)__builtin_ia32_alignd512_mask ((__v16si)(__m512i)(X), \ + (__v16si)(__m512i)(Y), (int)(C), (__v16si)_mm512_setzero_si512 (),\ + (__mmask16)(U))) + +#define _mm512_alignr_epi64(X, Y, C) \ + ((__m512i)__builtin_ia32_alignq512_mask ((__v8di)(__m512i)(X), \ + (__v8di)(__m512i)(Y), (int)(C), (__v8di)_mm512_undefined_epi32 (), \ + (__mmask8)-1)) + +#define _mm512_mask_alignr_epi64(W, U, X, Y, C) \ + ((__m512i)__builtin_ia32_alignq512_mask ((__v8di)(__m512i)(X), \ + (__v8di)(__m512i)(Y), (int)(C), (__v8di)(__m512i)(W), (__mmask8)(U))) + +#define _mm512_maskz_alignr_epi64(U, X, Y, C) \ + ((__m512i)__builtin_ia32_alignq512_mask ((__v8di)(__m512i)(X), \ + (__v8di)(__m512i)(Y), (int)(C), (__v8di)_mm512_setzero_si512 (),\ + (__mmask8)(U))) +#endif + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpeq_epi32_mask (__m512i __A, __m512i __B) +{ + return (__mmask16) __builtin_ia32_pcmpeqd512_mask ((__v16si) __A, + (__v16si) __B, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpeq_epi32_mask (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__mmask16) __builtin_ia32_pcmpeqd512_mask ((__v16si) __A, + (__v16si) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpeq_epi64_mask (__mmask8 __U, __m512i __A, __m512i __B) +{ + return (__mmask8) __builtin_ia32_pcmpeqq512_mask ((__v8di) __A, + (__v8di) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpeq_epi64_mask (__m512i __A, __m512i __B) +{ + return (__mmask8) __builtin_ia32_pcmpeqq512_mask ((__v8di) __A, + (__v8di) __B, + (__mmask8) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpgt_epi32_mask (__m512i __A, __m512i __B) +{ + return (__mmask16) __builtin_ia32_pcmpgtd512_mask ((__v16si) __A, + (__v16si) __B, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpgt_epi32_mask (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__mmask16) __builtin_ia32_pcmpgtd512_mask ((__v16si) __A, + (__v16si) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpgt_epi64_mask (__mmask8 __U, __m512i __A, __m512i __B) +{ + return (__mmask8) __builtin_ia32_pcmpgtq512_mask ((__v8di) __A, + (__v8di) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpgt_epi64_mask (__m512i __A, __m512i __B) +{ + return (__mmask8) __builtin_ia32_pcmpgtq512_mask ((__v8di) __A, + (__v8di) __B, + (__mmask8) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpge_epi32_mask (__m512i __X, __m512i __Y) +{ + return (__mmask16) __builtin_ia32_cmpd512_mask ((__v16si) __X, + (__v16si) __Y, 5, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpge_epi32_mask (__mmask16 __M, __m512i __X, __m512i __Y) +{ + return (__mmask16) __builtin_ia32_cmpd512_mask ((__v16si) __X, + (__v16si) __Y, 5, + (__mmask16) __M); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpge_epu32_mask (__mmask16 __M, __m512i __X, __m512i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpd512_mask ((__v16si) __X, + (__v16si) __Y, 5, + (__mmask16) __M); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpge_epu32_mask (__m512i __X, __m512i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpd512_mask ((__v16si) __X, + (__v16si) __Y, 5, + (__mmask16) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpge_epi64_mask (__mmask8 __M, __m512i __X, __m512i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq512_mask ((__v8di) __X, + (__v8di) __Y, 5, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpge_epi64_mask (__m512i __X, __m512i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq512_mask ((__v8di) __X, + (__v8di) __Y, 5, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpge_epu64_mask (__mmask8 __M, __m512i __X, __m512i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq512_mask ((__v8di) __X, + (__v8di) __Y, 5, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpge_epu64_mask (__m512i __X, __m512i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq512_mask ((__v8di) __X, + (__v8di) __Y, 5, + (__mmask8) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmple_epi32_mask (__mmask16 __M, __m512i __X, __m512i __Y) +{ + return (__mmask16) __builtin_ia32_cmpd512_mask ((__v16si) __X, + (__v16si) __Y, 2, + (__mmask16) __M); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmple_epi32_mask (__m512i __X, __m512i __Y) +{ + return (__mmask16) __builtin_ia32_cmpd512_mask ((__v16si) __X, + (__v16si) __Y, 2, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmple_epu32_mask (__mmask16 __M, __m512i __X, __m512i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpd512_mask ((__v16si) __X, + (__v16si) __Y, 2, + (__mmask16) __M); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmple_epu32_mask (__m512i __X, __m512i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpd512_mask ((__v16si) __X, + (__v16si) __Y, 2, + (__mmask16) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmple_epi64_mask (__mmask8 __M, __m512i __X, __m512i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq512_mask ((__v8di) __X, + (__v8di) __Y, 2, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmple_epi64_mask (__m512i __X, __m512i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq512_mask ((__v8di) __X, + (__v8di) __Y, 2, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmple_epu64_mask (__mmask8 __M, __m512i __X, __m512i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq512_mask ((__v8di) __X, + (__v8di) __Y, 2, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmple_epu64_mask (__m512i __X, __m512i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq512_mask ((__v8di) __X, + (__v8di) __Y, 2, + (__mmask8) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmplt_epi32_mask (__mmask16 __M, __m512i __X, __m512i __Y) +{ + return (__mmask16) __builtin_ia32_cmpd512_mask ((__v16si) __X, + (__v16si) __Y, 1, + (__mmask16) __M); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmplt_epi32_mask (__m512i __X, __m512i __Y) +{ + return (__mmask16) __builtin_ia32_cmpd512_mask ((__v16si) __X, + (__v16si) __Y, 1, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmplt_epu32_mask (__mmask16 __M, __m512i __X, __m512i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpd512_mask ((__v16si) __X, + (__v16si) __Y, 1, + (__mmask16) __M); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmplt_epu32_mask (__m512i __X, __m512i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpd512_mask ((__v16si) __X, + (__v16si) __Y, 1, + (__mmask16) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmplt_epi64_mask (__mmask8 __M, __m512i __X, __m512i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq512_mask ((__v8di) __X, + (__v8di) __Y, 1, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmplt_epi64_mask (__m512i __X, __m512i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq512_mask ((__v8di) __X, + (__v8di) __Y, 1, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmplt_epu64_mask (__mmask8 __M, __m512i __X, __m512i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq512_mask ((__v8di) __X, + (__v8di) __Y, 1, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmplt_epu64_mask (__m512i __X, __m512i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq512_mask ((__v8di) __X, + (__v8di) __Y, 1, + (__mmask8) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpneq_epi32_mask (__m512i __X, __m512i __Y) +{ + return (__mmask16) __builtin_ia32_cmpd512_mask ((__v16si) __X, + (__v16si) __Y, 4, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpneq_epi32_mask (__mmask16 __M, __m512i __X, __m512i __Y) +{ + return (__mmask16) __builtin_ia32_cmpd512_mask ((__v16si) __X, + (__v16si) __Y, 4, + (__mmask16) __M); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpneq_epu32_mask (__mmask16 __M, __m512i __X, __m512i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpd512_mask ((__v16si) __X, + (__v16si) __Y, 4, + (__mmask16) __M); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpneq_epu32_mask (__m512i __X, __m512i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpd512_mask ((__v16si) __X, + (__v16si) __Y, 4, + (__mmask16) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpneq_epi64_mask (__mmask16 __M, __m512i __X, __m512i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq512_mask ((__v8di) __X, + (__v8di) __Y, 4, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpneq_epi64_mask (__m512i __X, __m512i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq512_mask ((__v8di) __X, + (__v8di) __Y, 4, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpneq_epu64_mask (__mmask8 __M, __m512i __X, __m512i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq512_mask ((__v8di) __X, + (__v8di) __Y, 4, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpneq_epu64_mask (__m512i __X, __m512i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq512_mask ((__v8di) __X, + (__v8di) __Y, 4, + (__mmask8) -1); +} + +#define _MM_CMPINT_EQ 0x0 +#define _MM_CMPINT_LT 0x1 +#define _MM_CMPINT_LE 0x2 +#define _MM_CMPINT_UNUSED 0x3 +#define _MM_CMPINT_NE 0x4 +#define _MM_CMPINT_NLT 0x5 +#define _MM_CMPINT_GE 0x5 +#define _MM_CMPINT_NLE 0x6 +#define _MM_CMPINT_GT 0x6 + +#ifdef __OPTIMIZE__ +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kshiftli_mask16 (__mmask16 __A, unsigned int __B) +{ + return (__mmask16) __builtin_ia32_kshiftlihi ((__mmask16) __A, + (__mmask8) __B); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kshiftri_mask16 (__mmask16 __A, unsigned int __B) +{ + return (__mmask16) __builtin_ia32_kshiftrihi ((__mmask16) __A, + (__mmask8) __B); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmp_epi64_mask (__m512i __X, __m512i __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_cmpq512_mask ((__v8di) __X, + (__v8di) __Y, __P, + (__mmask8) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmp_epi32_mask (__m512i __X, __m512i __Y, const int __P) +{ + return (__mmask16) __builtin_ia32_cmpd512_mask ((__v16si) __X, + (__v16si) __Y, __P, + (__mmask16) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmp_epu64_mask (__m512i __X, __m512i __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_ucmpq512_mask ((__v8di) __X, + (__v8di) __Y, __P, + (__mmask8) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmp_epu32_mask (__m512i __X, __m512i __Y, const int __P) +{ + return (__mmask16) __builtin_ia32_ucmpd512_mask ((__v16si) __X, + (__v16si) __Y, __P, + (__mmask16) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmp_round_pd_mask (__m512d __X, __m512d __Y, const int __P, + const int __R) +{ + return (__mmask8) __builtin_ia32_cmppd512_mask ((__v8df) __X, + (__v8df) __Y, __P, + (__mmask8) -1, __R); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmp_round_ps_mask (__m512 __X, __m512 __Y, const int __P, const int __R) +{ + return (__mmask16) __builtin_ia32_cmpps512_mask ((__v16sf) __X, + (__v16sf) __Y, __P, + (__mmask16) -1, __R); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmp_epi64_mask (__mmask8 __U, __m512i __X, __m512i __Y, + const int __P) +{ + return (__mmask8) __builtin_ia32_cmpq512_mask ((__v8di) __X, + (__v8di) __Y, __P, + (__mmask8) __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmp_epi32_mask (__mmask16 __U, __m512i __X, __m512i __Y, + const int __P) +{ + return (__mmask16) __builtin_ia32_cmpd512_mask ((__v16si) __X, + (__v16si) __Y, __P, + (__mmask16) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmp_epu64_mask (__mmask8 __U, __m512i __X, __m512i __Y, + const int __P) +{ + return (__mmask8) __builtin_ia32_ucmpq512_mask ((__v8di) __X, + (__v8di) __Y, __P, + (__mmask8) __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmp_epu32_mask (__mmask16 __U, __m512i __X, __m512i __Y, + const int __P) +{ + return (__mmask16) __builtin_ia32_ucmpd512_mask ((__v16si) __X, + (__v16si) __Y, __P, + (__mmask16) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmp_round_pd_mask (__mmask8 __U, __m512d __X, __m512d __Y, + const int __P, const int __R) +{ + return (__mmask8) __builtin_ia32_cmppd512_mask ((__v8df) __X, + (__v8df) __Y, __P, + (__mmask8) __U, __R); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmp_round_ps_mask (__mmask16 __U, __m512 __X, __m512 __Y, + const int __P, const int __R) +{ + return (__mmask16) __builtin_ia32_cmpps512_mask ((__v16sf) __X, + (__v16sf) __Y, __P, + (__mmask16) __U, __R); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_round_sd_mask (__m128d __X, __m128d __Y, const int __P, const int __R) +{ + return (__mmask8) __builtin_ia32_cmpsd_mask ((__v2df) __X, + (__v2df) __Y, __P, + (__mmask8) -1, __R); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmp_round_sd_mask (__mmask8 __M, __m128d __X, __m128d __Y, + const int __P, const int __R) +{ + return (__mmask8) __builtin_ia32_cmpsd_mask ((__v2df) __X, + (__v2df) __Y, __P, + (__mmask8) __M, __R); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_round_ss_mask (__m128 __X, __m128 __Y, const int __P, const int __R) +{ + return (__mmask8) __builtin_ia32_cmpss_mask ((__v4sf) __X, + (__v4sf) __Y, __P, + (__mmask8) -1, __R); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmp_round_ss_mask (__mmask8 __M, __m128 __X, __m128 __Y, + const int __P, const int __R) +{ + return (__mmask8) __builtin_ia32_cmpss_mask ((__v4sf) __X, + (__v4sf) __Y, __P, + (__mmask8) __M, __R); +} + +#else +#define _kshiftli_mask16(X, Y) \ + ((__mmask16) __builtin_ia32_kshiftlihi ((__mmask16)(X), (__mmask8)(Y))) + +#define _kshiftri_mask16(X, Y) \ + ((__mmask16) __builtin_ia32_kshiftrihi ((__mmask16)(X), (__mmask8)(Y))) + +#define _mm512_cmp_epi64_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpq512_mask ((__v8di)(__m512i)(X), \ + (__v8di)(__m512i)(Y), (int)(P),\ + (__mmask8)-1)) + +#define _mm512_cmp_epi32_mask(X, Y, P) \ + ((__mmask16) __builtin_ia32_cmpd512_mask ((__v16si)(__m512i)(X), \ + (__v16si)(__m512i)(Y), (int)(P), \ + (__mmask16)-1)) + +#define _mm512_cmp_epu64_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_ucmpq512_mask ((__v8di)(__m512i)(X), \ + (__v8di)(__m512i)(Y), (int)(P),\ + (__mmask8)-1)) + +#define _mm512_cmp_epu32_mask(X, Y, P) \ + ((__mmask16) __builtin_ia32_ucmpd512_mask ((__v16si)(__m512i)(X), \ + (__v16si)(__m512i)(Y), (int)(P), \ + (__mmask16)-1)) + +#define _mm512_cmp_round_pd_mask(X, Y, P, R) \ + ((__mmask8) __builtin_ia32_cmppd512_mask ((__v8df)(__m512d)(X), \ + (__v8df)(__m512d)(Y), (int)(P),\ + (__mmask8)-1, R)) + +#define _mm512_cmp_round_ps_mask(X, Y, P, R) \ + ((__mmask16) __builtin_ia32_cmpps512_mask ((__v16sf)(__m512)(X), \ + (__v16sf)(__m512)(Y), (int)(P),\ + (__mmask16)-1, R)) + +#define _mm512_mask_cmp_epi64_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpq512_mask ((__v8di)(__m512i)(X), \ + (__v8di)(__m512i)(Y), (int)(P),\ + (__mmask8)M)) + +#define _mm512_mask_cmp_epi32_mask(M, X, Y, P) \ + ((__mmask16) __builtin_ia32_cmpd512_mask ((__v16si)(__m512i)(X), \ + (__v16si)(__m512i)(Y), (int)(P), \ + (__mmask16)M)) + +#define _mm512_mask_cmp_epu64_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_ucmpq512_mask ((__v8di)(__m512i)(X), \ + (__v8di)(__m512i)(Y), (int)(P),\ + (__mmask8)M)) + +#define _mm512_mask_cmp_epu32_mask(M, X, Y, P) \ + ((__mmask16) __builtin_ia32_ucmpd512_mask ((__v16si)(__m512i)(X), \ + (__v16si)(__m512i)(Y), (int)(P), \ + (__mmask16)M)) + +#define _mm512_mask_cmp_round_pd_mask(M, X, Y, P, R) \ + ((__mmask8) __builtin_ia32_cmppd512_mask ((__v8df)(__m512d)(X), \ + (__v8df)(__m512d)(Y), (int)(P),\ + (__mmask8)M, R)) + +#define _mm512_mask_cmp_round_ps_mask(M, X, Y, P, R) \ + ((__mmask16) __builtin_ia32_cmpps512_mask ((__v16sf)(__m512)(X), \ + (__v16sf)(__m512)(Y), (int)(P),\ + (__mmask16)M, R)) + +#define _mm_cmp_round_sd_mask(X, Y, P, R) \ + ((__mmask8) __builtin_ia32_cmpsd_mask ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (int)(P),\ + (__mmask8)-1, R)) + +#define _mm_mask_cmp_round_sd_mask(M, X, Y, P, R) \ + ((__mmask8) __builtin_ia32_cmpsd_mask ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (int)(P),\ + (M), R)) + +#define _mm_cmp_round_ss_mask(X, Y, P, R) \ + ((__mmask8) __builtin_ia32_cmpss_mask ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (int)(P), \ + (__mmask8)-1, R)) + +#define _mm_mask_cmp_round_ss_mask(M, X, Y, P, R) \ + ((__mmask8) __builtin_ia32_cmpss_mask ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (int)(P), \ + (M), R)) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_i32gather_ps (__m512i __index, void const *__addr, int __scale) +{ + __m512 __v1_old = _mm512_undefined_ps (); + __mmask16 __mask = 0xFFFF; + + return (__m512) __builtin_ia32_gathersiv16sf ((__v16sf) __v1_old, + __addr, + (__v16si) __index, + __mask, __scale); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_i32gather_ps (__m512 __v1_old, __mmask16 __mask, + __m512i __index, void const *__addr, int __scale) +{ + return (__m512) __builtin_ia32_gathersiv16sf ((__v16sf) __v1_old, + __addr, + (__v16si) __index, + __mask, __scale); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_i32gather_pd (__m256i __index, void const *__addr, int __scale) +{ + __m512d __v1_old = _mm512_undefined_pd (); + __mmask8 __mask = 0xFF; + + return (__m512d) __builtin_ia32_gathersiv8df ((__v8df) __v1_old, + __addr, + (__v8si) __index, __mask, + __scale); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_i32gather_pd (__m512d __v1_old, __mmask8 __mask, + __m256i __index, void const *__addr, int __scale) +{ + return (__m512d) __builtin_ia32_gathersiv8df ((__v8df) __v1_old, + __addr, + (__v8si) __index, + __mask, __scale); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_i64gather_ps (__m512i __index, void const *__addr, int __scale) +{ + __m256 __v1_old = _mm256_undefined_ps (); + __mmask8 __mask = 0xFF; + + return (__m256) __builtin_ia32_gatherdiv16sf ((__v8sf) __v1_old, + __addr, + (__v8di) __index, __mask, + __scale); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_i64gather_ps (__m256 __v1_old, __mmask8 __mask, + __m512i __index, void const *__addr, int __scale) +{ + return (__m256) __builtin_ia32_gatherdiv16sf ((__v8sf) __v1_old, + __addr, + (__v8di) __index, + __mask, __scale); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_i64gather_pd (__m512i __index, void const *__addr, int __scale) +{ + __m512d __v1_old = _mm512_undefined_pd (); + __mmask8 __mask = 0xFF; + + return (__m512d) __builtin_ia32_gatherdiv8df ((__v8df) __v1_old, + __addr, + (__v8di) __index, __mask, + __scale); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_i64gather_pd (__m512d __v1_old, __mmask8 __mask, + __m512i __index, void const *__addr, int __scale) +{ + return (__m512d) __builtin_ia32_gatherdiv8df ((__v8df) __v1_old, + __addr, + (__v8di) __index, + __mask, __scale); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_i32gather_epi32 (__m512i __index, void const *__addr, int __scale) +{ + __m512i __v1_old = _mm512_undefined_epi32 (); + __mmask16 __mask = 0xFFFF; + + return (__m512i) __builtin_ia32_gathersiv16si ((__v16si) __v1_old, + __addr, + (__v16si) __index, + __mask, __scale); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_i32gather_epi32 (__m512i __v1_old, __mmask16 __mask, + __m512i __index, void const *__addr, int __scale) +{ + return (__m512i) __builtin_ia32_gathersiv16si ((__v16si) __v1_old, + __addr, + (__v16si) __index, + __mask, __scale); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_i32gather_epi64 (__m256i __index, void const *__addr, int __scale) +{ + __m512i __v1_old = _mm512_undefined_epi32 (); + __mmask8 __mask = 0xFF; + + return (__m512i) __builtin_ia32_gathersiv8di ((__v8di) __v1_old, + __addr, + (__v8si) __index, __mask, + __scale); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_i32gather_epi64 (__m512i __v1_old, __mmask8 __mask, + __m256i __index, void const *__addr, + int __scale) +{ + return (__m512i) __builtin_ia32_gathersiv8di ((__v8di) __v1_old, + __addr, + (__v8si) __index, + __mask, __scale); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_i64gather_epi32 (__m512i __index, void const *__addr, int __scale) +{ + __m256i __v1_old = _mm256_undefined_si256 (); + __mmask8 __mask = 0xFF; + + return (__m256i) __builtin_ia32_gatherdiv16si ((__v8si) __v1_old, + __addr, + (__v8di) __index, + __mask, __scale); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_i64gather_epi32 (__m256i __v1_old, __mmask8 __mask, + __m512i __index, void const *__addr, int __scale) +{ + return (__m256i) __builtin_ia32_gatherdiv16si ((__v8si) __v1_old, + __addr, + (__v8di) __index, + __mask, __scale); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_i64gather_epi64 (__m512i __index, void const *__addr, int __scale) +{ + __m512i __v1_old = _mm512_undefined_epi32 (); + __mmask8 __mask = 0xFF; + + return (__m512i) __builtin_ia32_gatherdiv8di ((__v8di) __v1_old, + __addr, + (__v8di) __index, __mask, + __scale); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_i64gather_epi64 (__m512i __v1_old, __mmask8 __mask, + __m512i __index, void const *__addr, + int __scale) +{ + return (__m512i) __builtin_ia32_gatherdiv8di ((__v8di) __v1_old, + __addr, + (__v8di) __index, + __mask, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_i32scatter_ps (void *__addr, __m512i __index, __m512 __v1, int __scale) +{ + __builtin_ia32_scattersiv16sf (__addr, (__mmask16) 0xFFFF, + (__v16si) __index, (__v16sf) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_i32scatter_ps (void *__addr, __mmask16 __mask, + __m512i __index, __m512 __v1, int __scale) +{ + __builtin_ia32_scattersiv16sf (__addr, __mask, (__v16si) __index, + (__v16sf) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_i32scatter_pd (void *__addr, __m256i __index, __m512d __v1, + int __scale) +{ + __builtin_ia32_scattersiv8df (__addr, (__mmask8) 0xFF, + (__v8si) __index, (__v8df) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_i32scatter_pd (void *__addr, __mmask8 __mask, + __m256i __index, __m512d __v1, int __scale) +{ + __builtin_ia32_scattersiv8df (__addr, __mask, (__v8si) __index, + (__v8df) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_i64scatter_ps (void *__addr, __m512i __index, __m256 __v1, int __scale) +{ + __builtin_ia32_scatterdiv16sf (__addr, (__mmask8) 0xFF, + (__v8di) __index, (__v8sf) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_i64scatter_ps (void *__addr, __mmask8 __mask, + __m512i __index, __m256 __v1, int __scale) +{ + __builtin_ia32_scatterdiv16sf (__addr, __mask, (__v8di) __index, + (__v8sf) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_i64scatter_pd (void *__addr, __m512i __index, __m512d __v1, + int __scale) +{ + __builtin_ia32_scatterdiv8df (__addr, (__mmask8) 0xFF, + (__v8di) __index, (__v8df) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_i64scatter_pd (void *__addr, __mmask8 __mask, + __m512i __index, __m512d __v1, int __scale) +{ + __builtin_ia32_scatterdiv8df (__addr, __mask, (__v8di) __index, + (__v8df) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_i32scatter_epi32 (void *__addr, __m512i __index, + __m512i __v1, int __scale) +{ + __builtin_ia32_scattersiv16si (__addr, (__mmask16) 0xFFFF, + (__v16si) __index, (__v16si) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_i32scatter_epi32 (void *__addr, __mmask16 __mask, + __m512i __index, __m512i __v1, int __scale) +{ + __builtin_ia32_scattersiv16si (__addr, __mask, (__v16si) __index, + (__v16si) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_i32scatter_epi64 (void *__addr, __m256i __index, + __m512i __v1, int __scale) +{ + __builtin_ia32_scattersiv8di (__addr, (__mmask8) 0xFF, + (__v8si) __index, (__v8di) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_i32scatter_epi64 (void *__addr, __mmask8 __mask, + __m256i __index, __m512i __v1, int __scale) +{ + __builtin_ia32_scattersiv8di (__addr, __mask, (__v8si) __index, + (__v8di) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_i64scatter_epi32 (void *__addr, __m512i __index, + __m256i __v1, int __scale) +{ + __builtin_ia32_scatterdiv16si (__addr, (__mmask8) 0xFF, + (__v8di) __index, (__v8si) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_i64scatter_epi32 (void *__addr, __mmask8 __mask, + __m512i __index, __m256i __v1, int __scale) +{ + __builtin_ia32_scatterdiv16si (__addr, __mask, (__v8di) __index, + (__v8si) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_i64scatter_epi64 (void *__addr, __m512i __index, + __m512i __v1, int __scale) +{ + __builtin_ia32_scatterdiv8di (__addr, (__mmask8) 0xFF, + (__v8di) __index, (__v8di) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_i64scatter_epi64 (void *__addr, __mmask8 __mask, + __m512i __index, __m512i __v1, int __scale) +{ + __builtin_ia32_scatterdiv8di (__addr, __mask, (__v8di) __index, + (__v8di) __v1, __scale); +} +#else +#define _mm512_i32gather_ps(INDEX, ADDR, SCALE) \ + (__m512) __builtin_ia32_gathersiv16sf ((__v16sf)_mm512_undefined_ps(),\ + (void const *)ADDR, \ + (__v16si)(__m512i)INDEX, \ + (__mmask16)0xFFFF, (int)SCALE) + +#define _mm512_mask_i32gather_ps(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m512) __builtin_ia32_gathersiv16sf ((__v16sf)(__m512)V1OLD, \ + (void const *)ADDR, \ + (__v16si)(__m512i)INDEX, \ + (__mmask16)MASK, (int)SCALE) + +#define _mm512_i32gather_pd(INDEX, ADDR, SCALE) \ + (__m512d) __builtin_ia32_gathersiv8df ((__v8df)_mm512_undefined_pd(), \ + (void const *)ADDR, \ + (__v8si)(__m256i)INDEX, \ + (__mmask8)0xFF, (int)SCALE) + +#define _mm512_mask_i32gather_pd(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m512d) __builtin_ia32_gathersiv8df ((__v8df)(__m512d)V1OLD, \ + (void const *)ADDR, \ + (__v8si)(__m256i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm512_i64gather_ps(INDEX, ADDR, SCALE) \ + (__m256) __builtin_ia32_gatherdiv16sf ((__v8sf)_mm256_undefined_ps(), \ + (void const *)ADDR, \ + (__v8di)(__m512i)INDEX, \ + (__mmask8)0xFF, (int)SCALE) + +#define _mm512_mask_i64gather_ps(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m256) __builtin_ia32_gatherdiv16sf ((__v8sf)(__m256)V1OLD, \ + (void const *)ADDR, \ + (__v8di)(__m512i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm512_i64gather_pd(INDEX, ADDR, SCALE) \ + (__m512d) __builtin_ia32_gatherdiv8df ((__v8df)_mm512_undefined_pd(), \ + (void const *)ADDR, \ + (__v8di)(__m512i)INDEX, \ + (__mmask8)0xFF, (int)SCALE) + +#define _mm512_mask_i64gather_pd(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m512d) __builtin_ia32_gatherdiv8df ((__v8df)(__m512d)V1OLD, \ + (void const *)ADDR, \ + (__v8di)(__m512i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm512_i32gather_epi32(INDEX, ADDR, SCALE) \ + (__m512i) __builtin_ia32_gathersiv16si ((__v16si)_mm512_undefined_epi32 (), \ + (void const *)ADDR, \ + (__v16si)(__m512i)INDEX, \ + (__mmask16)0xFFFF, (int)SCALE) + +#define _mm512_mask_i32gather_epi32(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m512i) __builtin_ia32_gathersiv16si ((__v16si)(__m512i)V1OLD, \ + (void const *)ADDR, \ + (__v16si)(__m512i)INDEX, \ + (__mmask16)MASK, (int)SCALE) + +#define _mm512_i32gather_epi64(INDEX, ADDR, SCALE) \ + (__m512i) __builtin_ia32_gathersiv8di ((__v8di)_mm512_undefined_epi32 (), \ + (void const *)ADDR, \ + (__v8si)(__m256i)INDEX, \ + (__mmask8)0xFF, (int)SCALE) + +#define _mm512_mask_i32gather_epi64(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m512i) __builtin_ia32_gathersiv8di ((__v8di)(__m512i)V1OLD, \ + (void const *)ADDR, \ + (__v8si)(__m256i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm512_i64gather_epi32(INDEX, ADDR, SCALE) \ + (__m256i) __builtin_ia32_gatherdiv16si ((__v8si)_mm256_undefined_si256(), \ + (void const *)ADDR, \ + (__v8di)(__m512i)INDEX, \ + (__mmask8)0xFF, (int)SCALE) + +#define _mm512_mask_i64gather_epi32(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m256i) __builtin_ia32_gatherdiv16si ((__v8si)(__m256i)V1OLD, \ + (void const *)ADDR, \ + (__v8di)(__m512i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm512_i64gather_epi64(INDEX, ADDR, SCALE) \ + (__m512i) __builtin_ia32_gatherdiv8di ((__v8di)_mm512_undefined_epi32 (), \ + (void const *)ADDR, \ + (__v8di)(__m512i)INDEX, \ + (__mmask8)0xFF, (int)SCALE) + +#define _mm512_mask_i64gather_epi64(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m512i) __builtin_ia32_gatherdiv8di ((__v8di)(__m512i)V1OLD, \ + (void const *)ADDR, \ + (__v8di)(__m512i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm512_i32scatter_ps(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv16sf ((void *)ADDR, (__mmask16)0xFFFF, \ + (__v16si)(__m512i)INDEX, \ + (__v16sf)(__m512)V1, (int)SCALE) + +#define _mm512_mask_i32scatter_ps(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv16sf ((void *)ADDR, (__mmask16)MASK, \ + (__v16si)(__m512i)INDEX, \ + (__v16sf)(__m512)V1, (int)SCALE) + +#define _mm512_i32scatter_pd(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv8df ((void *)ADDR, (__mmask8)0xFF, \ + (__v8si)(__m256i)INDEX, \ + (__v8df)(__m512d)V1, (int)SCALE) + +#define _mm512_mask_i32scatter_pd(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv8df ((void *)ADDR, (__mmask8)MASK, \ + (__v8si)(__m256i)INDEX, \ + (__v8df)(__m512d)V1, (int)SCALE) + +#define _mm512_i64scatter_ps(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv16sf ((void *)ADDR, (__mmask8)0xFF, \ + (__v8di)(__m512i)INDEX, \ + (__v8sf)(__m256)V1, (int)SCALE) + +#define _mm512_mask_i64scatter_ps(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv16sf ((void *)ADDR, (__mmask16)MASK, \ + (__v8di)(__m512i)INDEX, \ + (__v8sf)(__m256)V1, (int)SCALE) + +#define _mm512_i64scatter_pd(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv8df ((void *)ADDR, (__mmask8)0xFF, \ + (__v8di)(__m512i)INDEX, \ + (__v8df)(__m512d)V1, (int)SCALE) + +#define _mm512_mask_i64scatter_pd(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv8df ((void *)ADDR, (__mmask8)MASK, \ + (__v8di)(__m512i)INDEX, \ + (__v8df)(__m512d)V1, (int)SCALE) + +#define _mm512_i32scatter_epi32(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv16si ((void *)ADDR, (__mmask16)0xFFFF, \ + (__v16si)(__m512i)INDEX, \ + (__v16si)(__m512i)V1, (int)SCALE) + +#define _mm512_mask_i32scatter_epi32(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv16si ((void *)ADDR, (__mmask16)MASK, \ + (__v16si)(__m512i)INDEX, \ + (__v16si)(__m512i)V1, (int)SCALE) + +#define _mm512_i32scatter_epi64(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv8di ((void *)ADDR, (__mmask8)0xFF, \ + (__v8si)(__m256i)INDEX, \ + (__v8di)(__m512i)V1, (int)SCALE) + +#define _mm512_mask_i32scatter_epi64(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv8di ((void *)ADDR, (__mmask8)MASK, \ + (__v8si)(__m256i)INDEX, \ + (__v8di)(__m512i)V1, (int)SCALE) + +#define _mm512_i64scatter_epi32(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv16si ((void *)ADDR, (__mmask8)0xFF, \ + (__v8di)(__m512i)INDEX, \ + (__v8si)(__m256i)V1, (int)SCALE) + +#define _mm512_mask_i64scatter_epi32(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv16si ((void *)ADDR, (__mmask8)MASK, \ + (__v8di)(__m512i)INDEX, \ + (__v8si)(__m256i)V1, (int)SCALE) + +#define _mm512_i64scatter_epi64(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv8di ((void *)ADDR, (__mmask8)0xFF, \ + (__v8di)(__m512i)INDEX, \ + (__v8di)(__m512i)V1, (int)SCALE) + +#define _mm512_mask_i64scatter_epi64(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv8di ((void *)ADDR, (__mmask8)MASK, \ + (__v8di)(__m512i)INDEX, \ + (__v8di)(__m512i)V1, (int)SCALE) +#endif + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_compress_pd (__m512d __W, __mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_compressdf512_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_compress_pd (__mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_compressdf512_mask ((__v8df) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_compressstoreu_pd (void *__P, __mmask8 __U, __m512d __A) +{ + __builtin_ia32_compressstoredf512_mask ((__v8df *) __P, (__v8df) __A, + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_compress_ps (__m512 __W, __mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_compresssf512_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_compress_ps (__mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_compresssf512_mask ((__v16sf) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_compressstoreu_ps (void *__P, __mmask16 __U, __m512 __A) +{ + __builtin_ia32_compressstoresf512_mask ((__v16sf *) __P, (__v16sf) __A, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_compress_epi64 (__m512i __W, __mmask8 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_compressdi512_mask ((__v8di) __A, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_compress_epi64 (__mmask8 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_compressdi512_mask ((__v8di) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_compressstoreu_epi64 (void *__P, __mmask8 __U, __m512i __A) +{ + __builtin_ia32_compressstoredi512_mask ((__v8di *) __P, (__v8di) __A, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_compress_epi32 (__m512i __W, __mmask16 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_compresssi512_mask ((__v16si) __A, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_compress_epi32 (__mmask16 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_compresssi512_mask ((__v16si) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_compressstoreu_epi32 (void *__P, __mmask16 __U, __m512i __A) +{ + __builtin_ia32_compressstoresi512_mask ((__v16si *) __P, (__v16si) __A, + (__mmask16) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_expand_pd (__m512d __W, __mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_expanddf512_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_expand_pd (__mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_expanddf512_maskz ((__v8df) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_expandloadu_pd (__m512d __W, __mmask8 __U, void const *__P) +{ + return (__m512d) __builtin_ia32_expandloaddf512_mask ((const __v8df *) __P, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_expandloadu_pd (__mmask8 __U, void const *__P) +{ + return (__m512d) __builtin_ia32_expandloaddf512_maskz ((const __v8df *) __P, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_expand_ps (__m512 __W, __mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_expandsf512_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_expand_ps (__mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_expandsf512_maskz ((__v16sf) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_expandloadu_ps (__m512 __W, __mmask16 __U, void const *__P) +{ + return (__m512) __builtin_ia32_expandloadsf512_mask ((const __v16sf *) __P, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_expandloadu_ps (__mmask16 __U, void const *__P) +{ + return (__m512) __builtin_ia32_expandloadsf512_maskz ((const __v16sf *) __P, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_expand_epi64 (__m512i __W, __mmask8 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_expanddi512_mask ((__v8di) __A, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_expand_epi64 (__mmask8 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_expanddi512_maskz ((__v8di) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_expandloadu_epi64 (__m512i __W, __mmask8 __U, void const *__P) +{ + return (__m512i) __builtin_ia32_expandloaddi512_mask ((const __v8di *) __P, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_expandloadu_epi64 (__mmask8 __U, void const *__P) +{ + return (__m512i) + __builtin_ia32_expandloaddi512_maskz ((const __v8di *) __P, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_expand_epi32 (__m512i __W, __mmask16 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_expandsi512_mask ((__v16si) __A, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_expand_epi32 (__mmask16 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_expandsi512_maskz ((__v16si) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_expandloadu_epi32 (__m512i __W, __mmask16 __U, void const *__P) +{ + return (__m512i) __builtin_ia32_expandloadsi512_mask ((const __v16si *) __P, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_expandloadu_epi32 (__mmask16 __U, void const *__P) +{ + return (__m512i) __builtin_ia32_expandloadsi512_maskz ((const __v16si *) __P, + (__v16si) + _mm512_setzero_si512 + (), (__mmask16) __U); +} + +/* Mask arithmetic operations */ +#define _kand_mask16 _mm512_kand +#define _kandn_mask16 _mm512_kandn +#define _knot_mask16 _mm512_knot +#define _kor_mask16 _mm512_kor +#define _kxnor_mask16 _mm512_kxnor +#define _kxor_mask16 _mm512_kxor + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kortest_mask16_u8 (__mmask16 __A, __mmask16 __B, unsigned char *__CF) +{ + *__CF = (unsigned char) __builtin_ia32_kortestchi (__A, __B); + return (unsigned char) __builtin_ia32_kortestzhi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kortestz_mask16_u8 (__mmask16 __A, __mmask16 __B) +{ + return (unsigned char) __builtin_ia32_kortestzhi ((__mmask16) __A, + (__mmask16) __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kortestc_mask16_u8 (__mmask16 __A, __mmask16 __B) +{ + return (unsigned char) __builtin_ia32_kortestchi ((__mmask16) __A, + (__mmask16) __B); +} + +extern __inline unsigned int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_cvtmask16_u32 (__mmask16 __A) +{ + return (unsigned int) __builtin_ia32_kmovw ((__mmask16 ) __A); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_cvtu32_mask16 (unsigned int __A) +{ + return (__mmask16) __builtin_ia32_kmovw ((__mmask16 ) __A); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_load_mask16 (__mmask16 *__A) +{ + return (__mmask16) __builtin_ia32_kmovw (*(__mmask16 *) __A); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_store_mask16 (__mmask16 *__A, __mmask16 __B) +{ + *(__mmask16 *) __A = __builtin_ia32_kmovw (__B); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_kand (__mmask16 __A, __mmask16 __B) +{ + return (__mmask16) __builtin_ia32_kandhi ((__mmask16) __A, (__mmask16) __B); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_kandn (__mmask16 __A, __mmask16 __B) +{ + return (__mmask16) __builtin_ia32_kandnhi ((__mmask16) __A, + (__mmask16) __B); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_kor (__mmask16 __A, __mmask16 __B) +{ + return (__mmask16) __builtin_ia32_korhi ((__mmask16) __A, (__mmask16) __B); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_kortestz (__mmask16 __A, __mmask16 __B) +{ + return (__mmask16) __builtin_ia32_kortestzhi ((__mmask16) __A, + (__mmask16) __B); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_kortestc (__mmask16 __A, __mmask16 __B) +{ + return (__mmask16) __builtin_ia32_kortestchi ((__mmask16) __A, + (__mmask16) __B); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_kxnor (__mmask16 __A, __mmask16 __B) +{ + return (__mmask16) __builtin_ia32_kxnorhi ((__mmask16) __A, (__mmask16) __B); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_kxor (__mmask16 __A, __mmask16 __B) +{ + return (__mmask16) __builtin_ia32_kxorhi ((__mmask16) __A, (__mmask16) __B); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_knot (__mmask16 __A) +{ + return (__mmask16) __builtin_ia32_knothi ((__mmask16) __A); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_kunpackb (__mmask16 __A, __mmask16 __B) +{ + return (__mmask16) __builtin_ia32_kunpckhi ((__mmask16) __A, (__mmask16) __B); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kunpackb_mask16 (__mmask8 __A, __mmask8 __B) +{ + return (__mmask16) __builtin_ia32_kunpckhi ((__mmask16) __A, (__mmask16) __B); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_inserti32x4 (__mmask16 __B, __m512i __C, __m128i __D, + const int __imm) +{ + return (__m512i) __builtin_ia32_inserti32x4_mask ((__v16si) __C, + (__v4si) __D, + __imm, + (__v16si) + _mm512_setzero_si512 (), + __B); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_insertf32x4 (__mmask16 __B, __m512 __C, __m128 __D, + const int __imm) +{ + return (__m512) __builtin_ia32_insertf32x4_mask ((__v16sf) __C, + (__v4sf) __D, + __imm, + (__v16sf) + _mm512_setzero_ps (), __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_inserti32x4 (__m512i __A, __mmask16 __B, __m512i __C, + __m128i __D, const int __imm) +{ + return (__m512i) __builtin_ia32_inserti32x4_mask ((__v16si) __C, + (__v4si) __D, + __imm, + (__v16si) __A, + __B); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_insertf32x4 (__m512 __A, __mmask16 __B, __m512 __C, + __m128 __D, const int __imm) +{ + return (__m512) __builtin_ia32_insertf32x4_mask ((__v16sf) __C, + (__v4sf) __D, + __imm, + (__v16sf) __A, __B); +} +#else +#define _mm512_maskz_insertf32x4(A, X, Y, C) \ + ((__m512) __builtin_ia32_insertf32x4_mask ((__v16sf)(__m512) (X), \ + (__v4sf)(__m128) (Y), (int) (C), (__v16sf)_mm512_setzero_ps(), \ + (__mmask8)(A))) + +#define _mm512_maskz_inserti32x4(A, X, Y, C) \ + ((__m512i) __builtin_ia32_inserti32x4_mask ((__v16si)(__m512i) (X), \ + (__v4si)(__m128i) (Y), (int) (C), (__v16si)_mm512_setzero_si512 (), \ + (__mmask8)(A))) + +#define _mm512_mask_insertf32x4(A, B, X, Y, C) \ + ((__m512) __builtin_ia32_insertf32x4_mask ((__v16sf)(__m512) (X), \ + (__v4sf)(__m128) (Y), (int) (C), (__v16sf)(__m512) (A), \ + (__mmask8)(B))) + +#define _mm512_mask_inserti32x4(A, B, X, Y, C) \ + ((__m512i) __builtin_ia32_inserti32x4_mask ((__v16si)(__m512i) (X), \ + (__v4si)(__m128i) (Y), (int) (C), (__v16si)(__m512i) (A), \ + (__mmask8)(B))) +#endif + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_max_epi64 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxsq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_max_epi64 (__mmask8 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxsq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_max_epi64 (__m512i __W, __mmask8 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxsq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __W, __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_min_epi64 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminsq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_min_epi64 (__m512i __W, __mmask8 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminsq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __W, __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_min_epi64 (__mmask8 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminsq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_max_epu64 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxuq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_max_epu64 (__mmask8 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxuq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_max_epu64 (__m512i __W, __mmask8 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxuq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __W, __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_min_epu64 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminuq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_min_epu64 (__m512i __W, __mmask8 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminuq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __W, __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_min_epu64 (__mmask8 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminuq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_max_epi32 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxsd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_max_epi32 (__mmask16 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxsd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_max_epi32 (__m512i __W, __mmask16 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxsd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __W, __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_min_epi32 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminsd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_min_epi32 (__mmask16 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminsd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_min_epi32 (__m512i __W, __mmask16 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminsd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __W, __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_max_epu32 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxud512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_max_epu32 (__mmask16 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxud512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_max_epu32 (__m512i __W, __mmask16 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxud512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __W, __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_min_epu32 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminud512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_min_epu32 (__mmask16 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminud512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_min_epu32 (__m512i __W, __mmask16 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminud512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __W, __M); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_unpacklo_ps (__m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_unpcklps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_unpacklo_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_unpcklps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_unpacklo_ps (__mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_unpcklps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +#ifdef __OPTIMIZE__ +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_max_round_sd (__m128d __A, __m128d __B, const int __R) +{ + return (__m128d) __builtin_ia32_maxsd_round ((__v2df) __A, + (__v2df) __B, + __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_max_round_ss (__m128 __A, __m128 __B, const int __R) +{ + return (__m128) __builtin_ia32_maxss_round ((__v4sf) __A, + (__v4sf) __B, + __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_min_round_sd (__m128d __A, __m128d __B, const int __R) +{ + return (__m128d) __builtin_ia32_minsd_round ((__v2df) __A, + (__v2df) __B, + __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_min_round_ss (__m128 __A, __m128 __B, const int __R) +{ + return (__m128) __builtin_ia32_minss_round ((__v4sf) __A, + (__v4sf) __B, + __R); +} + +#else +#define _mm_max_round_sd(A, B, C) \ + (__m128d)__builtin_ia32_addsd_round(A, B, C) + +#define _mm_max_round_ss(A, B, C) \ + (__m128)__builtin_ia32_addss_round(A, B, C) + +#define _mm_min_round_sd(A, B, C) \ + (__m128d)__builtin_ia32_subsd_round(A, B, C) + +#define _mm_min_round_ss(A, B, C) \ + (__m128)__builtin_ia32_subss_round(A, B, C) +#endif + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_blend_pd (__mmask8 __U, __m512d __A, __m512d __W) +{ + return (__m512d) __builtin_ia32_blendmpd_512_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_blend_ps (__mmask16 __U, __m512 __A, __m512 __W) +{ + return (__m512) __builtin_ia32_blendmps_512_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_blend_epi64 (__mmask8 __U, __m512i __A, __m512i __W) +{ + return (__m512i) __builtin_ia32_blendmq_512_mask ((__v8di) __A, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_blend_epi32 (__mmask16 __U, __m512i __A, __m512i __W) +{ + return (__m512i) __builtin_ia32_blendmd_512_mask ((__v16si) __A, + (__v16si) __W, + (__mmask16) __U); +} + +#ifdef __OPTIMIZE__ +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fmadd_round_sd (__m128d __W, __m128d __A, __m128d __B, const int __R) +{ + return (__m128d) __builtin_ia32_vfmaddsd3_round ((__v2df) __W, + (__v2df) __A, + (__v2df) __B, + __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fmadd_round_ss (__m128 __W, __m128 __A, __m128 __B, const int __R) +{ + return (__m128) __builtin_ia32_vfmaddss3_round ((__v4sf) __W, + (__v4sf) __A, + (__v4sf) __B, + __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fmsub_round_sd (__m128d __W, __m128d __A, __m128d __B, const int __R) +{ + return (__m128d) __builtin_ia32_vfmaddsd3_round ((__v2df) __W, + (__v2df) __A, + -(__v2df) __B, + __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fmsub_round_ss (__m128 __W, __m128 __A, __m128 __B, const int __R) +{ + return (__m128) __builtin_ia32_vfmaddss3_round ((__v4sf) __W, + (__v4sf) __A, + -(__v4sf) __B, + __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fnmadd_round_sd (__m128d __W, __m128d __A, __m128d __B, const int __R) +{ + return (__m128d) __builtin_ia32_vfmaddsd3_round ((__v2df) __W, + -(__v2df) __A, + (__v2df) __B, + __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fnmadd_round_ss (__m128 __W, __m128 __A, __m128 __B, const int __R) +{ + return (__m128) __builtin_ia32_vfmaddss3_round ((__v4sf) __W, + -(__v4sf) __A, + (__v4sf) __B, + __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fnmsub_round_sd (__m128d __W, __m128d __A, __m128d __B, const int __R) +{ + return (__m128d) __builtin_ia32_vfmaddsd3_round ((__v2df) __W, + -(__v2df) __A, + -(__v2df) __B, + __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fnmsub_round_ss (__m128 __W, __m128 __A, __m128 __B, const int __R) +{ + return (__m128) __builtin_ia32_vfmaddss3_round ((__v4sf) __W, + -(__v4sf) __A, + -(__v4sf) __B, + __R); +} +#else +#define _mm_fmadd_round_sd(A, B, C, R) \ + (__m128d)__builtin_ia32_vfmaddsd3_round(A, B, C, R) + +#define _mm_fmadd_round_ss(A, B, C, R) \ + (__m128)__builtin_ia32_vfmaddss3_round(A, B, C, R) + +#define _mm_fmsub_round_sd(A, B, C, R) \ + (__m128d)__builtin_ia32_vfmaddsd3_round(A, B, -(C), R) + +#define _mm_fmsub_round_ss(A, B, C, R) \ + (__m128)__builtin_ia32_vfmaddss3_round(A, B, -(C), R) + +#define _mm_fnmadd_round_sd(A, B, C, R) \ + (__m128d)__builtin_ia32_vfmaddsd3_round(A, -(B), C, R) + +#define _mm_fnmadd_round_ss(A, B, C, R) \ + (__m128)__builtin_ia32_vfmaddss3_round(A, -(B), C, R) + +#define _mm_fnmsub_round_sd(A, B, C, R) \ + (__m128d)__builtin_ia32_vfmaddsd3_round(A, -(B), -(C), R) + +#define _mm_fnmsub_round_ss(A, B, C, R) \ + (__m128)__builtin_ia32_vfmaddss3_round(A, -(B), -(C), R) +#endif + +#ifdef __OPTIMIZE__ +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comi_round_ss (__m128 __A, __m128 __B, const int __P, const int __R) +{ + return __builtin_ia32_vcomiss ((__v4sf) __A, (__v4sf) __B, __P, __R); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comi_round_sd (__m128d __A, __m128d __B, const int __P, const int __R) +{ + return __builtin_ia32_vcomisd ((__v2df) __A, (__v2df) __B, __P, __R); +} +#else +#define _mm_comi_round_ss(A, B, C, D)\ +__builtin_ia32_vcomiss(A, B, C, D) +#define _mm_comi_round_sd(A, B, C, D)\ +__builtin_ia32_vcomisd(A, B, C, D) +#endif + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sqrt_pd (__m512d __A) +{ + return (__m512d) __builtin_ia32_sqrtpd512_mask ((__v8df) __A, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sqrt_pd (__m512d __W, __mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_sqrtpd512_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sqrt_pd (__mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_sqrtpd512_mask ((__v8df) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sqrt_ps (__m512 __A) +{ + return (__m512) __builtin_ia32_sqrtps512_mask ((__v16sf) __A, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sqrt_ps (__m512 __W, __mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_sqrtps512_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sqrt_ps (__mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_sqrtps512_mask ((__v16sf) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_add_pd (__m512d __A, __m512d __B) +{ + return (__m512d) ((__v8df)__A + (__v8df)__B); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_add_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_addpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_add_pd (__mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_addpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_add_ps (__m512 __A, __m512 __B) +{ + return (__m512) ((__v16sf)__A + (__v16sf)__B); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_add_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_addps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_add_ps (__mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_addps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sub_pd (__m512d __A, __m512d __B) +{ + return (__m512d) ((__v8df)__A - (__v8df)__B); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sub_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_subpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sub_pd (__mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_subpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sub_ps (__m512 __A, __m512 __B) +{ + return (__m512) ((__v16sf)__A - (__v16sf)__B); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sub_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_subps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sub_ps (__mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_subps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mul_pd (__m512d __A, __m512d __B) +{ + return (__m512d) ((__v8df)__A * (__v8df)__B); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mul_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_mulpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mul_pd (__mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_mulpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mul_ps (__m512 __A, __m512 __B) +{ + return (__m512) ((__v16sf)__A * (__v16sf)__B); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mul_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_mulps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mul_ps (__mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_mulps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_div_pd (__m512d __M, __m512d __V) +{ + return (__m512d) ((__v8df)__M / (__v8df)__V); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_div_pd (__m512d __W, __mmask8 __U, __m512d __M, __m512d __V) +{ + return (__m512d) __builtin_ia32_divpd512_mask ((__v8df) __M, + (__v8df) __V, + (__v8df) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_div_pd (__mmask8 __U, __m512d __M, __m512d __V) +{ + return (__m512d) __builtin_ia32_divpd512_mask ((__v8df) __M, + (__v8df) __V, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_div_ps (__m512 __A, __m512 __B) +{ + return (__m512) ((__v16sf)__A / (__v16sf)__B); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_div_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_divps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_div_ps (__mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_divps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_max_pd (__m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_maxpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_max_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_maxpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_max_pd (__mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_maxpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_max_ps (__m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_maxps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_max_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_maxps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_max_ps (__mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_maxps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_min_pd (__m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_minpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_min_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_minpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_min_pd (__mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_minpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_min_ps (__m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_minps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_min_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_minps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_min_ps (__mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_minps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_scalef_pd (__m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_scalefpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_scalef_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_scalefpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_scalef_pd (__mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_scalefpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_scalef_ps (__m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_scalefps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_scalef_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_scalefps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_scalef_ps (__mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_scalefps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_scalef_sd (__m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_scalefsd_round ((__v2df) __A, + (__v2df) __B, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_scalef_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_scalefss_round ((__v4sf) __A, + (__v4sf) __B, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fmadd_pd (__m512d __A, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fmadd_pd (__m512d __A, __mmask8 __U, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fmadd_pd (__m512d __A, __m512d __B, __m512d __C, __mmask8 __U) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_mask3 ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fmadd_pd (__mmask8 __U, __m512d __A, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_maskz ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fmadd_ps (__m512 __A, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fmadd_ps (__m512 __A, __mmask16 __U, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fmadd_ps (__m512 __A, __m512 __B, __m512 __C, __mmask16 __U) +{ + return (__m512) __builtin_ia32_vfmaddps512_mask3 ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fmadd_ps (__mmask16 __U, __m512 __A, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfmaddps512_maskz ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fmsub_pd (__m512d __A, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A, + (__v8df) __B, + -(__v8df) __C, + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fmsub_pd (__m512d __A, __mmask8 __U, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A, + (__v8df) __B, + -(__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fmsub_pd (__m512d __A, __m512d __B, __m512d __C, __mmask8 __U) +{ + return (__m512d) __builtin_ia32_vfmsubpd512_mask3 ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fmsub_pd (__mmask8 __U, __m512d __A, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_maskz ((__v8df) __A, + (__v8df) __B, + -(__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fmsub_ps (__m512 __A, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A, + (__v16sf) __B, + -(__v16sf) __C, + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fmsub_ps (__m512 __A, __mmask16 __U, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A, + (__v16sf) __B, + -(__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fmsub_ps (__m512 __A, __m512 __B, __m512 __C, __mmask16 __U) +{ + return (__m512) __builtin_ia32_vfmsubps512_mask3 ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fmsub_ps (__mmask16 __U, __m512 __A, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfmaddps512_maskz ((__v16sf) __A, + (__v16sf) __B, + -(__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fmaddsub_pd (__m512d __A, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfmaddsubpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fmaddsub_pd (__m512d __A, __mmask8 __U, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfmaddsubpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fmaddsub_pd (__m512d __A, __m512d __B, __m512d __C, __mmask8 __U) +{ + return (__m512d) __builtin_ia32_vfmaddsubpd512_mask3 ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fmaddsub_pd (__mmask8 __U, __m512d __A, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfmaddsubpd512_maskz ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fmaddsub_ps (__m512 __A, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfmaddsubps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fmaddsub_ps (__m512 __A, __mmask16 __U, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfmaddsubps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fmaddsub_ps (__m512 __A, __m512 __B, __m512 __C, __mmask16 __U) +{ + return (__m512) __builtin_ia32_vfmaddsubps512_mask3 ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fmaddsub_ps (__mmask16 __U, __m512 __A, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfmaddsubps512_maskz ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fmsubadd_pd (__m512d __A, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfmaddsubpd512_mask ((__v8df) __A, + (__v8df) __B, + -(__v8df) __C, + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fmsubadd_pd (__m512d __A, __mmask8 __U, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfmaddsubpd512_mask ((__v8df) __A, + (__v8df) __B, + -(__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fmsubadd_pd (__m512d __A, __m512d __B, __m512d __C, __mmask8 __U) +{ + return (__m512d) __builtin_ia32_vfmsubaddpd512_mask3 ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fmsubadd_pd (__mmask8 __U, __m512d __A, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfmaddsubpd512_maskz ((__v8df) __A, + (__v8df) __B, + -(__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fmsubadd_ps (__m512 __A, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfmaddsubps512_mask ((__v16sf) __A, + (__v16sf) __B, + -(__v16sf) __C, + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fmsubadd_ps (__m512 __A, __mmask16 __U, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfmaddsubps512_mask ((__v16sf) __A, + (__v16sf) __B, + -(__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fmsubadd_ps (__m512 __A, __m512 __B, __m512 __C, __mmask16 __U) +{ + return (__m512) __builtin_ia32_vfmsubaddps512_mask3 ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fmsubadd_ps (__mmask16 __U, __m512 __A, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfmaddsubps512_maskz ((__v16sf) __A, + (__v16sf) __B, + -(__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fnmadd_pd (__m512d __A, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_mask (-(__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fnmadd_pd (__m512d __A, __mmask8 __U, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfnmaddpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fnmadd_pd (__m512d __A, __m512d __B, __m512d __C, __mmask8 __U) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_mask3 (-(__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fnmadd_pd (__mmask8 __U, __m512d __A, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_maskz (-(__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fnmadd_ps (__m512 __A, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfmaddps512_mask (-(__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fnmadd_ps (__m512 __A, __mmask16 __U, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfnmaddps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fnmadd_ps (__m512 __A, __m512 __B, __m512 __C, __mmask16 __U) +{ + return (__m512) __builtin_ia32_vfmaddps512_mask3 (-(__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fnmadd_ps (__mmask16 __U, __m512 __A, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfmaddps512_maskz (-(__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fnmsub_pd (__m512d __A, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_mask (-(__v8df) __A, + (__v8df) __B, + -(__v8df) __C, + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fnmsub_pd (__m512d __A, __mmask8 __U, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfnmsubpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fnmsub_pd (__m512d __A, __m512d __B, __m512d __C, __mmask8 __U) +{ + return (__m512d) __builtin_ia32_vfnmsubpd512_mask3 ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fnmsub_pd (__mmask8 __U, __m512d __A, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_maskz (-(__v8df) __A, + (__v8df) __B, + -(__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fnmsub_ps (__m512 __A, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfmaddps512_mask (-(__v16sf) __A, + (__v16sf) __B, + -(__v16sf) __C, + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fnmsub_ps (__m512 __A, __mmask16 __U, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfnmsubps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fnmsub_ps (__m512 __A, __m512 __B, __m512 __C, __mmask16 __U) +{ + return (__m512) __builtin_ia32_vfnmsubps512_mask3 ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fnmsub_ps (__mmask16 __U, __m512 __A, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfmaddps512_maskz (-(__v16sf) __A, + (__v16sf) __B, + -(__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvttpd_epi32 (__m512d __A) +{ + return (__m256i) __builtin_ia32_cvttpd2dq512_mask ((__v8df) __A, + (__v8si) + _mm256_undefined_si256 (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvttpd_epi32 (__m256i __W, __mmask8 __U, __m512d __A) +{ + return (__m256i) __builtin_ia32_cvttpd2dq512_mask ((__v8df) __A, + (__v8si) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvttpd_epi32 (__mmask8 __U, __m512d __A) +{ + return (__m256i) __builtin_ia32_cvttpd2dq512_mask ((__v8df) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvttpd_epu32 (__m512d __A) +{ + return (__m256i) __builtin_ia32_cvttpd2udq512_mask ((__v8df) __A, + (__v8si) + _mm256_undefined_si256 (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvttpd_epu32 (__m256i __W, __mmask8 __U, __m512d __A) +{ + return (__m256i) __builtin_ia32_cvttpd2udq512_mask ((__v8df) __A, + (__v8si) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvttpd_epu32 (__mmask8 __U, __m512d __A) +{ + return (__m256i) __builtin_ia32_cvttpd2udq512_mask ((__v8df) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtpd_epi32 (__m512d __A) +{ + return (__m256i) __builtin_ia32_cvtpd2dq512_mask ((__v8df) __A, + (__v8si) + _mm256_undefined_si256 (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtpd_epi32 (__m256i __W, __mmask8 __U, __m512d __A) +{ + return (__m256i) __builtin_ia32_cvtpd2dq512_mask ((__v8df) __A, + (__v8si) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtpd_epi32 (__mmask8 __U, __m512d __A) +{ + return (__m256i) __builtin_ia32_cvtpd2dq512_mask ((__v8df) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtpd_epu32 (__m512d __A) +{ + return (__m256i) __builtin_ia32_cvtpd2udq512_mask ((__v8df) __A, + (__v8si) + _mm256_undefined_si256 (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtpd_epu32 (__m256i __W, __mmask8 __U, __m512d __A) +{ + return (__m256i) __builtin_ia32_cvtpd2udq512_mask ((__v8df) __A, + (__v8si) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtpd_epu32 (__mmask8 __U, __m512d __A) +{ + return (__m256i) __builtin_ia32_cvtpd2udq512_mask ((__v8df) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvttps_epi32 (__m512 __A) +{ + return (__m512i) __builtin_ia32_cvttps2dq512_mask ((__v16sf) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvttps_epi32 (__m512i __W, __mmask16 __U, __m512 __A) +{ + return (__m512i) __builtin_ia32_cvttps2dq512_mask ((__v16sf) __A, + (__v16si) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvttps_epi32 (__mmask16 __U, __m512 __A) +{ + return (__m512i) __builtin_ia32_cvttps2dq512_mask ((__v16sf) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvttps_epu32 (__m512 __A) +{ + return (__m512i) __builtin_ia32_cvttps2udq512_mask ((__v16sf) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvttps_epu32 (__m512i __W, __mmask16 __U, __m512 __A) +{ + return (__m512i) __builtin_ia32_cvttps2udq512_mask ((__v16sf) __A, + (__v16si) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvttps_epu32 (__mmask16 __U, __m512 __A) +{ + return (__m512i) __builtin_ia32_cvttps2udq512_mask ((__v16sf) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtps_epi32 (__m512 __A) +{ + return (__m512i) __builtin_ia32_cvtps2dq512_mask ((__v16sf) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtps_epi32 (__m512i __W, __mmask16 __U, __m512 __A) +{ + return (__m512i) __builtin_ia32_cvtps2dq512_mask ((__v16sf) __A, + (__v16si) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtps_epi32 (__mmask16 __U, __m512 __A) +{ + return (__m512i) __builtin_ia32_cvtps2dq512_mask ((__v16sf) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtps_epu32 (__m512 __A) +{ + return (__m512i) __builtin_ia32_cvtps2udq512_mask ((__v16sf) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtps_epu32 (__m512i __W, __mmask16 __U, __m512 __A) +{ + return (__m512i) __builtin_ia32_cvtps2udq512_mask ((__v16sf) __A, + (__v16si) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtps_epu32 (__mmask16 __U, __m512 __A) +{ + return (__m512i) __builtin_ia32_cvtps2udq512_mask ((__v16sf) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline double +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtsd_f64 (__m512d __A) +{ + return __A[0]; +} + +extern __inline float +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtss_f32 (__m512 __A) +{ + return __A[0]; +} + +#ifdef __x86_64__ +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtu64_ss (__m128 __A, unsigned long long __B) +{ + return (__m128) __builtin_ia32_cvtusi2ss64 ((__v4sf) __A, __B, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtu64_sd (__m128d __A, unsigned long long __B) +{ + return (__m128d) __builtin_ia32_cvtusi2sd64 ((__v2df) __A, __B, + _MM_FROUND_CUR_DIRECTION); +} +#endif + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtu32_ss (__m128 __A, unsigned __B) +{ + return (__m128) __builtin_ia32_cvtusi2ss32 ((__v4sf) __A, __B, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepi32_ps (__m512i __A) +{ + return (__m512) __builtin_ia32_cvtdq2ps512_mask ((__v16si) __A, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi32_ps (__m512 __W, __mmask16 __U, __m512i __A) +{ + return (__m512) __builtin_ia32_cvtdq2ps512_mask ((__v16si) __A, + (__v16sf) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepi32_ps (__mmask16 __U, __m512i __A) +{ + return (__m512) __builtin_ia32_cvtdq2ps512_mask ((__v16si) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepu32_ps (__m512i __A) +{ + return (__m512) __builtin_ia32_cvtudq2ps512_mask ((__v16si) __A, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepu32_ps (__m512 __W, __mmask16 __U, __m512i __A) +{ + return (__m512) __builtin_ia32_cvtudq2ps512_mask ((__v16si) __A, + (__v16sf) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepu32_ps (__mmask16 __U, __m512i __A) +{ + return (__m512) __builtin_ia32_cvtudq2ps512_mask ((__v16si) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fixupimm_pd (__m512d __A, __m512d __B, __m512i __C, const int __imm) +{ + return (__m512d) __builtin_ia32_fixupimmpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8di) __C, + __imm, + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fixupimm_pd (__m512d __A, __mmask8 __U, __m512d __B, + __m512i __C, const int __imm) +{ + return (__m512d) __builtin_ia32_fixupimmpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8di) __C, + __imm, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fixupimm_pd (__mmask8 __U, __m512d __A, __m512d __B, + __m512i __C, const int __imm) +{ + return (__m512d) __builtin_ia32_fixupimmpd512_maskz ((__v8df) __A, + (__v8df) __B, + (__v8di) __C, + __imm, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fixupimm_ps (__m512 __A, __m512 __B, __m512i __C, const int __imm) +{ + return (__m512) __builtin_ia32_fixupimmps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16si) __C, + __imm, + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fixupimm_ps (__m512 __A, __mmask16 __U, __m512 __B, + __m512i __C, const int __imm) +{ + return (__m512) __builtin_ia32_fixupimmps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16si) __C, + __imm, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fixupimm_ps (__mmask16 __U, __m512 __A, __m512 __B, + __m512i __C, const int __imm) +{ + return (__m512) __builtin_ia32_fixupimmps512_maskz ((__v16sf) __A, + (__v16sf) __B, + (__v16si) __C, + __imm, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fixupimm_sd (__m128d __A, __m128d __B, __m128i __C, const int __imm) +{ + return (__m128d) __builtin_ia32_fixupimmsd_mask ((__v2df) __A, + (__v2df) __B, + (__v2di) __C, __imm, + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fixupimm_sd (__m128d __A, __mmask8 __U, __m128d __B, + __m128i __C, const int __imm) +{ + return (__m128d) __builtin_ia32_fixupimmsd_mask ((__v2df) __A, + (__v2df) __B, + (__v2di) __C, __imm, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fixupimm_sd (__mmask8 __U, __m128d __A, __m128d __B, + __m128i __C, const int __imm) +{ + return (__m128d) __builtin_ia32_fixupimmsd_maskz ((__v2df) __A, + (__v2df) __B, + (__v2di) __C, + __imm, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fixupimm_ss (__m128 __A, __m128 __B, __m128i __C, const int __imm) +{ + return (__m128) __builtin_ia32_fixupimmss_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4si) __C, __imm, + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fixupimm_ss (__m128 __A, __mmask8 __U, __m128 __B, + __m128i __C, const int __imm) +{ + return (__m128) __builtin_ia32_fixupimmss_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4si) __C, __imm, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fixupimm_ss (__mmask8 __U, __m128 __A, __m128 __B, + __m128i __C, const int __imm) +{ + return (__m128) __builtin_ia32_fixupimmss_maskz ((__v4sf) __A, + (__v4sf) __B, + (__v4si) __C, __imm, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} +#else +#define _mm512_fixupimm_pd(X, Y, Z, C) \ + ((__m512d)__builtin_ia32_fixupimmpd512_mask ((__v8df)(__m512d)(X), \ + (__v8df)(__m512d)(Y), (__v8di)(__m512i)(Z), (int)(C), \ + (__mmask8)(-1), _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_mask_fixupimm_pd(X, U, Y, Z, C) \ + ((__m512d)__builtin_ia32_fixupimmpd512_mask ((__v8df)(__m512d)(X), \ + (__v8df)(__m512d)(Y), (__v8di)(__m512i)(Z), (int)(C), \ + (__mmask8)(U), _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_maskz_fixupimm_pd(U, X, Y, Z, C) \ + ((__m512d)__builtin_ia32_fixupimmpd512_maskz ((__v8df)(__m512d)(X), \ + (__v8df)(__m512d)(Y), (__v8di)(__m512i)(Z), (int)(C), \ + (__mmask8)(U), _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_fixupimm_ps(X, Y, Z, C) \ + ((__m512)__builtin_ia32_fixupimmps512_mask ((__v16sf)(__m512)(X), \ + (__v16sf)(__m512)(Y), (__v16si)(__m512i)(Z), (int)(C), \ + (__mmask16)(-1), _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_mask_fixupimm_ps(X, U, Y, Z, C) \ + ((__m512)__builtin_ia32_fixupimmps512_mask ((__v16sf)(__m512)(X), \ + (__v16sf)(__m512)(Y), (__v16si)(__m512i)(Z), (int)(C), \ + (__mmask16)(U), _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_maskz_fixupimm_ps(U, X, Y, Z, C) \ + ((__m512)__builtin_ia32_fixupimmps512_maskz ((__v16sf)(__m512)(X), \ + (__v16sf)(__m512)(Y), (__v16si)(__m512i)(Z), (int)(C), \ + (__mmask16)(U), _MM_FROUND_CUR_DIRECTION)) + +#define _mm_fixupimm_sd(X, Y, Z, C) \ + ((__m128d)__builtin_ia32_fixupimmsd_mask ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (__v2di)(__m128i)(Z), (int)(C), \ + (__mmask8)(-1), _MM_FROUND_CUR_DIRECTION)) + +#define _mm_mask_fixupimm_sd(X, U, Y, Z, C) \ + ((__m128d)__builtin_ia32_fixupimmsd_mask ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (__v2di)(__m128i)(Z), (int)(C), \ + (__mmask8)(U), _MM_FROUND_CUR_DIRECTION)) + +#define _mm_maskz_fixupimm_sd(U, X, Y, Z, C) \ + ((__m128d)__builtin_ia32_fixupimmsd_maskz ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (__v2di)(__m128i)(Z), (int)(C), \ + (__mmask8)(U), _MM_FROUND_CUR_DIRECTION)) + +#define _mm_fixupimm_ss(X, Y, Z, C) \ + ((__m128)__builtin_ia32_fixupimmss_mask ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (__v4si)(__m128i)(Z), (int)(C), \ + (__mmask8)(-1), _MM_FROUND_CUR_DIRECTION)) + +#define _mm_mask_fixupimm_ss(X, U, Y, Z, C) \ + ((__m128)__builtin_ia32_fixupimmss_mask ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (__v4si)(__m128i)(Z), (int)(C), \ + (__mmask8)(U), _MM_FROUND_CUR_DIRECTION)) + +#define _mm_maskz_fixupimm_ss(U, X, Y, Z, C) \ + ((__m128)__builtin_ia32_fixupimmss_maskz ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (__v4si)(__m128i)(Z), (int)(C), \ + (__mmask8)(U), _MM_FROUND_CUR_DIRECTION)) +#endif + +#ifdef __x86_64__ +extern __inline unsigned long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtss_u64 (__m128 __A) +{ + return (unsigned long long) __builtin_ia32_vcvtss2usi64 ((__v4sf) + __A, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline unsigned long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttss_u64 (__m128 __A) +{ + return (unsigned long long) __builtin_ia32_vcvttss2usi64 ((__v4sf) + __A, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttss_i64 (__m128 __A) +{ + return (long long) __builtin_ia32_vcvttss2si64 ((__v4sf) __A, + _MM_FROUND_CUR_DIRECTION); +} +#endif /* __x86_64__ */ + +extern __inline unsigned +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtss_u32 (__m128 __A) +{ + return (unsigned) __builtin_ia32_vcvtss2usi32 ((__v4sf) __A, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline unsigned +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttss_u32 (__m128 __A) +{ + return (unsigned) __builtin_ia32_vcvttss2usi32 ((__v4sf) __A, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttss_i32 (__m128 __A) +{ + return (int) __builtin_ia32_vcvttss2si32 ((__v4sf) __A, + _MM_FROUND_CUR_DIRECTION); +} + +#ifdef __x86_64__ +extern __inline unsigned long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsd_u64 (__m128d __A) +{ + return (unsigned long long) __builtin_ia32_vcvtsd2usi64 ((__v2df) + __A, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline unsigned long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttsd_u64 (__m128d __A) +{ + return (unsigned long long) __builtin_ia32_vcvttsd2usi64 ((__v2df) + __A, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttsd_i64 (__m128d __A) +{ + return (long long) __builtin_ia32_vcvttsd2si64 ((__v2df) __A, + _MM_FROUND_CUR_DIRECTION); +} +#endif /* __x86_64__ */ + +extern __inline unsigned +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsd_u32 (__m128d __A) +{ + return (unsigned) __builtin_ia32_vcvtsd2usi32 ((__v2df) __A, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline unsigned +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttsd_u32 (__m128d __A) +{ + return (unsigned) __builtin_ia32_vcvttsd2usi32 ((__v2df) __A, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttsd_i32 (__m128d __A) +{ + return (int) __builtin_ia32_vcvttsd2si32 ((__v2df) __A, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtps_pd (__m256 __A) +{ + return (__m512d) __builtin_ia32_cvtps2pd512_mask ((__v8sf) __A, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtps_pd (__m512d __W, __mmask8 __U, __m256 __A) +{ + return (__m512d) __builtin_ia32_cvtps2pd512_mask ((__v8sf) __A, + (__v8df) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtps_pd (__mmask8 __U, __m256 __A) +{ + return (__m512d) __builtin_ia32_cvtps2pd512_mask ((__v8sf) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtph_ps (__m256i __A) +{ + return (__m512) __builtin_ia32_vcvtph2ps512_mask ((__v16hi) __A, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtph_ps (__m512 __W, __mmask16 __U, __m256i __A) +{ + return (__m512) __builtin_ia32_vcvtph2ps512_mask ((__v16hi) __A, + (__v16sf) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtph_ps (__mmask16 __U, __m256i __A) +{ + return (__m512) __builtin_ia32_vcvtph2ps512_mask ((__v16hi) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtpd_ps (__m512d __A) +{ + return (__m256) __builtin_ia32_cvtpd2ps512_mask ((__v8df) __A, + (__v8sf) + _mm256_undefined_ps (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtpd_ps (__m256 __W, __mmask8 __U, __m512d __A) +{ + return (__m256) __builtin_ia32_cvtpd2ps512_mask ((__v8df) __A, + (__v8sf) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtpd_ps (__mmask8 __U, __m512d __A) +{ + return (__m256) __builtin_ia32_cvtpd2ps512_mask ((__v8df) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_getexp_ps (__m512 __A) +{ + return (__m512) __builtin_ia32_getexpps512_mask ((__v16sf) __A, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_getexp_ps (__m512 __W, __mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_getexpps512_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_getexp_ps (__mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_getexpps512_mask ((__v16sf) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_getexp_pd (__m512d __A) +{ + return (__m512d) __builtin_ia32_getexppd512_mask ((__v8df) __A, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_getexp_pd (__m512d __W, __mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_getexppd512_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_getexp_pd (__mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_getexppd512_mask ((__v8df) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_getexp_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_getexpss128_round ((__v4sf) __A, + (__v4sf) __B, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_getexp_sd (__m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_getexpsd128_round ((__v2df) __A, + (__v2df) __B, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_getmant_pd (__m512d __A, _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m512d) __builtin_ia32_getmantpd512_mask ((__v8df) __A, + (__C << 2) | __B, + _mm512_undefined_pd (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_getmant_pd (__m512d __W, __mmask8 __U, __m512d __A, + _MM_MANTISSA_NORM_ENUM __B, _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m512d) __builtin_ia32_getmantpd512_mask ((__v8df) __A, + (__C << 2) | __B, + (__v8df) __W, __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_getmant_pd (__mmask8 __U, __m512d __A, + _MM_MANTISSA_NORM_ENUM __B, _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m512d) __builtin_ia32_getmantpd512_mask ((__v8df) __A, + (__C << 2) | __B, + (__v8df) + _mm512_setzero_pd (), + __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_getmant_ps (__m512 __A, _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m512) __builtin_ia32_getmantps512_mask ((__v16sf) __A, + (__C << 2) | __B, + _mm512_undefined_ps (), + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_getmant_ps (__m512 __W, __mmask16 __U, __m512 __A, + _MM_MANTISSA_NORM_ENUM __B, _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m512) __builtin_ia32_getmantps512_mask ((__v16sf) __A, + (__C << 2) | __B, + (__v16sf) __W, __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_getmant_ps (__mmask16 __U, __m512 __A, + _MM_MANTISSA_NORM_ENUM __B, _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m512) __builtin_ia32_getmantps512_mask ((__v16sf) __A, + (__C << 2) | __B, + (__v16sf) + _mm512_setzero_ps (), + __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_getmant_sd (__m128d __A, __m128d __B, _MM_MANTISSA_NORM_ENUM __C, + _MM_MANTISSA_SIGN_ENUM __D) +{ + return (__m128d) __builtin_ia32_getmantsd_round ((__v2df) __A, + (__v2df) __B, + (__D << 2) | __C, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_getmant_ss (__m128 __A, __m128 __B, _MM_MANTISSA_NORM_ENUM __C, + _MM_MANTISSA_SIGN_ENUM __D) +{ + return (__m128) __builtin_ia32_getmantss_round ((__v4sf) __A, + (__v4sf) __B, + (__D << 2) | __C, + _MM_FROUND_CUR_DIRECTION); +} + +#else +#define _mm512_getmant_pd(X, B, C) \ + ((__m512d)__builtin_ia32_getmantpd512_mask ((__v8df)(__m512d)(X), \ + (int)(((C)<<2) | (B)), \ + (__v8df)_mm512_undefined_pd(), \ + (__mmask8)-1,\ + _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_mask_getmant_pd(W, U, X, B, C) \ + ((__m512d)__builtin_ia32_getmantpd512_mask ((__v8df)(__m512d)(X), \ + (int)(((C)<<2) | (B)), \ + (__v8df)(__m512d)(W), \ + (__mmask8)(U),\ + _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_maskz_getmant_pd(U, X, B, C) \ + ((__m512d)__builtin_ia32_getmantpd512_mask ((__v8df)(__m512d)(X), \ + (int)(((C)<<2) | (B)), \ + (__v8df)_mm512_setzero_pd(), \ + (__mmask8)(U),\ + _MM_FROUND_CUR_DIRECTION)) +#define _mm512_getmant_ps(X, B, C) \ + ((__m512)__builtin_ia32_getmantps512_mask ((__v16sf)(__m512)(X), \ + (int)(((C)<<2) | (B)), \ + (__v16sf)_mm512_undefined_ps(), \ + (__mmask16)-1,\ + _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_mask_getmant_ps(W, U, X, B, C) \ + ((__m512)__builtin_ia32_getmantps512_mask ((__v16sf)(__m512)(X), \ + (int)(((C)<<2) | (B)), \ + (__v16sf)(__m512)(W), \ + (__mmask16)(U),\ + _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_maskz_getmant_ps(U, X, B, C) \ + ((__m512)__builtin_ia32_getmantps512_mask ((__v16sf)(__m512)(X), \ + (int)(((C)<<2) | (B)), \ + (__v16sf)_mm512_setzero_ps(), \ + (__mmask16)(U),\ + _MM_FROUND_CUR_DIRECTION)) +#define _mm_getmant_sd(X, Y, C, D) \ + ((__m128d)__builtin_ia32_getmantsd_round ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), \ + (int)(((D)<<2) | (C)), \ + _MM_FROUND_CUR_DIRECTION)) + +#define _mm_getmant_ss(X, Y, C, D) \ + ((__m128)__builtin_ia32_getmantss_round ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), \ + (int)(((D)<<2) | (C)), \ + _MM_FROUND_CUR_DIRECTION)) + +#define _mm_getexp_ss(A, B) \ + ((__m128)__builtin_ia32_getexpss128_mask((__v4sf)(__m128)(A), (__v4sf)(__m128)(B), \ + _MM_FROUND_CUR_DIRECTION)) + +#define _mm_getexp_sd(A, B) \ + ((__m128d)__builtin_ia32_getexpsd128_mask((__v2df)(__m128d)(A), (__v2df)(__m128d)(B),\ + _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_getexp_ps(A) \ + ((__m512)__builtin_ia32_getexpps512_mask((__v16sf)(__m512)(A), \ + (__v16sf)_mm512_undefined_ps(), (__mmask16)-1, _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_mask_getexp_ps(W, U, A) \ + ((__m512)__builtin_ia32_getexpps512_mask((__v16sf)(__m512)(A), \ + (__v16sf)(__m512)(W), (__mmask16)(U), _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_maskz_getexp_ps(U, A) \ + ((__m512)__builtin_ia32_getexpps512_mask((__v16sf)(__m512)(A), \ + (__v16sf)_mm512_setzero_ps(), (__mmask16)(U), _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_getexp_pd(A) \ + ((__m512d)__builtin_ia32_getexppd512_mask((__v8df)(__m512d)(A), \ + (__v8df)_mm512_undefined_pd(), (__mmask8)-1, _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_mask_getexp_pd(W, U, A) \ + ((__m512d)__builtin_ia32_getexppd512_mask((__v8df)(__m512d)(A), \ + (__v8df)(__m512d)(W), (__mmask8)(U), _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_maskz_getexp_pd(U, A) \ + ((__m512d)__builtin_ia32_getexppd512_mask((__v8df)(__m512d)(A), \ + (__v8df)_mm512_setzero_pd(), (__mmask8)(U), _MM_FROUND_CUR_DIRECTION)) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_roundscale_ps (__m512 __A, const int __imm) +{ + return (__m512) __builtin_ia32_rndscaleps_mask ((__v16sf) __A, __imm, + (__v16sf) + _mm512_undefined_ps (), + -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_roundscale_ps (__m512 __A, __mmask16 __B, __m512 __C, + const int __imm) +{ + return (__m512) __builtin_ia32_rndscaleps_mask ((__v16sf) __C, __imm, + (__v16sf) __A, + (__mmask16) __B, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_roundscale_ps (__mmask16 __A, __m512 __B, const int __imm) +{ + return (__m512) __builtin_ia32_rndscaleps_mask ((__v16sf) __B, + __imm, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __A, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_roundscale_pd (__m512d __A, const int __imm) +{ + return (__m512d) __builtin_ia32_rndscalepd_mask ((__v8df) __A, __imm, + (__v8df) + _mm512_undefined_pd (), + -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_roundscale_pd (__m512d __A, __mmask8 __B, __m512d __C, + const int __imm) +{ + return (__m512d) __builtin_ia32_rndscalepd_mask ((__v8df) __C, __imm, + (__v8df) __A, + (__mmask8) __B, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_roundscale_pd (__mmask8 __A, __m512d __B, const int __imm) +{ + return (__m512d) __builtin_ia32_rndscalepd_mask ((__v8df) __B, + __imm, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __A, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_roundscale_ss (__m128 __A, __m128 __B, const int __imm) +{ + return (__m128) __builtin_ia32_rndscaless_round ((__v4sf) __A, + (__v4sf) __B, __imm, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_roundscale_sd (__m128d __A, __m128d __B, const int __imm) +{ + return (__m128d) __builtin_ia32_rndscalesd_round ((__v2df) __A, + (__v2df) __B, __imm, + _MM_FROUND_CUR_DIRECTION); +} + +#else +#define _mm512_roundscale_ps(A, B) \ + ((__m512) __builtin_ia32_rndscaleps_mask ((__v16sf)(__m512)(A), (int)(B),\ + (__v16sf)_mm512_undefined_ps(), (__mmask16)(-1), _MM_FROUND_CUR_DIRECTION)) +#define _mm512_mask_roundscale_ps(A, B, C, D) \ + ((__m512) __builtin_ia32_rndscaleps_mask ((__v16sf)(__m512)(C), \ + (int)(D), \ + (__v16sf)(__m512)(A), \ + (__mmask16)(B), _MM_FROUND_CUR_DIRECTION)) +#define _mm512_maskz_roundscale_ps(A, B, C) \ + ((__m512) __builtin_ia32_rndscaleps_mask ((__v16sf)(__m512)(B), \ + (int)(C), \ + (__v16sf)_mm512_setzero_ps(),\ + (__mmask16)(A), _MM_FROUND_CUR_DIRECTION)) +#define _mm512_roundscale_pd(A, B) \ + ((__m512d) __builtin_ia32_rndscalepd_mask ((__v8df)(__m512d)(A), (int)(B),\ + (__v8df)_mm512_undefined_pd(), (__mmask8)(-1), _MM_FROUND_CUR_DIRECTION)) +#define _mm512_mask_roundscale_pd(A, B, C, D) \ + ((__m512d) __builtin_ia32_rndscalepd_mask ((__v8df)(__m512d)(C), \ + (int)(D), \ + (__v8df)(__m512d)(A), \ + (__mmask8)(B), _MM_FROUND_CUR_DIRECTION)) +#define _mm512_maskz_roundscale_pd(A, B, C) \ + ((__m512d) __builtin_ia32_rndscalepd_mask ((__v8df)(__m512d)(B), \ + (int)(C), \ + (__v8df)_mm512_setzero_pd(),\ + (__mmask8)(A), _MM_FROUND_CUR_DIRECTION)) +#define _mm_roundscale_ss(A, B, C) \ + ((__m128) __builtin_ia32_rndscaless_round ((__v4sf)(__m128)(A), \ + (__v4sf)(__m128)(B), (int)(C), _MM_FROUND_CUR_DIRECTION)) +#define _mm_roundscale_sd(A, B, C) \ + ((__m128d) __builtin_ia32_rndscalesd_round ((__v2df)(__m128d)(A), \ + (__v2df)(__m128d)(B), (int)(C), _MM_FROUND_CUR_DIRECTION)) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmp_pd_mask (__m512d __X, __m512d __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_cmppd512_mask ((__v8df) __X, + (__v8df) __Y, __P, + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmp_ps_mask (__m512 __X, __m512 __Y, const int __P) +{ + return (__mmask16) __builtin_ia32_cmpps512_mask ((__v16sf) __X, + (__v16sf) __Y, __P, + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmp_ps_mask (__mmask16 __U, __m512 __X, __m512 __Y, const int __P) +{ + return (__mmask16) __builtin_ia32_cmpps512_mask ((__v16sf) __X, + (__v16sf) __Y, __P, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmp_pd_mask (__mmask8 __U, __m512d __X, __m512d __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_cmppd512_mask ((__v8df) __X, + (__v8df) __Y, __P, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_sd_mask (__m128d __X, __m128d __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_cmpsd_mask ((__v2df) __X, + (__v2df) __Y, __P, + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmp_sd_mask (__mmask8 __M, __m128d __X, __m128d __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_cmpsd_mask ((__v2df) __X, + (__v2df) __Y, __P, + (__mmask8) __M, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_ss_mask (__m128 __X, __m128 __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_cmpss_mask ((__v4sf) __X, + (__v4sf) __Y, __P, + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmp_ss_mask (__mmask8 __M, __m128 __X, __m128 __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_cmpss_mask ((__v4sf) __X, + (__v4sf) __Y, __P, + (__mmask8) __M, + _MM_FROUND_CUR_DIRECTION); +} + +#else +#define _mm512_cmp_pd_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_cmppd512_mask ((__v8df)(__m512d)(X), \ + (__v8df)(__m512d)(Y), (int)(P),\ + (__mmask8)-1,_MM_FROUND_CUR_DIRECTION)) + +#define _mm512_cmp_ps_mask(X, Y, P) \ + ((__mmask16) __builtin_ia32_cmpps512_mask ((__v16sf)(__m512)(X), \ + (__v16sf)(__m512)(Y), (int)(P),\ + (__mmask16)-1,_MM_FROUND_CUR_DIRECTION)) + +#define _mm512_mask_cmp_pd_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_cmppd512_mask ((__v8df)(__m512d)(X), \ + (__v8df)(__m512d)(Y), (int)(P),\ + (__mmask8)M, _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_mask_cmp_ps_mask(M, X, Y, P) \ + ((__mmask16) __builtin_ia32_cmpps512_mask ((__v16sf)(__m512)(X), \ + (__v16sf)(__m512)(Y), (int)(P),\ + (__mmask16)M,_MM_FROUND_CUR_DIRECTION)) + +#define _mm_cmp_sd_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpsd_mask ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (int)(P),\ + (__mmask8)-1,_MM_FROUND_CUR_DIRECTION)) + +#define _mm_mask_cmp_sd_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpsd_mask ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (int)(P),\ + M,_MM_FROUND_CUR_DIRECTION)) + +#define _mm_cmp_ss_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpss_mask ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (int)(P), \ + (__mmask8)-1,_MM_FROUND_CUR_DIRECTION)) + +#define _mm_mask_cmp_ss_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpss_mask ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (int)(P), \ + M,_MM_FROUND_CUR_DIRECTION)) +#endif + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_kmov (__mmask16 __A) +{ + return __builtin_ia32_kmovw (__A); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castpd_ps (__m512d __A) +{ + return (__m512) (__A); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castpd_si512 (__m512d __A) +{ + return (__m512i) (__A); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castps_pd (__m512 __A) +{ + return (__m512d) (__A); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castps_si512 (__m512 __A) +{ + return (__m512i) (__A); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castsi512_ps (__m512i __A) +{ + return (__m512) (__A); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castsi512_pd (__m512i __A) +{ + return (__m512d) (__A); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castpd512_pd128 (__m512d __A) +{ + return (__m128d)_mm512_extractf32x4_ps((__m512)__A, 0); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castps512_ps128 (__m512 __A) +{ + return _mm512_extractf32x4_ps(__A, 0); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castsi512_si128 (__m512i __A) +{ + return (__m128i)_mm512_extracti32x4_epi32((__m512i)__A, 0); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castpd512_pd256 (__m512d __A) +{ + return _mm512_extractf64x4_pd(__A, 0); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castps512_ps256 (__m512 __A) +{ + return (__m256)_mm512_extractf64x4_pd((__m512d)__A, 0); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castsi512_si256 (__m512i __A) +{ + return (__m256i)_mm512_extractf64x4_pd((__m512d)__A, 0); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castpd128_pd512 (__m128d __A) +{ + return (__m512d) __builtin_ia32_pd512_pd((__m128d)__A); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castps128_ps512 (__m128 __A) +{ + return (__m512) __builtin_ia32_ps512_ps((__m128)__A); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castsi128_si512 (__m128i __A) +{ + return (__m512i) __builtin_ia32_si512_si((__v4si)__A); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castpd256_pd512 (__m256d __A) +{ + return __builtin_ia32_pd512_256pd (__A); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castps256_ps512 (__m256 __A) +{ + return __builtin_ia32_ps512_256ps (__A); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castsi256_si512 (__m256i __A) +{ + return (__m512i)__builtin_ia32_si512_256si ((__v8si)__A); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpeq_epu32_mask (__m512i __A, __m512i __B) +{ + return (__mmask16) __builtin_ia32_ucmpd512_mask ((__v16si) __A, + (__v16si) __B, 0, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpeq_epu32_mask (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__mmask16) __builtin_ia32_ucmpd512_mask ((__v16si) __A, + (__v16si) __B, 0, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpeq_epu64_mask (__mmask8 __U, __m512i __A, __m512i __B) +{ + return (__mmask8) __builtin_ia32_ucmpq512_mask ((__v8di) __A, + (__v8di) __B, 0, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpeq_epu64_mask (__m512i __A, __m512i __B) +{ + return (__mmask8) __builtin_ia32_ucmpq512_mask ((__v8di) __A, + (__v8di) __B, 0, + (__mmask8) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpgt_epu32_mask (__m512i __A, __m512i __B) +{ + return (__mmask16) __builtin_ia32_ucmpd512_mask ((__v16si) __A, + (__v16si) __B, 6, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpgt_epu32_mask (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__mmask16) __builtin_ia32_ucmpd512_mask ((__v16si) __A, + (__v16si) __B, 6, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpgt_epu64_mask (__mmask8 __U, __m512i __A, __m512i __B) +{ + return (__mmask8) __builtin_ia32_ucmpq512_mask ((__v8di) __A, + (__v8di) __B, 6, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpgt_epu64_mask (__m512i __A, __m512i __B) +{ + return (__mmask8) __builtin_ia32_ucmpq512_mask ((__v8di) __A, + (__v8di) __B, 6, + (__mmask8) -1); +} + +#undef __MM512_REDUCE_OP +#define __MM512_REDUCE_OP(op) \ + __v8si __T1 = (__v8si) _mm512_extracti64x4_epi64 (__A, 1); \ + __v8si __T2 = (__v8si) _mm512_extracti64x4_epi64 (__A, 0); \ + __m256i __T3 = (__m256i) (__T1 op __T2); \ + __v4si __T4 = (__v4si) _mm256_extracti128_si256 (__T3, 1); \ + __v4si __T5 = (__v4si) _mm256_extracti128_si256 (__T3, 0); \ + __v4si __T6 = __T4 op __T5; \ + __v4si __T7 = __builtin_shuffle (__T6, (__v4si) { 2, 3, 0, 1 }); \ + __v4si __T8 = __T6 op __T7; \ + return __T8[0] op __T8[1] + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_add_epi32 (__m512i __A) +{ + __MM512_REDUCE_OP (+); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_mul_epi32 (__m512i __A) +{ + __MM512_REDUCE_OP (*); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_and_epi32 (__m512i __A) +{ + __MM512_REDUCE_OP (&); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_or_epi32 (__m512i __A) +{ + __MM512_REDUCE_OP (|); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_add_epi32 (__mmask16 __U, __m512i __A) +{ + __A = _mm512_maskz_mov_epi32 (__U, __A); + __MM512_REDUCE_OP (+); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_mul_epi32 (__mmask16 __U, __m512i __A) +{ + __A = _mm512_mask_mov_epi32 (_mm512_set1_epi32 (1), __U, __A); + __MM512_REDUCE_OP (*); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_and_epi32 (__mmask16 __U, __m512i __A) +{ + __A = _mm512_mask_mov_epi32 (_mm512_set1_epi32 (~0), __U, __A); + __MM512_REDUCE_OP (&); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_or_epi32 (__mmask16 __U, __m512i __A) +{ + __A = _mm512_maskz_mov_epi32 (__U, __A); + __MM512_REDUCE_OP (|); +} + +#undef __MM512_REDUCE_OP +#define __MM512_REDUCE_OP(op) \ + __m256i __T1 = (__m256i) _mm512_extracti64x4_epi64 (__A, 1); \ + __m256i __T2 = (__m256i) _mm512_extracti64x4_epi64 (__A, 0); \ + __m256i __T3 = _mm256_##op (__T1, __T2); \ + __m128i __T4 = (__m128i) _mm256_extracti128_si256 (__T3, 1); \ + __m128i __T5 = (__m128i) _mm256_extracti128_si256 (__T3, 0); \ + __m128i __T6 = _mm_##op (__T4, __T5); \ + __m128i __T7 = (__m128i) __builtin_shuffle ((__v4si) __T6, \ + (__v4si) { 2, 3, 0, 1 }); \ + __m128i __T8 = _mm_##op (__T6, __T7); \ + __m128i __T9 = (__m128i) __builtin_shuffle ((__v4si) __T8, \ + (__v4si) { 1, 0, 1, 0 }); \ + __v4si __T10 = (__v4si) _mm_##op (__T8, __T9); \ + return __T10[0] + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_min_epi32 (__m512i __A) +{ + __MM512_REDUCE_OP (min_epi32); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_max_epi32 (__m512i __A) +{ + __MM512_REDUCE_OP (max_epi32); +} + +extern __inline unsigned int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_min_epu32 (__m512i __A) +{ + __MM512_REDUCE_OP (min_epu32); +} + +extern __inline unsigned int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_max_epu32 (__m512i __A) +{ + __MM512_REDUCE_OP (max_epu32); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_min_epi32 (__mmask16 __U, __m512i __A) +{ + __A = _mm512_mask_mov_epi32 (_mm512_set1_epi32 (__INT_MAX__), __U, __A); + __MM512_REDUCE_OP (min_epi32); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_max_epi32 (__mmask16 __U, __m512i __A) +{ + __A = _mm512_mask_mov_epi32 (_mm512_set1_epi32 (-__INT_MAX__ - 1), __U, __A); + __MM512_REDUCE_OP (max_epi32); +} + +extern __inline unsigned int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_min_epu32 (__mmask16 __U, __m512i __A) +{ + __A = _mm512_mask_mov_epi32 (_mm512_set1_epi32 (~0), __U, __A); + __MM512_REDUCE_OP (min_epu32); +} + +extern __inline unsigned int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_max_epu32 (__mmask16 __U, __m512i __A) +{ + __A = _mm512_maskz_mov_epi32 (__U, __A); + __MM512_REDUCE_OP (max_epu32); +} + +#undef __MM512_REDUCE_OP +#define __MM512_REDUCE_OP(op) \ + __m256 __T1 = (__m256) _mm512_extractf64x4_pd ((__m512d) __A, 1); \ + __m256 __T2 = (__m256) _mm512_extractf64x4_pd ((__m512d) __A, 0); \ + __m256 __T3 = __T1 op __T2; \ + __m128 __T4 = _mm256_extractf128_ps (__T3, 1); \ + __m128 __T5 = _mm256_extractf128_ps (__T3, 0); \ + __m128 __T6 = __T4 op __T5; \ + __m128 __T7 = __builtin_shuffle (__T6, (__v4si) { 2, 3, 0, 1 }); \ + __m128 __T8 = __T6 op __T7; \ + return __T8[0] op __T8[1] + +extern __inline float +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_add_ps (__m512 __A) +{ + __MM512_REDUCE_OP (+); +} + +extern __inline float +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_mul_ps (__m512 __A) +{ + __MM512_REDUCE_OP (*); +} + +extern __inline float +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_add_ps (__mmask16 __U, __m512 __A) +{ + __A = _mm512_maskz_mov_ps (__U, __A); + __MM512_REDUCE_OP (+); +} + +extern __inline float +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_mul_ps (__mmask16 __U, __m512 __A) +{ + __A = _mm512_mask_mov_ps (_mm512_set1_ps (1.0f), __U, __A); + __MM512_REDUCE_OP (*); +} + +#undef __MM512_REDUCE_OP +#define __MM512_REDUCE_OP(op) \ + __m256 __T1 = (__m256) _mm512_extractf64x4_pd ((__m512d) __A, 1); \ + __m256 __T2 = (__m256) _mm512_extractf64x4_pd ((__m512d) __A, 0); \ + __m256 __T3 = _mm256_##op (__T1, __T2); \ + __m128 __T4 = _mm256_extractf128_ps (__T3, 1); \ + __m128 __T5 = _mm256_extractf128_ps (__T3, 0); \ + __m128 __T6 = _mm_##op (__T4, __T5); \ + __m128 __T7 = __builtin_shuffle (__T6, (__v4si) { 2, 3, 0, 1 }); \ + __m128 __T8 = _mm_##op (__T6, __T7); \ + __m128 __T9 = __builtin_shuffle (__T8, (__v4si) { 1, 0, 1, 0 }); \ + __m128 __T10 = _mm_##op (__T8, __T9); \ + return __T10[0] + +extern __inline float +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_min_ps (__m512 __A) +{ + __MM512_REDUCE_OP (min_ps); +} + +extern __inline float +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_max_ps (__m512 __A) +{ + __MM512_REDUCE_OP (max_ps); +} + +extern __inline float +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_min_ps (__mmask16 __U, __m512 __A) +{ + __A = _mm512_mask_mov_ps (_mm512_set1_ps (__builtin_inff ()), __U, __A); + __MM512_REDUCE_OP (min_ps); +} + +extern __inline float +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_max_ps (__mmask16 __U, __m512 __A) +{ + __A = _mm512_mask_mov_ps (_mm512_set1_ps (-__builtin_inff ()), __U, __A); + __MM512_REDUCE_OP (max_ps); +} + +#undef __MM512_REDUCE_OP +#define __MM512_REDUCE_OP(op) \ + __v4di __T1 = (__v4di) _mm512_extracti64x4_epi64 (__A, 1); \ + __v4di __T2 = (__v4di) _mm512_extracti64x4_epi64 (__A, 0); \ + __m256i __T3 = (__m256i) (__T1 op __T2); \ + __v2di __T4 = (__v2di) _mm256_extracti128_si256 (__T3, 1); \ + __v2di __T5 = (__v2di) _mm256_extracti128_si256 (__T3, 0); \ + __v2di __T6 = __T4 op __T5; \ + return __T6[0] op __T6[1] + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_add_epi64 (__m512i __A) +{ + __MM512_REDUCE_OP (+); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_mul_epi64 (__m512i __A) +{ + __MM512_REDUCE_OP (*); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_and_epi64 (__m512i __A) +{ + __MM512_REDUCE_OP (&); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_or_epi64 (__m512i __A) +{ + __MM512_REDUCE_OP (|); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_add_epi64 (__mmask8 __U, __m512i __A) +{ + __A = _mm512_maskz_mov_epi64 (__U, __A); + __MM512_REDUCE_OP (+); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_mul_epi64 (__mmask8 __U, __m512i __A) +{ + __A = _mm512_mask_mov_epi64 (_mm512_set1_epi64 (1LL), __U, __A); + __MM512_REDUCE_OP (*); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_and_epi64 (__mmask8 __U, __m512i __A) +{ + __A = _mm512_mask_mov_epi64 (_mm512_set1_epi64 (~0LL), __U, __A); + __MM512_REDUCE_OP (&); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_or_epi64 (__mmask8 __U, __m512i __A) +{ + __A = _mm512_maskz_mov_epi64 (__U, __A); + __MM512_REDUCE_OP (|); +} + +#undef __MM512_REDUCE_OP +#define __MM512_REDUCE_OP(op) \ + __m512i __T1 = _mm512_shuffle_i64x2 (__A, __A, 0x4e); \ + __m512i __T2 = _mm512_##op (__A, __T1); \ + __m512i __T3 \ + = (__m512i) __builtin_shuffle ((__v8di) __T2, \ + (__v8di) { 2, 3, 0, 1, 6, 7, 4, 5 });\ + __m512i __T4 = _mm512_##op (__T2, __T3); \ + __m512i __T5 \ + = (__m512i) __builtin_shuffle ((__v8di) __T4, \ + (__v8di) { 1, 0, 3, 2, 5, 4, 7, 6 });\ + __v8di __T6 = (__v8di) _mm512_##op (__T4, __T5); \ + return __T6[0] + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_min_epi64 (__m512i __A) +{ + __MM512_REDUCE_OP (min_epi64); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_max_epi64 (__m512i __A) +{ + __MM512_REDUCE_OP (max_epi64); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_min_epi64 (__mmask8 __U, __m512i __A) +{ + __A = _mm512_mask_mov_epi64 (_mm512_set1_epi64 (__LONG_LONG_MAX__), + __U, __A); + __MM512_REDUCE_OP (min_epi64); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_max_epi64 (__mmask8 __U, __m512i __A) +{ + __A = _mm512_mask_mov_epi64 (_mm512_set1_epi64 (-__LONG_LONG_MAX__ - 1), + __U, __A); + __MM512_REDUCE_OP (max_epi64); +} + +extern __inline unsigned long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_min_epu64 (__m512i __A) +{ + __MM512_REDUCE_OP (min_epu64); +} + +extern __inline unsigned long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_max_epu64 (__m512i __A) +{ + __MM512_REDUCE_OP (max_epu64); +} + +extern __inline unsigned long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_min_epu64 (__mmask8 __U, __m512i __A) +{ + __A = _mm512_mask_mov_epi64 (_mm512_set1_epi64 (~0LL), __U, __A); + __MM512_REDUCE_OP (min_epu64); +} + +extern __inline unsigned long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_max_epu64 (__mmask8 __U, __m512i __A) +{ + __A = _mm512_maskz_mov_epi64 (__U, __A); + __MM512_REDUCE_OP (max_epu64); +} + +#undef __MM512_REDUCE_OP +#define __MM512_REDUCE_OP(op) \ + __m256d __T1 = (__m256d) _mm512_extractf64x4_pd (__A, 1); \ + __m256d __T2 = (__m256d) _mm512_extractf64x4_pd (__A, 0); \ + __m256d __T3 = __T1 op __T2; \ + __m128d __T4 = _mm256_extractf128_pd (__T3, 1); \ + __m128d __T5 = _mm256_extractf128_pd (__T3, 0); \ + __m128d __T6 = __T4 op __T5; \ + return __T6[0] op __T6[1] + +extern __inline double +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_add_pd (__m512d __A) +{ + __MM512_REDUCE_OP (+); +} + +extern __inline double +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_mul_pd (__m512d __A) +{ + __MM512_REDUCE_OP (*); +} + +extern __inline double +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_add_pd (__mmask8 __U, __m512d __A) +{ + __A = _mm512_maskz_mov_pd (__U, __A); + __MM512_REDUCE_OP (+); +} + +extern __inline double +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_mul_pd (__mmask8 __U, __m512d __A) +{ + __A = _mm512_mask_mov_pd (_mm512_set1_pd (1.0), __U, __A); + __MM512_REDUCE_OP (*); +} + +#undef __MM512_REDUCE_OP +#define __MM512_REDUCE_OP(op) \ + __m256d __T1 = (__m256d) _mm512_extractf64x4_pd (__A, 1); \ + __m256d __T2 = (__m256d) _mm512_extractf64x4_pd (__A, 0); \ + __m256d __T3 = _mm256_##op (__T1, __T2); \ + __m128d __T4 = _mm256_extractf128_pd (__T3, 1); \ + __m128d __T5 = _mm256_extractf128_pd (__T3, 0); \ + __m128d __T6 = _mm_##op (__T4, __T5); \ + __m128d __T7 = (__m128d) __builtin_shuffle (__T6, (__v2di) { 1, 0 }); \ + __m128d __T8 = _mm_##op (__T6, __T7); \ + return __T8[0] + +extern __inline double +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_min_pd (__m512d __A) +{ + __MM512_REDUCE_OP (min_pd); +} + +extern __inline double +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_max_pd (__m512d __A) +{ + __MM512_REDUCE_OP (max_pd); +} + +extern __inline double +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_min_pd (__mmask8 __U, __m512d __A) +{ + __A = _mm512_mask_mov_pd (_mm512_set1_pd (__builtin_inf ()), __U, __A); + __MM512_REDUCE_OP (min_pd); +} + +extern __inline double +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_max_pd (__mmask8 __U, __m512d __A) +{ + __A = _mm512_mask_mov_pd (_mm512_set1_pd (-__builtin_inf ()), __U, __A); + __MM512_REDUCE_OP (max_pd); +} + +#undef __MM512_REDUCE_OP + +#ifdef __DISABLE_AVX512F__ +#undef __DISABLE_AVX512F__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX512F__ */ + +#endif /* _AVX512FINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512ifmaintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512ifmaintrin.h new file mode 100644 index 0000000000..56502d29ba --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512ifmaintrin.h @@ -0,0 +1,104 @@ +/* Copyright (C) 2013-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +#error "Never use directly; include instead." +#endif + +#ifndef _AVX512IFMAINTRIN_H_INCLUDED +#define _AVX512IFMAINTRIN_H_INCLUDED + +#ifndef __AVX512IFMA__ +#pragma GCC push_options +//#pragma GCC target("avx512ifma") +#define __DISABLE_AVX512IFMA__ +#endif /* __AVX512IFMA__ */ + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_madd52lo_epu64 (__m512i __X, __m512i __Y, __m512i __Z) +{ + return (__m512i) __builtin_ia32_vpmadd52luq512_mask ((__v8di) __X, + (__v8di) __Y, + (__v8di) __Z, + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_madd52hi_epu64 (__m512i __X, __m512i __Y, __m512i __Z) +{ + return (__m512i) __builtin_ia32_vpmadd52huq512_mask ((__v8di) __X, + (__v8di) __Y, + (__v8di) __Z, + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_madd52lo_epu64 (__m512i __W, __mmask8 __M, __m512i __X, + __m512i __Y) +{ + return (__m512i) __builtin_ia32_vpmadd52luq512_mask ((__v8di) __W, + (__v8di) __X, + (__v8di) __Y, + (__mmask8) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_madd52hi_epu64 (__m512i __W, __mmask8 __M, __m512i __X, + __m512i __Y) +{ + return (__m512i) __builtin_ia32_vpmadd52huq512_mask ((__v8di) __W, + (__v8di) __X, + (__v8di) __Y, + (__mmask8) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_madd52lo_epu64 (__mmask8 __M, __m512i __X, __m512i __Y, __m512i __Z) +{ + return (__m512i) __builtin_ia32_vpmadd52luq512_maskz ((__v8di) __X, + (__v8di) __Y, + (__v8di) __Z, + (__mmask8) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_madd52hi_epu64 (__mmask8 __M, __m512i __X, __m512i __Y, __m512i __Z) +{ + return (__m512i) __builtin_ia32_vpmadd52huq512_maskz ((__v8di) __X, + (__v8di) __Y, + (__v8di) __Z, + (__mmask8) __M); +} + +#ifdef __DISABLE_AVX512IFMA__ +#undef __DISABLE_AVX512IFMA__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX512IFMA__ */ + +#endif /* _AVX512IFMAINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512ifmavlintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512ifmavlintrin.h new file mode 100644 index 0000000000..2522dd054f --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512ifmavlintrin.h @@ -0,0 +1,164 @@ +/* Copyright (C) 2013-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +#error "Never use directly; include instead." +#endif + +#ifndef _AVX512IFMAVLINTRIN_H_INCLUDED +#define _AVX512IFMAVLINTRIN_H_INCLUDED + +#if !defined(__AVX512VL__) || !defined(__AVX512IFMA__) +#pragma GCC push_options +//#pragma GCC target("avx512ifma,avx512vl") +#define __DISABLE_AVX512IFMAVL__ +#endif /* __AVX512IFMAVL__ */ + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_madd52lo_epu64 (__m128i __X, __m128i __Y, __m128i __Z) +{ + return (__m128i) __builtin_ia32_vpmadd52luq128_mask ((__v2di) __X, + (__v2di) __Y, + (__v2di) __Z, + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_madd52hi_epu64 (__m128i __X, __m128i __Y, __m128i __Z) +{ + return (__m128i) __builtin_ia32_vpmadd52huq128_mask ((__v2di) __X, + (__v2di) __Y, + (__v2di) __Z, + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_madd52lo_epu64 (__m256i __X, __m256i __Y, __m256i __Z) +{ + return (__m256i) __builtin_ia32_vpmadd52luq256_mask ((__v4di) __X, + (__v4di) __Y, + (__v4di) __Z, + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_madd52hi_epu64 (__m256i __X, __m256i __Y, __m256i __Z) +{ + return (__m256i) __builtin_ia32_vpmadd52huq256_mask ((__v4di) __X, + (__v4di) __Y, + (__v4di) __Z, + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_madd52lo_epu64 (__m128i __W, __mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_vpmadd52luq128_mask ((__v2di) __W, + (__v2di) __X, + (__v2di) __Y, + (__mmask8) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_madd52hi_epu64 (__m128i __W, __mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_vpmadd52huq128_mask ((__v2di) __W, + (__v2di) __X, + (__v2di) __Y, + (__mmask8) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_madd52lo_epu64 (__m256i __W, __mmask8 __M, __m256i __X, + __m256i __Y) +{ + return (__m256i) __builtin_ia32_vpmadd52luq256_mask ((__v4di) __W, + (__v4di) __X, + (__v4di) __Y, + (__mmask8) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_madd52hi_epu64 (__m256i __W, __mmask8 __M, __m256i __X, + __m256i __Y) +{ + return (__m256i) __builtin_ia32_vpmadd52huq256_mask ((__v4di) __W, + (__v4di) __X, + (__v4di) __Y, + (__mmask8) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_madd52lo_epu64 (__mmask8 __M, __m128i __X, __m128i __Y, __m128i __Z) +{ + return (__m128i) __builtin_ia32_vpmadd52luq128_maskz ((__v2di) __X, + (__v2di) __Y, + (__v2di) __Z, + (__mmask8) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_madd52hi_epu64 (__mmask8 __M, __m128i __X, __m128i __Y, __m128i __Z) +{ + return (__m128i) __builtin_ia32_vpmadd52huq128_maskz ((__v2di) __X, + (__v2di) __Y, + (__v2di) __Z, + (__mmask8) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_madd52lo_epu64 (__mmask8 __M, __m256i __X, __m256i __Y, __m256i __Z) +{ + return (__m256i) __builtin_ia32_vpmadd52luq256_maskz ((__v4di) __X, + (__v4di) __Y, + (__v4di) __Z, + (__mmask8) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_madd52hi_epu64 (__mmask8 __M, __m256i __X, __m256i __Y, __m256i __Z) +{ + return (__m256i) __builtin_ia32_vpmadd52huq256_maskz ((__v4di) __X, + (__v4di) __Y, + (__v4di) __Z, + (__mmask8) __M); +} + +#ifdef __DISABLE_AVX512IFMAVL__ +#undef __DISABLE_AVX512IFMAVL__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX512IFMAVL__ */ + +#endif /* _AVX512IFMAVLINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512pfintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512pfintrin.h new file mode 100644 index 0000000000..6e988954de --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512pfintrin.h @@ -0,0 +1,264 @@ +/* Copyright (C) 2013-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +#error "Never use directly; include instead." +#endif + +#ifndef _AVX512PFINTRIN_H_INCLUDED +#define _AVX512PFINTRIN_H_INCLUDED + +#ifndef __AVX512PF__ +#pragma GCC push_options +//#pragma GCC target("avx512pf") +#define __DISABLE_AVX512PF__ +#endif /* __AVX512PF__ */ + +/* Internal data types for implementing the intrinsics. */ +typedef long long __v8di __attribute__ ((__vector_size__ (64))); +typedef int __v16si __attribute__ ((__vector_size__ (64))); + +/* The Intel API is flexible enough that we must allow aliasing with other + vector types, and their scalar components. */ +typedef long long __m512i __attribute__ ((__vector_size__ (64), __may_alias__)); + +typedef unsigned char __mmask8; +typedef unsigned short __mmask16; + +#ifdef __OPTIMIZE__ +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_prefetch_i32gather_pd (__m256i __index, void const *__addr, + int __scale, int __hint) +{ + __builtin_ia32_gatherpfdpd ((__mmask8) 0xFF, (__v8si) __index, __addr, + __scale, __hint); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_prefetch_i32gather_ps (__m512i __index, void const *__addr, + int __scale, int __hint) +{ + __builtin_ia32_gatherpfdps ((__mmask16) 0xFFFF, (__v16si) __index, __addr, + __scale, __hint); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_prefetch_i32gather_pd (__m256i __index, __mmask8 __mask, + void const *__addr, int __scale, int __hint) +{ + __builtin_ia32_gatherpfdpd (__mask, (__v8si) __index, __addr, __scale, + __hint); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_prefetch_i32gather_ps (__m512i __index, __mmask16 __mask, + void const *__addr, int __scale, int __hint) +{ + __builtin_ia32_gatherpfdps (__mask, (__v16si) __index, __addr, __scale, + __hint); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_prefetch_i64gather_pd (__m512i __index, void const *__addr, + int __scale, int __hint) +{ + __builtin_ia32_gatherpfqpd ((__mmask8) 0xFF, (__v8di) __index, __addr, + __scale, __hint); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_prefetch_i64gather_ps (__m512i __index, void const *__addr, + int __scale, int __hint) +{ + __builtin_ia32_gatherpfqps ((__mmask8) 0xFF, (__v8di) __index, __addr, + __scale, __hint); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_prefetch_i64gather_pd (__m512i __index, __mmask8 __mask, + void const *__addr, int __scale, int __hint) +{ + __builtin_ia32_gatherpfqpd (__mask, (__v8di) __index, __addr, __scale, + __hint); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_prefetch_i64gather_ps (__m512i __index, __mmask8 __mask, + void const *__addr, int __scale, int __hint) +{ + __builtin_ia32_gatherpfqps (__mask, (__v8di) __index, __addr, __scale, + __hint); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_prefetch_i32scatter_pd (void *__addr, __m256i __index, int __scale, + int __hint) +{ + __builtin_ia32_scatterpfdpd ((__mmask8) 0xFF, (__v8si) __index, __addr, + __scale, __hint); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_prefetch_i32scatter_ps (void *__addr, __m512i __index, int __scale, + int __hint) +{ + __builtin_ia32_scatterpfdps ((__mmask16) 0xFFFF, (__v16si) __index, __addr, + __scale, __hint); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_prefetch_i32scatter_pd (void *__addr, __mmask8 __mask, + __m256i __index, int __scale, int __hint) +{ + __builtin_ia32_scatterpfdpd (__mask, (__v8si) __index, __addr, __scale, + __hint); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_prefetch_i32scatter_ps (void *__addr, __mmask16 __mask, + __m512i __index, int __scale, int __hint) +{ + __builtin_ia32_scatterpfdps (__mask, (__v16si) __index, __addr, __scale, + __hint); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_prefetch_i64scatter_pd (void *__addr, __m512i __index, int __scale, + int __hint) +{ + __builtin_ia32_scatterpfqpd ((__mmask8) 0xFF, (__v8di) __index,__addr, + __scale, __hint); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_prefetch_i64scatter_ps (void *__addr, __m512i __index, int __scale, + int __hint) +{ + __builtin_ia32_scatterpfqps ((__mmask8) 0xFF, (__v8di) __index, __addr, + __scale, __hint); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_prefetch_i64scatter_pd (void *__addr, __mmask16 __mask, + __m512i __index, int __scale, int __hint) +{ + __builtin_ia32_scatterpfqpd (__mask, (__v8di) __index, __addr, __scale, + __hint); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_prefetch_i64scatter_ps (void *__addr, __mmask16 __mask, + __m512i __index, int __scale, int __hint) +{ + __builtin_ia32_scatterpfqps (__mask, (__v8di) __index, __addr, __scale, + __hint); +} + +#else +#define _mm512_prefetch_i32gather_pd(INDEX, ADDR, SCALE, HINT) \ + __builtin_ia32_gatherpfdpd ((__mmask8)0xFF, (__v8si)(__m256i)INDEX, \ + (void const *)ADDR, (int)SCALE, (int)HINT) + +#define _mm512_prefetch_i32gather_ps(INDEX, ADDR, SCALE, HINT) \ + __builtin_ia32_gatherpfdps ((__mmask16)0xFFFF, (__v16si)(__m512i)INDEX, \ + (void const *)ADDR, (int)SCALE, (int)HINT) + +#define _mm512_mask_prefetch_i32gather_pd(INDEX, MASK, ADDR, SCALE, HINT) \ + __builtin_ia32_gatherpfdpd ((__mmask8)MASK, (__v8si)(__m256i)INDEX, \ + (void const *)ADDR, (int)SCALE, (int)HINT) + +#define _mm512_mask_prefetch_i32gather_ps(INDEX, MASK, ADDR, SCALE, HINT) \ + __builtin_ia32_gatherpfdps ((__mmask16)MASK, (__v16si)(__m512i)INDEX, \ + (void const *)ADDR, (int)SCALE, (int)HINT) + +#define _mm512_prefetch_i64gather_pd(INDEX, ADDR, SCALE, HINT) \ + __builtin_ia32_gatherpfqpd ((__mmask8)0xFF, (__v8di)(__m512i)INDEX, \ + (void *)ADDR, (int)SCALE, (int)HINT) + +#define _mm512_prefetch_i64gather_ps(INDEX, ADDR, SCALE, HINT) \ + __builtin_ia32_gatherpfqps ((__mmask8)0xFF, (__v8di)(__m512i)INDEX, \ + (void *)ADDR, (int)SCALE, (int)HINT) + +#define _mm512_mask_prefetch_i64gather_pd(INDEX, MASK, ADDR, SCALE, HINT) \ + __builtin_ia32_gatherpfqpd ((__mmask8)MASK, (__v8di)(__m512i)INDEX, \ + (void *)ADDR, (int)SCALE, (int)HINT) + +#define _mm512_mask_prefetch_i64gather_ps(INDEX, MASK, ADDR, SCALE, HINT) \ + __builtin_ia32_gatherpfqps ((__mmask8)MASK, (__v8di)(__m512i)INDEX, \ + (void *)ADDR, (int)SCALE, (int)HINT) + +#define _mm512_prefetch_i32scatter_pd(ADDR, INDEX, SCALE, HINT) \ + __builtin_ia32_scatterpfdpd ((__mmask8)0xFF, (__v8si)(__m256i)INDEX, \ + (void *)ADDR, (int)SCALE, (int)HINT) + +#define _mm512_prefetch_i32scatter_ps(ADDR, INDEX, SCALE, HINT) \ + __builtin_ia32_scatterpfdps ((__mmask16)0xFFFF, (__v16si)(__m512i)INDEX, \ + (void *)ADDR, (int)SCALE, (int)HINT) + +#define _mm512_mask_prefetch_i32scatter_pd(ADDR, MASK, INDEX, SCALE, HINT) \ + __builtin_ia32_scatterpfdpd ((__mmask8)MASK, (__v8si)(__m256i)INDEX, \ + (void *)ADDR, (int)SCALE, (int)HINT) + +#define _mm512_mask_prefetch_i32scatter_ps(ADDR, MASK, INDEX, SCALE, HINT) \ + __builtin_ia32_scatterpfdps ((__mmask16)MASK, (__v16si)(__m512i)INDEX, \ + (void *)ADDR, (int)SCALE, (int)HINT) + +#define _mm512_prefetch_i64scatter_pd(ADDR, INDEX, SCALE, HINT) \ + __builtin_ia32_scatterpfqpd ((__mmask8)0xFF, (__v8di)(__m512i)INDEX, \ + (void *)ADDR, (int)SCALE, (int)HINT) + +#define _mm512_prefetch_i64scatter_ps(ADDR, INDEX, SCALE, HINT) \ + __builtin_ia32_scatterpfqps ((__mmask8)0xFF, (__v8di)(__m512i)INDEX, \ + (void *)ADDR, (int)SCALE, (int)HINT) + +#define _mm512_mask_prefetch_i64scatter_pd(ADDR, MASK, INDEX, SCALE, HINT) \ + __builtin_ia32_scatterpfqpd ((__mmask8)MASK, (__v8di)(__m512i)INDEX, \ + (void *)ADDR, (int)SCALE, (int)HINT) + +#define _mm512_mask_prefetch_i64scatter_ps(ADDR, MASK, INDEX, SCALE, HINT) \ + __builtin_ia32_scatterpfqps ((__mmask8)MASK, (__v8di)(__m512i)INDEX, \ + (void *)ADDR, (int)SCALE, (int)HINT) +#endif + +#ifdef __DISABLE_AVX512PF__ +#undef __DISABLE_AVX512PF__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX512PF__ */ + +#endif /* _AVX512PFINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512vbmiintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512vbmiintrin.h new file mode 100644 index 0000000000..bb6d383a7c --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512vbmiintrin.h @@ -0,0 +1,158 @@ +/* Copyright (C) 2013-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +#error "Never use directly; include instead." +#endif + +#ifndef _AVX512VBMIINTRIN_H_INCLUDED +#define _AVX512VBMIINTRIN_H_INCLUDED + +#ifndef __AVX512VBMI__ +#pragma GCC push_options +//#pragma GCC target("avx512vbmi") +#define __DISABLE_AVX512VBMI__ +#endif /* __AVX512VBMI__ */ + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_multishift_epi64_epi8 (__m512i __W, __mmask64 __M, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_vpmultishiftqb512_mask ((__v64qi) __X, + (__v64qi) __Y, + (__v64qi) __W, + (__mmask64) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_multishift_epi64_epi8 (__mmask64 __M, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_vpmultishiftqb512_mask ((__v64qi) __X, + (__v64qi) __Y, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_multishift_epi64_epi8 (__m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_vpmultishiftqb512_mask ((__v64qi) __X, + (__v64qi) __Y, + (__v64qi) + _mm512_undefined_epi32 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permutexvar_epi8 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_permvarqi512_mask ((__v64qi) __B, + (__v64qi) __A, + (__v64qi) + _mm512_undefined_epi32 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permutexvar_epi8 (__mmask64 __M, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_permvarqi512_mask ((__v64qi) __B, + (__v64qi) __A, + (__v64qi) + _mm512_setzero_si512(), + (__mmask64) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permutexvar_epi8 (__m512i __W, __mmask64 __M, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_permvarqi512_mask ((__v64qi) __B, + (__v64qi) __A, + (__v64qi) __W, + (__mmask64) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permutex2var_epi8 (__m512i __A, __m512i __I, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpermt2varqi512_mask ((__v64qi) __I + /* idx */ , + (__v64qi) __A, + (__v64qi) __B, + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permutex2var_epi8 (__m512i __A, __mmask64 __U, + __m512i __I, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpermt2varqi512_mask ((__v64qi) __I + /* idx */ , + (__v64qi) __A, + (__v64qi) __B, + (__mmask64) + __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask2_permutex2var_epi8 (__m512i __A, __m512i __I, + __mmask64 __U, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpermi2varqi512_mask ((__v64qi) __A, + (__v64qi) __I + /* idx */ , + (__v64qi) __B, + (__mmask64) + __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permutex2var_epi8 (__mmask64 __U, __m512i __A, + __m512i __I, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpermt2varqi512_maskz ((__v64qi) __I + /* idx */ , + (__v64qi) __A, + (__v64qi) __B, + (__mmask64) + __U); +} + +#ifdef __DISABLE_AVX512VBMI__ +#undef __DISABLE_AVX512VBMI__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX512VBMI__ */ + +#endif /* _AVX512VBMIINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512vbmivlintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512vbmivlintrin.h new file mode 100644 index 0000000000..58f0cb26f2 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512vbmivlintrin.h @@ -0,0 +1,273 @@ +/* Copyright (C) 2013-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +#error "Never use directly; include instead." +#endif + +#ifndef _AVX512VBMIVLINTRIN_H_INCLUDED +#define _AVX512VBMIVLINTRIN_H_INCLUDED + +#if !defined(__AVX512VL__) || !defined(__AVX512VBMI__) +#pragma GCC push_options +//#pragma GCC target("avx512vbmi,avx512vl") +#define __DISABLE_AVX512VBMIVL__ +#endif /* __AVX512VBMIVL__ */ + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_multishift_epi64_epi8 (__m256i __W, __mmask32 __M, __m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_vpmultishiftqb256_mask ((__v32qi) __X, + (__v32qi) __Y, + (__v32qi) __W, + (__mmask32) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_multishift_epi64_epi8 (__mmask32 __M, __m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_vpmultishiftqb256_mask ((__v32qi) __X, + (__v32qi) __Y, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_multishift_epi64_epi8 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_vpmultishiftqb256_mask ((__v32qi) __X, + (__v32qi) __Y, + (__v32qi) + _mm256_undefined_si256 (), + (__mmask32) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_multishift_epi64_epi8 (__m128i __W, __mmask16 __M, __m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_vpmultishiftqb128_mask ((__v16qi) __X, + (__v16qi) __Y, + (__v16qi) __W, + (__mmask16) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_multishift_epi64_epi8 (__mmask16 __M, __m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_vpmultishiftqb128_mask ((__v16qi) __X, + (__v16qi) __Y, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_multishift_epi64_epi8 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_vpmultishiftqb128_mask ((__v16qi) __X, + (__v16qi) __Y, + (__v16qi) + _mm_undefined_si128 (), + (__mmask16) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permutexvar_epi8 (__m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_permvarqi256_mask ((__v32qi) __B, + (__v32qi) __A, + (__v32qi) + _mm256_undefined_si256 (), + (__mmask32) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permutexvar_epi8 (__mmask32 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_permvarqi256_mask ((__v32qi) __B, + (__v32qi) __A, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permutexvar_epi8 (__m256i __W, __mmask32 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_permvarqi256_mask ((__v32qi) __B, + (__v32qi) __A, + (__v32qi) __W, + (__mmask32) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permutexvar_epi8 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_permvarqi128_mask ((__v16qi) __B, + (__v16qi) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask16) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_permutexvar_epi8 (__mmask16 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_permvarqi128_mask ((__v16qi) __B, + (__v16qi) __A, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_permutexvar_epi8 (__m128i __W, __mmask16 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_permvarqi128_mask ((__v16qi) __B, + (__v16qi) __A, + (__v16qi) __W, + (__mmask16) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permutex2var_epi8 (__m256i __A, __m256i __I, __m256i __B) +{ + return (__m256i) __builtin_ia32_vpermt2varqi256_mask ((__v32qi) __I + /* idx */ , + (__v32qi) __A, + (__v32qi) __B, + (__mmask32) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permutex2var_epi8 (__m256i __A, __mmask32 __U, + __m256i __I, __m256i __B) +{ + return (__m256i) __builtin_ia32_vpermt2varqi256_mask ((__v32qi) __I + /* idx */ , + (__v32qi) __A, + (__v32qi) __B, + (__mmask32) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask2_permutex2var_epi8 (__m256i __A, __m256i __I, + __mmask32 __U, __m256i __B) +{ + return (__m256i) __builtin_ia32_vpermi2varqi256_mask ((__v32qi) __A, + (__v32qi) __I + /* idx */ , + (__v32qi) __B, + (__mmask32) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permutex2var_epi8 (__mmask32 __U, __m256i __A, + __m256i __I, __m256i __B) +{ + return (__m256i) __builtin_ia32_vpermt2varqi256_maskz ((__v32qi) __I + /* idx */ , + (__v32qi) __A, + (__v32qi) __B, + (__mmask32) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permutex2var_epi8 (__m128i __A, __m128i __I, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpermt2varqi128_mask ((__v16qi) __I + /* idx */ , + (__v16qi) __A, + (__v16qi) __B, + (__mmask16) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_permutex2var_epi8 (__m128i __A, __mmask16 __U, __m128i __I, + __m128i __B) +{ + return (__m128i) __builtin_ia32_vpermt2varqi128_mask ((__v16qi) __I + /* idx */ , + (__v16qi) __A, + (__v16qi) __B, + (__mmask16) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask2_permutex2var_epi8 (__m128i __A, __m128i __I, __mmask16 __U, + __m128i __B) +{ + return (__m128i) __builtin_ia32_vpermi2varqi128_mask ((__v16qi) __A, + (__v16qi) __I + /* idx */ , + (__v16qi) __B, + (__mmask16) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_permutex2var_epi8 (__mmask16 __U, __m128i __A, __m128i __I, + __m128i __B) +{ + return (__m128i) __builtin_ia32_vpermt2varqi128_maskz ((__v16qi) __I + /* idx */ , + (__v16qi) __A, + (__v16qi) __B, + (__mmask16) + __U); +} + +#ifdef __DISABLE_AVX512VBMIVL__ +#undef __DISABLE_AVX512VBMIVL__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX512VBMIVL__ */ + +#endif /* _AVX512VBMIVLINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512vlbwintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512vlbwintrin.h new file mode 100644 index 0000000000..6836814ff3 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512vlbwintrin.h @@ -0,0 +1,4651 @@ +/* Copyright (C) 2014-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +#error "Never use directly; include instead." +#endif + +#ifndef _AVX512VLBWINTRIN_H_INCLUDED +#define _AVX512VLBWINTRIN_H_INCLUDED + +#if !defined(__AVX512VL__) || !defined(__AVX512BW__) +#pragma GCC push_options +//#pragma GCC target("avx512vl,avx512bw") +#define __DISABLE_AVX512VLBW__ +#endif /* __AVX512VLBW__ */ + + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_mov_epi8 (__m256i __W, __mmask32 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_movdquqi256_mask ((__v32qi) __A, + (__v32qi) __W, + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_mov_epi8 (__mmask32 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_movdquqi256_mask ((__v32qi) __A, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_mov_epi8 (__m128i __W, __mmask16 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_movdquqi128_mask ((__v16qi) __A, + (__v16qi) __W, + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_mov_epi8 (__mmask16 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_movdquqi128_mask ((__v16qi) __A, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_storeu_epi8 (void *__P, __mmask32 __U, __m256i __A) +{ + __builtin_ia32_storedquqi256_mask ((char *) __P, + (__v32qi) __A, + (__mmask32) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_storeu_epi8 (void *__P, __mmask16 __U, __m128i __A) +{ + __builtin_ia32_storedquqi128_mask ((char *) __P, + (__v16qi) __A, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_loadu_epi16 (__m256i __W, __mmask16 __U, void const *__P) +{ + return (__m256i) __builtin_ia32_loaddquhi256_mask ((const short *) __P, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_loadu_epi16 (__mmask16 __U, void const *__P) +{ + return (__m256i) __builtin_ia32_loaddquhi256_mask ((const short *) __P, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_loadu_epi16 (__m128i __W, __mmask8 __U, void const *__P) +{ + return (__m128i) __builtin_ia32_loaddquhi128_mask ((const short *) __P, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_loadu_epi16 (__mmask8 __U, void const *__P) +{ + return (__m128i) __builtin_ia32_loaddquhi128_mask ((const short *) __P, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_mov_epi16 (__m256i __W, __mmask16 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_movdquhi256_mask ((__v16hi) __A, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_mov_epi16 (__mmask16 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_movdquhi256_mask ((__v16hi) __A, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_mov_epi16 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_movdquhi128_mask ((__v8hi) __A, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_mov_epi16 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_movdquhi128_mask ((__v8hi) __A, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_loadu_epi8 (__m256i __W, __mmask32 __U, void const *__P) +{ + return (__m256i) __builtin_ia32_loaddquqi256_mask ((const char *) __P, + (__v32qi) __W, + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_loadu_epi8 (__mmask32 __U, void const *__P) +{ + return (__m256i) __builtin_ia32_loaddquqi256_mask ((const char *) __P, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_loadu_epi8 (__m128i __W, __mmask16 __U, void const *__P) +{ + return (__m128i) __builtin_ia32_loaddquqi128_mask ((const char *) __P, + (__v16qi) __W, + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_loadu_epi8 (__mmask16 __U, void const *__P) +{ + return (__m128i) __builtin_ia32_loaddquqi128_mask ((const char *) __P, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi16_epi8 (__m256i __A) +{ + + return (__m128i) __builtin_ia32_pmovwb256_mask ((__v16hi) __A, + (__v16qi)_mm_undefined_si128(), + (__mmask16) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi16_epi8 (__m128i __O, __mmask16 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovwb256_mask ((__v16hi) __A, + (__v16qi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepi16_epi8 (__mmask16 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovwb256_mask ((__v16hi) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsepi16_epi8 (__m128i __A) +{ + + return (__m128i) __builtin_ia32_pmovswb128_mask ((__v8hi) __A, + (__v16qi)_mm_undefined_si128(), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtsepi16_epi8 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovswb128_mask ((__v8hi) __A, + (__v16qi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtsepi16_epi8 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovswb128_mask ((__v8hi) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtsepi16_epi8 (__m256i __A) +{ + + return (__m128i) __builtin_ia32_pmovswb256_mask ((__v16hi) __A, + (__v16qi)_mm_undefined_si128(), + (__mmask16) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtsepi16_epi8 (__m128i __O, __mmask16 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovswb256_mask ((__v16hi) __A, + (__v16qi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtsepi16_epi8 (__mmask16 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovswb256_mask ((__v16hi) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtusepi16_epi8 (__m128i __A) +{ + + return (__m128i) __builtin_ia32_pmovuswb128_mask ((__v8hi) __A, + (__v16qi)_mm_undefined_si128(), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtusepi16_epi8 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovuswb128_mask ((__v8hi) __A, + (__v16qi) __O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtusepi16_epi8 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovuswb128_mask ((__v8hi) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtusepi16_epi8 (__m256i __A) +{ + + return (__m128i) __builtin_ia32_pmovuswb256_mask ((__v16hi) __A, + (__v16qi)_mm_undefined_si128(), + (__mmask16) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtusepi16_epi8 (__m128i __O, __mmask16 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovuswb256_mask ((__v16hi) __A, + (__v16qi) __O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtusepi16_epi8 (__mmask16 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovuswb256_mask ((__v16hi) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_broadcastb_epi8 (__m256i __O, __mmask32 __M, __m128i __A) +{ + return (__m256i) __builtin_ia32_pbroadcastb256_mask ((__v16qi) __A, + (__v32qi) __O, + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_broadcastb_epi8 (__mmask32 __M, __m128i __A) +{ + return (__m256i) __builtin_ia32_pbroadcastb256_mask ((__v16qi) __A, + (__v32qi) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_set1_epi8 (__m256i __O, __mmask32 __M, char __A) +{ + return (__m256i) __builtin_ia32_pbroadcastb256_gpr_mask (__A, + (__v32qi) __O, + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_set1_epi8 (__mmask32 __M, char __A) +{ + return (__m256i) __builtin_ia32_pbroadcastb256_gpr_mask (__A, + (__v32qi) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_broadcastb_epi8 (__m128i __O, __mmask16 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pbroadcastb128_mask ((__v16qi) __A, + (__v16qi) __O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_broadcastb_epi8 (__mmask16 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pbroadcastb128_mask ((__v16qi) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_set1_epi8 (__m128i __O, __mmask16 __M, char __A) +{ + return (__m128i) __builtin_ia32_pbroadcastb128_gpr_mask (__A, + (__v16qi) __O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_set1_epi8 (__mmask16 __M, char __A) +{ + return (__m128i) __builtin_ia32_pbroadcastb128_gpr_mask (__A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_broadcastw_epi16 (__m256i __O, __mmask16 __M, __m128i __A) +{ + return (__m256i) __builtin_ia32_pbroadcastw256_mask ((__v8hi) __A, + (__v16hi) __O, + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_broadcastw_epi16 (__mmask16 __M, __m128i __A) +{ + return (__m256i) __builtin_ia32_pbroadcastw256_mask ((__v8hi) __A, + (__v16hi) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_set1_epi16 (__m256i __O, __mmask16 __M, short __A) +{ + return (__m256i) __builtin_ia32_pbroadcastw256_gpr_mask (__A, + (__v16hi) __O, + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_set1_epi16 (__mmask16 __M, short __A) +{ + return (__m256i) __builtin_ia32_pbroadcastw256_gpr_mask (__A, + (__v16hi) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_broadcastw_epi16 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pbroadcastw128_mask ((__v8hi) __A, + (__v8hi) __O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_broadcastw_epi16 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pbroadcastw128_mask ((__v8hi) __A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_set1_epi16 (__m128i __O, __mmask8 __M, short __A) +{ + return (__m128i) __builtin_ia32_pbroadcastw128_gpr_mask (__A, + (__v8hi) __O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_set1_epi16 (__mmask8 __M, short __A) +{ + return (__m128i) __builtin_ia32_pbroadcastw128_gpr_mask (__A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permutexvar_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_permvarhi256_mask ((__v16hi) __B, + (__v16hi) __A, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permutexvar_epi16 (__mmask16 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_permvarhi256_mask ((__v16hi) __B, + (__v16hi) __A, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permutexvar_epi16 (__m256i __W, __mmask16 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_permvarhi256_mask ((__v16hi) __B, + (__v16hi) __A, + (__v16hi) __W, + (__mmask16) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permutexvar_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_permvarhi128_mask ((__v8hi) __B, + (__v8hi) __A, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_permutexvar_epi16 (__mmask8 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_permvarhi128_mask ((__v8hi) __B, + (__v8hi) __A, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_permutexvar_epi16 (__m128i __W, __mmask8 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_permvarhi128_mask ((__v8hi) __B, + (__v8hi) __A, + (__v8hi) __W, + (__mmask8) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permutex2var_epi16 (__m256i __A, __m256i __I, __m256i __B) +{ + return (__m256i) __builtin_ia32_vpermt2varhi256_mask ((__v16hi) __I + /* idx */ , + (__v16hi) __A, + (__v16hi) __B, + (__mmask16) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permutex2var_epi16 (__m256i __A, __mmask16 __U, + __m256i __I, __m256i __B) +{ + return (__m256i) __builtin_ia32_vpermt2varhi256_mask ((__v16hi) __I + /* idx */ , + (__v16hi) __A, + (__v16hi) __B, + (__mmask16) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask2_permutex2var_epi16 (__m256i __A, __m256i __I, + __mmask16 __U, __m256i __B) +{ + return (__m256i) __builtin_ia32_vpermi2varhi256_mask ((__v16hi) __A, + (__v16hi) __I + /* idx */ , + (__v16hi) __B, + (__mmask16) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permutex2var_epi16 (__mmask16 __U, __m256i __A, + __m256i __I, __m256i __B) +{ + return (__m256i) __builtin_ia32_vpermt2varhi256_maskz ((__v16hi) __I + /* idx */ , + (__v16hi) __A, + (__v16hi) __B, + (__mmask16) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permutex2var_epi16 (__m128i __A, __m128i __I, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpermt2varhi128_mask ((__v8hi) __I + /* idx */ , + (__v8hi) __A, + (__v8hi) __B, + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_permutex2var_epi16 (__m128i __A, __mmask8 __U, __m128i __I, + __m128i __B) +{ + return (__m128i) __builtin_ia32_vpermt2varhi128_mask ((__v8hi) __I + /* idx */ , + (__v8hi) __A, + (__v8hi) __B, + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask2_permutex2var_epi16 (__m128i __A, __m128i __I, __mmask8 __U, + __m128i __B) +{ + return (__m128i) __builtin_ia32_vpermi2varhi128_mask ((__v8hi) __A, + (__v8hi) __I + /* idx */ , + (__v8hi) __B, + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_permutex2var_epi16 (__mmask8 __U, __m128i __A, __m128i __I, + __m128i __B) +{ + return (__m128i) __builtin_ia32_vpermt2varhi128_maskz ((__v8hi) __I + /* idx */ , + (__v8hi) __A, + (__v8hi) __B, + (__mmask8) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_maddubs_epi16 (__m256i __W, __mmask16 __U, __m256i __X, + __m256i __Y) +{ + return (__m256i) __builtin_ia32_pmaddubsw256_mask ((__v32qi) __X, + (__v32qi) __Y, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_maddubs_epi16 (__mmask16 __U, __m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_pmaddubsw256_mask ((__v32qi) __X, + (__v32qi) __Y, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_maddubs_epi16 (__m128i __W, __mmask8 __U, __m128i __X, + __m128i __Y) +{ + return (__m128i) __builtin_ia32_pmaddubsw128_mask ((__v16qi) __X, + (__v16qi) __Y, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_maddubs_epi16 (__mmask8 __U, __m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_pmaddubsw128_mask ((__v16qi) __X, + (__v16qi) __Y, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_madd_epi16 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaddwd256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_madd_epi16 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaddwd256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_madd_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaddwd128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_madd_epi16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaddwd128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movepi8_mask (__m128i __A) +{ + return (__mmask16) __builtin_ia32_cvtb2mask128 ((__v16qi) __A); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_movepi8_mask (__m256i __A) +{ + return (__mmask32) __builtin_ia32_cvtb2mask256 ((__v32qi) __A); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movepi16_mask (__m128i __A) +{ + return (__mmask8) __builtin_ia32_cvtw2mask128 ((__v8hi) __A); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_movepi16_mask (__m256i __A) +{ + return (__mmask16) __builtin_ia32_cvtw2mask256 ((__v16hi) __A); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movm_epi8 (__mmask16 __A) +{ + return (__m128i) __builtin_ia32_cvtmask2b128 (__A); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_movm_epi8 (__mmask32 __A) +{ + return (__m256i) __builtin_ia32_cvtmask2b256 (__A); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movm_epi16 (__mmask8 __A) +{ + return (__m128i) __builtin_ia32_cvtmask2w128 (__A); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_movm_epi16 (__mmask16 __A) +{ + return (__m256i) __builtin_ia32_cvtmask2w256 (__A); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_test_epi8_mask (__m128i __A, __m128i __B) +{ + return (__mmask16) __builtin_ia32_ptestmb128 ((__v16qi) __A, + (__v16qi) __B, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_test_epi8_mask (__mmask16 __U, __m128i __A, __m128i __B) +{ + return (__mmask16) __builtin_ia32_ptestmb128 ((__v16qi) __A, + (__v16qi) __B, __U); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_test_epi8_mask (__m256i __A, __m256i __B) +{ + return (__mmask32) __builtin_ia32_ptestmb256 ((__v32qi) __A, + (__v32qi) __B, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_test_epi8_mask (__mmask32 __U, __m256i __A, __m256i __B) +{ + return (__mmask32) __builtin_ia32_ptestmb256 ((__v32qi) __A, + (__v32qi) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_test_epi16_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ptestmw128 ((__v8hi) __A, + (__v8hi) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_test_epi16_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ptestmw128 ((__v8hi) __A, + (__v8hi) __B, __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_test_epi16_mask (__m256i __A, __m256i __B) +{ + return (__mmask16) __builtin_ia32_ptestmw256 ((__v16hi) __A, + (__v16hi) __B, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_test_epi16_mask (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__mmask16) __builtin_ia32_ptestmw256 ((__v16hi) __A, + (__v16hi) __B, __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_min_epu16 (__mmask16 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pminuw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_min_epu16 (__m256i __W, __mmask16 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pminuw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_min_epu16 (__mmask8 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pminuw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_min_epu16 (__m128i __W, __mmask8 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pminuw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_min_epi16 (__mmask16 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pminsw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_min_epi16 (__m256i __W, __mmask16 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pminsw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_max_epu8 (__mmask32 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxub256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_max_epu8 (__m256i __W, __mmask32 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxub256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) __W, + (__mmask32) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_max_epu8 (__mmask16 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxub128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_max_epu8 (__m128i __W, __mmask16 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxub128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) __W, + (__mmask16) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_max_epi8 (__mmask32 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxsb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_max_epi8 (__m256i __W, __mmask32 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxsb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) __W, + (__mmask32) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_max_epi8 (__mmask16 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxsb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_max_epi8 (__m128i __W, __mmask16 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxsb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) __W, + (__mmask16) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_min_epu8 (__mmask32 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pminub256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_min_epu8 (__m256i __W, __mmask32 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pminub256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) __W, + (__mmask32) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_min_epu8 (__mmask16 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pminub128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_min_epu8 (__m128i __W, __mmask16 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pminub128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) __W, + (__mmask16) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_min_epi8 (__mmask32 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pminsb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_min_epi8 (__m256i __W, __mmask32 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pminsb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) __W, + (__mmask32) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_min_epi8 (__mmask16 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pminsb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_min_epi8 (__m128i __W, __mmask16 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pminsb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) __W, + (__mmask16) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_max_epi16 (__mmask16 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxsw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_max_epi16 (__m256i __W, __mmask16 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxsw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_max_epi16 (__mmask8 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxsw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_max_epi16 (__m128i __W, __mmask8 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxsw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_max_epu16 (__mmask16 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxuw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_max_epu16 (__m256i __W, __mmask16 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxuw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_max_epu16 (__mmask8 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxuw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_max_epu16 (__m128i __W, __mmask8 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxuw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_min_epi16 (__mmask8 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pminsw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_min_epi16 (__m128i __W, __mmask8 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pminsw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __M); +} + +#ifdef __OPTIMIZE__ +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_alignr_epi8 (__m256i __W, __mmask32 __U, __m256i __A, + __m256i __B, const int __N) +{ + return (__m256i) __builtin_ia32_palignr256_mask ((__v4di) __A, + (__v4di) __B, + __N * 8, + (__v4di) __W, + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_alignr_epi8 (__mmask32 __U, __m256i __A, __m256i __B, + const int __N) +{ + return (__m256i) __builtin_ia32_palignr256_mask ((__v4di) __A, + (__v4di) __B, + __N * 8, + (__v4di) + _mm256_setzero_si256 (), + (__mmask32) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_alignr_epi8 (__m128i __W, __mmask16 __U, __m128i __A, + __m128i __B, const int __N) +{ + return (__m128i) __builtin_ia32_palignr128_mask ((__v2di) __A, + (__v2di) __B, + __N * 8, + (__v2di) __W, + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_alignr_epi8 (__mmask16 __U, __m128i __A, __m128i __B, + const int __N) +{ + return (__m128i) __builtin_ia32_palignr128_mask ((__v2di) __A, + (__v2di) __B, + __N * 8, + (__v2di) + _mm_setzero_si128 (), + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_dbsad_epu8 (__m256i __A, __m256i __B, const int __imm) +{ + return (__m256i) __builtin_ia32_dbpsadbw256_mask ((__v32qi) __A, + (__v32qi) __B, + __imm, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_dbsad_epu8 (__m256i __W, __mmask16 __U, __m256i __A, + __m256i __B, const int __imm) +{ + return (__m256i) __builtin_ia32_dbpsadbw256_mask ((__v32qi) __A, + (__v32qi) __B, + __imm, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_dbsad_epu8 (__mmask16 __U, __m256i __A, __m256i __B, + const int __imm) +{ + return (__m256i) __builtin_ia32_dbpsadbw256_mask ((__v32qi) __A, + (__v32qi) __B, + __imm, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_dbsad_epu8 (__m128i __A, __m128i __B, const int __imm) +{ + return (__m128i) __builtin_ia32_dbpsadbw128_mask ((__v16qi) __A, + (__v16qi) __B, + __imm, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_dbsad_epu8 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B, const int __imm) +{ + return (__m128i) __builtin_ia32_dbpsadbw128_mask ((__v16qi) __A, + (__v16qi) __B, + __imm, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_dbsad_epu8 (__mmask8 __U, __m128i __A, __m128i __B, + const int __imm) +{ + return (__m128i) __builtin_ia32_dbpsadbw128_mask ((__v16qi) __A, + (__v16qi) __B, + __imm, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_blend_epi16 (__mmask8 __U, __m128i __A, __m128i __W) +{ + return (__m128i) __builtin_ia32_blendmw_128_mask ((__v8hi) __A, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_blend_epi8 (__mmask16 __U, __m128i __A, __m128i __W) +{ + return (__m128i) __builtin_ia32_blendmb_128_mask ((__v16qi) __A, + (__v16qi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_blend_epi16 (__mmask16 __U, __m256i __A, __m256i __W) +{ + return (__m256i) __builtin_ia32_blendmw_256_mask ((__v16hi) __A, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_blend_epi8 (__mmask32 __U, __m256i __A, __m256i __W) +{ + return (__m256i) __builtin_ia32_blendmb_256_mask ((__v32qi) __A, + (__v32qi) __W, + (__mmask32) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmp_epi16_mask (__mmask8 __U, __m128i __X, __m128i __Y, + const int __P) +{ + return (__mmask8) __builtin_ia32_cmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, __P, + (__mmask8) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_epi16_mask (__m128i __X, __m128i __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_cmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, __P, + (__mmask8) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmp_epi16_mask (__mmask16 __U, __m256i __X, __m256i __Y, + const int __P) +{ + return (__mmask16) __builtin_ia32_cmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, __P, + (__mmask16) __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmp_epi16_mask (__m256i __X, __m256i __Y, const int __P) +{ + return (__mmask16) __builtin_ia32_cmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, __P, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmp_epi8_mask (__mmask8 __U, __m128i __X, __m128i __Y, + const int __P) +{ + return (__mmask16) __builtin_ia32_cmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, __P, + (__mmask16) __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_epi8_mask (__m128i __X, __m128i __Y, const int __P) +{ + return (__mmask16) __builtin_ia32_cmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, __P, + (__mmask16) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmp_epi8_mask (__mmask16 __U, __m256i __X, __m256i __Y, + const int __P) +{ + return (__mmask32) __builtin_ia32_cmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, __P, + (__mmask32) __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmp_epi8_mask (__m256i __X, __m256i __Y, const int __P) +{ + return (__mmask32) __builtin_ia32_cmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, __P, + (__mmask32) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmp_epu16_mask (__mmask8 __U, __m128i __X, __m128i __Y, + const int __P) +{ + return (__mmask8) __builtin_ia32_ucmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, __P, + (__mmask8) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_epu16_mask (__m128i __X, __m128i __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_ucmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, __P, + (__mmask8) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmp_epu16_mask (__mmask16 __U, __m256i __X, __m256i __Y, + const int __P) +{ + return (__mmask16) __builtin_ia32_ucmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, __P, + (__mmask16) __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmp_epu16_mask (__m256i __X, __m256i __Y, const int __P) +{ + return (__mmask16) __builtin_ia32_ucmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, __P, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmp_epu8_mask (__mmask8 __U, __m128i __X, __m128i __Y, + const int __P) +{ + return (__mmask16) __builtin_ia32_ucmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, __P, + (__mmask16) __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_epu8_mask (__m128i __X, __m128i __Y, const int __P) +{ + return (__mmask16) __builtin_ia32_ucmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, __P, + (__mmask16) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmp_epu8_mask (__mmask16 __U, __m256i __X, __m256i __Y, + const int __P) +{ + return (__mmask32) __builtin_ia32_ucmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, __P, + (__mmask32) __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmp_epu8_mask (__m256i __X, __m256i __Y, const int __P) +{ + return (__mmask32) __builtin_ia32_ucmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, __P, + (__mmask32) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_srli_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + const int __imm) +{ + return (__m256i) __builtin_ia32_psrlwi256_mask ((__v16hi) __A, __imm, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_srli_epi16 (__mmask16 __U, __m256i __A, const int __imm) +{ + return (__m256i) __builtin_ia32_psrlwi256_mask ((__v16hi) __A, __imm, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_srli_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + const int __imm) +{ + return (__m128i) __builtin_ia32_psrlwi128_mask ((__v8hi) __A, __imm, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_srli_epi16 (__mmask8 __U, __m128i __A, const int __imm) +{ + return (__m128i) __builtin_ia32_psrlwi128_mask ((__v8hi) __A, __imm, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_shufflehi_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + const int __imm) +{ + return (__m256i) __builtin_ia32_pshufhw256_mask ((__v16hi) __A, + __imm, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_shufflehi_epi16 (__mmask16 __U, __m256i __A, + const int __imm) +{ + return (__m256i) __builtin_ia32_pshufhw256_mask ((__v16hi) __A, + __imm, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_shufflehi_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + const int __imm) +{ + return (__m128i) __builtin_ia32_pshufhw128_mask ((__v8hi) __A, __imm, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_shufflehi_epi16 (__mmask8 __U, __m128i __A, const int __imm) +{ + return (__m128i) __builtin_ia32_pshufhw128_mask ((__v8hi) __A, __imm, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_shufflelo_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + const int __imm) +{ + return (__m256i) __builtin_ia32_pshuflw256_mask ((__v16hi) __A, + __imm, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_shufflelo_epi16 (__mmask16 __U, __m256i __A, + const int __imm) +{ + return (__m256i) __builtin_ia32_pshuflw256_mask ((__v16hi) __A, + __imm, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_shufflelo_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + const int __imm) +{ + return (__m128i) __builtin_ia32_pshuflw128_mask ((__v8hi) __A, __imm, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_shufflelo_epi16 (__mmask8 __U, __m128i __A, const int __imm) +{ + return (__m128i) __builtin_ia32_pshuflw128_mask ((__v8hi) __A, __imm, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_srai_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + const int __imm) +{ + return (__m256i) __builtin_ia32_psrawi256_mask ((__v16hi) __A, __imm, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_srai_epi16 (__mmask16 __U, __m256i __A, const int __imm) +{ + return (__m256i) __builtin_ia32_psrawi256_mask ((__v16hi) __A, __imm, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_srai_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + const int __imm) +{ + return (__m128i) __builtin_ia32_psrawi128_mask ((__v8hi) __A, __imm, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_srai_epi16 (__mmask8 __U, __m128i __A, const int __imm) +{ + return (__m128i) __builtin_ia32_psrawi128_mask ((__v8hi) __A, __imm, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_slli_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + int __B) +{ + return (__m256i) __builtin_ia32_psllwi256_mask ((__v16hi) __A, __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_slli_epi16 (__mmask16 __U, __m256i __A, int __B) +{ + return (__m256i) __builtin_ia32_psllwi256_mask ((__v16hi) __A, __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_slli_epi16 (__m128i __W, __mmask8 __U, __m128i __A, int __B) +{ + return (__m128i) __builtin_ia32_psllwi128_mask ((__v8hi) __A, __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_slli_epi16 (__mmask8 __U, __m128i __A, int __B) +{ + return (__m128i) __builtin_ia32_psllwi128_mask ((__v8hi) __A, __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +#else +#define _mm256_mask_alignr_epi8(W, U, X, Y, N) \ + ((__m256i) __builtin_ia32_palignr256_mask ((__v4di)(__m256i)(X), \ + (__v4di)(__m256i)(Y), (int)(N * 8), \ + (__v4di)(__m256i)(X), (__mmask32)(U))) + +#define _mm256_mask_srli_epi16(W, U, A, B) \ + ((__m256i) __builtin_ia32_psrlwi256_mask ((__v16hi)(__m256i)(A), \ + (int)(B), (__v16hi)(__m256i)(W), (__mmask16)(U))) + +#define _mm256_maskz_srli_epi16(U, A, B) \ + ((__m256i) __builtin_ia32_psrlwi256_mask ((__v16hi)(__m256i)(A), \ + (int)(B), (__v16hi)_mm256_setzero_si256 (), (__mmask16)(U))) + +#define _mm_mask_srli_epi16(W, U, A, B) \ + ((__m128i) __builtin_ia32_psrlwi128_mask ((__v8hi)(__m128i)(A), \ + (int)(B), (__v8hi)(__m128i)(W), (__mmask8)(U))) + +#define _mm_maskz_srli_epi16(U, A, B) \ + ((__m128i) __builtin_ia32_psrlwi128_mask ((__v8hi)(__m128i)(A), \ + (int)(B), (__v8hi)_mm_setzero_si128(), (__mmask8)(U))) + +#define _mm256_mask_srai_epi16(W, U, A, B) \ + ((__m256i) __builtin_ia32_psrawi256_mask ((__v16hi)(__m256i)(A), \ + (int)(B), (__v16hi)(__m256i)(W), (__mmask16)(U))) + +#define _mm256_maskz_srai_epi16(U, A, B) \ + ((__m256i) __builtin_ia32_psrawi256_mask ((__v16hi)(__m256i)(A), \ + (int)(B), (__v16hi)_mm256_setzero_si256 (), (__mmask16)(U))) + +#define _mm_mask_srai_epi16(W, U, A, B) \ + ((__m128i) __builtin_ia32_psrawi128_mask ((__v8hi)(__m128i)(A), \ + (int)(B), (__v8hi)(__m128i)(W), (__mmask8)(U))) + +#define _mm_maskz_srai_epi16(U, A, B) \ + ((__m128i) __builtin_ia32_psrawi128_mask ((__v8hi)(__m128i)(A), \ + (int)(B), (__v8hi)_mm_setzero_si128(), (__mmask8)(U))) + +#define _mm256_mask_shufflehi_epi16(W, U, A, B) \ + ((__m256i) __builtin_ia32_pshufhw256_mask ((__v16hi)(__m256i)(A), (int)(B), \ + (__v16hi)(__m256i)(W), \ + (__mmask16)(U))) + +#define _mm256_maskz_shufflehi_epi16(U, A, B) \ + ((__m256i) __builtin_ia32_pshufhw256_mask ((__v16hi)(__m256i)(A), (int)(B), \ + (__v16hi)(__m256i)_mm256_setzero_si256 (), \ + (__mmask16)(U))) + +#define _mm_mask_shufflehi_epi16(W, U, A, B) \ + ((__m128i) __builtin_ia32_pshufhw128_mask ((__v8hi)(__m128i)(A), (int)(B), \ + (__v8hi)(__m128i)(W), \ + (__mmask8)(U))) + +#define _mm_maskz_shufflehi_epi16(U, A, B) \ + ((__m128i) __builtin_ia32_pshufhw128_mask ((__v8hi)(__m128i)(A), (int)(B), \ + (__v8hi)(__m128i)_mm_setzero_si128 (), \ + (__mmask8)(U))) + +#define _mm256_mask_shufflelo_epi16(W, U, A, B) \ + ((__m256i) __builtin_ia32_pshuflw256_mask ((__v16hi)(__m256i)(A), (int)(B), \ + (__v16hi)(__m256i)(W), \ + (__mmask16)(U))) + +#define _mm256_maskz_shufflelo_epi16(U, A, B) \ + ((__m256i) __builtin_ia32_pshuflw256_mask ((__v16hi)(__m256i)(A), (int)(B), \ + (__v16hi)(__m256i)_mm256_setzero_si256 (), \ + (__mmask16)(U))) + +#define _mm_mask_shufflelo_epi16(W, U, A, B) \ + ((__m128i) __builtin_ia32_pshuflw128_mask ((__v8hi)(__m128i)(A), (int)(B), \ + (__v8hi)(__m128i)(W), \ + (__mmask8)(U))) + +#define _mm_maskz_shufflelo_epi16(U, A, B) \ + ((__m128i) __builtin_ia32_pshuflw128_mask ((__v8hi)(__m128i)(A), (int)(B), \ + (__v8hi)(__m128i)_mm_setzero_si128 (), \ + (__mmask8)(U))) + +#define _mm256_maskz_alignr_epi8(U, X, Y, N) \ + ((__m256i) __builtin_ia32_palignr256_mask ((__v4di)(__m256i)(X), \ + (__v4di)(__m256i)(Y), (int)(N * 8), \ + (__v4di)(__m256i)_mm256_setzero_si256 (), \ + (__mmask32)(U))) + +#define _mm_mask_alignr_epi8(W, U, X, Y, N) \ + ((__m128i) __builtin_ia32_palignr128_mask ((__v2di)(__m128i)(X), \ + (__v2di)(__m128i)(Y), (int)(N * 8), \ + (__v2di)(__m128i)(X), (__mmask16)(U))) + +#define _mm_maskz_alignr_epi8(U, X, Y, N) \ + ((__m128i) __builtin_ia32_palignr128_mask ((__v2di)(__m128i)(X), \ + (__v2di)(__m128i)(Y), (int)(N * 8), \ + (__v2di)(__m128i)_mm_setzero_si128 (), \ + (__mmask16)(U))) + +#define _mm_mask_slli_epi16(W, U, X, C) \ + ((__m128i)__builtin_ia32_psllwi128_mask ((__v8hi)(__m128i)(X), (int)(C),\ + (__v8hi)(__m128i)(W),\ + (__mmask8)(U))) + +#define _mm_maskz_slli_epi16(U, X, C) \ + ((__m128i)__builtin_ia32_psllwi128_mask ((__v8hi)(__m128i)(X), (int)(C),\ + (__v8hi)(__m128i)_mm_setzero_si128 (),\ + (__mmask8)(U))) + +#define _mm256_dbsad_epu8(X, Y, C) \ + ((__m256i) __builtin_ia32_dbpsadbw256_mask ((__v32qi)(__m256i) (X), \ + (__v32qi)(__m256i) (Y), (int) (C), \ + (__v16hi)(__m256i)_mm256_setzero_si256(),\ + (__mmask16)-1)) + +#define _mm256_mask_slli_epi16(W, U, X, C) \ + ((__m256i)__builtin_ia32_psllwi256_mask ((__v16hi)(__m256i)(X), (int)(C),\ + (__v16hi)(__m256i)(W),\ + (__mmask16)(U))) + +#define _mm256_maskz_slli_epi16(U, X, C) \ + ((__m256i)__builtin_ia32_psllwi256_mask ((__v16hi)(__m256i)(X), (int)(C),\ + (__v16hi)(__m256i)_mm256_setzero_si256 (),\ + (__mmask16)(U))) + +#define _mm256_mask_dbsad_epu8(W, U, X, Y, C) \ + ((__m256i) __builtin_ia32_dbpsadbw256_mask ((__v32qi)(__m256i) (X), \ + (__v32qi)(__m256i) (Y), (int) (C), \ + (__v16hi)(__m256i)(W), \ + (__mmask16)(U))) + +#define _mm256_maskz_dbsad_epu8(U, X, Y, C) \ + ((__m256i) __builtin_ia32_dbpsadbw256_mask ((__v32qi)(__m256i) (X), \ + (__v32qi)(__m256i) (Y), (int) (C), \ + (__v16hi)(__m256i)_mm256_setzero_si256(),\ + (__mmask16)(U))) + +#define _mm_dbsad_epu8(X, Y, C) \ + ((__m128i) __builtin_ia32_dbpsadbw128_mask ((__v16qi)(__m128i) (X), \ + (__v16qi)(__m128i) (Y), (int) (C), \ + (__v8hi)(__m128i)_mm_setzero_si128(), \ + (__mmask8)-1)) + +#define _mm_mask_dbsad_epu8(W, U, X, Y, C) \ + ((__m128i) __builtin_ia32_dbpsadbw128_mask ((__v16qi)(__m128i) (X), \ + (__v16qi)(__m128i) (Y), (int) (C), \ + (__v8hi)(__m128i)(W), \ + (__mmask8)(U))) + +#define _mm_maskz_dbsad_epu8(U, X, Y, C) \ + ((__m128i) __builtin_ia32_dbpsadbw128_mask ((__v16qi)(__m128i) (X), \ + (__v16qi)(__m128i) (Y), (int) (C), \ + (__v8hi)(__m128i)_mm_setzero_si128(), \ + (__mmask8)(U))) + +#define _mm_mask_blend_epi16(__U, __A, __W) \ + ((__m128i) __builtin_ia32_blendmw_128_mask ((__v8hi) (__A), \ + (__v8hi) (__W), \ + (__mmask8) (__U))) + +#define _mm_mask_blend_epi8(__U, __A, __W) \ + ((__m128i) __builtin_ia32_blendmb_128_mask ((__v16qi) (__A), \ + (__v16qi) (__W), \ + (__mmask16) (__U))) + +#define _mm256_mask_blend_epi16(__U, __A, __W) \ + ((__m256i) __builtin_ia32_blendmw_256_mask ((__v16hi) (__A), \ + (__v16hi) (__W), \ + (__mmask16) (__U))) + +#define _mm256_mask_blend_epi8(__U, __A, __W) \ + ((__m256i) __builtin_ia32_blendmb_256_mask ((__v32qi) (__A), \ + (__v32qi) (__W), \ + (__mmask32) (__U))) + +#define _mm_cmp_epi16_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpw128_mask ((__v8hi)(__m128i)(X), \ + (__v8hi)(__m128i)(Y), (int)(P),\ + (__mmask8)(-1))) + +#define _mm_cmp_epi8_mask(X, Y, P) \ + ((__mmask16) __builtin_ia32_cmpb128_mask ((__v16qi)(__m128i)(X), \ + (__v16qi)(__m128i)(Y), (int)(P),\ + (__mmask16)(-1))) + +#define _mm256_cmp_epi16_mask(X, Y, P) \ + ((__mmask16) __builtin_ia32_cmpw256_mask ((__v16hi)(__m256i)(X), \ + (__v16hi)(__m256i)(Y), (int)(P),\ + (__mmask16)(-1))) + +#define _mm256_cmp_epi8_mask(X, Y, P) \ + ((__mmask32) __builtin_ia32_cmpb256_mask ((__v32qi)(__m256i)(X), \ + (__v32qi)(__m256i)(Y), (int)(P),\ + (__mmask32)(-1))) + +#define _mm_cmp_epu16_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_ucmpw128_mask ((__v8hi)(__m128i)(X), \ + (__v8hi)(__m128i)(Y), (int)(P),\ + (__mmask8)(-1))) + +#define _mm_cmp_epu8_mask(X, Y, P) \ + ((__mmask16) __builtin_ia32_ucmpb128_mask ((__v16qi)(__m128i)(X), \ + (__v16qi)(__m128i)(Y), (int)(P),\ + (__mmask16)(-1))) + +#define _mm256_cmp_epu16_mask(X, Y, P) \ + ((__mmask16) __builtin_ia32_ucmpw256_mask ((__v16hi)(__m256i)(X), \ + (__v16hi)(__m256i)(Y), (int)(P),\ + (__mmask16)(-1))) + +#define _mm256_cmp_epu8_mask(X, Y, P) \ + ((__mmask32) __builtin_ia32_ucmpb256_mask ((__v32qi)(__m256i)(X), \ + (__v32qi)(__m256i)(Y), (int)(P),\ + (__mmask32)-1)) + +#define _mm_mask_cmp_epi16_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpw128_mask ((__v8hi)(__m128i)(X), \ + (__v8hi)(__m128i)(Y), (int)(P),\ + (__mmask16)(M))) + +#define _mm_mask_cmp_epi8_mask(M, X, Y, P) \ + ((__mmask16) __builtin_ia32_cmpb128_mask ((__v16qi)(__m128i)(X), \ + (__v16qi)(__m128i)(Y), (int)(P),\ + (__mmask16)(M))) + +#define _mm256_mask_cmp_epi16_mask(M, X, Y, P) \ + ((__mmask16) __builtin_ia32_cmpw256_mask ((__v16hi)(__m256i)(X), \ + (__v16hi)(__m256i)(Y), (int)(P),\ + (__mmask16)(M))) + +#define _mm256_mask_cmp_epi8_mask(M, X, Y, P) \ + ((__mmask32) __builtin_ia32_cmpb256_mask ((__v32qi)(__m256i)(X), \ + (__v32qi)(__m256i)(Y), (int)(P),\ + (__mmask32)(M))) + +#define _mm_mask_cmp_epu16_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_ucmpw128_mask ((__v8hi)(__m128i)(X), \ + (__v8hi)(__m128i)(Y), (int)(P),\ + (__mmask8)(M))) + +#define _mm_mask_cmp_epu8_mask(M, X, Y, P) \ + ((__mmask16) __builtin_ia32_ucmpb128_mask ((__v16qi)(__m128i)(X), \ + (__v16qi)(__m128i)(Y), (int)(P),\ + (__mmask16)(M))) + +#define _mm256_mask_cmp_epu16_mask(M, X, Y, P) \ + ((__mmask16) __builtin_ia32_ucmpw256_mask ((__v16hi)(__m256i)(X), \ + (__v16hi)(__m256i)(Y), (int)(P),\ + (__mmask16)(M))) + +#define _mm256_mask_cmp_epu8_mask(M, X, Y, P) \ + ((__mmask32) __builtin_ia32_ucmpb256_mask ((__v32qi)(__m256i)(X), \ + (__v32qi)(__m256i)(Y), (int)(P),\ + (__mmask32)M)) +#endif + +extern __inline __mmask32 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpneq_epi8_mask (__m256i __X, __m256i __Y) +{ + return (__mmask32) __builtin_ia32_cmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, 4, + (__mmask32) -1); +} + +extern __inline __mmask32 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmplt_epi8_mask (__m256i __X, __m256i __Y) +{ + return (__mmask32) __builtin_ia32_cmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, 1, + (__mmask32) -1); +} + +extern __inline __mmask32 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpge_epi8_mask (__m256i __X, __m256i __Y) +{ + return (__mmask32) __builtin_ia32_cmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, 5, + (__mmask32) -1); +} + +extern __inline __mmask32 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmple_epi8_mask (__m256i __X, __m256i __Y) +{ + return (__mmask32) __builtin_ia32_cmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, 2, + (__mmask32) -1); +} + +extern __inline __mmask16 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpneq_epi16_mask (__m256i __X, __m256i __Y) +{ + return (__mmask16) __builtin_ia32_cmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, 4, + (__mmask16) -1); +} + +extern __inline __mmask16 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmplt_epi16_mask (__m256i __X, __m256i __Y) +{ + return (__mmask16) __builtin_ia32_cmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, 1, + (__mmask16) -1); +} + +extern __inline __mmask16 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpge_epi16_mask (__m256i __X, __m256i __Y) +{ + return (__mmask16) __builtin_ia32_cmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, 5, + (__mmask16) -1); +} + +extern __inline __mmask16 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmple_epi16_mask (__m256i __X, __m256i __Y) +{ + return (__mmask16) __builtin_ia32_cmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, 2, + (__mmask16) -1); +} + +extern __inline __mmask16 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpneq_epu8_mask (__m128i __X, __m128i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, 4, + (__mmask16) -1); +} + +extern __inline __mmask16 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmplt_epu8_mask (__m128i __X, __m128i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, 1, + (__mmask16) -1); +} + +extern __inline __mmask16 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpge_epu8_mask (__m128i __X, __m128i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, 5, + (__mmask16) -1); +} + +extern __inline __mmask16 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmple_epu8_mask (__m128i __X, __m128i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, 2, + (__mmask16) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpneq_epu16_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, 4, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmplt_epu16_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, 1, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpge_epu16_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, 5, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmple_epu16_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, 2, + (__mmask8) -1); +} + +extern __inline __mmask16 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpneq_epi8_mask (__m128i __X, __m128i __Y) +{ + return (__mmask16) __builtin_ia32_cmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, 4, + (__mmask16) -1); +} + +extern __inline __mmask16 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmplt_epi8_mask (__m128i __X, __m128i __Y) +{ + return (__mmask16) __builtin_ia32_cmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, 1, + (__mmask16) -1); +} + +extern __inline __mmask16 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpge_epi8_mask (__m128i __X, __m128i __Y) +{ + return (__mmask16) __builtin_ia32_cmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, 5, + (__mmask16) -1); +} + +extern __inline __mmask16 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmple_epi8_mask (__m128i __X, __m128i __Y) +{ + return (__mmask16) __builtin_ia32_cmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, 2, + (__mmask16) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpneq_epi16_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, 4, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmplt_epi16_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, 1, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpge_epi16_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, 5, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmple_epi16_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, 2, + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_mulhrs_epi16 (__m256i __W, __mmask16 __U, __m256i __X, + __m256i __Y) +{ + return (__m256i) __builtin_ia32_pmulhrsw256_mask ((__v16hi) __X, + (__v16hi) __Y, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_mulhrs_epi16 (__mmask16 __U, __m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_pmulhrsw256_mask ((__v16hi) __X, + (__v16hi) __Y, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_mulhi_epu16 (__m256i __W, __mmask16 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pmulhuw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_mulhi_epu16 (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pmulhuw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_mulhi_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pmulhw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_mulhi_epi16 (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pmulhw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_mulhi_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pmulhw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_mulhi_epi16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pmulhw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_mulhi_epu16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pmulhuw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_mulhi_epu16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pmulhuw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_mulhrs_epi16 (__m128i __W, __mmask8 __U, __m128i __X, + __m128i __Y) +{ + return (__m128i) __builtin_ia32_pmulhrsw128_mask ((__v8hi) __X, + (__v8hi) __Y, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_mulhrs_epi16 (__mmask8 __U, __m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_pmulhrsw128_mask ((__v8hi) __X, + (__v8hi) __Y, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_mullo_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pmullw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_mullo_epi16 (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pmullw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_mullo_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pmullw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_mullo_epi16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pmullw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi8_epi16 (__m256i __W, __mmask32 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovsxbw256_mask ((__v16qi) __A, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepi8_epi16 (__mmask16 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovsxbw256_mask ((__v16qi) __A, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi8_epi16 (__m128i __W, __mmask32 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsxbw128_mask ((__v16qi) __A, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepi8_epi16 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsxbw128_mask ((__v16qi) __A, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepu8_epi16 (__m256i __W, __mmask32 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovzxbw256_mask ((__v16qi) __A, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepu8_epi16 (__mmask16 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovzxbw256_mask ((__v16qi) __A, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepu8_epi16 (__m128i __W, __mmask32 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovzxbw128_mask ((__v16qi) __A, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepu8_epi16 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovzxbw128_mask ((__v16qi) __A, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_avg_epu8 (__m256i __W, __mmask32 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pavgb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) __W, + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_avg_epu8 (__mmask32 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pavgb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_avg_epu8 (__m128i __W, __mmask16 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pavgb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) __W, + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_avg_epu8 (__mmask16 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pavgb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_avg_epu16 (__m256i __W, __mmask16 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pavgw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_avg_epu16 (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pavgw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_avg_epu16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pavgw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_avg_epu16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pavgw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_add_epi8 (__m256i __W, __mmask32 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_paddb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) __W, + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_add_epi8 (__mmask32 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_paddb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_add_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_paddw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_add_epi16 (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_paddw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_adds_epi8 (__m256i __W, __mmask32 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_paddsb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) __W, + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_adds_epi8 (__mmask32 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_paddsb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_adds_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_paddsw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_adds_epi16 (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_paddsw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_adds_epu8 (__m256i __W, __mmask32 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_paddusb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) __W, + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_adds_epu8 (__mmask32 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_paddusb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_adds_epu16 (__m256i __W, __mmask16 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_paddusw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_adds_epu16 (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_paddusw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sub_epi8 (__m256i __W, __mmask32 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_psubb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) __W, + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sub_epi8 (__mmask32 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_psubb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sub_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_psubw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sub_epi16 (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_psubw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_subs_epi8 (__m256i __W, __mmask32 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_psubsb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) __W, + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_subs_epi8 (__mmask32 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_psubsb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_subs_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_psubsw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_subs_epi16 (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_psubsw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_subs_epu8 (__m256i __W, __mmask32 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_psubusb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) __W, + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_subs_epu8 (__mmask32 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_psubusb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_subs_epu16 (__m256i __W, __mmask16 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_psubusw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_subs_epu16 (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_psubusw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_add_epi8 (__m128i __W, __mmask16 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_paddb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) __W, + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_add_epi8 (__mmask16 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_paddb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_add_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_paddw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_add_epi16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_paddw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_unpackhi_epi8 (__m256i __W, __mmask32 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_punpckhbw256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) __W, + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_unpackhi_epi8 (__mmask32 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_punpckhbw256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_unpackhi_epi8 (__m128i __W, __mmask16 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_punpckhbw128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) __W, + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_unpackhi_epi8 (__mmask16 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_punpckhbw128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_unpackhi_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_punpckhwd256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_unpackhi_epi16 (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_punpckhwd256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_unpackhi_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_punpckhwd128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_unpackhi_epi16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_punpckhwd128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_unpacklo_epi8 (__m256i __W, __mmask32 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_punpcklbw256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) __W, + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_unpacklo_epi8 (__mmask32 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_punpcklbw256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_unpacklo_epi8 (__m128i __W, __mmask16 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_punpcklbw128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) __W, + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_unpacklo_epi8 (__mmask16 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_punpcklbw128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_unpacklo_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_punpcklwd256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_unpacklo_epi16 (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_punpcklwd256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_unpacklo_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_punpcklwd128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_unpacklo_epi16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_punpcklwd128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_epi8_mask (__m128i __A, __m128i __B) +{ + return (__mmask16) __builtin_ia32_pcmpeqb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_epu8_mask (__m128i __A, __m128i __B) +{ + return (__mmask16) __builtin_ia32_ucmpb128_mask ((__v16qi) __A, + (__v16qi) __B, 0, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpeq_epu8_mask (__mmask16 __U, __m128i __A, __m128i __B) +{ + return (__mmask16) __builtin_ia32_ucmpb128_mask ((__v16qi) __A, + (__v16qi) __B, 0, + __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpeq_epi8_mask (__mmask16 __U, __m128i __A, __m128i __B) +{ + return (__mmask16) __builtin_ia32_pcmpeqb128_mask ((__v16qi) __A, + (__v16qi) __B, + __U); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpeq_epu8_mask (__m256i __A, __m256i __B) +{ + return (__mmask32) __builtin_ia32_ucmpb256_mask ((__v32qi) __A, + (__v32qi) __B, 0, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpeq_epi8_mask (__m256i __A, __m256i __B) +{ + return (__mmask32) __builtin_ia32_pcmpeqb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpeq_epu8_mask (__mmask32 __U, __m256i __A, __m256i __B) +{ + return (__mmask32) __builtin_ia32_ucmpb256_mask ((__v32qi) __A, + (__v32qi) __B, 0, + __U); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpeq_epi8_mask (__mmask32 __U, __m256i __A, __m256i __B) +{ + return (__mmask32) __builtin_ia32_pcmpeqb256_mask ((__v32qi) __A, + (__v32qi) __B, + __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_epu16_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ucmpw128_mask ((__v8hi) __A, + (__v8hi) __B, 0, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_epi16_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_pcmpeqw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpeq_epu16_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ucmpw128_mask ((__v8hi) __A, + (__v8hi) __B, 0, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpeq_epi16_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_pcmpeqw128_mask ((__v8hi) __A, + (__v8hi) __B, __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpeq_epu16_mask (__m256i __A, __m256i __B) +{ + return (__mmask16) __builtin_ia32_ucmpw256_mask ((__v16hi) __A, + (__v16hi) __B, 0, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpeq_epi16_mask (__m256i __A, __m256i __B) +{ + return (__mmask16) __builtin_ia32_pcmpeqw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpeq_epu16_mask (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__mmask16) __builtin_ia32_ucmpw256_mask ((__v16hi) __A, + (__v16hi) __B, 0, + __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpeq_epi16_mask (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__mmask16) __builtin_ia32_pcmpeqw256_mask ((__v16hi) __A, + (__v16hi) __B, + __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_epu8_mask (__m128i __A, __m128i __B) +{ + return (__mmask16) __builtin_ia32_ucmpb128_mask ((__v16qi) __A, + (__v16qi) __B, 6, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_epi8_mask (__m128i __A, __m128i __B) +{ + return (__mmask16) __builtin_ia32_pcmpgtb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpgt_epu8_mask (__mmask16 __U, __m128i __A, __m128i __B) +{ + return (__mmask16) __builtin_ia32_ucmpb128_mask ((__v16qi) __A, + (__v16qi) __B, 6, + __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpgt_epi8_mask (__mmask16 __U, __m128i __A, __m128i __B) +{ + return (__mmask16) __builtin_ia32_pcmpgtb128_mask ((__v16qi) __A, + (__v16qi) __B, + __U); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpgt_epu8_mask (__m256i __A, __m256i __B) +{ + return (__mmask32) __builtin_ia32_ucmpb256_mask ((__v32qi) __A, + (__v32qi) __B, 6, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpgt_epi8_mask (__m256i __A, __m256i __B) +{ + return (__mmask32) __builtin_ia32_pcmpgtb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpgt_epu8_mask (__mmask32 __U, __m256i __A, __m256i __B) +{ + return (__mmask32) __builtin_ia32_ucmpb256_mask ((__v32qi) __A, + (__v32qi) __B, 6, + __U); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpgt_epi8_mask (__mmask32 __U, __m256i __A, __m256i __B) +{ + return (__mmask32) __builtin_ia32_pcmpgtb256_mask ((__v32qi) __A, + (__v32qi) __B, + __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_epu16_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ucmpw128_mask ((__v8hi) __A, + (__v8hi) __B, 6, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_epi16_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_pcmpgtw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpgt_epu16_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ucmpw128_mask ((__v8hi) __A, + (__v8hi) __B, 6, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpgt_epi16_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_pcmpgtw128_mask ((__v8hi) __A, + (__v8hi) __B, __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpgt_epu16_mask (__m256i __A, __m256i __B) +{ + return (__mmask16) __builtin_ia32_ucmpw256_mask ((__v16hi) __A, + (__v16hi) __B, 6, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpgt_epi16_mask (__m256i __A, __m256i __B) +{ + return (__mmask16) __builtin_ia32_pcmpgtw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpgt_epu16_mask (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__mmask16) __builtin_ia32_ucmpw256_mask ((__v16hi) __A, + (__v16hi) __B, 6, + __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpgt_epi16_mask (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__mmask16) __builtin_ia32_pcmpgtw256_mask ((__v16hi) __A, + (__v16hi) __B, + __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_testn_epi8_mask (__m128i __A, __m128i __B) +{ + return (__mmask16) __builtin_ia32_ptestnmb128 ((__v16qi) __A, + (__v16qi) __B, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_testn_epi8_mask (__mmask16 __U, __m128i __A, __m128i __B) +{ + return (__mmask16) __builtin_ia32_ptestnmb128 ((__v16qi) __A, + (__v16qi) __B, __U); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testn_epi8_mask (__m256i __A, __m256i __B) +{ + return (__mmask32) __builtin_ia32_ptestnmb256 ((__v32qi) __A, + (__v32qi) __B, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_testn_epi8_mask (__mmask32 __U, __m256i __A, __m256i __B) +{ + return (__mmask32) __builtin_ia32_ptestnmb256 ((__v32qi) __A, + (__v32qi) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_testn_epi16_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ptestnmw128 ((__v8hi) __A, + (__v8hi) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_testn_epi16_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ptestnmw128 ((__v8hi) __A, + (__v8hi) __B, __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testn_epi16_mask (__m256i __A, __m256i __B) +{ + return (__mmask16) __builtin_ia32_ptestnmw256 ((__v16hi) __A, + (__v16hi) __B, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_testn_epi16_mask (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__mmask16) __builtin_ia32_ptestnmw256 ((__v16hi) __A, + (__v16hi) __B, __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_shuffle_epi8 (__m256i __W, __mmask32 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pshufb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) __W, + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_shuffle_epi8 (__mmask32 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pshufb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_shuffle_epi8 (__m128i __W, __mmask16 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pshufb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) __W, + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_shuffle_epi8 (__mmask16 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pshufb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_packs_epi16 (__mmask32 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_packsswb256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v32qi) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_packs_epi16 (__m256i __W, __mmask32 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_packsswb256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v32qi) __W, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_packs_epi16 (__mmask16 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_packsswb128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_packs_epi16 (__m128i __W, __mmask16 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_packsswb128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v16qi) __W, + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_packus_epi16 (__mmask32 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_packuswb256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v32qi) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_packus_epi16 (__m256i __W, __mmask32 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_packuswb256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v32qi) __W, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_packus_epi16 (__mmask16 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_packuswb128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_packus_epi16 (__m128i __W, __mmask16 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_packuswb128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v16qi) __W, + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_abs_epi8 (__m256i __W, __mmask32 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_pabsb256_mask ((__v32qi) __A, + (__v32qi) __W, + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_abs_epi8 (__mmask32 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_pabsb256_mask ((__v32qi) __A, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_abs_epi8 (__m128i __W, __mmask16 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pabsb128_mask ((__v16qi) __A, + (__v16qi) __W, + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_abs_epi8 (__mmask16 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pabsb128_mask ((__v16qi) __A, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_abs_epi16 (__m256i __W, __mmask16 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_pabsw256_mask ((__v16hi) __A, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_abs_epi16 (__mmask16 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_pabsw256_mask ((__v16hi) __A, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_abs_epi16 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pabsw128_mask ((__v8hi) __A, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_abs_epi16 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pabsw128_mask ((__v8hi) __A, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __mmask32 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpneq_epu8_mask (__m256i __X, __m256i __Y) +{ + return (__mmask32) __builtin_ia32_ucmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, 4, + (__mmask32) -1); +} + +extern __inline __mmask32 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmplt_epu8_mask (__m256i __X, __m256i __Y) +{ + return (__mmask32) __builtin_ia32_ucmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, 1, + (__mmask32) -1); +} + +extern __inline __mmask32 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpge_epu8_mask (__m256i __X, __m256i __Y) +{ + return (__mmask32) __builtin_ia32_ucmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, 5, + (__mmask32) -1); +} + +extern __inline __mmask32 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmple_epu8_mask (__m256i __X, __m256i __Y) +{ + return (__mmask32) __builtin_ia32_ucmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, 2, + (__mmask32) -1); +} + +extern __inline __mmask16 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpneq_epu16_mask (__m256i __X, __m256i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, 4, + (__mmask16) -1); +} + +extern __inline __mmask16 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmplt_epu16_mask (__m256i __X, __m256i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, 1, + (__mmask16) -1); +} + +extern __inline __mmask16 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpge_epu16_mask (__m256i __X, __m256i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, 5, + (__mmask16) -1); +} + +extern __inline __mmask16 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmple_epu16_mask (__m256i __X, __m256i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, 2, + (__mmask16) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_storeu_epi16 (void *__P, __mmask16 __U, __m256i __A) +{ + __builtin_ia32_storedquhi256_mask ((short *) __P, + (__v16hi) __A, + (__mmask16) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_storeu_epi16 (void *__P, __mmask8 __U, __m128i __A) +{ + __builtin_ia32_storedquhi128_mask ((short *) __P, + (__v8hi) __A, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_adds_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_paddsw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_subs_epi8 (__m128i __W, __mmask16 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psubsb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) __W, + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_subs_epi8 (__mmask16 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psubsb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_subs_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psubsw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_subs_epi16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psubsw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_subs_epu8 (__m128i __W, __mmask16 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psubusb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) __W, + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_subs_epu8 (__mmask16 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psubusb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_subs_epu16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psubusw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_subs_epu16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psubusw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_srl_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + __m128i __B) +{ + return (__m256i) __builtin_ia32_psrlw256_mask ((__v16hi) __A, + (__v8hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_srl_epi16 (__mmask16 __U, __m256i __A, __m128i __B) +{ + return (__m256i) __builtin_ia32_psrlw256_mask ((__v16hi) __A, + (__v8hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_srl_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psrlw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_srl_epi16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psrlw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sra_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + __m128i __B) +{ + return (__m256i) __builtin_ia32_psraw256_mask ((__v16hi) __A, + (__v8hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sra_epi16 (__mmask16 __U, __m256i __A, __m128i __B) +{ + return (__m256i) __builtin_ia32_psraw256_mask ((__v16hi) __A, + (__v8hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sra_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psraw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sra_epi16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psraw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_adds_epi16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_paddsw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_adds_epu8 (__m128i __W, __mmask16 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_paddusb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) __W, + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_adds_epu8 (__mmask16 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_paddusb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_adds_epu16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_paddusw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_adds_epu16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_paddusw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sub_epi8 (__m128i __W, __mmask16 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psubb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) __W, + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sub_epi8 (__mmask16 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psubb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sub_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psubw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sub_epi16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psubw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_adds_epi8 (__m128i __W, __mmask16 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_paddsb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) __W, + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_adds_epi8 (__mmask16 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_paddsb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepi16_epi8 (__m128i __A) +{ + + return (__m128i) __builtin_ia32_pmovwb128_mask ((__v8hi) __A, + (__v16qi)_mm_undefined_si128(), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi16_epi8 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovwb128_mask ((__v8hi) __A, + (__v16qi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepi16_epi8 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovwb128_mask ((__v8hi) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_srav_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_psrav16hi_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_srav_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_psrav16hi_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_srav_epi16 (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_psrav16hi_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srav_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psrav8hi_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_srav_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psrav8hi_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_srav_epi16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psrav8hi_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_srlv_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_psrlv16hi_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_srlv_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_psrlv16hi_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_srlv_epi16 (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_psrlv16hi_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srlv_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psrlv8hi_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_srlv_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psrlv8hi_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_srlv_epi16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psrlv8hi_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sllv_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_psllv16hi_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sllv_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_psllv16hi_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sllv_epi16 (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_psllv16hi_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sllv_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psllv8hi_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sllv_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psllv8hi_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sllv_epi16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psllv8hi_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sll_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psllw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sll_epi16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psllw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sll_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + __m128i __B) +{ + return (__m256i) __builtin_ia32_psllw256_mask ((__v16hi) __A, + (__v8hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sll_epi16 (__mmask16 __U, __m256i __A, __m128i __B) +{ + return (__m256i) __builtin_ia32_psllw256_mask ((__v16hi) __A, + (__v8hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_packus_epi32 (__mmask16 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_packusdw256_mask ((__v8si) __A, + (__v8si) __B, + (__v16hi) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_packus_epi32 (__m256i __W, __mmask16 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_packusdw256_mask ((__v8si) __A, + (__v8si) __B, + (__v16hi) __W, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_packus_epi32 (__mmask8 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_packusdw128_mask ((__v4si) __A, + (__v4si) __B, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_packus_epi32 (__m128i __W, __mmask16 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_packusdw128_mask ((__v4si) __A, + (__v4si) __B, + (__v8hi) __W, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_packs_epi32 (__mmask16 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_packssdw256_mask ((__v8si) __A, + (__v8si) __B, + (__v16hi) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_packs_epi32 (__m256i __W, __mmask16 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_packssdw256_mask ((__v8si) __A, + (__v8si) __B, + (__v16hi) __W, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_packs_epi32 (__mmask8 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_packssdw128_mask ((__v4si) __A, + (__v4si) __B, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_packs_epi32 (__m128i __W, __mmask16 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_packssdw128_mask ((__v4si) __A, + (__v4si) __B, + (__v8hi) __W, __M); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpneq_epu8_mask (__mmask16 __M, __m128i __X, __m128i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, 4, + (__mmask16) __M); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmplt_epu8_mask (__mmask16 __M, __m128i __X, __m128i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, 1, + (__mmask16) __M); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpge_epu8_mask (__mmask16 __M, __m128i __X, __m128i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, 5, + (__mmask16) __M); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmple_epu8_mask (__mmask16 __M, __m128i __X, __m128i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, 2, + (__mmask16) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpneq_epu16_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, 4, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmplt_epu16_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, 1, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpge_epu16_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, 5, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmple_epu16_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, 2, + (__mmask8) __M); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpneq_epi8_mask (__mmask16 __M, __m128i __X, __m128i __Y) +{ + return (__mmask16) __builtin_ia32_cmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, 4, + (__mmask16) __M); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmplt_epi8_mask (__mmask16 __M, __m128i __X, __m128i __Y) +{ + return (__mmask16) __builtin_ia32_cmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, 1, + (__mmask16) __M); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpge_epi8_mask (__mmask16 __M, __m128i __X, __m128i __Y) +{ + return (__mmask16) __builtin_ia32_cmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, 5, + (__mmask16) __M); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmple_epi8_mask (__mmask16 __M, __m128i __X, __m128i __Y) +{ + return (__mmask16) __builtin_ia32_cmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, 2, + (__mmask16) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpneq_epi16_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, 4, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmplt_epi16_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, 1, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpge_epi16_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, 5, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmple_epi16_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, 2, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpneq_epu8_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, 4, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmplt_epu8_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, 1, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpge_epu8_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, 5, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmple_epu8_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, 2, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpneq_epu16_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, 4, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmplt_epu16_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, 1, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpge_epu16_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, 5, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmple_epu16_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, 2, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpneq_epi8_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, 4, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmplt_epi8_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, 1, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpge_epi8_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, 5, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmple_epi8_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, 2, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpneq_epi16_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, 4, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmplt_epi16_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, 1, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpge_epi16_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, 5, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmple_epi16_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, 2, + (__mmask8) __M); +} + +#ifdef __DISABLE_AVX512VLBW__ +#undef __DISABLE_AVX512VLBW__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX512VLBW__ */ + +#endif /* _AVX512VLBWINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512vldqintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512vldqintrin.h new file mode 100644 index 0000000000..dee74e4f4a --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512vldqintrin.h @@ -0,0 +1,2016 @@ +/* Copyright (C) 2014-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +#error "Never use directly; include instead." +#endif + +#ifndef _AVX512VLDQINTRIN_H_INCLUDED +#define _AVX512VLDQINTRIN_H_INCLUDED + +#if !defined(__AVX512VL__) || !defined(__AVX512DQ__) +#pragma GCC push_options +//#pragma GCC target("avx512vl,avx512dq") +#define __DISABLE_AVX512VLDQ__ +#endif /* __AVX512VLDQ__ */ + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvttpd_epi64 (__m256d __A) +{ + return (__m256i) __builtin_ia32_cvttpd2qq256_mask ((__v4df) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvttpd_epi64 (__m256i __W, __mmask8 __U, __m256d __A) +{ + return (__m256i) __builtin_ia32_cvttpd2qq256_mask ((__v4df) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvttpd_epi64 (__mmask8 __U, __m256d __A) +{ + return (__m256i) __builtin_ia32_cvttpd2qq256_mask ((__v4df) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttpd_epi64 (__m128d __A) +{ + return (__m128i) __builtin_ia32_cvttpd2qq128_mask ((__v2df) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvttpd_epi64 (__m128i __W, __mmask8 __U, __m128d __A) +{ + return (__m128i) __builtin_ia32_cvttpd2qq128_mask ((__v2df) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvttpd_epi64 (__mmask8 __U, __m128d __A) +{ + return (__m128i) __builtin_ia32_cvttpd2qq128_mask ((__v2df) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvttpd_epu64 (__m256d __A) +{ + return (__m256i) __builtin_ia32_cvttpd2uqq256_mask ((__v4df) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvttpd_epu64 (__m256i __W, __mmask8 __U, __m256d __A) +{ + return (__m256i) __builtin_ia32_cvttpd2uqq256_mask ((__v4df) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvttpd_epu64 (__mmask8 __U, __m256d __A) +{ + return (__m256i) __builtin_ia32_cvttpd2uqq256_mask ((__v4df) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttpd_epu64 (__m128d __A) +{ + return (__m128i) __builtin_ia32_cvttpd2uqq128_mask ((__v2df) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvttpd_epu64 (__m128i __W, __mmask8 __U, __m128d __A) +{ + return (__m128i) __builtin_ia32_cvttpd2uqq128_mask ((__v2df) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvttpd_epu64 (__mmask8 __U, __m128d __A) +{ + return (__m128i) __builtin_ia32_cvttpd2uqq128_mask ((__v2df) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtpd_epi64 (__m256d __A) +{ + return (__m256i) __builtin_ia32_cvtpd2qq256_mask ((__v4df) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtpd_epi64 (__m256i __W, __mmask8 __U, __m256d __A) +{ + return (__m256i) __builtin_ia32_cvtpd2qq256_mask ((__v4df) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtpd_epi64 (__mmask8 __U, __m256d __A) +{ + return (__m256i) __builtin_ia32_cvtpd2qq256_mask ((__v4df) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtpd_epi64 (__m128d __A) +{ + return (__m128i) __builtin_ia32_cvtpd2qq128_mask ((__v2df) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtpd_epi64 (__m128i __W, __mmask8 __U, __m128d __A) +{ + return (__m128i) __builtin_ia32_cvtpd2qq128_mask ((__v2df) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtpd_epi64 (__mmask8 __U, __m128d __A) +{ + return (__m128i) __builtin_ia32_cvtpd2qq128_mask ((__v2df) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtpd_epu64 (__m256d __A) +{ + return (__m256i) __builtin_ia32_cvtpd2uqq256_mask ((__v4df) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtpd_epu64 (__m256i __W, __mmask8 __U, __m256d __A) +{ + return (__m256i) __builtin_ia32_cvtpd2uqq256_mask ((__v4df) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtpd_epu64 (__mmask8 __U, __m256d __A) +{ + return (__m256i) __builtin_ia32_cvtpd2uqq256_mask ((__v4df) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtpd_epu64 (__m128d __A) +{ + return (__m128i) __builtin_ia32_cvtpd2uqq128_mask ((__v2df) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtpd_epu64 (__m128i __W, __mmask8 __U, __m128d __A) +{ + return (__m128i) __builtin_ia32_cvtpd2uqq128_mask ((__v2df) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtpd_epu64 (__mmask8 __U, __m128d __A) +{ + return (__m128i) __builtin_ia32_cvtpd2uqq128_mask ((__v2df) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvttps_epi64 (__m128 __A) +{ + return (__m256i) __builtin_ia32_cvttps2qq256_mask ((__v4sf) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvttps_epi64 (__m256i __W, __mmask8 __U, __m128 __A) +{ + return (__m256i) __builtin_ia32_cvttps2qq256_mask ((__v4sf) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvttps_epi64 (__mmask8 __U, __m128 __A) +{ + return (__m256i) __builtin_ia32_cvttps2qq256_mask ((__v4sf) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttps_epi64 (__m128 __A) +{ + return (__m128i) __builtin_ia32_cvttps2qq128_mask ((__v4sf) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvttps_epi64 (__m128i __W, __mmask8 __U, __m128 __A) +{ + return (__m128i) __builtin_ia32_cvttps2qq128_mask ((__v4sf) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvttps_epi64 (__mmask8 __U, __m128 __A) +{ + return (__m128i) __builtin_ia32_cvttps2qq128_mask ((__v4sf) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvttps_epu64 (__m128 __A) +{ + return (__m256i) __builtin_ia32_cvttps2uqq256_mask ((__v4sf) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvttps_epu64 (__m256i __W, __mmask8 __U, __m128 __A) +{ + return (__m256i) __builtin_ia32_cvttps2uqq256_mask ((__v4sf) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvttps_epu64 (__mmask8 __U, __m128 __A) +{ + return (__m256i) __builtin_ia32_cvttps2uqq256_mask ((__v4sf) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttps_epu64 (__m128 __A) +{ + return (__m128i) __builtin_ia32_cvttps2uqq128_mask ((__v4sf) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvttps_epu64 (__m128i __W, __mmask8 __U, __m128 __A) +{ + return (__m128i) __builtin_ia32_cvttps2uqq128_mask ((__v4sf) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvttps_epu64 (__mmask8 __U, __m128 __A) +{ + return (__m128i) __builtin_ia32_cvttps2uqq128_mask ((__v4sf) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcast_f64x2 (__m128d __A) +{ + return (__m256d) __builtin_ia32_broadcastf64x2_256_mask ((__v2df) + __A, + (__v4df)_mm256_undefined_pd(), + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_broadcast_f64x2 (__m256d __O, __mmask8 __M, __m128d __A) +{ + return (__m256d) __builtin_ia32_broadcastf64x2_256_mask ((__v2df) + __A, + (__v4df) + __O, __M); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_broadcast_f64x2 (__mmask8 __M, __m128d __A) +{ + return (__m256d) __builtin_ia32_broadcastf64x2_256_mask ((__v2df) + __A, + (__v4df) + _mm256_setzero_ps (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcast_i64x2 (__m128i __A) +{ + return (__m256i) __builtin_ia32_broadcasti64x2_256_mask ((__v2di) + __A, + (__v4di)_mm256_undefined_si256(), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_broadcast_i64x2 (__m256i __O, __mmask8 __M, __m128i __A) +{ + return (__m256i) __builtin_ia32_broadcasti64x2_256_mask ((__v2di) + __A, + (__v4di) + __O, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_broadcast_i64x2 (__mmask8 __M, __m128i __A) +{ + return (__m256i) __builtin_ia32_broadcasti64x2_256_mask ((__v2di) + __A, + (__v4di) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcast_f32x2 (__m128 __A) +{ + return (__m256) __builtin_ia32_broadcastf32x2_256_mask ((__v4sf) __A, + (__v8sf)_mm256_undefined_ps(), + (__mmask8) -1); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_broadcast_f32x2 (__m256 __O, __mmask8 __M, __m128 __A) +{ + return (__m256) __builtin_ia32_broadcastf32x2_256_mask ((__v4sf) __A, + (__v8sf) __O, + __M); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_broadcast_f32x2 (__mmask8 __M, __m128 __A) +{ + return (__m256) __builtin_ia32_broadcastf32x2_256_mask ((__v4sf) __A, + (__v8sf) + _mm256_setzero_ps (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcast_i32x2 (__m128i __A) +{ + return (__m256i) __builtin_ia32_broadcasti32x2_256_mask ((__v4si) + __A, + (__v8si)_mm256_undefined_si256(), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_broadcast_i32x2 (__m256i __O, __mmask8 __M, __m128i __A) +{ + return (__m256i) __builtin_ia32_broadcasti32x2_256_mask ((__v4si) + __A, + (__v8si) + __O, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_broadcast_i32x2 (__mmask8 __M, __m128i __A) +{ + return (__m256i) __builtin_ia32_broadcasti32x2_256_mask ((__v4si) + __A, + (__v8si) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_broadcast_i32x2 (__m128i __A) +{ + return (__m128i) __builtin_ia32_broadcasti32x2_128_mask ((__v4si) + __A, + (__v4si)_mm_undefined_si128(), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_broadcast_i32x2 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_broadcasti32x2_128_mask ((__v4si) + __A, + (__v4si) + __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_broadcast_i32x2 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_broadcasti32x2_128_mask ((__v4si) + __A, + (__v4si) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mullo_epi64 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v4du) __A * (__v4du) __B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_mullo_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pmullq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_mullo_epi64 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pmullq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mullo_epi64 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v2du) __A * (__v2du) __B); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_mullo_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pmullq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_mullo_epi64 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pmullq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_andnot_pd (__m256d __W, __mmask8 __U, __m256d __A, + __m256d __B) +{ + return (__m256d) __builtin_ia32_andnpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_andnot_pd (__mmask8 __U, __m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_andnpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_andnot_pd (__m128d __W, __mmask8 __U, __m128d __A, + __m128d __B) +{ + return (__m128d) __builtin_ia32_andnpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_andnot_pd (__mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_andnpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_andnot_ps (__m256 __W, __mmask8 __U, __m256 __A, + __m256 __B) +{ + return (__m256) __builtin_ia32_andnps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_andnot_ps (__mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_andnps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_andnot_ps (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_andnps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_andnot_ps (__mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_andnps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtps_epi64 (__m128 __A) +{ + return (__m256i) __builtin_ia32_cvtps2qq256_mask ((__v4sf) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtps_epi64 (__m256i __W, __mmask8 __U, __m128 __A) +{ + return (__m256i) __builtin_ia32_cvtps2qq256_mask ((__v4sf) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtps_epi64 (__mmask8 __U, __m128 __A) +{ + return (__m256i) __builtin_ia32_cvtps2qq256_mask ((__v4sf) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtps_epi64 (__m128 __A) +{ + return (__m128i) __builtin_ia32_cvtps2qq128_mask ((__v4sf) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtps_epi64 (__m128i __W, __mmask8 __U, __m128 __A) +{ + return (__m128i) __builtin_ia32_cvtps2qq128_mask ((__v4sf) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtps_epi64 (__mmask8 __U, __m128 __A) +{ + return (__m128i) __builtin_ia32_cvtps2qq128_mask ((__v4sf) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtps_epu64 (__m128 __A) +{ + return (__m256i) __builtin_ia32_cvtps2uqq256_mask ((__v4sf) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtps_epu64 (__m256i __W, __mmask8 __U, __m128 __A) +{ + return (__m256i) __builtin_ia32_cvtps2uqq256_mask ((__v4sf) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtps_epu64 (__mmask8 __U, __m128 __A) +{ + return (__m256i) __builtin_ia32_cvtps2uqq256_mask ((__v4sf) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtps_epu64 (__m128 __A) +{ + return (__m128i) __builtin_ia32_cvtps2uqq128_mask ((__v4sf) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtps_epu64 (__m128i __W, __mmask8 __U, __m128 __A) +{ + return (__m128i) __builtin_ia32_cvtps2uqq128_mask ((__v4sf) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtps_epu64 (__mmask8 __U, __m128 __A) +{ + return (__m128i) __builtin_ia32_cvtps2uqq128_mask ((__v4sf) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi64_ps (__m256i __A) +{ + return (__m128) __builtin_ia32_cvtqq2ps256_mask ((__v4di) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi64_ps (__m128 __W, __mmask8 __U, __m256i __A) +{ + return (__m128) __builtin_ia32_cvtqq2ps256_mask ((__v4di) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepi64_ps (__mmask8 __U, __m256i __A) +{ + return (__m128) __builtin_ia32_cvtqq2ps256_mask ((__v4di) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepi64_ps (__m128i __A) +{ + return (__m128) __builtin_ia32_cvtqq2ps128_mask ((__v2di) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi64_ps (__m128 __W, __mmask8 __U, __m128i __A) +{ + return (__m128) __builtin_ia32_cvtqq2ps128_mask ((__v2di) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepi64_ps (__mmask8 __U, __m128i __A) +{ + return (__m128) __builtin_ia32_cvtqq2ps128_mask ((__v2di) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepu64_ps (__m256i __A) +{ + return (__m128) __builtin_ia32_cvtuqq2ps256_mask ((__v4di) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepu64_ps (__m128 __W, __mmask8 __U, __m256i __A) +{ + return (__m128) __builtin_ia32_cvtuqq2ps256_mask ((__v4di) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepu64_ps (__mmask8 __U, __m256i __A) +{ + return (__m128) __builtin_ia32_cvtuqq2ps256_mask ((__v4di) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepu64_ps (__m128i __A) +{ + return (__m128) __builtin_ia32_cvtuqq2ps128_mask ((__v2di) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepu64_ps (__m128 __W, __mmask8 __U, __m128i __A) +{ + return (__m128) __builtin_ia32_cvtuqq2ps128_mask ((__v2di) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepu64_ps (__mmask8 __U, __m128i __A) +{ + return (__m128) __builtin_ia32_cvtuqq2ps128_mask ((__v2di) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi64_pd (__m256i __A) +{ + return (__m256d) __builtin_ia32_cvtqq2pd256_mask ((__v4di) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi64_pd (__m256d __W, __mmask8 __U, __m256i __A) +{ + return (__m256d) __builtin_ia32_cvtqq2pd256_mask ((__v4di) __A, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepi64_pd (__mmask8 __U, __m256i __A) +{ + return (__m256d) __builtin_ia32_cvtqq2pd256_mask ((__v4di) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepi64_pd (__m128i __A) +{ + return (__m128d) __builtin_ia32_cvtqq2pd128_mask ((__v2di) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi64_pd (__m128d __W, __mmask8 __U, __m128i __A) +{ + return (__m128d) __builtin_ia32_cvtqq2pd128_mask ((__v2di) __A, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepi64_pd (__mmask8 __U, __m128i __A) +{ + return (__m128d) __builtin_ia32_cvtqq2pd128_mask ((__v2di) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepu64_pd (__m256i __A) +{ + return (__m256d) __builtin_ia32_cvtuqq2pd256_mask ((__v4di) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepu64_pd (__m256d __W, __mmask8 __U, __m256i __A) +{ + return (__m256d) __builtin_ia32_cvtuqq2pd256_mask ((__v4di) __A, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepu64_pd (__mmask8 __U, __m256i __A) +{ + return (__m256d) __builtin_ia32_cvtuqq2pd256_mask ((__v4di) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_and_pd (__m256d __W, __mmask8 __U, __m256d __A, + __m256d __B) +{ + return (__m256d) __builtin_ia32_andpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_and_pd (__mmask8 __U, __m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_andpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_and_pd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_andpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_and_pd (__mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_andpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_and_ps (__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_andps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_and_ps (__mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_andps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_and_ps (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_andps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_and_ps (__mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_andps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepu64_pd (__m128i __A) +{ + return (__m128d) __builtin_ia32_cvtuqq2pd128_mask ((__v2di) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepu64_pd (__m128d __W, __mmask8 __U, __m128i __A) +{ + return (__m128d) __builtin_ia32_cvtuqq2pd128_mask ((__v2di) __A, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepu64_pd (__mmask8 __U, __m128i __A) +{ + return (__m128d) __builtin_ia32_cvtuqq2pd128_mask ((__v2di) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_xor_pd (__m256d __W, __mmask8 __U, __m256d __A, + __m256d __B) +{ + return (__m256d) __builtin_ia32_xorpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_xor_pd (__mmask8 __U, __m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_xorpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_xor_pd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_xorpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_xor_pd (__mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_xorpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_xor_ps (__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_xorps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_xor_ps (__mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_xorps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_xor_ps (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_xorps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_xor_ps (__mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_xorps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_or_pd (__m256d __W, __mmask8 __U, __m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_orpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_or_pd (__mmask8 __U, __m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_orpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_or_pd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_orpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_or_pd (__mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_orpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_or_ps (__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_orps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_or_ps (__mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_orps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_or_ps (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_orps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_or_ps (__mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_orps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movm_epi32 (__mmask8 __A) +{ + return (__m128i) __builtin_ia32_cvtmask2d128 (__A); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_movm_epi32 (__mmask8 __A) +{ + return (__m256i) __builtin_ia32_cvtmask2d256 (__A); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movm_epi64 (__mmask8 __A) +{ + return (__m128i) __builtin_ia32_cvtmask2q128 (__A); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_movm_epi64 (__mmask8 __A) +{ + return (__m256i) __builtin_ia32_cvtmask2q256 (__A); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movepi32_mask (__m128i __A) +{ + return (__mmask8) __builtin_ia32_cvtd2mask128 ((__v4si) __A); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_movepi32_mask (__m256i __A) +{ + return (__mmask8) __builtin_ia32_cvtd2mask256 ((__v8si) __A); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movepi64_mask (__m128i __A) +{ + return (__mmask8) __builtin_ia32_cvtq2mask128 ((__v2di) __A); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_movepi64_mask (__m256i __A) +{ + return (__mmask8) __builtin_ia32_cvtq2mask256 ((__v4di) __A); +} + +#ifdef __OPTIMIZE__ +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_extractf64x2_pd (__m256d __A, const int __imm) +{ + return (__m128d) __builtin_ia32_extractf64x2_256_mask ((__v4df) __A, + __imm, + (__v2df) + _mm_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_extractf64x2_pd (__m128d __W, __mmask8 __U, __m256d __A, + const int __imm) +{ + return (__m128d) __builtin_ia32_extractf64x2_256_mask ((__v4df) __A, + __imm, + (__v2df) __W, + (__mmask8) + __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_extractf64x2_pd (__mmask8 __U, __m256d __A, + const int __imm) +{ + return (__m128d) __builtin_ia32_extractf64x2_256_mask ((__v4df) __A, + __imm, + (__v2df) + _mm_setzero_pd (), + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_extracti64x2_epi64 (__m256i __A, const int __imm) +{ + return (__m128i) __builtin_ia32_extracti64x2_256_mask ((__v4di) __A, + __imm, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_extracti64x2_epi64 (__m128i __W, __mmask8 __U, __m256i __A, + const int __imm) +{ + return (__m128i) __builtin_ia32_extracti64x2_256_mask ((__v4di) __A, + __imm, + (__v2di) __W, + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_extracti64x2_epi64 (__mmask8 __U, __m256i __A, + const int __imm) +{ + return (__m128i) __builtin_ia32_extracti64x2_256_mask ((__v4di) __A, + __imm, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) + __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_reduce_pd (__m256d __A, int __B) +{ + return (__m256d) __builtin_ia32_reducepd256_mask ((__v4df) __A, __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_reduce_pd (__m256d __W, __mmask8 __U, __m256d __A, int __B) +{ + return (__m256d) __builtin_ia32_reducepd256_mask ((__v4df) __A, __B, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_reduce_pd (__mmask8 __U, __m256d __A, int __B) +{ + return (__m256d) __builtin_ia32_reducepd256_mask ((__v4df) __A, __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_reduce_pd (__m128d __A, int __B) +{ + return (__m128d) __builtin_ia32_reducepd128_mask ((__v2df) __A, __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_reduce_pd (__m128d __W, __mmask8 __U, __m128d __A, int __B) +{ + return (__m128d) __builtin_ia32_reducepd128_mask ((__v2df) __A, __B, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_reduce_pd (__mmask8 __U, __m128d __A, int __B) +{ + return (__m128d) __builtin_ia32_reducepd128_mask ((__v2df) __A, __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_reduce_ps (__m256 __A, int __B) +{ + return (__m256) __builtin_ia32_reduceps256_mask ((__v8sf) __A, __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_reduce_ps (__m256 __W, __mmask8 __U, __m256 __A, int __B) +{ + return (__m256) __builtin_ia32_reduceps256_mask ((__v8sf) __A, __B, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_reduce_ps (__mmask8 __U, __m256 __A, int __B) +{ + return (__m256) __builtin_ia32_reduceps256_mask ((__v8sf) __A, __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_reduce_ps (__m128 __A, int __B) +{ + return (__m128) __builtin_ia32_reduceps128_mask ((__v4sf) __A, __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_reduce_ps (__m128 __W, __mmask8 __U, __m128 __A, int __B) +{ + return (__m128) __builtin_ia32_reduceps128_mask ((__v4sf) __A, __B, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_reduce_ps (__mmask8 __U, __m128 __A, int __B) +{ + return (__m128) __builtin_ia32_reduceps128_mask ((__v4sf) __A, __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_range_pd (__m256d __A, __m256d __B, int __C) +{ + return (__m256d) __builtin_ia32_rangepd256_mask ((__v4df) __A, + (__v4df) __B, __C, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_range_pd (__m256d __W, __mmask8 __U, + __m256d __A, __m256d __B, int __C) +{ + return (__m256d) __builtin_ia32_rangepd256_mask ((__v4df) __A, + (__v4df) __B, __C, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_range_pd (__mmask8 __U, __m256d __A, __m256d __B, int __C) +{ + return (__m256d) __builtin_ia32_rangepd256_mask ((__v4df) __A, + (__v4df) __B, __C, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_range_pd (__m128d __A, __m128d __B, int __C) +{ + return (__m128d) __builtin_ia32_rangepd128_mask ((__v2df) __A, + (__v2df) __B, __C, + (__v2df) + _mm_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_range_pd (__m128d __W, __mmask8 __U, + __m128d __A, __m128d __B, int __C) +{ + return (__m128d) __builtin_ia32_rangepd128_mask ((__v2df) __A, + (__v2df) __B, __C, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_range_pd (__mmask8 __U, __m128d __A, __m128d __B, int __C) +{ + return (__m128d) __builtin_ia32_rangepd128_mask ((__v2df) __A, + (__v2df) __B, __C, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_range_ps (__m256 __A, __m256 __B, int __C) +{ + return (__m256) __builtin_ia32_rangeps256_mask ((__v8sf) __A, + (__v8sf) __B, __C, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_range_ps (__m256 __W, __mmask8 __U, __m256 __A, __m256 __B, + int __C) +{ + return (__m256) __builtin_ia32_rangeps256_mask ((__v8sf) __A, + (__v8sf) __B, __C, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_range_ps (__mmask8 __U, __m256 __A, __m256 __B, int __C) +{ + return (__m256) __builtin_ia32_rangeps256_mask ((__v8sf) __A, + (__v8sf) __B, __C, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_range_ps (__m128 __A, __m128 __B, int __C) +{ + return (__m128) __builtin_ia32_rangeps128_mask ((__v4sf) __A, + (__v4sf) __B, __C, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_range_ps (__m128 __W, __mmask8 __U, + __m128 __A, __m128 __B, int __C) +{ + return (__m128) __builtin_ia32_rangeps128_mask ((__v4sf) __A, + (__v4sf) __B, __C, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_range_ps (__mmask8 __U, __m128 __A, __m128 __B, int __C) +{ + return (__m128) __builtin_ia32_rangeps128_mask ((__v4sf) __A, + (__v4sf) __B, __C, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_fpclass_pd_mask (__mmask8 __U, __m256d __A, + const int __imm) +{ + return (__mmask8) __builtin_ia32_fpclasspd256_mask ((__v4df) __A, + __imm, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_fpclass_pd_mask (__m256d __A, const int __imm) +{ + return (__mmask8) __builtin_ia32_fpclasspd256_mask ((__v4df) __A, + __imm, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_fpclass_ps_mask (__mmask8 __U, __m256 __A, const int __imm) +{ + return (__mmask8) __builtin_ia32_fpclassps256_mask ((__v8sf) __A, + __imm, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_fpclass_ps_mask (__m256 __A, const int __imm) +{ + return (__mmask8) __builtin_ia32_fpclassps256_mask ((__v8sf) __A, + __imm, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fpclass_pd_mask (__mmask8 __U, __m128d __A, const int __imm) +{ + return (__mmask8) __builtin_ia32_fpclasspd128_mask ((__v2df) __A, + __imm, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fpclass_pd_mask (__m128d __A, const int __imm) +{ + return (__mmask8) __builtin_ia32_fpclasspd128_mask ((__v2df) __A, + __imm, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fpclass_ps_mask (__mmask8 __U, __m128 __A, const int __imm) +{ + return (__mmask8) __builtin_ia32_fpclassps128_mask ((__v4sf) __A, + __imm, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fpclass_ps_mask (__m128 __A, const int __imm) +{ + return (__mmask8) __builtin_ia32_fpclassps128_mask ((__v4sf) __A, + __imm, + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_inserti64x2 (__m256i __A, __m128i __B, const int __imm) +{ + return (__m256i) __builtin_ia32_inserti64x2_256_mask ((__v4di) __A, + (__v2di) __B, + __imm, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_inserti64x2 (__m256i __W, __mmask8 __U, __m256i __A, + __m128i __B, const int __imm) +{ + return (__m256i) __builtin_ia32_inserti64x2_256_mask ((__v4di) __A, + (__v2di) __B, + __imm, + (__v4di) __W, + (__mmask8) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_inserti64x2 (__mmask8 __U, __m256i __A, __m128i __B, + const int __imm) +{ + return (__m256i) __builtin_ia32_inserti64x2_256_mask ((__v4di) __A, + (__v2di) __B, + __imm, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) + __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_insertf64x2 (__m256d __A, __m128d __B, const int __imm) +{ + return (__m256d) __builtin_ia32_insertf64x2_256_mask ((__v4df) __A, + (__v2df) __B, + __imm, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_insertf64x2 (__m256d __W, __mmask8 __U, __m256d __A, + __m128d __B, const int __imm) +{ + return (__m256d) __builtin_ia32_insertf64x2_256_mask ((__v4df) __A, + (__v2df) __B, + __imm, + (__v4df) __W, + (__mmask8) + __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_insertf64x2 (__mmask8 __U, __m256d __A, __m128d __B, + const int __imm) +{ + return (__m256d) __builtin_ia32_insertf64x2_256_mask ((__v4df) __A, + (__v2df) __B, + __imm, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) + __U); +} + +#else +#define _mm256_insertf64x2(X, Y, C) \ + ((__m256d) __builtin_ia32_insertf64x2_256_mask ((__v4df)(__m256d) (X),\ + (__v2df)(__m128d) (Y), (int) (C), \ + (__v4df)(__m256d)_mm256_setzero_pd(), \ + (__mmask8)-1)) + +#define _mm256_mask_insertf64x2(W, U, X, Y, C) \ + ((__m256d) __builtin_ia32_insertf64x2_256_mask ((__v4df)(__m256d) (X),\ + (__v2df)(__m128d) (Y), (int) (C), \ + (__v4df)(__m256d)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_insertf64x2(U, X, Y, C) \ + ((__m256d) __builtin_ia32_insertf64x2_256_mask ((__v4df)(__m256d) (X),\ + (__v2df)(__m128d) (Y), (int) (C), \ + (__v4df)(__m256d)_mm256_setzero_pd(), \ + (__mmask8)(U))) + +#define _mm256_inserti64x2(X, Y, C) \ + ((__m256i) __builtin_ia32_inserti64x2_256_mask ((__v4di)(__m256i) (X),\ + (__v2di)(__m128i) (Y), (int) (C), \ + (__v4di)(__m256i)_mm256_setzero_si256 (), \ + (__mmask8)-1)) + +#define _mm256_mask_inserti64x2(W, U, X, Y, C) \ + ((__m256i) __builtin_ia32_inserti64x2_256_mask ((__v4di)(__m256i) (X),\ + (__v2di)(__m128i) (Y), (int) (C), \ + (__v4di)(__m256i)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_inserti64x2(U, X, Y, C) \ + ((__m256i) __builtin_ia32_inserti64x2_256_mask ((__v4di)(__m256i) (X),\ + (__v2di)(__m128i) (Y), (int) (C), \ + (__v4di)(__m256i)_mm256_setzero_si256 (), \ + (__mmask8)(U))) + +#define _mm256_extractf64x2_pd(X, C) \ + ((__m128d) __builtin_ia32_extractf64x2_256_mask ((__v4df)(__m256d) (X),\ + (int) (C), (__v2df)(__m128d) _mm_setzero_pd(), (__mmask8)-1)) + +#define _mm256_mask_extractf64x2_pd(W, U, X, C) \ + ((__m128d) __builtin_ia32_extractf64x2_256_mask ((__v4df)(__m256d) (X),\ + (int) (C), (__v2df)(__m128d) (W), (__mmask8) (U))) + +#define _mm256_maskz_extractf64x2_pd(U, X, C) \ + ((__m128d) __builtin_ia32_extractf64x2_256_mask ((__v4df)(__m256d) (X),\ + (int) (C), (__v2df)(__m128d) _mm_setzero_pd(), (__mmask8) (U))) + +#define _mm256_extracti64x2_epi64(X, C) \ + ((__m128i) __builtin_ia32_extracti64x2_256_mask ((__v4di)(__m256i) (X),\ + (int) (C), (__v2di)(__m128i) _mm_setzero_si128 (), (__mmask8)-1)) + +#define _mm256_mask_extracti64x2_epi64(W, U, X, C) \ + ((__m128i) __builtin_ia32_extracti64x2_256_mask ((__v4di)(__m256i) (X),\ + (int) (C), (__v2di)(__m128i) (W), (__mmask8) (U))) + +#define _mm256_maskz_extracti64x2_epi64(U, X, C) \ + ((__m128i) __builtin_ia32_extracti64x2_256_mask ((__v4di)(__m256i) (X),\ + (int) (C), (__v2di)(__m128i) _mm_setzero_si128 (), (__mmask8) (U))) + +#define _mm256_reduce_pd(A, B) \ + ((__m256d) __builtin_ia32_reducepd256_mask ((__v4df)(__m256d)(A), \ + (int)(B), (__v4df)_mm256_setzero_pd(), (__mmask8)-1)) + +#define _mm256_mask_reduce_pd(W, U, A, B) \ + ((__m256d) __builtin_ia32_reducepd256_mask ((__v4df)(__m256d)(A), \ + (int)(B), (__v4df)(__m256d)(W), (__mmask8)(U))) + +#define _mm256_maskz_reduce_pd(U, A, B) \ + ((__m256d) __builtin_ia32_reducepd256_mask ((__v4df)(__m256d)(A), \ + (int)(B), (__v4df)_mm256_setzero_pd(), (__mmask8)(U))) + +#define _mm_reduce_pd(A, B) \ + ((__m128d) __builtin_ia32_reducepd128_mask ((__v2df)(__m128d)(A), \ + (int)(B), (__v2df)_mm_setzero_pd(), (__mmask8)-1)) + +#define _mm_mask_reduce_pd(W, U, A, B) \ + ((__m128d) __builtin_ia32_reducepd128_mask ((__v2df)(__m128d)(A), \ + (int)(B), (__v2df)(__m128d)(W), (__mmask8)(U))) + +#define _mm_maskz_reduce_pd(U, A, B) \ + ((__m128d) __builtin_ia32_reducepd128_mask ((__v2df)(__m128d)(A), \ + (int)(B), (__v2df)_mm_setzero_pd(), (__mmask8)(U))) + +#define _mm256_reduce_ps(A, B) \ + ((__m256) __builtin_ia32_reduceps256_mask ((__v8sf)(__m256)(A), \ + (int)(B), (__v8sf)_mm256_setzero_ps(), (__mmask8)-1)) + +#define _mm256_mask_reduce_ps(W, U, A, B) \ + ((__m256) __builtin_ia32_reduceps256_mask ((__v8sf)(__m256)(A), \ + (int)(B), (__v8sf)(__m256)(W), (__mmask8)(U))) + +#define _mm256_maskz_reduce_ps(U, A, B) \ + ((__m256) __builtin_ia32_reduceps256_mask ((__v8sf)(__m256)(A), \ + (int)(B), (__v8sf)_mm256_setzero_ps(), (__mmask8)(U))) + +#define _mm_reduce_ps(A, B) \ + ((__m128) __builtin_ia32_reduceps128_mask ((__v4sf)(__m128)(A), \ + (int)(B), (__v4sf)_mm_setzero_ps(), (__mmask8)-1)) + +#define _mm_mask_reduce_ps(W, U, A, B) \ + ((__m128) __builtin_ia32_reduceps128_mask ((__v4sf)(__m128)(A), \ + (int)(B), (__v4sf)(__m128)(W), (__mmask8)(U))) + +#define _mm_maskz_reduce_ps(U, A, B) \ + ((__m128) __builtin_ia32_reduceps128_mask ((__v4sf)(__m128)(A), \ + (int)(B), (__v4sf)_mm_setzero_ps(), (__mmask8)(U))) + +#define _mm256_range_pd(A, B, C) \ + ((__m256d) __builtin_ia32_rangepd256_mask ((__v4df)(__m256d)(A), \ + (__v4df)(__m256d)(B), (int)(C), \ + (__v4df)_mm256_setzero_pd(), (__mmask8)-1)) + +#define _mm256_maskz_range_pd(U, A, B, C) \ + ((__m256d) __builtin_ia32_rangepd256_mask ((__v4df)(__m256d)(A), \ + (__v4df)(__m256d)(B), (int)(C), \ + (__v4df)_mm256_setzero_pd(), (__mmask8)(U))) + +#define _mm_range_pd(A, B, C) \ + ((__m128d) __builtin_ia32_rangepd128_mask ((__v2df)(__m128d)(A), \ + (__v2df)(__m128d)(B), (int)(C), \ + (__v2df)_mm_setzero_pd(), (__mmask8)-1)) + +#define _mm256_range_ps(A, B, C) \ + ((__m256) __builtin_ia32_rangeps256_mask ((__v8sf)(__m256)(A), \ + (__v8sf)(__m256)(B), (int)(C), \ + (__v8sf)_mm256_setzero_ps(), (__mmask8)-1)) + +#define _mm256_mask_range_ps(W, U, A, B, C) \ + ((__m256) __builtin_ia32_rangeps256_mask ((__v8sf)(__m256)(A), \ + (__v8sf)(__m256)(B), (int)(C), \ + (__v8sf)(__m256)(W), (__mmask8)(U))) + +#define _mm256_maskz_range_ps(U, A, B, C) \ + ((__m256) __builtin_ia32_rangeps256_mask ((__v8sf)(__m256)(A), \ + (__v8sf)(__m256)(B), (int)(C), \ + (__v8sf)_mm256_setzero_ps(), (__mmask8)(U))) + +#define _mm_range_ps(A, B, C) \ + ((__m128) __builtin_ia32_rangeps128_mask ((__v4sf)(__m128)(A), \ + (__v4sf)(__m128)(B), (int)(C), \ + (__v4sf)_mm_setzero_ps(), (__mmask8)-1)) + +#define _mm_mask_range_ps(W, U, A, B, C) \ + ((__m128) __builtin_ia32_rangeps128_mask ((__v4sf)(__m128)(A), \ + (__v4sf)(__m128)(B), (int)(C), \ + (__v4sf)(__m128)(W), (__mmask8)(U))) + +#define _mm_maskz_range_ps(U, A, B, C) \ + ((__m128) __builtin_ia32_rangeps128_mask ((__v4sf)(__m128)(A), \ + (__v4sf)(__m128)(B), (int)(C), \ + (__v4sf)_mm_setzero_ps(), (__mmask8)(U))) + +#define _mm256_mask_range_pd(W, U, A, B, C) \ + ((__m256d) __builtin_ia32_rangepd256_mask ((__v4df)(__m256d)(A), \ + (__v4df)(__m256d)(B), (int)(C), \ + (__v4df)(__m256d)(W), (__mmask8)(U))) + +#define _mm_mask_range_pd(W, U, A, B, C) \ + ((__m128d) __builtin_ia32_rangepd128_mask ((__v2df)(__m128d)(A), \ + (__v2df)(__m128d)(B), (int)(C), \ + (__v2df)(__m128d)(W), (__mmask8)(U))) + +#define _mm_maskz_range_pd(U, A, B, C) \ + ((__m128d) __builtin_ia32_rangepd128_mask ((__v2df)(__m128d)(A), \ + (__v2df)(__m128d)(B), (int)(C), \ + (__v2df)_mm_setzero_pd(), (__mmask8)(U))) + +#define _mm256_mask_fpclass_pd_mask(u, X, C) \ + ((__mmask8) __builtin_ia32_fpclasspd256_mask ((__v4df) (__m256d) (X), \ + (int) (C),(__mmask8)(u))) + +#define _mm256_mask_fpclass_ps_mask(u, X, C) \ + ((__mmask8) __builtin_ia32_fpclassps256_mask ((__v8sf) (__m256) (X), \ + (int) (C),(__mmask8)(u))) + +#define _mm_mask_fpclass_pd_mask(u, X, C) \ + ((__mmask8) __builtin_ia32_fpclasspd128_mask ((__v2df) (__m128d) (X), \ + (int) (C),(__mmask8)(u))) + +#define _mm_mask_fpclass_ps_mask(u, X, C) \ + ((__mmask8) __builtin_ia32_fpclassps128_mask ((__v4sf) (__m128) (X), \ + (int) (C),(__mmask8)(u))) + +#define _mm256_fpclass_pd_mask(X, C) \ + ((__mmask8) __builtin_ia32_fpclasspd256_mask ((__v4df) (__m256d) (X), \ + (int) (C),(__mmask8)-1)) + +#define _mm256_fpclass_ps_mask(X, C) \ + ((__mmask8) __builtin_ia32_fpclassps256_mask ((__v8sf) (__m256) (X), \ + (int) (C),(__mmask8)-1)) + +#define _mm_fpclass_pd_mask(X, C) \ + ((__mmask8) __builtin_ia32_fpclasspd128_mask ((__v2df) (__m128d) (X), \ + (int) (C),(__mmask8)-1)) + +#define _mm_fpclass_ps_mask(X, C) \ + ((__mmask8) __builtin_ia32_fpclassps128_mask ((__v4sf) (__m128) (X), \ + (int) (C),(__mmask8)-1)) + +#endif + +#ifdef __DISABLE_AVX512VLDQ__ +#undef __DISABLE_AVX512VLDQ__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX512VLDQ__ */ + +#endif /* _AVX512VLDQINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512vlintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512vlintrin.h new file mode 100644 index 0000000000..c9df31839a --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512vlintrin.h @@ -0,0 +1,13606 @@ +/* Copyright (C) 2014-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +#error "Never use directly; include instead." +#endif + +#ifndef _AVX512VLINTRIN_H_INCLUDED +#define _AVX512VLINTRIN_H_INCLUDED + +#ifndef __AVX512VL__ +#pragma GCC push_options +//#pragma GCC target("avx512vl") +#define __DISABLE_AVX512VL__ +#endif /* __AVX512VL__ */ + +/* Internal data types for implementing the intrinsics. */ +typedef unsigned int __mmask32; + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_mov_pd (__m256d __W, __mmask8 __U, __m256d __A) +{ + return (__m256d) __builtin_ia32_movapd256_mask ((__v4df) __A, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_mov_pd (__mmask8 __U, __m256d __A) +{ + return (__m256d) __builtin_ia32_movapd256_mask ((__v4df) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_mov_pd (__m128d __W, __mmask8 __U, __m128d __A) +{ + return (__m128d) __builtin_ia32_movapd128_mask ((__v2df) __A, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_mov_pd (__mmask8 __U, __m128d __A) +{ + return (__m128d) __builtin_ia32_movapd128_mask ((__v2df) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_load_pd (__m256d __W, __mmask8 __U, void const *__P) +{ + return (__m256d) __builtin_ia32_loadapd256_mask ((__v4df *) __P, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_load_pd (__mmask8 __U, void const *__P) +{ + return (__m256d) __builtin_ia32_loadapd256_mask ((__v4df *) __P, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_load_pd (__m128d __W, __mmask8 __U, void const *__P) +{ + return (__m128d) __builtin_ia32_loadapd128_mask ((__v2df *) __P, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_load_pd (__mmask8 __U, void const *__P) +{ + return (__m128d) __builtin_ia32_loadapd128_mask ((__v2df *) __P, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_store_pd (void *__P, __mmask8 __U, __m256d __A) +{ + __builtin_ia32_storeapd256_mask ((__v4df *) __P, + (__v4df) __A, + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_store_pd (void *__P, __mmask8 __U, __m128d __A) +{ + __builtin_ia32_storeapd128_mask ((__v2df *) __P, + (__v2df) __A, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_mov_ps (__m256 __W, __mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_movaps256_mask ((__v8sf) __A, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_mov_ps (__mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_movaps256_mask ((__v8sf) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_mov_ps (__m128 __W, __mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_movaps128_mask ((__v4sf) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_mov_ps (__mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_movaps128_mask ((__v4sf) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_load_ps (__m256 __W, __mmask8 __U, void const *__P) +{ + return (__m256) __builtin_ia32_loadaps256_mask ((__v8sf *) __P, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_load_ps (__mmask8 __U, void const *__P) +{ + return (__m256) __builtin_ia32_loadaps256_mask ((__v8sf *) __P, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_load_ps (__m128 __W, __mmask8 __U, void const *__P) +{ + return (__m128) __builtin_ia32_loadaps128_mask ((__v4sf *) __P, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_load_ps (__mmask8 __U, void const *__P) +{ + return (__m128) __builtin_ia32_loadaps128_mask ((__v4sf *) __P, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_store_ps (void *__P, __mmask8 __U, __m256 __A) +{ + __builtin_ia32_storeaps256_mask ((__v8sf *) __P, + (__v8sf) __A, + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_store_ps (void *__P, __mmask8 __U, __m128 __A) +{ + __builtin_ia32_storeaps128_mask ((__v4sf *) __P, + (__v4sf) __A, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_mov_epi64 (__m256i __W, __mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_movdqa64_256_mask ((__v4di) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_mov_epi64 (__mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_movdqa64_256_mask ((__v4di) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_mov_epi64 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_movdqa64_128_mask ((__v2di) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_mov_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_movdqa64_128_mask ((__v2di) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_load_epi64 (__m256i __W, __mmask8 __U, void const *__P) +{ + return (__m256i) __builtin_ia32_movdqa64load256_mask ((__v4di *) __P, + (__v4di) __W, + (__mmask8) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_load_epi64 (__mmask8 __U, void const *__P) +{ + return (__m256i) __builtin_ia32_movdqa64load256_mask ((__v4di *) __P, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_load_epi64 (__m128i __W, __mmask8 __U, void const *__P) +{ + return (__m128i) __builtin_ia32_movdqa64load128_mask ((__v2di *) __P, + (__v2di) __W, + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_load_epi64 (__mmask8 __U, void const *__P) +{ + return (__m128i) __builtin_ia32_movdqa64load128_mask ((__v2di *) __P, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) + __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_store_epi64 (void *__P, __mmask8 __U, __m256i __A) +{ + __builtin_ia32_movdqa64store256_mask ((__v4di *) __P, + (__v4di) __A, + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_store_epi64 (void *__P, __mmask8 __U, __m128i __A) +{ + __builtin_ia32_movdqa64store128_mask ((__v2di *) __P, + (__v2di) __A, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_mov_epi32 (__m256i __W, __mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_movdqa32_256_mask ((__v8si) __A, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_mov_epi32 (__mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_movdqa32_256_mask ((__v8si) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_mov_epi32 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_movdqa32_128_mask ((__v4si) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_mov_epi32 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_movdqa32_128_mask ((__v4si) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_load_epi32 (__m256i __W, __mmask8 __U, void const *__P) +{ + return (__m256i) __builtin_ia32_movdqa32load256_mask ((__v8si *) __P, + (__v8si) __W, + (__mmask8) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_load_epi32 (__mmask8 __U, void const *__P) +{ + return (__m256i) __builtin_ia32_movdqa32load256_mask ((__v8si *) __P, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_load_epi32 (__m128i __W, __mmask8 __U, void const *__P) +{ + return (__m128i) __builtin_ia32_movdqa32load128_mask ((__v4si *) __P, + (__v4si) __W, + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_load_epi32 (__mmask8 __U, void const *__P) +{ + return (__m128i) __builtin_ia32_movdqa32load128_mask ((__v4si *) __P, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) + __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_store_epi32 (void *__P, __mmask8 __U, __m256i __A) +{ + __builtin_ia32_movdqa32store256_mask ((__v8si *) __P, + (__v8si) __A, + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_store_epi32 (void *__P, __mmask8 __U, __m128i __A) +{ + __builtin_ia32_movdqa32store128_mask ((__v4si *) __P, + (__v4si) __A, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_add_pd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_addpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_add_pd (__mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_addpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_add_pd (__m256d __W, __mmask8 __U, __m256d __A, + __m256d __B) +{ + return (__m256d) __builtin_ia32_addpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_add_pd (__mmask8 __U, __m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_addpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_add_ps (__m128 __W, __mmask16 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_addps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_add_ps (__mmask16 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_addps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_add_ps (__m256 __W, __mmask16 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_addps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_add_ps (__mmask16 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_addps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sub_pd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_subpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sub_pd (__mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_subpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sub_pd (__m256d __W, __mmask8 __U, __m256d __A, + __m256d __B) +{ + return (__m256d) __builtin_ia32_subpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sub_pd (__mmask8 __U, __m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_subpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sub_ps (__m128 __W, __mmask16 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_subps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sub_ps (__mmask16 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_subps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sub_ps (__m256 __W, __mmask16 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_subps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sub_ps (__mmask16 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_subps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_store_epi64 (void *__P, __m256i __A) +{ + *(__m256i *) __P = __A; +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_store_epi64 (void *__P, __m128i __A) +{ + *(__m128i *) __P = __A; +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_loadu_pd (__m256d __W, __mmask8 __U, void const *__P) +{ + return (__m256d) __builtin_ia32_loadupd256_mask ((const double *) __P, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_loadu_pd (__mmask8 __U, void const *__P) +{ + return (__m256d) __builtin_ia32_loadupd256_mask ((const double *) __P, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_loadu_pd (__m128d __W, __mmask8 __U, void const *__P) +{ + return (__m128d) __builtin_ia32_loadupd128_mask ((const double *) __P, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_loadu_pd (__mmask8 __U, void const *__P) +{ + return (__m128d) __builtin_ia32_loadupd128_mask ((const double *) __P, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_storeu_pd (void *__P, __mmask8 __U, __m256d __A) +{ + __builtin_ia32_storeupd256_mask ((double *) __P, + (__v4df) __A, + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_storeu_pd (void *__P, __mmask8 __U, __m128d __A) +{ + __builtin_ia32_storeupd128_mask ((double *) __P, + (__v2df) __A, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_loadu_ps (__m256 __W, __mmask8 __U, void const *__P) +{ + return (__m256) __builtin_ia32_loadups256_mask ((const float *) __P, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_loadu_ps (__mmask8 __U, void const *__P) +{ + return (__m256) __builtin_ia32_loadups256_mask ((const float *) __P, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_loadu_ps (__m128 __W, __mmask8 __U, void const *__P) +{ + return (__m128) __builtin_ia32_loadups128_mask ((const float *) __P, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_loadu_ps (__mmask8 __U, void const *__P) +{ + return (__m128) __builtin_ia32_loadups128_mask ((const float *) __P, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_storeu_ps (void *__P, __mmask8 __U, __m256 __A) +{ + __builtin_ia32_storeups256_mask ((float *) __P, + (__v8sf) __A, + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_storeu_ps (void *__P, __mmask8 __U, __m128 __A) +{ + __builtin_ia32_storeups128_mask ((float *) __P, + (__v4sf) __A, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_loadu_epi64 (__m256i __W, __mmask8 __U, void const *__P) +{ + return (__m256i) __builtin_ia32_loaddqudi256_mask ((const long long *) __P, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_loadu_epi64 (__mmask8 __U, void const *__P) +{ + return (__m256i) __builtin_ia32_loaddqudi256_mask ((const long long *) __P, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_loadu_epi64 (__m128i __W, __mmask8 __U, void const *__P) +{ + return (__m128i) __builtin_ia32_loaddqudi128_mask ((const long long *) __P, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_loadu_epi64 (__mmask8 __U, void const *__P) +{ + return (__m128i) __builtin_ia32_loaddqudi128_mask ((const long long *) __P, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_storeu_epi64 (void *__P, __mmask8 __U, __m256i __A) +{ + __builtin_ia32_storedqudi256_mask ((long long *) __P, + (__v4di) __A, + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_storeu_epi64 (void *__P, __mmask8 __U, __m128i __A) +{ + __builtin_ia32_storedqudi128_mask ((long long *) __P, + (__v2di) __A, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_loadu_epi32 (__m256i __W, __mmask8 __U, void const *__P) +{ + return (__m256i) __builtin_ia32_loaddqusi256_mask ((const int *) __P, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_loadu_epi32 (__mmask8 __U, void const *__P) +{ + return (__m256i) __builtin_ia32_loaddqusi256_mask ((const int *) __P, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_loadu_epi32 (__m128i __W, __mmask8 __U, void const *__P) +{ + return (__m128i) __builtin_ia32_loaddqusi128_mask ((const int *) __P, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_loadu_epi32 (__mmask8 __U, void const *__P) +{ + return (__m128i) __builtin_ia32_loaddqusi128_mask ((const int *) __P, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_storeu_epi32 (void *__P, __mmask8 __U, __m256i __A) +{ + __builtin_ia32_storedqusi256_mask ((int *) __P, + (__v8si) __A, + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_storeu_epi32 (void *__P, __mmask8 __U, __m128i __A) +{ + __builtin_ia32_storedqusi128_mask ((int *) __P, + (__v4si) __A, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_abs_epi32 (__m256i __W, __mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_pabsd256_mask ((__v8si) __A, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_abs_epi32 (__mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_pabsd256_mask ((__v8si) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_abs_epi32 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pabsd128_mask ((__v4si) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_abs_epi32 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pabsd128_mask ((__v4si) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_abs_epi64 (__m256i __A) +{ + return (__m256i) __builtin_ia32_pabsq256_mask ((__v4di) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_abs_epi64 (__m256i __W, __mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_pabsq256_mask ((__v4di) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_abs_epi64 (__mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_pabsq256_mask ((__v4di) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_abs_epi64 (__m128i __A) +{ + return (__m128i) __builtin_ia32_pabsq128_mask ((__v2di) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_abs_epi64 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pabsq128_mask ((__v2di) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_abs_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pabsq128_mask ((__v2di) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtpd_epu32 (__m256d __A) +{ + return (__m128i) __builtin_ia32_cvtpd2udq256_mask ((__v4df) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtpd_epu32 (__m128i __W, __mmask8 __U, __m256d __A) +{ + return (__m128i) __builtin_ia32_cvtpd2udq256_mask ((__v4df) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtpd_epu32 (__mmask8 __U, __m256d __A) +{ + return (__m128i) __builtin_ia32_cvtpd2udq256_mask ((__v4df) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtpd_epu32 (__m128d __A) +{ + return (__m128i) __builtin_ia32_cvtpd2udq128_mask ((__v2df) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtpd_epu32 (__m128i __W, __mmask8 __U, __m128d __A) +{ + return (__m128i) __builtin_ia32_cvtpd2udq128_mask ((__v2df) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtpd_epu32 (__mmask8 __U, __m128d __A) +{ + return (__m128i) __builtin_ia32_cvtpd2udq128_mask ((__v2df) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvttps_epi32 (__m256i __W, __mmask8 __U, __m256 __A) +{ + return (__m256i) __builtin_ia32_cvttps2dq256_mask ((__v8sf) __A, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvttps_epi32 (__mmask8 __U, __m256 __A) +{ + return (__m256i) __builtin_ia32_cvttps2dq256_mask ((__v8sf) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvttps_epi32 (__m128i __W, __mmask8 __U, __m128 __A) +{ + return (__m128i) __builtin_ia32_cvttps2dq128_mask ((__v4sf) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvttps_epi32 (__mmask8 __U, __m128 __A) +{ + return (__m128i) __builtin_ia32_cvttps2dq128_mask ((__v4sf) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvttps_epu32 (__m256 __A) +{ + return (__m256i) __builtin_ia32_cvttps2udq256_mask ((__v8sf) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvttps_epu32 (__m256i __W, __mmask8 __U, __m256 __A) +{ + return (__m256i) __builtin_ia32_cvttps2udq256_mask ((__v8sf) __A, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvttps_epu32 (__mmask8 __U, __m256 __A) +{ + return (__m256i) __builtin_ia32_cvttps2udq256_mask ((__v8sf) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttps_epu32 (__m128 __A) +{ + return (__m128i) __builtin_ia32_cvttps2udq128_mask ((__v4sf) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvttps_epu32 (__m128i __W, __mmask8 __U, __m128 __A) +{ + return (__m128i) __builtin_ia32_cvttps2udq128_mask ((__v4sf) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvttps_epu32 (__mmask8 __U, __m128 __A) +{ + return (__m128i) __builtin_ia32_cvttps2udq128_mask ((__v4sf) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvttpd_epi32 (__m128i __W, __mmask8 __U, __m256d __A) +{ + return (__m128i) __builtin_ia32_cvttpd2dq256_mask ((__v4df) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvttpd_epi32 (__mmask8 __U, __m256d __A) +{ + return (__m128i) __builtin_ia32_cvttpd2dq256_mask ((__v4df) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvttpd_epi32 (__m128i __W, __mmask8 __U, __m128d __A) +{ + return (__m128i) __builtin_ia32_cvttpd2dq128_mask ((__v2df) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvttpd_epi32 (__mmask8 __U, __m128d __A) +{ + return (__m128i) __builtin_ia32_cvttpd2dq128_mask ((__v2df) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvttpd_epu32 (__m256d __A) +{ + return (__m128i) __builtin_ia32_cvttpd2udq256_mask ((__v4df) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvttpd_epu32 (__m128i __W, __mmask8 __U, __m256d __A) +{ + return (__m128i) __builtin_ia32_cvttpd2udq256_mask ((__v4df) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvttpd_epu32 (__mmask8 __U, __m256d __A) +{ + return (__m128i) __builtin_ia32_cvttpd2udq256_mask ((__v4df) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttpd_epu32 (__m128d __A) +{ + return (__m128i) __builtin_ia32_cvttpd2udq128_mask ((__v2df) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvttpd_epu32 (__m128i __W, __mmask8 __U, __m128d __A) +{ + return (__m128i) __builtin_ia32_cvttpd2udq128_mask ((__v2df) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvttpd_epu32 (__mmask8 __U, __m128d __A) +{ + return (__m128i) __builtin_ia32_cvttpd2udq128_mask ((__v2df) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtpd_epi32 (__m128i __W, __mmask8 __U, __m256d __A) +{ + return (__m128i) __builtin_ia32_cvtpd2dq256_mask ((__v4df) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtpd_epi32 (__mmask8 __U, __m256d __A) +{ + return (__m128i) __builtin_ia32_cvtpd2dq256_mask ((__v4df) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtpd_epi32 (__m128i __W, __mmask8 __U, __m128d __A) +{ + return (__m128i) __builtin_ia32_cvtpd2dq128_mask ((__v2df) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtpd_epi32 (__mmask8 __U, __m128d __A) +{ + return (__m128i) __builtin_ia32_cvtpd2dq128_mask ((__v2df) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi32_pd (__m256d __W, __mmask8 __U, __m128i __A) +{ + return (__m256d) __builtin_ia32_cvtdq2pd256_mask ((__v4si) __A, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepi32_pd (__mmask8 __U, __m128i __A) +{ + return (__m256d) __builtin_ia32_cvtdq2pd256_mask ((__v4si) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi32_pd (__m128d __W, __mmask8 __U, __m128i __A) +{ + return (__m128d) __builtin_ia32_cvtdq2pd128_mask ((__v4si) __A, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepi32_pd (__mmask8 __U, __m128i __A) +{ + return (__m128d) __builtin_ia32_cvtdq2pd128_mask ((__v4si) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepu32_pd (__m128i __A) +{ + return (__m256d) __builtin_ia32_cvtudq2pd256_mask ((__v4si) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepu32_pd (__m256d __W, __mmask8 __U, __m128i __A) +{ + return (__m256d) __builtin_ia32_cvtudq2pd256_mask ((__v4si) __A, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepu32_pd (__mmask8 __U, __m128i __A) +{ + return (__m256d) __builtin_ia32_cvtudq2pd256_mask ((__v4si) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepu32_pd (__m128i __A) +{ + return (__m128d) __builtin_ia32_cvtudq2pd128_mask ((__v4si) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepu32_pd (__m128d __W, __mmask8 __U, __m128i __A) +{ + return (__m128d) __builtin_ia32_cvtudq2pd128_mask ((__v4si) __A, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepu32_pd (__mmask8 __U, __m128i __A) +{ + return (__m128d) __builtin_ia32_cvtudq2pd128_mask ((__v4si) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi32_ps (__m256 __W, __mmask8 __U, __m256i __A) +{ + return (__m256) __builtin_ia32_cvtdq2ps256_mask ((__v8si) __A, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepi32_ps (__mmask16 __U, __m256i __A) +{ + return (__m256) __builtin_ia32_cvtdq2ps256_mask ((__v8si) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi32_ps (__m128 __W, __mmask8 __U, __m128i __A) +{ + return (__m128) __builtin_ia32_cvtdq2ps128_mask ((__v4si) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepi32_ps (__mmask16 __U, __m128i __A) +{ + return (__m128) __builtin_ia32_cvtdq2ps128_mask ((__v4si) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepu32_ps (__m256i __A) +{ + return (__m256) __builtin_ia32_cvtudq2ps256_mask ((__v8si) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepu32_ps (__m256 __W, __mmask8 __U, __m256i __A) +{ + return (__m256) __builtin_ia32_cvtudq2ps256_mask ((__v8si) __A, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepu32_ps (__mmask8 __U, __m256i __A) +{ + return (__m256) __builtin_ia32_cvtudq2ps256_mask ((__v8si) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepu32_ps (__m128i __A) +{ + return (__m128) __builtin_ia32_cvtudq2ps128_mask ((__v4si) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepu32_ps (__m128 __W, __mmask8 __U, __m128i __A) +{ + return (__m128) __builtin_ia32_cvtudq2ps128_mask ((__v4si) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepu32_ps (__mmask8 __U, __m128i __A) +{ + return (__m128) __builtin_ia32_cvtudq2ps128_mask ((__v4si) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtps_pd (__m256d __W, __mmask8 __U, __m128 __A) +{ + return (__m256d) __builtin_ia32_cvtps2pd256_mask ((__v4sf) __A, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtps_pd (__mmask8 __U, __m128 __A) +{ + return (__m256d) __builtin_ia32_cvtps2pd256_mask ((__v4sf) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtps_pd (__m128d __W, __mmask8 __U, __m128 __A) +{ + return (__m128d) __builtin_ia32_cvtps2pd128_mask ((__v4sf) __A, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtps_pd (__mmask8 __U, __m128 __A) +{ + return (__m128d) __builtin_ia32_cvtps2pd128_mask ((__v4sf) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepi32_epi8 (__m128i __A) +{ + return (__m128i) __builtin_ia32_pmovdb128_mask ((__v4si) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi32_storeu_epi8 (void * __P, __mmask8 __M, __m128i __A) +{ + __builtin_ia32_pmovdb128mem_mask ((__v16qi *) __P, (__v4si) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi32_epi8 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovdb128_mask ((__v4si) __A, + (__v16qi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepi32_epi8 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovdb128_mask ((__v4si) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi32_epi8 (__m256i __A) +{ + return (__m128i) __builtin_ia32_pmovdb256_mask ((__v8si) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi32_epi8 (__m128i __O, __mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovdb256_mask ((__v8si) __A, + (__v16qi) __O, __M); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi32_storeu_epi8 (void * __P, __mmask8 __M, __m256i __A) +{ + __builtin_ia32_pmovdb256mem_mask ((__v16qi *) __P, (__v8si) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepi32_epi8 (__mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovdb256_mask ((__v8si) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsepi32_epi8 (__m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsdb128_mask ((__v4si) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtsepi32_storeu_epi8 (void * __P, __mmask8 __M, __m128i __A) +{ + __builtin_ia32_pmovsdb128mem_mask ((__v16qi *) __P, (__v4si) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtsepi32_epi8 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsdb128_mask ((__v4si) __A, + (__v16qi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtsepi32_epi8 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsdb128_mask ((__v4si) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtsepi32_epi8 (__m256i __A) +{ + return (__m128i) __builtin_ia32_pmovsdb256_mask ((__v8si) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtsepi32_storeu_epi8 (void * __P, __mmask8 __M, __m256i __A) +{ + __builtin_ia32_pmovsdb256mem_mask ((__v16qi *) __P, (__v8si) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtsepi32_epi8 (__m128i __O, __mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovsdb256_mask ((__v8si) __A, + (__v16qi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtsepi32_epi8 (__mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovsdb256_mask ((__v8si) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtusepi32_epi8 (__m128i __A) +{ + return (__m128i) __builtin_ia32_pmovusdb128_mask ((__v4si) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtusepi32_storeu_epi8 (void * __P, __mmask8 __M, __m128i __A) +{ + __builtin_ia32_pmovusdb128mem_mask ((__v16qi *) __P, (__v4si) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtusepi32_epi8 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovusdb128_mask ((__v4si) __A, + (__v16qi) __O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtusepi32_epi8 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovusdb128_mask ((__v4si) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtusepi32_epi8 (__m256i __A) +{ + return (__m128i) __builtin_ia32_pmovusdb256_mask ((__v8si) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtusepi32_storeu_epi8 (void * __P, __mmask8 __M, __m256i __A) +{ + __builtin_ia32_pmovusdb256mem_mask ((__v16qi*) __P, (__v8si) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtusepi32_epi8 (__m128i __O, __mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovusdb256_mask ((__v8si) __A, + (__v16qi) __O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtusepi32_epi8 (__mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovusdb256_mask ((__v8si) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepi32_epi16 (__m128i __A) +{ + return (__m128i) __builtin_ia32_pmovdw128_mask ((__v4si) __A, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi32_storeu_epi16 (void * __P, __mmask8 __M, __m128i __A) +{ + __builtin_ia32_pmovdw128mem_mask ((__v8hi *) __P, (__v4si) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi32_epi16 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovdw128_mask ((__v4si) __A, + (__v8hi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepi32_epi16 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovdw128_mask ((__v4si) __A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi32_epi16 (__m256i __A) +{ + return (__m128i) __builtin_ia32_pmovdw256_mask ((__v8si) __A, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi32_storeu_epi16 (void * __P, __mmask8 __M, __m256i __A) +{ + __builtin_ia32_pmovdw256mem_mask ((__v8hi *) __P, (__v8si) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi32_epi16 (__m128i __O, __mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovdw256_mask ((__v8si) __A, + (__v8hi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepi32_epi16 (__mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovdw256_mask ((__v8si) __A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsepi32_epi16 (__m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsdw128_mask ((__v4si) __A, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtsepi32_storeu_epi16 (void * __P, __mmask8 __M, __m128i __A) +{ + __builtin_ia32_pmovsdw128mem_mask ((__v8hi *) __P, (__v4si) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtsepi32_epi16 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsdw128_mask ((__v4si) __A, + (__v8hi)__O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtsepi32_epi16 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsdw128_mask ((__v4si) __A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtsepi32_epi16 (__m256i __A) +{ + return (__m128i) __builtin_ia32_pmovsdw256_mask ((__v8si) __A, + (__v8hi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtsepi32_storeu_epi16 (void * __P, __mmask8 __M, __m256i __A) +{ + __builtin_ia32_pmovsdw256mem_mask ((__v8hi *) __P, (__v8si) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtsepi32_epi16 (__m128i __O, __mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovsdw256_mask ((__v8si) __A, + (__v8hi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtsepi32_epi16 (__mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovsdw256_mask ((__v8si) __A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtusepi32_epi16 (__m128i __A) +{ + return (__m128i) __builtin_ia32_pmovusdw128_mask ((__v4si) __A, + (__v8hi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtusepi32_storeu_epi16 (void * __P, __mmask8 __M, __m128i __A) +{ + __builtin_ia32_pmovusdw128mem_mask ((__v8hi *) __P, (__v4si) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtusepi32_epi16 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovusdw128_mask ((__v4si) __A, + (__v8hi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtusepi32_epi16 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovusdw128_mask ((__v4si) __A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtusepi32_epi16 (__m256i __A) +{ + return (__m128i) __builtin_ia32_pmovusdw256_mask ((__v8si) __A, + (__v8hi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtusepi32_storeu_epi16 (void * __P, __mmask8 __M, __m256i __A) +{ + __builtin_ia32_pmovusdw256mem_mask ((__v8hi *) __P, (__v8si) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtusepi32_epi16 (__m128i __O, __mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovusdw256_mask ((__v8si) __A, + (__v8hi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtusepi32_epi16 (__mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovusdw256_mask ((__v8si) __A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepi64_epi8 (__m128i __A) +{ + return (__m128i) __builtin_ia32_pmovqb128_mask ((__v2di) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi64_storeu_epi8 (void * __P, __mmask8 __M, __m128i __A) +{ + __builtin_ia32_pmovqb128mem_mask ((__v16qi *) __P, (__v2di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi64_epi8 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovqb128_mask ((__v2di) __A, + (__v16qi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepi64_epi8 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovqb128_mask ((__v2di) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi64_epi8 (__m256i __A) +{ + return (__m128i) __builtin_ia32_pmovqb256_mask ((__v4di) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi64_storeu_epi8 (void * __P, __mmask8 __M, __m256i __A) +{ + __builtin_ia32_pmovqb256mem_mask ((__v16qi *) __P, (__v4di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi64_epi8 (__m128i __O, __mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovqb256_mask ((__v4di) __A, + (__v16qi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepi64_epi8 (__mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovqb256_mask ((__v4di) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsepi64_epi8 (__m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsqb128_mask ((__v2di) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtsepi64_storeu_epi8 (void * __P, __mmask8 __M, __m128i __A) +{ + __builtin_ia32_pmovsqb128mem_mask ((__v16qi *) __P, (__v2di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtsepi64_epi8 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsqb128_mask ((__v2di) __A, + (__v16qi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtsepi64_epi8 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsqb128_mask ((__v2di) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtsepi64_epi8 (__m256i __A) +{ + return (__m128i) __builtin_ia32_pmovsqb256_mask ((__v4di) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtsepi64_storeu_epi8 (void * __P, __mmask8 __M, __m256i __A) +{ + __builtin_ia32_pmovsqb256mem_mask ((__v16qi *) __P, (__v4di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtsepi64_epi8 (__m128i __O, __mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovsqb256_mask ((__v4di) __A, + (__v16qi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtsepi64_epi8 (__mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovsqb256_mask ((__v4di) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtusepi64_epi8 (__m128i __A) +{ + return (__m128i) __builtin_ia32_pmovusqb128_mask ((__v2di) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtusepi64_storeu_epi8 (void * __P, __mmask8 __M, __m128i __A) +{ + __builtin_ia32_pmovusqb128mem_mask ((__v16qi *) __P, (__v2di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtusepi64_epi8 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovusqb128_mask ((__v2di) __A, + (__v16qi) __O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtusepi64_epi8 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovusqb128_mask ((__v2di) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtusepi64_epi8 (__m256i __A) +{ + return (__m128i) __builtin_ia32_pmovusqb256_mask ((__v4di) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtusepi64_storeu_epi8 (void * __P, __mmask8 __M, __m256i __A) +{ + __builtin_ia32_pmovusqb256mem_mask ((__v16qi *) __P, (__v4di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtusepi64_epi8 (__m128i __O, __mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovusqb256_mask ((__v4di) __A, + (__v16qi) __O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtusepi64_epi8 (__mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovusqb256_mask ((__v4di) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepi64_epi16 (__m128i __A) +{ + return (__m128i) __builtin_ia32_pmovqw128_mask ((__v2di) __A, + (__v8hi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi64_storeu_epi16 (void * __P, __mmask8 __M, __m128i __A) +{ + __builtin_ia32_pmovqw128mem_mask ((__v8hi *) __P, (__v2di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi64_epi16 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovqw128_mask ((__v2di) __A, + (__v8hi)__O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepi64_epi16 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovqw128_mask ((__v2di) __A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi64_epi16 (__m256i __A) +{ + return (__m128i) __builtin_ia32_pmovqw256_mask ((__v4di) __A, + (__v8hi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi64_storeu_epi16 (void * __P, __mmask8 __M, __m256i __A) +{ + __builtin_ia32_pmovqw256mem_mask ((__v8hi *) __P, (__v4di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi64_epi16 (__m128i __O, __mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovqw256_mask ((__v4di) __A, + (__v8hi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepi64_epi16 (__mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovqw256_mask ((__v4di) __A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsepi64_epi16 (__m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsqw128_mask ((__v2di) __A, + (__v8hi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtsepi64_storeu_epi16 (void * __P, __mmask8 __M, __m128i __A) +{ + __builtin_ia32_pmovsqw128mem_mask ((__v8hi *) __P, (__v2di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtsepi64_epi16 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsqw128_mask ((__v2di) __A, + (__v8hi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtsepi64_epi16 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsqw128_mask ((__v2di) __A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtsepi64_epi16 (__m256i __A) +{ + return (__m128i) __builtin_ia32_pmovsqw256_mask ((__v4di) __A, + (__v8hi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtsepi64_storeu_epi16 (void * __P, __mmask8 __M, __m256i __A) +{ + __builtin_ia32_pmovsqw256mem_mask ((__v8hi *) __P, (__v4di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtsepi64_epi16 (__m128i __O, __mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovsqw256_mask ((__v4di) __A, + (__v8hi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtsepi64_epi16 (__mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovsqw256_mask ((__v4di) __A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtusepi64_epi16 (__m128i __A) +{ + return (__m128i) __builtin_ia32_pmovusqw128_mask ((__v2di) __A, + (__v8hi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtusepi64_storeu_epi16 (void * __P, __mmask8 __M, __m128i __A) +{ + __builtin_ia32_pmovusqw128mem_mask ((__v8hi *) __P, (__v2di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtusepi64_epi16 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovusqw128_mask ((__v2di) __A, + (__v8hi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtusepi64_epi16 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovusqw128_mask ((__v2di) __A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtusepi64_epi16 (__m256i __A) +{ + return (__m128i) __builtin_ia32_pmovusqw256_mask ((__v4di) __A, + (__v8hi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtusepi64_storeu_epi16 (void * __P, __mmask8 __M, __m256i __A) +{ + __builtin_ia32_pmovusqw256mem_mask ((__v8hi *) __P, (__v4di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtusepi64_epi16 (__m128i __O, __mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovusqw256_mask ((__v4di) __A, + (__v8hi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtusepi64_epi16 (__mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovusqw256_mask ((__v4di) __A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepi64_epi32 (__m128i __A) +{ + return (__m128i) __builtin_ia32_pmovqd128_mask ((__v2di) __A, + (__v4si) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi64_storeu_epi32 (void * __P, __mmask8 __M, __m128i __A) +{ + __builtin_ia32_pmovqd128mem_mask ((__v4si *) __P, (__v2di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi64_epi32 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovqd128_mask ((__v2di) __A, + (__v4si) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepi64_epi32 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovqd128_mask ((__v2di) __A, + (__v4si) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi64_epi32 (__m256i __A) +{ + return (__m128i) __builtin_ia32_pmovqd256_mask ((__v4di) __A, + (__v4si) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi64_storeu_epi32 (void * __P, __mmask8 __M, __m256i __A) +{ + __builtin_ia32_pmovqd256mem_mask ((__v4si *) __P, (__v4di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi64_epi32 (__m128i __O, __mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovqd256_mask ((__v4di) __A, + (__v4si) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepi64_epi32 (__mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovqd256_mask ((__v4di) __A, + (__v4si) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsepi64_epi32 (__m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsqd128_mask ((__v2di) __A, + (__v4si) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtsepi64_storeu_epi32 (void * __P, __mmask8 __M, __m128i __A) +{ + __builtin_ia32_pmovsqd128mem_mask ((__v4si *) __P, (__v2di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtsepi64_epi32 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsqd128_mask ((__v2di) __A, + (__v4si) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtsepi64_epi32 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsqd128_mask ((__v2di) __A, + (__v4si) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtsepi64_epi32 (__m256i __A) +{ + return (__m128i) __builtin_ia32_pmovsqd256_mask ((__v4di) __A, + (__v4si) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtsepi64_storeu_epi32 (void * __P, __mmask8 __M, __m256i __A) +{ + __builtin_ia32_pmovsqd256mem_mask ((__v4si *) __P, (__v4di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtsepi64_epi32 (__m128i __O, __mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovsqd256_mask ((__v4di) __A, + (__v4si)__O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtsepi64_epi32 (__mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovsqd256_mask ((__v4di) __A, + (__v4si) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtusepi64_epi32 (__m128i __A) +{ + return (__m128i) __builtin_ia32_pmovusqd128_mask ((__v2di) __A, + (__v4si) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtusepi64_storeu_epi32 (void * __P, __mmask8 __M, __m128i __A) +{ + __builtin_ia32_pmovusqd128mem_mask ((__v4si *) __P, (__v2di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtusepi64_epi32 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovusqd128_mask ((__v2di) __A, + (__v4si) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtusepi64_epi32 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovusqd128_mask ((__v2di) __A, + (__v4si) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtusepi64_epi32 (__m256i __A) +{ + return (__m128i) __builtin_ia32_pmovusqd256_mask ((__v4di) __A, + (__v4si) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtusepi64_storeu_epi32 (void * __P, __mmask8 __M, __m256i __A) +{ + __builtin_ia32_pmovusqd256mem_mask ((__v4si *) __P, (__v4di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtusepi64_epi32 (__m128i __O, __mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovusqd256_mask ((__v4di) __A, + (__v4si) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtusepi64_epi32 (__mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovusqd256_mask ((__v4di) __A, + (__v4si) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_broadcastss_ps (__m256 __O, __mmask8 __M, __m128 __A) +{ + return (__m256) __builtin_ia32_broadcastss256_mask ((__v4sf) __A, + (__v8sf) __O, + __M); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_broadcastss_ps (__mmask8 __M, __m128 __A) +{ + return (__m256) __builtin_ia32_broadcastss256_mask ((__v4sf) __A, + (__v8sf) + _mm256_setzero_ps (), + __M); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_broadcastss_ps (__m128 __O, __mmask8 __M, __m128 __A) +{ + return (__m128) __builtin_ia32_broadcastss128_mask ((__v4sf) __A, + (__v4sf) __O, + __M); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_broadcastss_ps (__mmask8 __M, __m128 __A) +{ + return (__m128) __builtin_ia32_broadcastss128_mask ((__v4sf) __A, + (__v4sf) + _mm_setzero_ps (), + __M); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_broadcastsd_pd (__m256d __O, __mmask8 __M, __m128d __A) +{ + return (__m256d) __builtin_ia32_broadcastsd256_mask ((__v2df) __A, + (__v4df) __O, + __M); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_broadcastsd_pd (__mmask8 __M, __m128d __A) +{ + return (__m256d) __builtin_ia32_broadcastsd256_mask ((__v2df) __A, + (__v4df) + _mm256_setzero_pd (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_broadcastd_epi32 (__m256i __O, __mmask8 __M, __m128i __A) +{ + return (__m256i) __builtin_ia32_pbroadcastd256_mask ((__v4si) __A, + (__v8si) __O, + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_broadcastd_epi32 (__mmask8 __M, __m128i __A) +{ + return (__m256i) __builtin_ia32_pbroadcastd256_mask ((__v4si) __A, + (__v8si) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_set1_epi32 (__m256i __O, __mmask8 __M, int __A) +{ + return (__m256i) __builtin_ia32_pbroadcastd256_gpr_mask (__A, (__v8si) __O, + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_set1_epi32 (__mmask8 __M, int __A) +{ + return (__m256i) __builtin_ia32_pbroadcastd256_gpr_mask (__A, + (__v8si) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_broadcastd_epi32 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pbroadcastd128_mask ((__v4si) __A, + (__v4si) __O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_broadcastd_epi32 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pbroadcastd128_mask ((__v4si) __A, + (__v4si) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_set1_epi32 (__m128i __O, __mmask8 __M, int __A) +{ + return (__m128i) __builtin_ia32_pbroadcastd128_gpr_mask (__A, (__v4si) __O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_set1_epi32 (__mmask8 __M, int __A) +{ + return (__m128i) + __builtin_ia32_pbroadcastd128_gpr_mask (__A, + (__v4si) _mm_setzero_si128 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_broadcastq_epi64 (__m256i __O, __mmask8 __M, __m128i __A) +{ + return (__m256i) __builtin_ia32_pbroadcastq256_mask ((__v2di) __A, + (__v4di) __O, + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_broadcastq_epi64 (__mmask8 __M, __m128i __A) +{ + return (__m256i) __builtin_ia32_pbroadcastq256_mask ((__v2di) __A, + (__v4di) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_set1_epi64 (__m256i __O, __mmask8 __M, long long __A) +{ + return (__m256i) __builtin_ia32_pbroadcastq256_gpr_mask (__A, (__v4di) __O, + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_set1_epi64 (__mmask8 __M, long long __A) +{ + return (__m256i) __builtin_ia32_pbroadcastq256_gpr_mask (__A, + (__v4di) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_broadcastq_epi64 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pbroadcastq128_mask ((__v2di) __A, + (__v2di) __O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_broadcastq_epi64 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pbroadcastq128_mask ((__v2di) __A, + (__v2di) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_set1_epi64 (__m128i __O, __mmask8 __M, long long __A) +{ + return (__m128i) __builtin_ia32_pbroadcastq128_gpr_mask (__A, (__v2di) __O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_set1_epi64 (__mmask8 __M, long long __A) +{ + return (__m128i) + __builtin_ia32_pbroadcastq128_gpr_mask (__A, + (__v2di) _mm_setzero_si128 (), + __M); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcast_f32x4 (__m128 __A) +{ + return (__m256) __builtin_ia32_broadcastf32x4_256_mask ((__v4sf) __A, + (__v8sf)_mm256_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_broadcast_f32x4 (__m256 __O, __mmask8 __M, __m128 __A) +{ + return (__m256) __builtin_ia32_broadcastf32x4_256_mask ((__v4sf) __A, + (__v8sf) __O, + __M); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_broadcast_f32x4 (__mmask8 __M, __m128 __A) +{ + return (__m256) __builtin_ia32_broadcastf32x4_256_mask ((__v4sf) __A, + (__v8sf) + _mm256_setzero_ps (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcast_i32x4 (__m128i __A) +{ + return (__m256i) __builtin_ia32_broadcasti32x4_256_mask ((__v4si) + __A, + (__v8si)_mm256_undefined_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_broadcast_i32x4 (__m256i __O, __mmask8 __M, __m128i __A) +{ + return (__m256i) __builtin_ia32_broadcasti32x4_256_mask ((__v4si) + __A, + (__v8si) + __O, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_broadcast_i32x4 (__mmask8 __M, __m128i __A) +{ + return (__m256i) __builtin_ia32_broadcasti32x4_256_mask ((__v4si) + __A, + (__v8si) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi8_epi32 (__m256i __W, __mmask8 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovsxbd256_mask ((__v16qi) __A, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepi8_epi32 (__mmask8 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovsxbd256_mask ((__v16qi) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi8_epi32 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsxbd128_mask ((__v16qi) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepi8_epi32 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsxbd128_mask ((__v16qi) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi8_epi64 (__m256i __W, __mmask8 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovsxbq256_mask ((__v16qi) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepi8_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovsxbq256_mask ((__v16qi) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi8_epi64 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsxbq128_mask ((__v16qi) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepi8_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsxbq128_mask ((__v16qi) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi16_epi32 (__m256i __W, __mmask8 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovsxwd256_mask ((__v8hi) __A, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepi16_epi32 (__mmask8 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovsxwd256_mask ((__v8hi) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi16_epi32 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsxwd128_mask ((__v8hi) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepi16_epi32 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsxwd128_mask ((__v8hi) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi16_epi64 (__m256i __W, __mmask8 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovsxwq256_mask ((__v8hi) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepi16_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovsxwq256_mask ((__v8hi) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi16_epi64 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsxwq128_mask ((__v8hi) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepi16_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsxwq128_mask ((__v8hi) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi32_epi64 (__m256i __W, __mmask8 __U, __m128i __X) +{ + return (__m256i) __builtin_ia32_pmovsxdq256_mask ((__v4si) __X, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepi32_epi64 (__mmask8 __U, __m128i __X) +{ + return (__m256i) __builtin_ia32_pmovsxdq256_mask ((__v4si) __X, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi32_epi64 (__m128i __W, __mmask8 __U, __m128i __X) +{ + return (__m128i) __builtin_ia32_pmovsxdq128_mask ((__v4si) __X, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepi32_epi64 (__mmask8 __U, __m128i __X) +{ + return (__m128i) __builtin_ia32_pmovsxdq128_mask ((__v4si) __X, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepu8_epi32 (__m256i __W, __mmask8 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovzxbd256_mask ((__v16qi) __A, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepu8_epi32 (__mmask8 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovzxbd256_mask ((__v16qi) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepu8_epi32 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovzxbd128_mask ((__v16qi) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepu8_epi32 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovzxbd128_mask ((__v16qi) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepu8_epi64 (__m256i __W, __mmask8 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovzxbq256_mask ((__v16qi) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepu8_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovzxbq256_mask ((__v16qi) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepu8_epi64 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovzxbq128_mask ((__v16qi) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepu8_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovzxbq128_mask ((__v16qi) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepu16_epi32 (__m256i __W, __mmask8 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovzxwd256_mask ((__v8hi) __A, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepu16_epi32 (__mmask8 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovzxwd256_mask ((__v8hi) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepu16_epi32 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovzxwd128_mask ((__v8hi) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepu16_epi32 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovzxwd128_mask ((__v8hi) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepu16_epi64 (__m256i __W, __mmask8 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovzxwq256_mask ((__v8hi) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepu16_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovzxwq256_mask ((__v8hi) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepu16_epi64 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovzxwq128_mask ((__v8hi) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepu16_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovzxwq128_mask ((__v8hi) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepu32_epi64 (__m256i __W, __mmask8 __U, __m128i __X) +{ + return (__m256i) __builtin_ia32_pmovzxdq256_mask ((__v4si) __X, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepu32_epi64 (__mmask8 __U, __m128i __X) +{ + return (__m256i) __builtin_ia32_pmovzxdq256_mask ((__v4si) __X, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepu32_epi64 (__m128i __W, __mmask8 __U, __m128i __X) +{ + return (__m128i) __builtin_ia32_pmovzxdq128_mask ((__v4si) __X, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepu32_epi64 (__mmask8 __U, __m128i __X) +{ + return (__m128i) __builtin_ia32_pmovzxdq128_mask ((__v4si) __X, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_rcp14_pd (__m256d __A) +{ + return (__m256d) __builtin_ia32_rcp14pd256_mask ((__v4df) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_rcp14_pd (__m256d __W, __mmask8 __U, __m256d __A) +{ + return (__m256d) __builtin_ia32_rcp14pd256_mask ((__v4df) __A, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_rcp14_pd (__mmask8 __U, __m256d __A) +{ + return (__m256d) __builtin_ia32_rcp14pd256_mask ((__v4df) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rcp14_pd (__m128d __A) +{ + return (__m128d) __builtin_ia32_rcp14pd128_mask ((__v2df) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_rcp14_pd (__m128d __W, __mmask8 __U, __m128d __A) +{ + return (__m128d) __builtin_ia32_rcp14pd128_mask ((__v2df) __A, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_rcp14_pd (__mmask8 __U, __m128d __A) +{ + return (__m128d) __builtin_ia32_rcp14pd128_mask ((__v2df) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_rcp14_ps (__m256 __A) +{ + return (__m256) __builtin_ia32_rcp14ps256_mask ((__v8sf) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_rcp14_ps (__m256 __W, __mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_rcp14ps256_mask ((__v8sf) __A, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_rcp14_ps (__mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_rcp14ps256_mask ((__v8sf) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rcp14_ps (__m128 __A) +{ + return (__m128) __builtin_ia32_rcp14ps128_mask ((__v4sf) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_rcp14_ps (__m128 __W, __mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_rcp14ps128_mask ((__v4sf) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_rcp14_ps (__mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_rcp14ps128_mask ((__v4sf) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_rsqrt14_pd (__m256d __A) +{ + return (__m256d) __builtin_ia32_rsqrt14pd256_mask ((__v4df) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_rsqrt14_pd (__m256d __W, __mmask8 __U, __m256d __A) +{ + return (__m256d) __builtin_ia32_rsqrt14pd256_mask ((__v4df) __A, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_rsqrt14_pd (__mmask8 __U, __m256d __A) +{ + return (__m256d) __builtin_ia32_rsqrt14pd256_mask ((__v4df) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rsqrt14_pd (__m128d __A) +{ + return (__m128d) __builtin_ia32_rsqrt14pd128_mask ((__v2df) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_rsqrt14_pd (__m128d __W, __mmask8 __U, __m128d __A) +{ + return (__m128d) __builtin_ia32_rsqrt14pd128_mask ((__v2df) __A, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_rsqrt14_pd (__mmask8 __U, __m128d __A) +{ + return (__m128d) __builtin_ia32_rsqrt14pd128_mask ((__v2df) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_rsqrt14_ps (__m256 __A) +{ + return (__m256) __builtin_ia32_rsqrt14ps256_mask ((__v8sf) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_rsqrt14_ps (__m256 __W, __mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_rsqrt14ps256_mask ((__v8sf) __A, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_rsqrt14_ps (__mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_rsqrt14ps256_mask ((__v8sf) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rsqrt14_ps (__m128 __A) +{ + return (__m128) __builtin_ia32_rsqrt14ps128_mask ((__v4sf) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_rsqrt14_ps (__m128 __W, __mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_rsqrt14ps128_mask ((__v4sf) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_rsqrt14_ps (__mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_rsqrt14ps128_mask ((__v4sf) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sqrt_pd (__m256d __W, __mmask8 __U, __m256d __A) +{ + return (__m256d) __builtin_ia32_sqrtpd256_mask ((__v4df) __A, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sqrt_pd (__mmask8 __U, __m256d __A) +{ + return (__m256d) __builtin_ia32_sqrtpd256_mask ((__v4df) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sqrt_pd (__m128d __W, __mmask8 __U, __m128d __A) +{ + return (__m128d) __builtin_ia32_sqrtpd128_mask ((__v2df) __A, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sqrt_pd (__mmask8 __U, __m128d __A) +{ + return (__m128d) __builtin_ia32_sqrtpd128_mask ((__v2df) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sqrt_ps (__m256 __W, __mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_sqrtps256_mask ((__v8sf) __A, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sqrt_ps (__mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_sqrtps256_mask ((__v8sf) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sqrt_ps (__m128 __W, __mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_sqrtps128_mask ((__v4sf) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sqrt_ps (__mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_sqrtps128_mask ((__v4sf) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_add_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_paddd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_add_epi32 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_paddd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_add_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_paddq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_add_epi64 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_paddq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sub_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_psubd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sub_epi32 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_psubd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sub_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_psubq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sub_epi64 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_psubq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_add_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_paddd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_add_epi32 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_paddd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_add_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_paddq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_add_epi64 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_paddq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sub_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psubd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sub_epi32 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psubd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sub_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psubq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sub_epi64 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psubq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_getexp_ps (__m256 __A) +{ + return (__m256) __builtin_ia32_getexpps256_mask ((__v8sf) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_getexp_ps (__m256 __W, __mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_getexpps256_mask ((__v8sf) __A, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_getexp_ps (__mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_getexpps256_mask ((__v8sf) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_getexp_pd (__m256d __A) +{ + return (__m256d) __builtin_ia32_getexppd256_mask ((__v4df) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_getexp_pd (__m256d __W, __mmask8 __U, __m256d __A) +{ + return (__m256d) __builtin_ia32_getexppd256_mask ((__v4df) __A, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_getexp_pd (__mmask8 __U, __m256d __A) +{ + return (__m256d) __builtin_ia32_getexppd256_mask ((__v4df) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_getexp_ps (__m128 __A) +{ + return (__m128) __builtin_ia32_getexpps128_mask ((__v4sf) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_getexp_ps (__m128 __W, __mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_getexpps128_mask ((__v4sf) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_getexp_ps (__mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_getexpps128_mask ((__v4sf) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_getexp_pd (__m128d __A) +{ + return (__m128d) __builtin_ia32_getexppd128_mask ((__v2df) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_getexp_pd (__m128d __W, __mmask8 __U, __m128d __A) +{ + return (__m128d) __builtin_ia32_getexppd128_mask ((__v2df) __A, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_getexp_pd (__mmask8 __U, __m128d __A) +{ + return (__m128d) __builtin_ia32_getexppd128_mask ((__v2df) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_srl_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + __m128i __B) +{ + return (__m256i) __builtin_ia32_psrld256_mask ((__v8si) __A, + (__v4si) __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_srl_epi32 (__mmask8 __U, __m256i __A, __m128i __B) +{ + return (__m256i) __builtin_ia32_psrld256_mask ((__v8si) __A, + (__v4si) __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_srl_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psrld128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_srl_epi32 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psrld128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_srl_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + __m128i __B) +{ + return (__m256i) __builtin_ia32_psrlq256_mask ((__v4di) __A, + (__v2di) __B, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_srl_epi64 (__mmask8 __U, __m256i __A, __m128i __B) +{ + return (__m256i) __builtin_ia32_psrlq256_mask ((__v4di) __A, + (__v2di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_srl_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psrlq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_srl_epi64 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psrlq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_and_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pandd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_and_epi32 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pandd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_scalef_pd (__m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_scalefpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_scalef_pd (__m256d __W, __mmask8 __U, __m256d __A, + __m256d __B) +{ + return (__m256d) __builtin_ia32_scalefpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_scalef_pd (__mmask8 __U, __m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_scalefpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_scalef_ps (__m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_scalefps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_scalef_ps (__m256 __W, __mmask8 __U, __m256 __A, + __m256 __B) +{ + return (__m256) __builtin_ia32_scalefps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_scalef_ps (__mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_scalefps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_scalef_pd (__m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_scalefpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_scalef_pd (__m128d __W, __mmask8 __U, __m128d __A, + __m128d __B) +{ + return (__m128d) __builtin_ia32_scalefpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_scalef_pd (__mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_scalefpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_scalef_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_scalefps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_scalef_ps (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_scalefps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_scalef_ps (__mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_scalefps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_fmadd_pd (__m256d __A, __mmask8 __U, __m256d __B, + __m256d __C) +{ + return (__m256d) __builtin_ia32_vfmaddpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __C, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask3_fmadd_pd (__m256d __A, __m256d __B, __m256d __C, + __mmask8 __U) +{ + return (__m256d) __builtin_ia32_vfmaddpd256_mask3 ((__v4df) __A, + (__v4df) __B, + (__v4df) __C, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_fmadd_pd (__mmask8 __U, __m256d __A, __m256d __B, + __m256d __C) +{ + return (__m256d) __builtin_ia32_vfmaddpd256_maskz ((__v4df) __A, + (__v4df) __B, + (__v4df) __C, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fmadd_pd (__m128d __A, __mmask8 __U, __m128d __B, __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __C, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask3_fmadd_pd (__m128d __A, __m128d __B, __m128d __C, + __mmask8 __U) +{ + return (__m128d) __builtin_ia32_vfmaddpd128_mask3 ((__v2df) __A, + (__v2df) __B, + (__v2df) __C, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fmadd_pd (__mmask8 __U, __m128d __A, __m128d __B, + __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddpd128_maskz ((__v2df) __A, + (__v2df) __B, + (__v2df) __C, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_fmadd_ps (__m256 __A, __mmask8 __U, __m256 __B, __m256 __C) +{ + return (__m256) __builtin_ia32_vfmaddps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask3_fmadd_ps (__m256 __A, __m256 __B, __m256 __C, + __mmask8 __U) +{ + return (__m256) __builtin_ia32_vfmaddps256_mask3 ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_fmadd_ps (__mmask8 __U, __m256 __A, __m256 __B, + __m256 __C) +{ + return (__m256) __builtin_ia32_vfmaddps256_maskz ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fmadd_ps (__m128 __A, __mmask8 __U, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask3_fmadd_ps (__m128 __A, __m128 __B, __m128 __C, __mmask8 __U) +{ + return (__m128) __builtin_ia32_vfmaddps128_mask3 ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fmadd_ps (__mmask8 __U, __m128 __A, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddps128_maskz ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_fmsub_pd (__m256d __A, __mmask8 __U, __m256d __B, + __m256d __C) +{ + return (__m256d) __builtin_ia32_vfmaddpd256_mask ((__v4df) __A, + (__v4df) __B, + -(__v4df) __C, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask3_fmsub_pd (__m256d __A, __m256d __B, __m256d __C, + __mmask8 __U) +{ + return (__m256d) __builtin_ia32_vfmsubpd256_mask3 ((__v4df) __A, + (__v4df) __B, + (__v4df) __C, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_fmsub_pd (__mmask8 __U, __m256d __A, __m256d __B, + __m256d __C) +{ + return (__m256d) __builtin_ia32_vfmaddpd256_maskz ((__v4df) __A, + (__v4df) __B, + -(__v4df) __C, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fmsub_pd (__m128d __A, __mmask8 __U, __m128d __B, __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddpd128_mask ((__v2df) __A, + (__v2df) __B, + -(__v2df) __C, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask3_fmsub_pd (__m128d __A, __m128d __B, __m128d __C, + __mmask8 __U) +{ + return (__m128d) __builtin_ia32_vfmsubpd128_mask3 ((__v2df) __A, + (__v2df) __B, + (__v2df) __C, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fmsub_pd (__mmask8 __U, __m128d __A, __m128d __B, + __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddpd128_maskz ((__v2df) __A, + (__v2df) __B, + -(__v2df) __C, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_fmsub_ps (__m256 __A, __mmask8 __U, __m256 __B, __m256 __C) +{ + return (__m256) __builtin_ia32_vfmaddps256_mask ((__v8sf) __A, + (__v8sf) __B, + -(__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask3_fmsub_ps (__m256 __A, __m256 __B, __m256 __C, + __mmask8 __U) +{ + return (__m256) __builtin_ia32_vfmsubps256_mask3 ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_fmsub_ps (__mmask8 __U, __m256 __A, __m256 __B, + __m256 __C) +{ + return (__m256) __builtin_ia32_vfmaddps256_maskz ((__v8sf) __A, + (__v8sf) __B, + -(__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fmsub_ps (__m128 __A, __mmask8 __U, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddps128_mask ((__v4sf) __A, + (__v4sf) __B, + -(__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask3_fmsub_ps (__m128 __A, __m128 __B, __m128 __C, __mmask8 __U) +{ + return (__m128) __builtin_ia32_vfmsubps128_mask3 ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fmsub_ps (__mmask8 __U, __m128 __A, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddps128_maskz ((__v4sf) __A, + (__v4sf) __B, + -(__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_fmaddsub_pd (__m256d __A, __mmask8 __U, __m256d __B, + __m256d __C) +{ + return (__m256d) __builtin_ia32_vfmaddsubpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __C, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask3_fmaddsub_pd (__m256d __A, __m256d __B, __m256d __C, + __mmask8 __U) +{ + return (__m256d) __builtin_ia32_vfmaddsubpd256_mask3 ((__v4df) __A, + (__v4df) __B, + (__v4df) __C, + (__mmask8) + __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_fmaddsub_pd (__mmask8 __U, __m256d __A, __m256d __B, + __m256d __C) +{ + return (__m256d) __builtin_ia32_vfmaddsubpd256_maskz ((__v4df) __A, + (__v4df) __B, + (__v4df) __C, + (__mmask8) + __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fmaddsub_pd (__m128d __A, __mmask8 __U, __m128d __B, + __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddsubpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __C, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask3_fmaddsub_pd (__m128d __A, __m128d __B, __m128d __C, + __mmask8 __U) +{ + return (__m128d) __builtin_ia32_vfmaddsubpd128_mask3 ((__v2df) __A, + (__v2df) __B, + (__v2df) __C, + (__mmask8) + __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fmaddsub_pd (__mmask8 __U, __m128d __A, __m128d __B, + __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddsubpd128_maskz ((__v2df) __A, + (__v2df) __B, + (__v2df) __C, + (__mmask8) + __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_fmaddsub_ps (__m256 __A, __mmask8 __U, __m256 __B, + __m256 __C) +{ + return (__m256) __builtin_ia32_vfmaddsubps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask3_fmaddsub_ps (__m256 __A, __m256 __B, __m256 __C, + __mmask8 __U) +{ + return (__m256) __builtin_ia32_vfmaddsubps256_mask3 ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_fmaddsub_ps (__mmask8 __U, __m256 __A, __m256 __B, + __m256 __C) +{ + return (__m256) __builtin_ia32_vfmaddsubps256_maskz ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fmaddsub_ps (__m128 __A, __mmask8 __U, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddsubps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask3_fmaddsub_ps (__m128 __A, __m128 __B, __m128 __C, + __mmask8 __U) +{ + return (__m128) __builtin_ia32_vfmaddsubps128_mask3 ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fmaddsub_ps (__mmask8 __U, __m128 __A, __m128 __B, + __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddsubps128_maskz ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_fmsubadd_pd (__m256d __A, __mmask8 __U, __m256d __B, + __m256d __C) +{ + return (__m256d) __builtin_ia32_vfmaddsubpd256_mask ((__v4df) __A, + (__v4df) __B, + -(__v4df) __C, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask3_fmsubadd_pd (__m256d __A, __m256d __B, __m256d __C, + __mmask8 __U) +{ + return (__m256d) __builtin_ia32_vfmsubaddpd256_mask3 ((__v4df) __A, + (__v4df) __B, + (__v4df) __C, + (__mmask8) + __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_fmsubadd_pd (__mmask8 __U, __m256d __A, __m256d __B, + __m256d __C) +{ + return (__m256d) __builtin_ia32_vfmaddsubpd256_maskz ((__v4df) __A, + (__v4df) __B, + -(__v4df) __C, + (__mmask8) + __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fmsubadd_pd (__m128d __A, __mmask8 __U, __m128d __B, + __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddsubpd128_mask ((__v2df) __A, + (__v2df) __B, + -(__v2df) __C, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask3_fmsubadd_pd (__m128d __A, __m128d __B, __m128d __C, + __mmask8 __U) +{ + return (__m128d) __builtin_ia32_vfmsubaddpd128_mask3 ((__v2df) __A, + (__v2df) __B, + (__v2df) __C, + (__mmask8) + __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fmsubadd_pd (__mmask8 __U, __m128d __A, __m128d __B, + __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddsubpd128_maskz ((__v2df) __A, + (__v2df) __B, + -(__v2df) __C, + (__mmask8) + __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_fmsubadd_ps (__m256 __A, __mmask8 __U, __m256 __B, + __m256 __C) +{ + return (__m256) __builtin_ia32_vfmaddsubps256_mask ((__v8sf) __A, + (__v8sf) __B, + -(__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask3_fmsubadd_ps (__m256 __A, __m256 __B, __m256 __C, + __mmask8 __U) +{ + return (__m256) __builtin_ia32_vfmsubaddps256_mask3 ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_fmsubadd_ps (__mmask8 __U, __m256 __A, __m256 __B, + __m256 __C) +{ + return (__m256) __builtin_ia32_vfmaddsubps256_maskz ((__v8sf) __A, + (__v8sf) __B, + -(__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fmsubadd_ps (__m128 __A, __mmask8 __U, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddsubps128_mask ((__v4sf) __A, + (__v4sf) __B, + -(__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask3_fmsubadd_ps (__m128 __A, __m128 __B, __m128 __C, + __mmask8 __U) +{ + return (__m128) __builtin_ia32_vfmsubaddps128_mask3 ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fmsubadd_ps (__mmask8 __U, __m128 __A, __m128 __B, + __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddsubps128_maskz ((__v4sf) __A, + (__v4sf) __B, + -(__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_fnmadd_pd (__m256d __A, __mmask8 __U, __m256d __B, + __m256d __C) +{ + return (__m256d) __builtin_ia32_vfnmaddpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __C, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask3_fnmadd_pd (__m256d __A, __m256d __B, __m256d __C, + __mmask8 __U) +{ + return (__m256d) __builtin_ia32_vfmaddpd256_mask3 (-(__v4df) __A, + (__v4df) __B, + (__v4df) __C, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_fnmadd_pd (__mmask8 __U, __m256d __A, __m256d __B, + __m256d __C) +{ + return (__m256d) __builtin_ia32_vfmaddpd256_maskz (-(__v4df) __A, + (__v4df) __B, + (__v4df) __C, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fnmadd_pd (__m128d __A, __mmask8 __U, __m128d __B, + __m128d __C) +{ + return (__m128d) __builtin_ia32_vfnmaddpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __C, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask3_fnmadd_pd (__m128d __A, __m128d __B, __m128d __C, + __mmask8 __U) +{ + return (__m128d) __builtin_ia32_vfmaddpd128_mask3 (-(__v2df) __A, + (__v2df) __B, + (__v2df) __C, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fnmadd_pd (__mmask8 __U, __m128d __A, __m128d __B, + __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddpd128_maskz (-(__v2df) __A, + (__v2df) __B, + (__v2df) __C, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_fnmadd_ps (__m256 __A, __mmask8 __U, __m256 __B, + __m256 __C) +{ + return (__m256) __builtin_ia32_vfnmaddps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask3_fnmadd_ps (__m256 __A, __m256 __B, __m256 __C, + __mmask8 __U) +{ + return (__m256) __builtin_ia32_vfmaddps256_mask3 (-(__v8sf) __A, + (__v8sf) __B, + (__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_fnmadd_ps (__mmask8 __U, __m256 __A, __m256 __B, + __m256 __C) +{ + return (__m256) __builtin_ia32_vfmaddps256_maskz (-(__v8sf) __A, + (__v8sf) __B, + (__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fnmadd_ps (__m128 __A, __mmask8 __U, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfnmaddps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask3_fnmadd_ps (__m128 __A, __m128 __B, __m128 __C, __mmask8 __U) +{ + return (__m128) __builtin_ia32_vfmaddps128_mask3 (-(__v4sf) __A, + (__v4sf) __B, + (__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fnmadd_ps (__mmask8 __U, __m128 __A, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddps128_maskz (-(__v4sf) __A, + (__v4sf) __B, + (__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_fnmsub_pd (__m256d __A, __mmask8 __U, __m256d __B, + __m256d __C) +{ + return (__m256d) __builtin_ia32_vfnmsubpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __C, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask3_fnmsub_pd (__m256d __A, __m256d __B, __m256d __C, + __mmask8 __U) +{ + return (__m256d) __builtin_ia32_vfnmsubpd256_mask3 ((__v4df) __A, + (__v4df) __B, + (__v4df) __C, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_fnmsub_pd (__mmask8 __U, __m256d __A, __m256d __B, + __m256d __C) +{ + return (__m256d) __builtin_ia32_vfmaddpd256_maskz (-(__v4df) __A, + (__v4df) __B, + -(__v4df) __C, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fnmsub_pd (__m128d __A, __mmask8 __U, __m128d __B, + __m128d __C) +{ + return (__m128d) __builtin_ia32_vfnmsubpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __C, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask3_fnmsub_pd (__m128d __A, __m128d __B, __m128d __C, + __mmask8 __U) +{ + return (__m128d) __builtin_ia32_vfnmsubpd128_mask3 ((__v2df) __A, + (__v2df) __B, + (__v2df) __C, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fnmsub_pd (__mmask8 __U, __m128d __A, __m128d __B, + __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddpd128_maskz (-(__v2df) __A, + (__v2df) __B, + -(__v2df) __C, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_fnmsub_ps (__m256 __A, __mmask8 __U, __m256 __B, + __m256 __C) +{ + return (__m256) __builtin_ia32_vfnmsubps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask3_fnmsub_ps (__m256 __A, __m256 __B, __m256 __C, + __mmask8 __U) +{ + return (__m256) __builtin_ia32_vfnmsubps256_mask3 ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_fnmsub_ps (__mmask8 __U, __m256 __A, __m256 __B, + __m256 __C) +{ + return (__m256) __builtin_ia32_vfmaddps256_maskz (-(__v8sf) __A, + (__v8sf) __B, + -(__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fnmsub_ps (__m128 __A, __mmask8 __U, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfnmsubps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask3_fnmsub_ps (__m128 __A, __m128 __B, __m128 __C, __mmask8 __U) +{ + return (__m128) __builtin_ia32_vfnmsubps128_mask3 ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fnmsub_ps (__mmask8 __U, __m128 __A, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddps128_maskz (-(__v4sf) __A, + (__v4sf) __B, + -(__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_and_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pandd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_and_epi32 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pandd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_andnot_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pandnd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_andnot_epi32 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pandnd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_andnot_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pandnd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_andnot_epi32 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pandnd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_or_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pord256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_or_epi32 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pord256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_or_epi32 (__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pord128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_or_epi32 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pord128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_xor_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pxord256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_xor_epi32 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pxord256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_xor_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pxord128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_xor_epi32 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pxord128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtpd_ps (__m128 __W, __mmask8 __U, __m128d __A) +{ + return (__m128) __builtin_ia32_cvtpd2ps_mask ((__v2df) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtpd_ps (__mmask8 __U, __m128d __A) +{ + return (__m128) __builtin_ia32_cvtpd2ps_mask ((__v2df) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtpd_ps (__m128 __W, __mmask8 __U, __m256d __A) +{ + return (__m128) __builtin_ia32_cvtpd2ps256_mask ((__v4df) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtpd_ps (__mmask8 __U, __m256d __A) +{ + return (__m128) __builtin_ia32_cvtpd2ps256_mask ((__v4df) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtps_epi32 (__m256i __W, __mmask8 __U, __m256 __A) +{ + return (__m256i) __builtin_ia32_cvtps2dq256_mask ((__v8sf) __A, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtps_epi32 (__mmask8 __U, __m256 __A) +{ + return (__m256i) __builtin_ia32_cvtps2dq256_mask ((__v8sf) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtps_epi32 (__m128i __W, __mmask8 __U, __m128 __A) +{ + return (__m128i) __builtin_ia32_cvtps2dq128_mask ((__v4sf) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtps_epi32 (__mmask8 __U, __m128 __A) +{ + return (__m128i) __builtin_ia32_cvtps2dq128_mask ((__v4sf) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtps_epu32 (__m256 __A) +{ + return (__m256i) __builtin_ia32_cvtps2udq256_mask ((__v8sf) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtps_epu32 (__m256i __W, __mmask8 __U, __m256 __A) +{ + return (__m256i) __builtin_ia32_cvtps2udq256_mask ((__v8sf) __A, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtps_epu32 (__mmask8 __U, __m256 __A) +{ + return (__m256i) __builtin_ia32_cvtps2udq256_mask ((__v8sf) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtps_epu32 (__m128 __A) +{ + return (__m128i) __builtin_ia32_cvtps2udq128_mask ((__v4sf) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtps_epu32 (__m128i __W, __mmask8 __U, __m128 __A) +{ + return (__m128i) __builtin_ia32_cvtps2udq128_mask ((__v4sf) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtps_epu32 (__mmask8 __U, __m128 __A) +{ + return (__m128i) __builtin_ia32_cvtps2udq128_mask ((__v4sf) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_movedup_pd (__m256d __W, __mmask8 __U, __m256d __A) +{ + return (__m256d) __builtin_ia32_movddup256_mask ((__v4df) __A, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_movedup_pd (__mmask8 __U, __m256d __A) +{ + return (__m256d) __builtin_ia32_movddup256_mask ((__v4df) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_movedup_pd (__m128d __W, __mmask8 __U, __m128d __A) +{ + return (__m128d) __builtin_ia32_movddup128_mask ((__v2df) __A, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_movedup_pd (__mmask8 __U, __m128d __A) +{ + return (__m128d) __builtin_ia32_movddup128_mask ((__v2df) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_movehdup_ps (__m256 __W, __mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_movshdup256_mask ((__v8sf) __A, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_movehdup_ps (__mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_movshdup256_mask ((__v8sf) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_movehdup_ps (__m128 __W, __mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_movshdup128_mask ((__v4sf) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_movehdup_ps (__mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_movshdup128_mask ((__v4sf) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_moveldup_ps (__m256 __W, __mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_movsldup256_mask ((__v8sf) __A, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_moveldup_ps (__mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_movsldup256_mask ((__v8sf) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_moveldup_ps (__m128 __W, __mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_movsldup128_mask ((__v4sf) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_moveldup_ps (__mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_movsldup128_mask ((__v4sf) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_unpackhi_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_punpckhdq128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_unpackhi_epi32 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_punpckhdq128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_unpackhi_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_punpckhdq256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_unpackhi_epi32 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_punpckhdq256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_unpackhi_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_punpckhqdq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_unpackhi_epi64 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_punpckhqdq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_unpackhi_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_punpckhqdq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_unpackhi_epi64 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_punpckhqdq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_unpacklo_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_punpckldq128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_unpacklo_epi32 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_punpckldq128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_unpacklo_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_punpckldq256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_unpacklo_epi32 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_punpckldq256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_unpacklo_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_punpcklqdq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_unpacklo_epi64 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_punpcklqdq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_unpacklo_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_punpcklqdq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_unpacklo_epi64 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_punpcklqdq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_epu32_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ucmpd128_mask ((__v4si) __A, + (__v4si) __B, 0, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_epi32_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_pcmpeqd128_mask ((__v4si) __A, + (__v4si) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpeq_epu32_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ucmpd128_mask ((__v4si) __A, + (__v4si) __B, 0, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpeq_epi32_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_pcmpeqd128_mask ((__v4si) __A, + (__v4si) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpeq_epu32_mask (__m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_ucmpd256_mask ((__v8si) __A, + (__v8si) __B, 0, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpeq_epi32_mask (__m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_pcmpeqd256_mask ((__v8si) __A, + (__v8si) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpeq_epu32_mask (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_ucmpd256_mask ((__v8si) __A, + (__v8si) __B, 0, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpeq_epi32_mask (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_pcmpeqd256_mask ((__v8si) __A, + (__v8si) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_epu64_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ucmpq128_mask ((__v2di) __A, + (__v2di) __B, 0, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_epi64_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_pcmpeqq128_mask ((__v2di) __A, + (__v2di) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpeq_epu64_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ucmpq128_mask ((__v2di) __A, + (__v2di) __B, 0, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpeq_epi64_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_pcmpeqq128_mask ((__v2di) __A, + (__v2di) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpeq_epu64_mask (__m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_ucmpq256_mask ((__v4di) __A, + (__v4di) __B, 0, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpeq_epi64_mask (__m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_pcmpeqq256_mask ((__v4di) __A, + (__v4di) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpeq_epu64_mask (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_ucmpq256_mask ((__v4di) __A, + (__v4di) __B, 0, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpeq_epi64_mask (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_pcmpeqq256_mask ((__v4di) __A, + (__v4di) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_epu32_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ucmpd128_mask ((__v4si) __A, + (__v4si) __B, 6, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_epi32_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_pcmpgtd128_mask ((__v4si) __A, + (__v4si) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpgt_epu32_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ucmpd128_mask ((__v4si) __A, + (__v4si) __B, 6, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpgt_epi32_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_pcmpgtd128_mask ((__v4si) __A, + (__v4si) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpgt_epu32_mask (__m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_ucmpd256_mask ((__v8si) __A, + (__v8si) __B, 6, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpgt_epi32_mask (__m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_pcmpgtd256_mask ((__v8si) __A, + (__v8si) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpgt_epu32_mask (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_ucmpd256_mask ((__v8si) __A, + (__v8si) __B, 6, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpgt_epi32_mask (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_pcmpgtd256_mask ((__v8si) __A, + (__v8si) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_epu64_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ucmpq128_mask ((__v2di) __A, + (__v2di) __B, 6, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_epi64_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_pcmpgtq128_mask ((__v2di) __A, + (__v2di) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpgt_epu64_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ucmpq128_mask ((__v2di) __A, + (__v2di) __B, 6, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpgt_epi64_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_pcmpgtq128_mask ((__v2di) __A, + (__v2di) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpgt_epu64_mask (__m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_ucmpq256_mask ((__v4di) __A, + (__v4di) __B, 6, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpgt_epi64_mask (__m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_pcmpgtq256_mask ((__v4di) __A, + (__v4di) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpgt_epu64_mask (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_ucmpq256_mask ((__v4di) __A, + (__v4di) __B, 6, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpgt_epi64_mask (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_pcmpgtq256_mask ((__v4di) __A, + (__v4di) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_test_epi32_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ptestmd128 ((__v4si) __A, + (__v4si) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_test_epi32_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ptestmd128 ((__v4si) __A, + (__v4si) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_test_epi32_mask (__m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_ptestmd256 ((__v8si) __A, + (__v8si) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_test_epi32_mask (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_ptestmd256 ((__v8si) __A, + (__v8si) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_test_epi64_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ptestmq128 ((__v2di) __A, + (__v2di) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_test_epi64_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ptestmq128 ((__v2di) __A, + (__v2di) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_test_epi64_mask (__m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_ptestmq256 ((__v4di) __A, + (__v4di) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_test_epi64_mask (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_ptestmq256 ((__v4di) __A, + (__v4di) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_testn_epi32_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ptestnmd128 ((__v4si) __A, + (__v4si) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_testn_epi32_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ptestnmd128 ((__v4si) __A, + (__v4si) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testn_epi32_mask (__m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_ptestnmd256 ((__v8si) __A, + (__v8si) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_testn_epi32_mask (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_ptestnmd256 ((__v8si) __A, + (__v8si) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_testn_epi64_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ptestnmq128 ((__v2di) __A, + (__v2di) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_testn_epi64_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ptestnmq128 ((__v2di) __A, + (__v2di) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testn_epi64_mask (__m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_ptestnmq256 ((__v4di) __A, + (__v4di) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_testn_epi64_mask (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_ptestnmq256 ((__v4di) __A, + (__v4di) __B, __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_compress_pd (__m256d __W, __mmask8 __U, __m256d __A) +{ + return (__m256d) __builtin_ia32_compressdf256_mask ((__v4df) __A, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_compress_pd (__mmask8 __U, __m256d __A) +{ + return (__m256d) __builtin_ia32_compressdf256_mask ((__v4df) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_compressstoreu_pd (void *__P, __mmask8 __U, __m256d __A) +{ + __builtin_ia32_compressstoredf256_mask ((__v4df *) __P, + (__v4df) __A, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_compress_pd (__m128d __W, __mmask8 __U, __m128d __A) +{ + return (__m128d) __builtin_ia32_compressdf128_mask ((__v2df) __A, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_compress_pd (__mmask8 __U, __m128d __A) +{ + return (__m128d) __builtin_ia32_compressdf128_mask ((__v2df) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_compressstoreu_pd (void *__P, __mmask8 __U, __m128d __A) +{ + __builtin_ia32_compressstoredf128_mask ((__v2df *) __P, + (__v2df) __A, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_compress_ps (__m256 __W, __mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_compresssf256_mask ((__v8sf) __A, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_compress_ps (__mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_compresssf256_mask ((__v8sf) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_compressstoreu_ps (void *__P, __mmask8 __U, __m256 __A) +{ + __builtin_ia32_compressstoresf256_mask ((__v8sf *) __P, + (__v8sf) __A, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_compress_ps (__m128 __W, __mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_compresssf128_mask ((__v4sf) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_compress_ps (__mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_compresssf128_mask ((__v4sf) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_compressstoreu_ps (void *__P, __mmask8 __U, __m128 __A) +{ + __builtin_ia32_compressstoresf128_mask ((__v4sf *) __P, + (__v4sf) __A, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_compress_epi64 (__m256i __W, __mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_compressdi256_mask ((__v4di) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_compress_epi64 (__mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_compressdi256_mask ((__v4di) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_compressstoreu_epi64 (void *__P, __mmask8 __U, __m256i __A) +{ + __builtin_ia32_compressstoredi256_mask ((__v4di *) __P, + (__v4di) __A, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_compress_epi64 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_compressdi128_mask ((__v2di) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_compress_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_compressdi128_mask ((__v2di) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_compressstoreu_epi64 (void *__P, __mmask8 __U, __m128i __A) +{ + __builtin_ia32_compressstoredi128_mask ((__v2di *) __P, + (__v2di) __A, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_compress_epi32 (__m256i __W, __mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_compresssi256_mask ((__v8si) __A, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_compress_epi32 (__mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_compresssi256_mask ((__v8si) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_compressstoreu_epi32 (void *__P, __mmask8 __U, __m256i __A) +{ + __builtin_ia32_compressstoresi256_mask ((__v8si *) __P, + (__v8si) __A, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_compress_epi32 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_compresssi128_mask ((__v4si) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_compress_epi32 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_compresssi128_mask ((__v4si) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_compressstoreu_epi32 (void *__P, __mmask8 __U, __m128i __A) +{ + __builtin_ia32_compressstoresi128_mask ((__v4si *) __P, + (__v4si) __A, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_expand_pd (__m256d __W, __mmask8 __U, __m256d __A) +{ + return (__m256d) __builtin_ia32_expanddf256_mask ((__v4df) __A, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_expand_pd (__mmask8 __U, __m256d __A) +{ + return (__m256d) __builtin_ia32_expanddf256_maskz ((__v4df) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_expandloadu_pd (__m256d __W, __mmask8 __U, void const *__P) +{ + return (__m256d) __builtin_ia32_expandloaddf256_mask ((__v4df *) __P, + (__v4df) __W, + (__mmask8) + __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_expandloadu_pd (__mmask8 __U, void const *__P) +{ + return (__m256d) __builtin_ia32_expandloaddf256_maskz ((__v4df *) __P, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) + __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_expand_pd (__m128d __W, __mmask8 __U, __m128d __A) +{ + return (__m128d) __builtin_ia32_expanddf128_mask ((__v2df) __A, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_expand_pd (__mmask8 __U, __m128d __A) +{ + return (__m128d) __builtin_ia32_expanddf128_maskz ((__v2df) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_expandloadu_pd (__m128d __W, __mmask8 __U, void const *__P) +{ + return (__m128d) __builtin_ia32_expandloaddf128_mask ((__v2df *) __P, + (__v2df) __W, + (__mmask8) + __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_expandloadu_pd (__mmask8 __U, void const *__P) +{ + return (__m128d) __builtin_ia32_expandloaddf128_maskz ((__v2df *) __P, + (__v2df) + _mm_setzero_pd (), + (__mmask8) + __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_expand_ps (__m256 __W, __mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_expandsf256_mask ((__v8sf) __A, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_expand_ps (__mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_expandsf256_maskz ((__v8sf) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_expandloadu_ps (__m256 __W, __mmask8 __U, void const *__P) +{ + return (__m256) __builtin_ia32_expandloadsf256_mask ((__v8sf *) __P, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_expandloadu_ps (__mmask8 __U, void const *__P) +{ + return (__m256) __builtin_ia32_expandloadsf256_maskz ((__v8sf *) __P, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) + __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_expand_ps (__m128 __W, __mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_expandsf128_mask ((__v4sf) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_expand_ps (__mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_expandsf128_maskz ((__v4sf) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_expandloadu_ps (__m128 __W, __mmask8 __U, void const *__P) +{ + return (__m128) __builtin_ia32_expandloadsf128_mask ((__v4sf *) __P, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_expandloadu_ps (__mmask8 __U, void const *__P) +{ + return (__m128) __builtin_ia32_expandloadsf128_maskz ((__v4sf *) __P, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_expand_epi64 (__m256i __W, __mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_expanddi256_mask ((__v4di) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_expand_epi64 (__mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_expanddi256_maskz ((__v4di) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_expandloadu_epi64 (__m256i __W, __mmask8 __U, + void const *__P) +{ + return (__m256i) __builtin_ia32_expandloaddi256_mask ((__v4di *) __P, + (__v4di) __W, + (__mmask8) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_expandloadu_epi64 (__mmask8 __U, void const *__P) +{ + return (__m256i) __builtin_ia32_expandloaddi256_maskz ((__v4di *) __P, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_expand_epi64 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_expanddi128_mask ((__v2di) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_expand_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_expanddi128_maskz ((__v2di) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_expandloadu_epi64 (__m128i __W, __mmask8 __U, void const *__P) +{ + return (__m128i) __builtin_ia32_expandloaddi128_mask ((__v2di *) __P, + (__v2di) __W, + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_expandloadu_epi64 (__mmask8 __U, void const *__P) +{ + return (__m128i) __builtin_ia32_expandloaddi128_maskz ((__v2di *) __P, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_expand_epi32 (__m256i __W, __mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_expandsi256_mask ((__v8si) __A, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_expand_epi32 (__mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_expandsi256_maskz ((__v8si) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_expandloadu_epi32 (__m256i __W, __mmask8 __U, + void const *__P) +{ + return (__m256i) __builtin_ia32_expandloadsi256_mask ((__v8si *) __P, + (__v8si) __W, + (__mmask8) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_expandloadu_epi32 (__mmask8 __U, void const *__P) +{ + return (__m256i) __builtin_ia32_expandloadsi256_maskz ((__v8si *) __P, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_expand_epi32 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_expandsi128_mask ((__v4si) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_expand_epi32 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_expandsi128_maskz ((__v4si) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_expandloadu_epi32 (__m128i __W, __mmask8 __U, void const *__P) +{ + return (__m128i) __builtin_ia32_expandloadsi128_mask ((__v4si *) __P, + (__v4si) __W, + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_expandloadu_epi32 (__mmask8 __U, void const *__P) +{ + return (__m128i) __builtin_ia32_expandloadsi128_maskz ((__v4si *) __P, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) + __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permutex2var_pd (__m256d __A, __m256i __I, __m256d __B) +{ + return (__m256d) __builtin_ia32_vpermt2varpd256_mask ((__v4di) __I + /* idx */ , + (__v4df) __A, + (__v4df) __B, + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permutex2var_pd (__m256d __A, __mmask8 __U, __m256i __I, + __m256d __B) +{ + return (__m256d) __builtin_ia32_vpermt2varpd256_mask ((__v4di) __I + /* idx */ , + (__v4df) __A, + (__v4df) __B, + (__mmask8) + __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask2_permutex2var_pd (__m256d __A, __m256i __I, __mmask8 __U, + __m256d __B) +{ + return (__m256d) __builtin_ia32_vpermi2varpd256_mask ((__v4df) __A, + (__v4di) __I + /* idx */ , + (__v4df) __B, + (__mmask8) + __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permutex2var_pd (__mmask8 __U, __m256d __A, __m256i __I, + __m256d __B) +{ + return (__m256d) __builtin_ia32_vpermt2varpd256_maskz ((__v4di) __I + /* idx */ , + (__v4df) __A, + (__v4df) __B, + (__mmask8) + __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permutex2var_ps (__m256 __A, __m256i __I, __m256 __B) +{ + return (__m256) __builtin_ia32_vpermt2varps256_mask ((__v8si) __I + /* idx */ , + (__v8sf) __A, + (__v8sf) __B, + (__mmask8) -1); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permutex2var_ps (__m256 __A, __mmask8 __U, __m256i __I, + __m256 __B) +{ + return (__m256) __builtin_ia32_vpermt2varps256_mask ((__v8si) __I + /* idx */ , + (__v8sf) __A, + (__v8sf) __B, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask2_permutex2var_ps (__m256 __A, __m256i __I, __mmask8 __U, + __m256 __B) +{ + return (__m256) __builtin_ia32_vpermi2varps256_mask ((__v8sf) __A, + (__v8si) __I + /* idx */ , + (__v8sf) __B, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permutex2var_ps (__mmask8 __U, __m256 __A, __m256i __I, + __m256 __B) +{ + return (__m256) __builtin_ia32_vpermt2varps256_maskz ((__v8si) __I + /* idx */ , + (__v8sf) __A, + (__v8sf) __B, + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permutex2var_epi64 (__m128i __A, __m128i __I, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpermt2varq128_mask ((__v2di) __I + /* idx */ , + (__v2di) __A, + (__v2di) __B, + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_permutex2var_epi64 (__m128i __A, __mmask8 __U, __m128i __I, + __m128i __B) +{ + return (__m128i) __builtin_ia32_vpermt2varq128_mask ((__v2di) __I + /* idx */ , + (__v2di) __A, + (__v2di) __B, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask2_permutex2var_epi64 (__m128i __A, __m128i __I, __mmask8 __U, + __m128i __B) +{ + return (__m128i) __builtin_ia32_vpermi2varq128_mask ((__v2di) __A, + (__v2di) __I + /* idx */ , + (__v2di) __B, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_permutex2var_epi64 (__mmask8 __U, __m128i __A, __m128i __I, + __m128i __B) +{ + return (__m128i) __builtin_ia32_vpermt2varq128_maskz ((__v2di) __I + /* idx */ , + (__v2di) __A, + (__v2di) __B, + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permutex2var_epi32 (__m128i __A, __m128i __I, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpermt2vard128_mask ((__v4si) __I + /* idx */ , + (__v4si) __A, + (__v4si) __B, + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_permutex2var_epi32 (__m128i __A, __mmask8 __U, __m128i __I, + __m128i __B) +{ + return (__m128i) __builtin_ia32_vpermt2vard128_mask ((__v4si) __I + /* idx */ , + (__v4si) __A, + (__v4si) __B, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask2_permutex2var_epi32 (__m128i __A, __m128i __I, __mmask8 __U, + __m128i __B) +{ + return (__m128i) __builtin_ia32_vpermi2vard128_mask ((__v4si) __A, + (__v4si) __I + /* idx */ , + (__v4si) __B, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_permutex2var_epi32 (__mmask8 __U, __m128i __A, __m128i __I, + __m128i __B) +{ + return (__m128i) __builtin_ia32_vpermt2vard128_maskz ((__v4si) __I + /* idx */ , + (__v4si) __A, + (__v4si) __B, + (__mmask8) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permutex2var_epi64 (__m256i __A, __m256i __I, __m256i __B) +{ + return (__m256i) __builtin_ia32_vpermt2varq256_mask ((__v4di) __I + /* idx */ , + (__v4di) __A, + (__v4di) __B, + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permutex2var_epi64 (__m256i __A, __mmask8 __U, __m256i __I, + __m256i __B) +{ + return (__m256i) __builtin_ia32_vpermt2varq256_mask ((__v4di) __I + /* idx */ , + (__v4di) __A, + (__v4di) __B, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask2_permutex2var_epi64 (__m256i __A, __m256i __I, + __mmask8 __U, __m256i __B) +{ + return (__m256i) __builtin_ia32_vpermi2varq256_mask ((__v4di) __A, + (__v4di) __I + /* idx */ , + (__v4di) __B, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permutex2var_epi64 (__mmask8 __U, __m256i __A, + __m256i __I, __m256i __B) +{ + return (__m256i) __builtin_ia32_vpermt2varq256_maskz ((__v4di) __I + /* idx */ , + (__v4di) __A, + (__v4di) __B, + (__mmask8) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permutex2var_epi32 (__m256i __A, __m256i __I, __m256i __B) +{ + return (__m256i) __builtin_ia32_vpermt2vard256_mask ((__v8si) __I + /* idx */ , + (__v8si) __A, + (__v8si) __B, + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permutex2var_epi32 (__m256i __A, __mmask8 __U, __m256i __I, + __m256i __B) +{ + return (__m256i) __builtin_ia32_vpermt2vard256_mask ((__v8si) __I + /* idx */ , + (__v8si) __A, + (__v8si) __B, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask2_permutex2var_epi32 (__m256i __A, __m256i __I, + __mmask8 __U, __m256i __B) +{ + return (__m256i) __builtin_ia32_vpermi2vard256_mask ((__v8si) __A, + (__v8si) __I + /* idx */ , + (__v8si) __B, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permutex2var_epi32 (__mmask8 __U, __m256i __A, + __m256i __I, __m256i __B) +{ + return (__m256i) __builtin_ia32_vpermt2vard256_maskz ((__v8si) __I + /* idx */ , + (__v8si) __A, + (__v8si) __B, + (__mmask8) + __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permutex2var_pd (__m128d __A, __m128i __I, __m128d __B) +{ + return (__m128d) __builtin_ia32_vpermt2varpd128_mask ((__v2di) __I + /* idx */ , + (__v2df) __A, + (__v2df) __B, + (__mmask8) -1); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_permutex2var_pd (__m128d __A, __mmask8 __U, __m128i __I, + __m128d __B) +{ + return (__m128d) __builtin_ia32_vpermt2varpd128_mask ((__v2di) __I + /* idx */ , + (__v2df) __A, + (__v2df) __B, + (__mmask8) + __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask2_permutex2var_pd (__m128d __A, __m128i __I, __mmask8 __U, + __m128d __B) +{ + return (__m128d) __builtin_ia32_vpermi2varpd128_mask ((__v2df) __A, + (__v2di) __I + /* idx */ , + (__v2df) __B, + (__mmask8) + __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_permutex2var_pd (__mmask8 __U, __m128d __A, __m128i __I, + __m128d __B) +{ + return (__m128d) __builtin_ia32_vpermt2varpd128_maskz ((__v2di) __I + /* idx */ , + (__v2df) __A, + (__v2df) __B, + (__mmask8) + __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permutex2var_ps (__m128 __A, __m128i __I, __m128 __B) +{ + return (__m128) __builtin_ia32_vpermt2varps128_mask ((__v4si) __I + /* idx */ , + (__v4sf) __A, + (__v4sf) __B, + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_permutex2var_ps (__m128 __A, __mmask8 __U, __m128i __I, + __m128 __B) +{ + return (__m128) __builtin_ia32_vpermt2varps128_mask ((__v4si) __I + /* idx */ , + (__v4sf) __A, + (__v4sf) __B, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask2_permutex2var_ps (__m128 __A, __m128i __I, __mmask8 __U, + __m128 __B) +{ + return (__m128) __builtin_ia32_vpermi2varps128_mask ((__v4sf) __A, + (__v4si) __I + /* idx */ , + (__v4sf) __B, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_permutex2var_ps (__mmask8 __U, __m128 __A, __m128i __I, + __m128 __B) +{ + return (__m128) __builtin_ia32_vpermt2varps128_maskz ((__v4si) __I + /* idx */ , + (__v4sf) __A, + (__v4sf) __B, + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srav_epi64 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_psravq128_mask ((__v2di) __X, + (__v2di) __Y, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_srav_epi64 (__m128i __W, __mmask8 __U, __m128i __X, + __m128i __Y) +{ + return (__m128i) __builtin_ia32_psravq128_mask ((__v2di) __X, + (__v2di) __Y, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_srav_epi64 (__mmask8 __U, __m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_psravq128_mask ((__v2di) __X, + (__v2di) __Y, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sllv_epi32 (__m256i __W, __mmask8 __U, __m256i __X, + __m256i __Y) +{ + return (__m256i) __builtin_ia32_psllv8si_mask ((__v8si) __X, + (__v8si) __Y, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sllv_epi32 (__mmask8 __U, __m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_psllv8si_mask ((__v8si) __X, + (__v8si) __Y, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sllv_epi32 (__m128i __W, __mmask8 __U, __m128i __X, + __m128i __Y) +{ + return (__m128i) __builtin_ia32_psllv4si_mask ((__v4si) __X, + (__v4si) __Y, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sllv_epi32 (__mmask8 __U, __m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_psllv4si_mask ((__v4si) __X, + (__v4si) __Y, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sllv_epi64 (__m256i __W, __mmask8 __U, __m256i __X, + __m256i __Y) +{ + return (__m256i) __builtin_ia32_psllv4di_mask ((__v4di) __X, + (__v4di) __Y, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sllv_epi64 (__mmask8 __U, __m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_psllv4di_mask ((__v4di) __X, + (__v4di) __Y, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sllv_epi64 (__m128i __W, __mmask8 __U, __m128i __X, + __m128i __Y) +{ + return (__m128i) __builtin_ia32_psllv2di_mask ((__v2di) __X, + (__v2di) __Y, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sllv_epi64 (__mmask8 __U, __m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_psllv2di_mask ((__v2di) __X, + (__v2di) __Y, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_srav_epi32 (__m256i __W, __mmask8 __U, __m256i __X, + __m256i __Y) +{ + return (__m256i) __builtin_ia32_psrav8si_mask ((__v8si) __X, + (__v8si) __Y, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_srav_epi32 (__mmask8 __U, __m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_psrav8si_mask ((__v8si) __X, + (__v8si) __Y, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_srav_epi32 (__m128i __W, __mmask8 __U, __m128i __X, + __m128i __Y) +{ + return (__m128i) __builtin_ia32_psrav4si_mask ((__v4si) __X, + (__v4si) __Y, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_srav_epi32 (__mmask8 __U, __m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_psrav4si_mask ((__v4si) __X, + (__v4si) __Y, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_srlv_epi32 (__m256i __W, __mmask8 __U, __m256i __X, + __m256i __Y) +{ + return (__m256i) __builtin_ia32_psrlv8si_mask ((__v8si) __X, + (__v8si) __Y, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_srlv_epi32 (__mmask8 __U, __m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_psrlv8si_mask ((__v8si) __X, + (__v8si) __Y, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_srlv_epi32 (__m128i __W, __mmask8 __U, __m128i __X, + __m128i __Y) +{ + return (__m128i) __builtin_ia32_psrlv4si_mask ((__v4si) __X, + (__v4si) __Y, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_srlv_epi32 (__mmask8 __U, __m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_psrlv4si_mask ((__v4si) __X, + (__v4si) __Y, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_srlv_epi64 (__m256i __W, __mmask8 __U, __m256i __X, + __m256i __Y) +{ + return (__m256i) __builtin_ia32_psrlv4di_mask ((__v4di) __X, + (__v4di) __Y, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_srlv_epi64 (__mmask8 __U, __m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_psrlv4di_mask ((__v4di) __X, + (__v4di) __Y, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_srlv_epi64 (__m128i __W, __mmask8 __U, __m128i __X, + __m128i __Y) +{ + return (__m128i) __builtin_ia32_psrlv2di_mask ((__v2di) __X, + (__v2di) __Y, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_srlv_epi64 (__mmask8 __U, __m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_psrlv2di_mask ((__v2di) __X, + (__v2di) __Y, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_rolv_epi32 (__m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_prolvd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_rolv_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_prolvd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_rolv_epi32 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_prolvd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rolv_epi32 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_prolvd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_rolv_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_prolvd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_rolv_epi32 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_prolvd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_rorv_epi32 (__m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_prorvd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_rorv_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_prorvd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_rorv_epi32 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_prorvd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rorv_epi32 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_prorvd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_rorv_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_prorvd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_rorv_epi32 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_prorvd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_rolv_epi64 (__m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_prolvq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_rolv_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_prolvq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_rolv_epi64 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_prolvq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rolv_epi64 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_prolvq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_rolv_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_prolvq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_rolv_epi64 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_prolvq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_rorv_epi64 (__m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_prorvq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_rorv_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_prorvq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_rorv_epi64 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_prorvq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rorv_epi64 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_prorvq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_rorv_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_prorvq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_rorv_epi64 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_prorvq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_srav_epi64 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_psravq256_mask ((__v4di) __X, + (__v4di) __Y, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_srav_epi64 (__m256i __W, __mmask8 __U, __m256i __X, + __m256i __Y) +{ + return (__m256i) __builtin_ia32_psravq256_mask ((__v4di) __X, + (__v4di) __Y, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_srav_epi64 (__mmask8 __U, __m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_psravq256_mask ((__v4di) __X, + (__v4di) __Y, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_and_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pandq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __W, __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_and_epi64 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pandq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_pd (), + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_and_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pandq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_and_epi64 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pandq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_pd (), + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_andnot_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pandnq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __W, __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_andnot_epi64 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pandnq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_pd (), + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_andnot_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pandnq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_andnot_epi64 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pandnq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_pd (), + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_or_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_porq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_or_epi64 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_porq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_or_epi64 (__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_porq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_or_epi64 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_porq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_xor_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pxorq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_xor_epi64 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pxorq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_xor_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pxorq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_xor_epi64 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pxorq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_max_pd (__m256d __W, __mmask8 __U, __m256d __A, + __m256d __B) +{ + return (__m256d) __builtin_ia32_maxpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_max_pd (__mmask8 __U, __m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_maxpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_max_ps (__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_maxps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_max_ps (__mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_maxps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_div_ps (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_divps_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_div_ps (__mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_divps_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_div_pd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_divpd_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_div_pd (__mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_divpd_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_min_pd (__m256d __W, __mmask8 __U, __m256d __A, + __m256d __B) +{ + return (__m256d) __builtin_ia32_minpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_div_pd (__m256d __W, __mmask8 __U, __m256d __A, + __m256d __B) +{ + return (__m256d) __builtin_ia32_divpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_min_pd (__mmask8 __U, __m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_minpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_min_ps (__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_minps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_div_pd (__mmask8 __U, __m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_divpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_div_ps (__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_divps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_min_ps (__mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_minps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_div_ps (__mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_divps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_min_ps (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_minps_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_mul_ps (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_mulps_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_min_ps (__mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_minps_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_mul_ps (__mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_mulps_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_max_ps (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_maxps_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_max_ps (__mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_maxps_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_min_pd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_minpd_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_min_pd (__mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_minpd_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_max_pd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_maxpd_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_max_pd (__mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_maxpd_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_mul_pd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_mulpd_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_mul_pd (__mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_mulpd_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_mul_ps (__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_mulps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_mul_ps (__mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_mulps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_mul_pd (__m256d __W, __mmask8 __U, __m256d __A, + __m256d __B) +{ + return (__m256d) __builtin_ia32_mulpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_mul_pd (__mmask8 __U, __m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_mulpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_max_epi64 (__mmask8 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxsq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_max_epi64 (__m256i __W, __mmask8 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxsq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __W, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_min_epi64 (__m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pminsq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_min_epi64 (__m256i __W, __mmask8 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pminsq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __W, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_min_epi64 (__mmask8 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pminsq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_max_epu64 (__mmask8 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxuq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_max_epi64 (__m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxsq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_max_epu64 (__m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxuq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_max_epu64 (__m256i __W, __mmask8 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxuq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __W, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_min_epu64 (__m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pminuq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_min_epu64 (__m256i __W, __mmask8 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pminuq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __W, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_min_epu64 (__mmask8 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pminuq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_max_epi32 (__mmask8 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxsd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_max_epi32 (__m256i __W, __mmask8 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxsd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __W, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_min_epi32 (__mmask8 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pminsd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_min_epi32 (__m256i __W, __mmask8 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pminsd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __W, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_max_epu32 (__mmask8 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxud256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_max_epu32 (__m256i __W, __mmask8 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxud256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __W, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_min_epu32 (__mmask8 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pminud256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_min_epu32 (__m256i __W, __mmask8 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pminud256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __W, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_max_epi64 (__mmask8 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxsq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_max_epi64 (__m128i __W, __mmask8 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxsq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_min_epi64 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pminsq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_min_epi64 (__m128i __W, __mmask8 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pminsq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_min_epi64 (__mmask8 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pminsq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_max_epu64 (__mmask8 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxuq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_max_epi64 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxsq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_max_epu64 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxuq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_max_epu64 (__m128i __W, __mmask8 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxuq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_min_epu64 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pminuq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_min_epu64 (__m128i __W, __mmask8 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pminuq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_min_epu64 (__mmask8 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pminuq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_max_epi32 (__mmask8 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxsd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_max_epi32 (__m128i __W, __mmask8 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxsd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_min_epi32 (__mmask8 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pminsd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_min_epi32 (__m128i __W, __mmask8 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pminsd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_max_epu32 (__mmask8 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxud128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_max_epu32 (__m128i __W, __mmask8 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxud128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_min_epu32 (__mmask8 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pminud128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_min_epu32 (__m128i __W, __mmask8 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pminud128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, __M); +} + +#ifndef __AVX512CD__ +#pragma GCC push_options +#pragma GCC target("avx512vl,avx512cd") +#define __DISABLE_AVX512VLCD__ +#endif + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_broadcastmb_epi64 (__mmask8 __A) +{ + return (__m128i) __builtin_ia32_broadcastmb128 (__A); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcastmb_epi64 (__mmask8 __A) +{ + return (__m256i) __builtin_ia32_broadcastmb256 (__A); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_broadcastmw_epi32 (__mmask16 __A) +{ + return (__m128i) __builtin_ia32_broadcastmw128 (__A); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcastmw_epi32 (__mmask16 __A) +{ + return (__m256i) __builtin_ia32_broadcastmw256 (__A); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_lzcnt_epi32 (__m256i __A) +{ + return (__m256i) __builtin_ia32_vplzcntd_256_mask ((__v8si) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_lzcnt_epi32 (__m256i __W, __mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_vplzcntd_256_mask ((__v8si) __A, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_lzcnt_epi32 (__mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_vplzcntd_256_mask ((__v8si) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_lzcnt_epi64 (__m256i __A) +{ + return (__m256i) __builtin_ia32_vplzcntq_256_mask ((__v4di) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_lzcnt_epi64 (__m256i __W, __mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_vplzcntq_256_mask ((__v4di) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_lzcnt_epi64 (__mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_vplzcntq_256_mask ((__v4di) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_conflict_epi64 (__m256i __A) +{ + return (__m256i) __builtin_ia32_vpconflictdi_256_mask ((__v4di) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_conflict_epi64 (__m256i __W, __mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_vpconflictdi_256_mask ((__v4di) __A, + (__v4di) __W, + (__mmask8) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_conflict_epi64 (__mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_vpconflictdi_256_mask ((__v4di) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_conflict_epi32 (__m256i __A) +{ + return (__m256i) __builtin_ia32_vpconflictsi_256_mask ((__v8si) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_conflict_epi32 (__m256i __W, __mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_vpconflictsi_256_mask ((__v8si) __A, + (__v8si) __W, + (__mmask8) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_conflict_epi32 (__mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_vpconflictsi_256_mask ((__v8si) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_lzcnt_epi32 (__m128i __A) +{ + return (__m128i) __builtin_ia32_vplzcntd_128_mask ((__v4si) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_lzcnt_epi32 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_vplzcntd_128_mask ((__v4si) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_lzcnt_epi32 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_vplzcntd_128_mask ((__v4si) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_lzcnt_epi64 (__m128i __A) +{ + return (__m128i) __builtin_ia32_vplzcntq_128_mask ((__v2di) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_lzcnt_epi64 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_vplzcntq_128_mask ((__v2di) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_lzcnt_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_vplzcntq_128_mask ((__v2di) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_conflict_epi64 (__m128i __A) +{ + return (__m128i) __builtin_ia32_vpconflictdi_128_mask ((__v2di) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_conflict_epi64 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_vpconflictdi_128_mask ((__v2di) __A, + (__v2di) __W, + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_conflict_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_vpconflictdi_128_mask ((__v2di) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_conflict_epi32 (__m128i __A) +{ + return (__m128i) __builtin_ia32_vpconflictsi_128_mask ((__v4si) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_conflict_epi32 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_vpconflictsi_128_mask ((__v4si) __A, + (__v4si) __W, + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_conflict_epi32 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_vpconflictsi_128_mask ((__v4si) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) + __U); +} + +#ifdef __DISABLE_AVX512VLCD__ +#pragma GCC pop_options +#endif + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_unpacklo_pd (__m256d __W, __mmask8 __U, __m256d __A, + __m256d __B) +{ + return (__m256d) __builtin_ia32_unpcklpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_unpacklo_pd (__mmask8 __U, __m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_unpcklpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_unpacklo_pd (__m128d __W, __mmask8 __U, __m128d __A, + __m128d __B) +{ + return (__m128d) __builtin_ia32_unpcklpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_unpacklo_pd (__mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_unpcklpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_unpacklo_ps (__m256 __W, __mmask8 __U, __m256 __A, + __m256 __B) +{ + return (__m256) __builtin_ia32_unpcklps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_unpackhi_pd (__m256d __W, __mmask8 __U, __m256d __A, + __m256d __B) +{ + return (__m256d) __builtin_ia32_unpckhpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_unpackhi_pd (__mmask8 __U, __m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_unpckhpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_unpackhi_pd (__m128d __W, __mmask8 __U, __m128d __A, + __m128d __B) +{ + return (__m128d) __builtin_ia32_unpckhpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_unpackhi_pd (__mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_unpckhpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_unpackhi_ps (__m256 __W, __mmask8 __U, __m256 __A, + __m256 __B) +{ + return (__m256) __builtin_ia32_unpckhps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_unpackhi_ps (__mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_unpckhps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_unpackhi_ps (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_unpckhps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_unpackhi_ps (__mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_unpckhps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtph_ps (__m128 __W, __mmask8 __U, __m128i __A) +{ + return (__m128) __builtin_ia32_vcvtph2ps_mask ((__v8hi) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtph_ps (__mmask8 __U, __m128i __A) +{ + return (__m128) __builtin_ia32_vcvtph2ps_mask ((__v8hi) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_unpacklo_ps (__mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_unpcklps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtph_ps (__m256 __W, __mmask8 __U, __m128i __A) +{ + return (__m256) __builtin_ia32_vcvtph2ps256_mask ((__v8hi) __A, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtph_ps (__mmask8 __U, __m128i __A) +{ + return (__m256) __builtin_ia32_vcvtph2ps256_mask ((__v8hi) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_unpacklo_ps (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_unpcklps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_unpacklo_ps (__mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_unpcklps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sra_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + __m128i __B) +{ + return (__m256i) __builtin_ia32_psrad256_mask ((__v8si) __A, + (__v4si) __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sra_epi32 (__mmask8 __U, __m256i __A, __m128i __B) +{ + return (__m256i) __builtin_ia32_psrad256_mask ((__v8si) __A, + (__v4si) __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sra_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psrad128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sra_epi32 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psrad128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sra_epi64 (__m256i __A, __m128i __B) +{ + return (__m256i) __builtin_ia32_psraq256_mask ((__v4di) __A, + (__v2di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sra_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + __m128i __B) +{ + return (__m256i) __builtin_ia32_psraq256_mask ((__v4di) __A, + (__v2di) __B, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sra_epi64 (__mmask8 __U, __m256i __A, __m128i __B) +{ + return (__m256i) __builtin_ia32_psraq256_mask ((__v4di) __A, + (__v2di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sra_epi64 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psraq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sra_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psraq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sra_epi64 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psraq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sll_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pslld128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sll_epi32 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pslld128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sll_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psllq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sll_epi64 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psllq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sll_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + __m128i __B) +{ + return (__m256i) __builtin_ia32_pslld256_mask ((__v8si) __A, + (__v4si) __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sll_epi32 (__mmask8 __U, __m256i __A, __m128i __B) +{ + return (__m256i) __builtin_ia32_pslld256_mask ((__v8si) __A, + (__v4si) __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sll_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + __m128i __B) +{ + return (__m256i) __builtin_ia32_psllq256_mask ((__v4di) __A, + (__v2di) __B, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sll_epi64 (__mmask8 __U, __m256i __A, __m128i __B) +{ + return (__m256i) __builtin_ia32_psllq256_mask ((__v4di) __A, + (__v2di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permutexvar_ps (__m256 __W, __mmask8 __U, __m256i __X, + __m256 __Y) +{ + return (__m256) __builtin_ia32_permvarsf256_mask ((__v8sf) __Y, + (__v8si) __X, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permutexvar_ps (__mmask8 __U, __m256i __X, __m256 __Y) +{ + return (__m256) __builtin_ia32_permvarsf256_mask ((__v8sf) __Y, + (__v8si) __X, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permutexvar_pd (__m256i __X, __m256d __Y) +{ + return (__m256d) __builtin_ia32_permvardf256_mask ((__v4df) __Y, + (__v4di) __X, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permutexvar_pd (__m256d __W, __mmask8 __U, __m256i __X, + __m256d __Y) +{ + return (__m256d) __builtin_ia32_permvardf256_mask ((__v4df) __Y, + (__v4di) __X, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permutexvar_pd (__mmask8 __U, __m256i __X, __m256d __Y) +{ + return (__m256d) __builtin_ia32_permvardf256_mask ((__v4df) __Y, + (__v4di) __X, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permutevar_pd (__m256d __W, __mmask8 __U, __m256d __A, + __m256i __C) +{ + return (__m256d) __builtin_ia32_vpermilvarpd256_mask ((__v4df) __A, + (__v4di) __C, + (__v4df) __W, + (__mmask8) + __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permutevar_pd (__mmask8 __U, __m256d __A, __m256i __C) +{ + return (__m256d) __builtin_ia32_vpermilvarpd256_mask ((__v4df) __A, + (__v4di) __C, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) + __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permutevar_ps (__m256 __W, __mmask8 __U, __m256 __A, + __m256i __C) +{ + return (__m256) __builtin_ia32_vpermilvarps256_mask ((__v8sf) __A, + (__v8si) __C, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permutevar_ps (__mmask8 __U, __m256 __A, __m256i __C) +{ + return (__m256) __builtin_ia32_vpermilvarps256_mask ((__v8sf) __A, + (__v8si) __C, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_permutevar_pd (__m128d __W, __mmask8 __U, __m128d __A, + __m128i __C) +{ + return (__m128d) __builtin_ia32_vpermilvarpd_mask ((__v2df) __A, + (__v2di) __C, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_permutevar_pd (__mmask8 __U, __m128d __A, __m128i __C) +{ + return (__m128d) __builtin_ia32_vpermilvarpd_mask ((__v2df) __A, + (__v2di) __C, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_permutevar_ps (__m128 __W, __mmask8 __U, __m128 __A, + __m128i __C) +{ + return (__m128) __builtin_ia32_vpermilvarps_mask ((__v4sf) __A, + (__v4si) __C, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_permutevar_ps (__mmask8 __U, __m128 __A, __m128i __C) +{ + return (__m128) __builtin_ia32_vpermilvarps_mask ((__v4sf) __A, + (__v4si) __C, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_mullo_epi32 (__mmask8 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pmulld256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permutexvar_epi64 (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_permvardi256_mask ((__v4di) __Y, + (__v4di) __X, + (__v4di) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_mullo_epi32 (__m256i __W, __mmask8 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pmulld256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __W, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_mullo_epi32 (__mmask8 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pmulld128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_mullo_epi32 (__m128i __W, __mmask16 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pmulld128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_mul_epi32 (__m256i __W, __mmask8 __M, __m256i __X, + __m256i __Y) +{ + return (__m256i) __builtin_ia32_pmuldq256_mask ((__v8si) __X, + (__v8si) __Y, + (__v4di) __W, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_mul_epi32 (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_pmuldq256_mask ((__v8si) __X, + (__v8si) __Y, + (__v4di) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_mul_epi32 (__m128i __W, __mmask8 __M, __m128i __X, + __m128i __Y) +{ + return (__m128i) __builtin_ia32_pmuldq128_mask ((__v4si) __X, + (__v4si) __Y, + (__v2di) __W, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_mul_epi32 (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_pmuldq128_mask ((__v4si) __X, + (__v4si) __Y, + (__v2di) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permutexvar_epi64 (__m256i __W, __mmask8 __M, __m256i __X, + __m256i __Y) +{ + return (__m256i) __builtin_ia32_permvardi256_mask ((__v4di) __Y, + (__v4di) __X, + (__v4di) __W, + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_mul_epu32 (__m256i __W, __mmask8 __M, __m256i __X, + __m256i __Y) +{ + return (__m256i) __builtin_ia32_pmuludq256_mask ((__v8si) __X, + (__v8si) __Y, + (__v4di) __W, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permutexvar_epi32 (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_permvarsi256_mask ((__v8si) __Y, + (__v8si) __X, + (__v8si) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_mul_epu32 (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_pmuludq256_mask ((__v8si) __X, + (__v8si) __Y, + (__v4di) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_mul_epu32 (__m128i __W, __mmask8 __M, __m128i __X, + __m128i __Y) +{ + return (__m128i) __builtin_ia32_pmuludq128_mask ((__v4si) __X, + (__v4si) __Y, + (__v2di) __W, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_mul_epu32 (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_pmuludq128_mask ((__v4si) __X, + (__v4si) __Y, + (__v2di) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permutexvar_epi32 (__m256i __W, __mmask8 __M, __m256i __X, + __m256i __Y) +{ + return (__m256i) __builtin_ia32_permvarsi256_mask ((__v8si) __Y, + (__v8si) __X, + (__v8si) __W, + __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpneq_epu32_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpd256_mask ((__v8si) __X, + (__v8si) __Y, 4, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpneq_epu32_mask (__m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpd256_mask ((__v8si) __X, + (__v8si) __Y, 4, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmplt_epu32_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpd256_mask ((__v8si) __X, + (__v8si) __Y, 1, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmplt_epu32_mask (__m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpd256_mask ((__v8si) __X, + (__v8si) __Y, 1, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpge_epu32_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpd256_mask ((__v8si) __X, + (__v8si) __Y, 5, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpge_epu32_mask (__m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpd256_mask ((__v8si) __X, + (__v8si) __Y, 5, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmple_epu32_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpd256_mask ((__v8si) __X, + (__v8si) __Y, 2, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmple_epu32_mask (__m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpd256_mask ((__v8si) __X, + (__v8si) __Y, 2, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpneq_epu64_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq256_mask ((__v4di) __X, + (__v4di) __Y, 4, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpneq_epu64_mask (__m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq256_mask ((__v4di) __X, + (__v4di) __Y, 4, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmplt_epu64_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq256_mask ((__v4di) __X, + (__v4di) __Y, 1, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmplt_epu64_mask (__m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq256_mask ((__v4di) __X, + (__v4di) __Y, 1, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpge_epu64_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq256_mask ((__v4di) __X, + (__v4di) __Y, 5, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpge_epu64_mask (__m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq256_mask ((__v4di) __X, + (__v4di) __Y, 5, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmple_epu64_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq256_mask ((__v4di) __X, + (__v4di) __Y, 2, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmple_epu64_mask (__m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq256_mask ((__v4di) __X, + (__v4di) __Y, 2, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpneq_epi32_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpd256_mask ((__v8si) __X, + (__v8si) __Y, 4, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpneq_epi32_mask (__m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpd256_mask ((__v8si) __X, + (__v8si) __Y, 4, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmplt_epi32_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpd256_mask ((__v8si) __X, + (__v8si) __Y, 1, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmplt_epi32_mask (__m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpd256_mask ((__v8si) __X, + (__v8si) __Y, 1, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpge_epi32_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpd256_mask ((__v8si) __X, + (__v8si) __Y, 5, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpge_epi32_mask (__m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpd256_mask ((__v8si) __X, + (__v8si) __Y, 5, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmple_epi32_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpd256_mask ((__v8si) __X, + (__v8si) __Y, 2, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmple_epi32_mask (__m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpd256_mask ((__v8si) __X, + (__v8si) __Y, 2, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpneq_epi64_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq256_mask ((__v4di) __X, + (__v4di) __Y, 4, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpneq_epi64_mask (__m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq256_mask ((__v4di) __X, + (__v4di) __Y, 4, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmplt_epi64_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq256_mask ((__v4di) __X, + (__v4di) __Y, 1, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmplt_epi64_mask (__m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq256_mask ((__v4di) __X, + (__v4di) __Y, 1, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpge_epi64_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq256_mask ((__v4di) __X, + (__v4di) __Y, 5, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpge_epi64_mask (__m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq256_mask ((__v4di) __X, + (__v4di) __Y, 5, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmple_epi64_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq256_mask ((__v4di) __X, + (__v4di) __Y, 2, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmple_epi64_mask (__m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq256_mask ((__v4di) __X, + (__v4di) __Y, 2, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpneq_epu32_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpd128_mask ((__v4si) __X, + (__v4si) __Y, 4, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpneq_epu32_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpd128_mask ((__v4si) __X, + (__v4si) __Y, 4, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmplt_epu32_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpd128_mask ((__v4si) __X, + (__v4si) __Y, 1, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmplt_epu32_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpd128_mask ((__v4si) __X, + (__v4si) __Y, 1, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpge_epu32_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpd128_mask ((__v4si) __X, + (__v4si) __Y, 5, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpge_epu32_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpd128_mask ((__v4si) __X, + (__v4si) __Y, 5, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmple_epu32_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpd128_mask ((__v4si) __X, + (__v4si) __Y, 2, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmple_epu32_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpd128_mask ((__v4si) __X, + (__v4si) __Y, 2, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpneq_epu64_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq128_mask ((__v2di) __X, + (__v2di) __Y, 4, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpneq_epu64_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq128_mask ((__v2di) __X, + (__v2di) __Y, 4, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmplt_epu64_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq128_mask ((__v2di) __X, + (__v2di) __Y, 1, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmplt_epu64_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq128_mask ((__v2di) __X, + (__v2di) __Y, 1, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpge_epu64_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq128_mask ((__v2di) __X, + (__v2di) __Y, 5, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpge_epu64_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq128_mask ((__v2di) __X, + (__v2di) __Y, 5, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmple_epu64_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq128_mask ((__v2di) __X, + (__v2di) __Y, 2, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmple_epu64_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq128_mask ((__v2di) __X, + (__v2di) __Y, 2, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpneq_epi32_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpd128_mask ((__v4si) __X, + (__v4si) __Y, 4, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpneq_epi32_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpd128_mask ((__v4si) __X, + (__v4si) __Y, 4, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmplt_epi32_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpd128_mask ((__v4si) __X, + (__v4si) __Y, 1, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmplt_epi32_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpd128_mask ((__v4si) __X, + (__v4si) __Y, 1, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpge_epi32_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpd128_mask ((__v4si) __X, + (__v4si) __Y, 5, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpge_epi32_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpd128_mask ((__v4si) __X, + (__v4si) __Y, 5, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmple_epi32_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpd128_mask ((__v4si) __X, + (__v4si) __Y, 2, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmple_epi32_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpd128_mask ((__v4si) __X, + (__v4si) __Y, 2, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpneq_epi64_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq128_mask ((__v2di) __X, + (__v2di) __Y, 4, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpneq_epi64_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq128_mask ((__v2di) __X, + (__v2di) __Y, 4, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmplt_epi64_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq128_mask ((__v2di) __X, + (__v2di) __Y, 1, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmplt_epi64_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq128_mask ((__v2di) __X, + (__v2di) __Y, 1, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpge_epi64_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq128_mask ((__v2di) __X, + (__v2di) __Y, 5, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpge_epi64_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq128_mask ((__v2di) __X, + (__v2di) __Y, 5, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmple_epi64_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq128_mask ((__v2di) __X, + (__v2di) __Y, 2, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmple_epi64_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq128_mask ((__v2di) __X, + (__v2di) __Y, 2, + (__mmask8) -1); +} + +#ifdef __OPTIMIZE__ +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permutex_epi64 (__m256i __W, __mmask8 __M, + __m256i __X, const int __I) +{ + return (__m256i) __builtin_ia32_permdi256_mask ((__v4di) __X, + __I, + (__v4di) __W, + (__mmask8) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permutex_epi64 (__mmask8 __M, __m256i __X, const int __I) +{ + return (__m256i) __builtin_ia32_permdi256_mask ((__v4di) __X, + __I, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __M); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_shuffle_pd (__m256d __W, __mmask8 __U, __m256d __A, + __m256d __B, const int __imm) +{ + return (__m256d) __builtin_ia32_shufpd256_mask ((__v4df) __A, + (__v4df) __B, __imm, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_shuffle_pd (__mmask8 __U, __m256d __A, __m256d __B, + const int __imm) +{ + return (__m256d) __builtin_ia32_shufpd256_mask ((__v4df) __A, + (__v4df) __B, __imm, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_shuffle_pd (__m128d __W, __mmask8 __U, __m128d __A, + __m128d __B, const int __imm) +{ + return (__m128d) __builtin_ia32_shufpd128_mask ((__v2df) __A, + (__v2df) __B, __imm, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_shuffle_pd (__mmask8 __U, __m128d __A, __m128d __B, + const int __imm) +{ + return (__m128d) __builtin_ia32_shufpd128_mask ((__v2df) __A, + (__v2df) __B, __imm, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_shuffle_ps (__m256 __W, __mmask8 __U, __m256 __A, + __m256 __B, const int __imm) +{ + return (__m256) __builtin_ia32_shufps256_mask ((__v8sf) __A, + (__v8sf) __B, __imm, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_shuffle_ps (__mmask8 __U, __m256 __A, __m256 __B, + const int __imm) +{ + return (__m256) __builtin_ia32_shufps256_mask ((__v8sf) __A, + (__v8sf) __B, __imm, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_shuffle_ps (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B, + const int __imm) +{ + return (__m128) __builtin_ia32_shufps128_mask ((__v4sf) __A, + (__v4sf) __B, __imm, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_shuffle_ps (__mmask8 __U, __m128 __A, __m128 __B, + const int __imm) +{ + return (__m128) __builtin_ia32_shufps128_mask ((__v4sf) __A, + (__v4sf) __B, __imm, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_inserti32x4 (__m256i __A, __m128i __B, const int __imm) +{ + return (__m256i) __builtin_ia32_inserti32x4_256_mask ((__v8si) __A, + (__v4si) __B, + __imm, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_inserti32x4 (__m256i __W, __mmask8 __U, __m256i __A, + __m128i __B, const int __imm) +{ + return (__m256i) __builtin_ia32_inserti32x4_256_mask ((__v8si) __A, + (__v4si) __B, + __imm, + (__v8si) __W, + (__mmask8) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_inserti32x4 (__mmask8 __U, __m256i __A, __m128i __B, + const int __imm) +{ + return (__m256i) __builtin_ia32_inserti32x4_256_mask ((__v8si) __A, + (__v4si) __B, + __imm, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) + __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_insertf32x4 (__m256 __A, __m128 __B, const int __imm) +{ + return (__m256) __builtin_ia32_insertf32x4_256_mask ((__v8sf) __A, + (__v4sf) __B, + __imm, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_insertf32x4 (__m256 __W, __mmask8 __U, __m256 __A, + __m128 __B, const int __imm) +{ + return (__m256) __builtin_ia32_insertf32x4_256_mask ((__v8sf) __A, + (__v4sf) __B, + __imm, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_insertf32x4 (__mmask8 __U, __m256 __A, __m128 __B, + const int __imm) +{ + return (__m256) __builtin_ia32_insertf32x4_256_mask ((__v8sf) __A, + (__v4sf) __B, + __imm, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_extracti32x4_epi32 (__m256i __A, const int __imm) +{ + return (__m128i) __builtin_ia32_extracti32x4_256_mask ((__v8si) __A, + __imm, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_extracti32x4_epi32 (__m128i __W, __mmask8 __U, __m256i __A, + const int __imm) +{ + return (__m128i) __builtin_ia32_extracti32x4_256_mask ((__v8si) __A, + __imm, + (__v4si) __W, + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_extracti32x4_epi32 (__mmask8 __U, __m256i __A, + const int __imm) +{ + return (__m128i) __builtin_ia32_extracti32x4_256_mask ((__v8si) __A, + __imm, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) + __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_extractf32x4_ps (__m256 __A, const int __imm) +{ + return (__m128) __builtin_ia32_extractf32x4_256_mask ((__v8sf) __A, + __imm, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_extractf32x4_ps (__m128 __W, __mmask8 __U, __m256 __A, + const int __imm) +{ + return (__m128) __builtin_ia32_extractf32x4_256_mask ((__v8sf) __A, + __imm, + (__v4sf) __W, + (__mmask8) + __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_extractf32x4_ps (__mmask8 __U, __m256 __A, + const int __imm) +{ + return (__m128) __builtin_ia32_extractf32x4_256_mask ((__v8sf) __A, + __imm, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_shuffle_i64x2 (__m256i __A, __m256i __B, const int __imm) +{ + return (__m256i) __builtin_ia32_shuf_i64x2_256_mask ((__v4di) __A, + (__v4di) __B, + __imm, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_shuffle_i64x2 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B, const int __imm) +{ + return (__m256i) __builtin_ia32_shuf_i64x2_256_mask ((__v4di) __A, + (__v4di) __B, + __imm, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_shuffle_i64x2 (__mmask8 __U, __m256i __A, __m256i __B, + const int __imm) +{ + return (__m256i) __builtin_ia32_shuf_i64x2_256_mask ((__v4di) __A, + (__v4di) __B, + __imm, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_shuffle_i32x4 (__m256i __A, __m256i __B, const int __imm) +{ + return (__m256i) __builtin_ia32_shuf_i32x4_256_mask ((__v8si) __A, + (__v8si) __B, + __imm, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_shuffle_i32x4 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B, const int __imm) +{ + return (__m256i) __builtin_ia32_shuf_i32x4_256_mask ((__v8si) __A, + (__v8si) __B, + __imm, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_shuffle_i32x4 (__mmask8 __U, __m256i __A, __m256i __B, + const int __imm) +{ + return (__m256i) __builtin_ia32_shuf_i32x4_256_mask ((__v8si) __A, + (__v8si) __B, + __imm, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_shuffle_f64x2 (__m256d __A, __m256d __B, const int __imm) +{ + return (__m256d) __builtin_ia32_shuf_f64x2_256_mask ((__v4df) __A, + (__v4df) __B, + __imm, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_shuffle_f64x2 (__m256d __W, __mmask8 __U, __m256d __A, + __m256d __B, const int __imm) +{ + return (__m256d) __builtin_ia32_shuf_f64x2_256_mask ((__v4df) __A, + (__v4df) __B, + __imm, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_shuffle_f64x2 (__mmask8 __U, __m256d __A, __m256d __B, + const int __imm) +{ + return (__m256d) __builtin_ia32_shuf_f64x2_256_mask ((__v4df) __A, + (__v4df) __B, + __imm, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_shuffle_f32x4 (__m256 __A, __m256 __B, const int __imm) +{ + return (__m256) __builtin_ia32_shuf_f32x4_256_mask ((__v8sf) __A, + (__v8sf) __B, + __imm, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_shuffle_f32x4 (__m256 __W, __mmask8 __U, __m256 __A, + __m256 __B, const int __imm) +{ + return (__m256) __builtin_ia32_shuf_f32x4_256_mask ((__v8sf) __A, + (__v8sf) __B, + __imm, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_shuffle_f32x4 (__mmask8 __U, __m256 __A, __m256 __B, + const int __imm) +{ + return (__m256) __builtin_ia32_shuf_f32x4_256_mask ((__v8sf) __A, + (__v8sf) __B, + __imm, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_fixupimm_pd (__m256d __A, __m256d __B, __m256i __C, + const int __imm) +{ + return (__m256d) __builtin_ia32_fixupimmpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4di) __C, + __imm, + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_fixupimm_pd (__m256d __A, __mmask8 __U, __m256d __B, + __m256i __C, const int __imm) +{ + return (__m256d) __builtin_ia32_fixupimmpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4di) __C, + __imm, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_fixupimm_pd (__mmask8 __U, __m256d __A, __m256d __B, + __m256i __C, const int __imm) +{ + return (__m256d) __builtin_ia32_fixupimmpd256_maskz ((__v4df) __A, + (__v4df) __B, + (__v4di) __C, + __imm, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_fixupimm_ps (__m256 __A, __m256 __B, __m256i __C, + const int __imm) +{ + return (__m256) __builtin_ia32_fixupimmps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8si) __C, + __imm, + (__mmask8) -1); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_fixupimm_ps (__m256 __A, __mmask8 __U, __m256 __B, + __m256i __C, const int __imm) +{ + return (__m256) __builtin_ia32_fixupimmps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8si) __C, + __imm, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_fixupimm_ps (__mmask8 __U, __m256 __A, __m256 __B, + __m256i __C, const int __imm) +{ + return (__m256) __builtin_ia32_fixupimmps256_maskz ((__v8sf) __A, + (__v8sf) __B, + (__v8si) __C, + __imm, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fixupimm_pd (__m128d __A, __m128d __B, __m128i __C, + const int __imm) +{ + return (__m128d) __builtin_ia32_fixupimmpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2di) __C, + __imm, + (__mmask8) -1); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fixupimm_pd (__m128d __A, __mmask8 __U, __m128d __B, + __m128i __C, const int __imm) +{ + return (__m128d) __builtin_ia32_fixupimmpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2di) __C, + __imm, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fixupimm_pd (__mmask8 __U, __m128d __A, __m128d __B, + __m128i __C, const int __imm) +{ + return (__m128d) __builtin_ia32_fixupimmpd128_maskz ((__v2df) __A, + (__v2df) __B, + (__v2di) __C, + __imm, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fixupimm_ps (__m128 __A, __m128 __B, __m128i __C, const int __imm) +{ + return (__m128) __builtin_ia32_fixupimmps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4si) __C, + __imm, + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fixupimm_ps (__m128 __A, __mmask8 __U, __m128 __B, + __m128i __C, const int __imm) +{ + return (__m128) __builtin_ia32_fixupimmps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4si) __C, + __imm, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fixupimm_ps (__mmask8 __U, __m128 __A, __m128 __B, + __m128i __C, const int __imm) +{ + return (__m128) __builtin_ia32_fixupimmps128_maskz ((__v4sf) __A, + (__v4sf) __B, + (__v4si) __C, + __imm, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_srli_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + const int __imm) +{ + return (__m256i) __builtin_ia32_psrldi256_mask ((__v8si) __A, __imm, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_srli_epi32 (__mmask8 __U, __m256i __A, const int __imm) +{ + return (__m256i) __builtin_ia32_psrldi256_mask ((__v8si) __A, __imm, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_srli_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + const int __imm) +{ + return (__m128i) __builtin_ia32_psrldi128_mask ((__v4si) __A, __imm, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_srli_epi32 (__mmask8 __U, __m128i __A, const int __imm) +{ + return (__m128i) __builtin_ia32_psrldi128_mask ((__v4si) __A, __imm, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_srli_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + const int __imm) +{ + return (__m256i) __builtin_ia32_psrlqi256_mask ((__v4di) __A, __imm, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_srli_epi64 (__mmask8 __U, __m256i __A, const int __imm) +{ + return (__m256i) __builtin_ia32_psrlqi256_mask ((__v4di) __A, __imm, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_srli_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + const int __imm) +{ + return (__m128i) __builtin_ia32_psrlqi128_mask ((__v2di) __A, __imm, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_srli_epi64 (__mmask8 __U, __m128i __A, const int __imm) +{ + return (__m128i) __builtin_ia32_psrlqi128_mask ((__v2di) __A, __imm, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_ternarylogic_epi64 (__m256i __A, __m256i __B, __m256i __C, + const int __imm) +{ + return (__m256i) __builtin_ia32_pternlogq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __C, __imm, + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_ternarylogic_epi64 (__m256i __A, __mmask8 __U, + __m256i __B, __m256i __C, + const int __imm) +{ + return (__m256i) __builtin_ia32_pternlogq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __C, __imm, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_ternarylogic_epi64 (__mmask8 __U, __m256i __A, + __m256i __B, __m256i __C, + const int __imm) +{ + return (__m256i) __builtin_ia32_pternlogq256_maskz ((__v4di) __A, + (__v4di) __B, + (__v4di) __C, + __imm, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_ternarylogic_epi32 (__m256i __A, __m256i __B, __m256i __C, + const int __imm) +{ + return (__m256i) __builtin_ia32_pternlogd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __C, __imm, + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_ternarylogic_epi32 (__m256i __A, __mmask8 __U, + __m256i __B, __m256i __C, + const int __imm) +{ + return (__m256i) __builtin_ia32_pternlogd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __C, __imm, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_ternarylogic_epi32 (__mmask8 __U, __m256i __A, + __m256i __B, __m256i __C, + const int __imm) +{ + return (__m256i) __builtin_ia32_pternlogd256_maskz ((__v8si) __A, + (__v8si) __B, + (__v8si) __C, + __imm, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_ternarylogic_epi64 (__m128i __A, __m128i __B, __m128i __C, + const int __imm) +{ + return (__m128i) __builtin_ia32_pternlogq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __C, __imm, + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_ternarylogic_epi64 (__m128i __A, __mmask8 __U, + __m128i __B, __m128i __C, const int __imm) +{ + return (__m128i) __builtin_ia32_pternlogq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __C, __imm, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_ternarylogic_epi64 (__mmask8 __U, __m128i __A, + __m128i __B, __m128i __C, const int __imm) +{ + return (__m128i) __builtin_ia32_pternlogq128_maskz ((__v2di) __A, + (__v2di) __B, + (__v2di) __C, + __imm, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_ternarylogic_epi32 (__m128i __A, __m128i __B, __m128i __C, + const int __imm) +{ + return (__m128i) __builtin_ia32_pternlogd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __C, __imm, + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_ternarylogic_epi32 (__m128i __A, __mmask8 __U, + __m128i __B, __m128i __C, const int __imm) +{ + return (__m128i) __builtin_ia32_pternlogd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __C, __imm, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_ternarylogic_epi32 (__mmask8 __U, __m128i __A, + __m128i __B, __m128i __C, const int __imm) +{ + return (__m128i) __builtin_ia32_pternlogd128_maskz ((__v4si) __A, + (__v4si) __B, + (__v4si) __C, + __imm, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_roundscale_ps (__m256 __A, const int __imm) +{ + return (__m256) __builtin_ia32_rndscaleps_256_mask ((__v8sf) __A, + __imm, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_roundscale_ps (__m256 __W, __mmask8 __U, __m256 __A, + const int __imm) +{ + return (__m256) __builtin_ia32_rndscaleps_256_mask ((__v8sf) __A, + __imm, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_roundscale_ps (__mmask8 __U, __m256 __A, const int __imm) +{ + return (__m256) __builtin_ia32_rndscaleps_256_mask ((__v8sf) __A, + __imm, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_roundscale_pd (__m256d __A, const int __imm) +{ + return (__m256d) __builtin_ia32_rndscalepd_256_mask ((__v4df) __A, + __imm, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_roundscale_pd (__m256d __W, __mmask8 __U, __m256d __A, + const int __imm) +{ + return (__m256d) __builtin_ia32_rndscalepd_256_mask ((__v4df) __A, + __imm, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_roundscale_pd (__mmask8 __U, __m256d __A, const int __imm) +{ + return (__m256d) __builtin_ia32_rndscalepd_256_mask ((__v4df) __A, + __imm, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_roundscale_ps (__m128 __A, const int __imm) +{ + return (__m128) __builtin_ia32_rndscaleps_128_mask ((__v4sf) __A, + __imm, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_roundscale_ps (__m128 __W, __mmask8 __U, __m128 __A, + const int __imm) +{ + return (__m128) __builtin_ia32_rndscaleps_128_mask ((__v4sf) __A, + __imm, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_roundscale_ps (__mmask8 __U, __m128 __A, const int __imm) +{ + return (__m128) __builtin_ia32_rndscaleps_128_mask ((__v4sf) __A, + __imm, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_roundscale_pd (__m128d __A, const int __imm) +{ + return (__m128d) __builtin_ia32_rndscalepd_128_mask ((__v2df) __A, + __imm, + (__v2df) + _mm_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_roundscale_pd (__m128d __W, __mmask8 __U, __m128d __A, + const int __imm) +{ + return (__m128d) __builtin_ia32_rndscalepd_128_mask ((__v2df) __A, + __imm, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_roundscale_pd (__mmask8 __U, __m128d __A, const int __imm) +{ + return (__m128d) __builtin_ia32_rndscalepd_128_mask ((__v2df) __A, + __imm, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_getmant_ps (__m256 __A, _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m256) __builtin_ia32_getmantps256_mask ((__v8sf) __A, + (__C << 2) | __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_getmant_ps (__m256 __W, __mmask8 __U, __m256 __A, + _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m256) __builtin_ia32_getmantps256_mask ((__v8sf) __A, + (__C << 2) | __B, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_getmant_ps (__mmask8 __U, __m256 __A, + _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m256) __builtin_ia32_getmantps256_mask ((__v8sf) __A, + (__C << 2) | __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_getmant_ps (__m128 __A, _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m128) __builtin_ia32_getmantps128_mask ((__v4sf) __A, + (__C << 2) | __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_getmant_ps (__m128 __W, __mmask8 __U, __m128 __A, + _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m128) __builtin_ia32_getmantps128_mask ((__v4sf) __A, + (__C << 2) | __B, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_getmant_ps (__mmask8 __U, __m128 __A, + _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m128) __builtin_ia32_getmantps128_mask ((__v4sf) __A, + (__C << 2) | __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_getmant_pd (__m256d __A, _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m256d) __builtin_ia32_getmantpd256_mask ((__v4df) __A, + (__C << 2) | __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_getmant_pd (__m256d __W, __mmask8 __U, __m256d __A, + _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m256d) __builtin_ia32_getmantpd256_mask ((__v4df) __A, + (__C << 2) | __B, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_getmant_pd (__mmask8 __U, __m256d __A, + _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m256d) __builtin_ia32_getmantpd256_mask ((__v4df) __A, + (__C << 2) | __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_getmant_pd (__m128d __A, _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m128d) __builtin_ia32_getmantpd128_mask ((__v2df) __A, + (__C << 2) | __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_getmant_pd (__m128d __W, __mmask8 __U, __m128d __A, + _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m128d) __builtin_ia32_getmantpd128_mask ((__v2df) __A, + (__C << 2) | __B, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_getmant_pd (__mmask8 __U, __m128d __A, + _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m128d) __builtin_ia32_getmantpd128_mask ((__v2df) __A, + (__C << 2) | __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mmask_i32gather_ps (__m256 __v1_old, __mmask8 __mask, + __m256i __index, void const *__addr, + int __scale) +{ + return (__m256) __builtin_ia32_gather3siv8sf ((__v8sf) __v1_old, + __addr, + (__v8si) __index, + __mask, __scale); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mmask_i32gather_ps (__m128 __v1_old, __mmask8 __mask, + __m128i __index, void const *__addr, + int __scale) +{ + return (__m128) __builtin_ia32_gather3siv4sf ((__v4sf) __v1_old, + __addr, + (__v4si) __index, + __mask, __scale); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mmask_i32gather_pd (__m256d __v1_old, __mmask8 __mask, + __m128i __index, void const *__addr, + int __scale) +{ + return (__m256d) __builtin_ia32_gather3siv4df ((__v4df) __v1_old, + __addr, + (__v4si) __index, + __mask, __scale); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mmask_i32gather_pd (__m128d __v1_old, __mmask8 __mask, + __m128i __index, void const *__addr, + int __scale) +{ + return (__m128d) __builtin_ia32_gather3siv2df ((__v2df) __v1_old, + __addr, + (__v4si) __index, + __mask, __scale); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mmask_i64gather_ps (__m128 __v1_old, __mmask8 __mask, + __m256i __index, void const *__addr, + int __scale) +{ + return (__m128) __builtin_ia32_gather3div8sf ((__v4sf) __v1_old, + __addr, + (__v4di) __index, + __mask, __scale); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mmask_i64gather_ps (__m128 __v1_old, __mmask8 __mask, + __m128i __index, void const *__addr, + int __scale) +{ + return (__m128) __builtin_ia32_gather3div4sf ((__v4sf) __v1_old, + __addr, + (__v2di) __index, + __mask, __scale); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mmask_i64gather_pd (__m256d __v1_old, __mmask8 __mask, + __m256i __index, void const *__addr, + int __scale) +{ + return (__m256d) __builtin_ia32_gather3div4df ((__v4df) __v1_old, + __addr, + (__v4di) __index, + __mask, __scale); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mmask_i64gather_pd (__m128d __v1_old, __mmask8 __mask, + __m128i __index, void const *__addr, + int __scale) +{ + return (__m128d) __builtin_ia32_gather3div2df ((__v2df) __v1_old, + __addr, + (__v2di) __index, + __mask, __scale); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mmask_i32gather_epi32 (__m256i __v1_old, __mmask8 __mask, + __m256i __index, void const *__addr, + int __scale) +{ + return (__m256i) __builtin_ia32_gather3siv8si ((__v8si) __v1_old, + __addr, + (__v8si) __index, + __mask, __scale); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mmask_i32gather_epi32 (__m128i __v1_old, __mmask8 __mask, + __m128i __index, void const *__addr, + int __scale) +{ + return (__m128i) __builtin_ia32_gather3siv4si ((__v4si) __v1_old, + __addr, + (__v4si) __index, + __mask, __scale); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mmask_i32gather_epi64 (__m256i __v1_old, __mmask8 __mask, + __m128i __index, void const *__addr, + int __scale) +{ + return (__m256i) __builtin_ia32_gather3siv4di ((__v4di) __v1_old, + __addr, + (__v4si) __index, + __mask, __scale); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mmask_i32gather_epi64 (__m128i __v1_old, __mmask8 __mask, + __m128i __index, void const *__addr, + int __scale) +{ + return (__m128i) __builtin_ia32_gather3siv2di ((__v2di) __v1_old, + __addr, + (__v4si) __index, + __mask, __scale); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mmask_i64gather_epi32 (__m128i __v1_old, __mmask8 __mask, + __m256i __index, void const *__addr, + int __scale) +{ + return (__m128i) __builtin_ia32_gather3div8si ((__v4si) __v1_old, + __addr, + (__v4di) __index, + __mask, __scale); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mmask_i64gather_epi32 (__m128i __v1_old, __mmask8 __mask, + __m128i __index, void const *__addr, + int __scale) +{ + return (__m128i) __builtin_ia32_gather3div4si ((__v4si) __v1_old, + __addr, + (__v2di) __index, + __mask, __scale); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mmask_i64gather_epi64 (__m256i __v1_old, __mmask8 __mask, + __m256i __index, void const *__addr, + int __scale) +{ + return (__m256i) __builtin_ia32_gather3div4di ((__v4di) __v1_old, + __addr, + (__v4di) __index, + __mask, __scale); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mmask_i64gather_epi64 (__m128i __v1_old, __mmask8 __mask, + __m128i __index, void const *__addr, + int __scale) +{ + return (__m128i) __builtin_ia32_gather3div2di ((__v2di) __v1_old, + __addr, + (__v2di) __index, + __mask, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_i32scatter_ps (void *__addr, __m256i __index, + __m256 __v1, const int __scale) +{ + __builtin_ia32_scattersiv8sf (__addr, (__mmask8) 0xFF, + (__v8si) __index, (__v8sf) __v1, + __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_i32scatter_ps (void *__addr, __mmask8 __mask, + __m256i __index, __m256 __v1, + const int __scale) +{ + __builtin_ia32_scattersiv8sf (__addr, __mask, (__v8si) __index, + (__v8sf) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_i32scatter_ps (void *__addr, __m128i __index, __m128 __v1, + const int __scale) +{ + __builtin_ia32_scattersiv4sf (__addr, (__mmask8) 0xFF, + (__v4si) __index, (__v4sf) __v1, + __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_i32scatter_ps (void *__addr, __mmask8 __mask, + __m128i __index, __m128 __v1, + const int __scale) +{ + __builtin_ia32_scattersiv4sf (__addr, __mask, (__v4si) __index, + (__v4sf) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_i32scatter_pd (void *__addr, __m128i __index, + __m256d __v1, const int __scale) +{ + __builtin_ia32_scattersiv4df (__addr, (__mmask8) 0xFF, + (__v4si) __index, (__v4df) __v1, + __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_i32scatter_pd (void *__addr, __mmask8 __mask, + __m128i __index, __m256d __v1, + const int __scale) +{ + __builtin_ia32_scattersiv4df (__addr, __mask, (__v4si) __index, + (__v4df) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_i32scatter_pd (void *__addr, __m128i __index, + __m128d __v1, const int __scale) +{ + __builtin_ia32_scattersiv2df (__addr, (__mmask8) 0xFF, + (__v4si) __index, (__v2df) __v1, + __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_i32scatter_pd (void *__addr, __mmask8 __mask, + __m128i __index, __m128d __v1, + const int __scale) +{ + __builtin_ia32_scattersiv2df (__addr, __mask, (__v4si) __index, + (__v2df) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_i64scatter_ps (void *__addr, __m256i __index, + __m128 __v1, const int __scale) +{ + __builtin_ia32_scatterdiv8sf (__addr, (__mmask8) 0xFF, + (__v4di) __index, (__v4sf) __v1, + __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_i64scatter_ps (void *__addr, __mmask8 __mask, + __m256i __index, __m128 __v1, + const int __scale) +{ + __builtin_ia32_scatterdiv8sf (__addr, __mask, (__v4di) __index, + (__v4sf) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_i64scatter_ps (void *__addr, __m128i __index, __m128 __v1, + const int __scale) +{ + __builtin_ia32_scatterdiv4sf (__addr, (__mmask8) 0xFF, + (__v2di) __index, (__v4sf) __v1, + __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_i64scatter_ps (void *__addr, __mmask8 __mask, + __m128i __index, __m128 __v1, + const int __scale) +{ + __builtin_ia32_scatterdiv4sf (__addr, __mask, (__v2di) __index, + (__v4sf) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_i64scatter_pd (void *__addr, __m256i __index, + __m256d __v1, const int __scale) +{ + __builtin_ia32_scatterdiv4df (__addr, (__mmask8) 0xFF, + (__v4di) __index, (__v4df) __v1, + __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_i64scatter_pd (void *__addr, __mmask8 __mask, + __m256i __index, __m256d __v1, + const int __scale) +{ + __builtin_ia32_scatterdiv4df (__addr, __mask, (__v4di) __index, + (__v4df) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_i64scatter_pd (void *__addr, __m128i __index, + __m128d __v1, const int __scale) +{ + __builtin_ia32_scatterdiv2df (__addr, (__mmask8) 0xFF, + (__v2di) __index, (__v2df) __v1, + __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_i64scatter_pd (void *__addr, __mmask8 __mask, + __m128i __index, __m128d __v1, + const int __scale) +{ + __builtin_ia32_scatterdiv2df (__addr, __mask, (__v2di) __index, + (__v2df) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_i32scatter_epi32 (void *__addr, __m256i __index, + __m256i __v1, const int __scale) +{ + __builtin_ia32_scattersiv8si (__addr, (__mmask8) 0xFF, + (__v8si) __index, (__v8si) __v1, + __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_i32scatter_epi32 (void *__addr, __mmask8 __mask, + __m256i __index, __m256i __v1, + const int __scale) +{ + __builtin_ia32_scattersiv8si (__addr, __mask, (__v8si) __index, + (__v8si) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_i32scatter_epi32 (void *__addr, __m128i __index, + __m128i __v1, const int __scale) +{ + __builtin_ia32_scattersiv4si (__addr, (__mmask8) 0xFF, + (__v4si) __index, (__v4si) __v1, + __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_i32scatter_epi32 (void *__addr, __mmask8 __mask, + __m128i __index, __m128i __v1, + const int __scale) +{ + __builtin_ia32_scattersiv4si (__addr, __mask, (__v4si) __index, + (__v4si) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_i32scatter_epi64 (void *__addr, __m128i __index, + __m256i __v1, const int __scale) +{ + __builtin_ia32_scattersiv4di (__addr, (__mmask8) 0xFF, + (__v4si) __index, (__v4di) __v1, + __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_i32scatter_epi64 (void *__addr, __mmask8 __mask, + __m128i __index, __m256i __v1, + const int __scale) +{ + __builtin_ia32_scattersiv4di (__addr, __mask, (__v4si) __index, + (__v4di) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_i32scatter_epi64 (void *__addr, __m128i __index, + __m128i __v1, const int __scale) +{ + __builtin_ia32_scattersiv2di (__addr, (__mmask8) 0xFF, + (__v4si) __index, (__v2di) __v1, + __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_i32scatter_epi64 (void *__addr, __mmask8 __mask, + __m128i __index, __m128i __v1, + const int __scale) +{ + __builtin_ia32_scattersiv2di (__addr, __mask, (__v4si) __index, + (__v2di) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_i64scatter_epi32 (void *__addr, __m256i __index, + __m128i __v1, const int __scale) +{ + __builtin_ia32_scatterdiv8si (__addr, (__mmask8) 0xFF, + (__v4di) __index, (__v4si) __v1, + __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_i64scatter_epi32 (void *__addr, __mmask8 __mask, + __m256i __index, __m128i __v1, + const int __scale) +{ + __builtin_ia32_scatterdiv8si (__addr, __mask, (__v4di) __index, + (__v4si) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_i64scatter_epi32 (void *__addr, __m128i __index, + __m128i __v1, const int __scale) +{ + __builtin_ia32_scatterdiv4si (__addr, (__mmask8) 0xFF, + (__v2di) __index, (__v4si) __v1, + __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_i64scatter_epi32 (void *__addr, __mmask8 __mask, + __m128i __index, __m128i __v1, + const int __scale) +{ + __builtin_ia32_scatterdiv4si (__addr, __mask, (__v2di) __index, + (__v4si) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_i64scatter_epi64 (void *__addr, __m256i __index, + __m256i __v1, const int __scale) +{ + __builtin_ia32_scatterdiv4di (__addr, (__mmask8) 0xFF, + (__v4di) __index, (__v4di) __v1, + __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_i64scatter_epi64 (void *__addr, __mmask8 __mask, + __m256i __index, __m256i __v1, + const int __scale) +{ + __builtin_ia32_scatterdiv4di (__addr, __mask, (__v4di) __index, + (__v4di) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_i64scatter_epi64 (void *__addr, __m128i __index, + __m128i __v1, const int __scale) +{ + __builtin_ia32_scatterdiv2di (__addr, (__mmask8) 0xFF, + (__v2di) __index, (__v2di) __v1, + __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_i64scatter_epi64 (void *__addr, __mmask8 __mask, + __m128i __index, __m128i __v1, + const int __scale) +{ + __builtin_ia32_scatterdiv2di (__addr, __mask, (__v2di) __index, + (__v2di) __v1, __scale); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_shuffle_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + _MM_PERM_ENUM __mask) +{ + return (__m256i) __builtin_ia32_pshufd256_mask ((__v8si) __A, __mask, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_shuffle_epi32 (__mmask8 __U, __m256i __A, + _MM_PERM_ENUM __mask) +{ + return (__m256i) __builtin_ia32_pshufd256_mask ((__v8si) __A, __mask, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_shuffle_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + _MM_PERM_ENUM __mask) +{ + return (__m128i) __builtin_ia32_pshufd128_mask ((__v4si) __A, __mask, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_shuffle_epi32 (__mmask8 __U, __m128i __A, + _MM_PERM_ENUM __mask) +{ + return (__m128i) __builtin_ia32_pshufd128_mask ((__v4si) __A, __mask, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_rol_epi32 (__m256i __A, const int __B) +{ + return (__m256i) __builtin_ia32_prold256_mask ((__v8si) __A, __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_rol_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + const int __B) +{ + return (__m256i) __builtin_ia32_prold256_mask ((__v8si) __A, __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_rol_epi32 (__mmask8 __U, __m256i __A, const int __B) +{ + return (__m256i) __builtin_ia32_prold256_mask ((__v8si) __A, __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rol_epi32 (__m128i __A, const int __B) +{ + return (__m128i) __builtin_ia32_prold128_mask ((__v4si) __A, __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_rol_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + const int __B) +{ + return (__m128i) __builtin_ia32_prold128_mask ((__v4si) __A, __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_rol_epi32 (__mmask8 __U, __m128i __A, const int __B) +{ + return (__m128i) __builtin_ia32_prold128_mask ((__v4si) __A, __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_ror_epi32 (__m256i __A, const int __B) +{ + return (__m256i) __builtin_ia32_prord256_mask ((__v8si) __A, __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_ror_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + const int __B) +{ + return (__m256i) __builtin_ia32_prord256_mask ((__v8si) __A, __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_ror_epi32 (__mmask8 __U, __m256i __A, const int __B) +{ + return (__m256i) __builtin_ia32_prord256_mask ((__v8si) __A, __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_ror_epi32 (__m128i __A, const int __B) +{ + return (__m128i) __builtin_ia32_prord128_mask ((__v4si) __A, __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_ror_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + const int __B) +{ + return (__m128i) __builtin_ia32_prord128_mask ((__v4si) __A, __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_ror_epi32 (__mmask8 __U, __m128i __A, const int __B) +{ + return (__m128i) __builtin_ia32_prord128_mask ((__v4si) __A, __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_rol_epi64 (__m256i __A, const int __B) +{ + return (__m256i) __builtin_ia32_prolq256_mask ((__v4di) __A, __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_rol_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + const int __B) +{ + return (__m256i) __builtin_ia32_prolq256_mask ((__v4di) __A, __B, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_rol_epi64 (__mmask8 __U, __m256i __A, const int __B) +{ + return (__m256i) __builtin_ia32_prolq256_mask ((__v4di) __A, __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rol_epi64 (__m128i __A, const int __B) +{ + return (__m128i) __builtin_ia32_prolq128_mask ((__v2di) __A, __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_rol_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + const int __B) +{ + return (__m128i) __builtin_ia32_prolq128_mask ((__v2di) __A, __B, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_rol_epi64 (__mmask8 __U, __m128i __A, const int __B) +{ + return (__m128i) __builtin_ia32_prolq128_mask ((__v2di) __A, __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_ror_epi64 (__m256i __A, const int __B) +{ + return (__m256i) __builtin_ia32_prorq256_mask ((__v4di) __A, __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_ror_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + const int __B) +{ + return (__m256i) __builtin_ia32_prorq256_mask ((__v4di) __A, __B, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_ror_epi64 (__mmask8 __U, __m256i __A, const int __B) +{ + return (__m256i) __builtin_ia32_prorq256_mask ((__v4di) __A, __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_ror_epi64 (__m128i __A, const int __B) +{ + return (__m128i) __builtin_ia32_prorq128_mask ((__v2di) __A, __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_ror_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + const int __B) +{ + return (__m128i) __builtin_ia32_prorq128_mask ((__v2di) __A, __B, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_ror_epi64 (__mmask8 __U, __m128i __A, const int __B) +{ + return (__m128i) __builtin_ia32_prorq128_mask ((__v2di) __A, __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_alignr_epi32 (__m128i __A, __m128i __B, const int __imm) +{ + return (__m128i) __builtin_ia32_alignd128_mask ((__v4si) __A, + (__v4si) __B, __imm, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_alignr_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B, const int __imm) +{ + return (__m128i) __builtin_ia32_alignd128_mask ((__v4si) __A, + (__v4si) __B, __imm, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_alignr_epi32 (__mmask8 __U, __m128i __A, __m128i __B, + const int __imm) +{ + return (__m128i) __builtin_ia32_alignd128_mask ((__v4si) __A, + (__v4si) __B, __imm, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_alignr_epi64 (__m128i __A, __m128i __B, const int __imm) +{ + return (__m128i) __builtin_ia32_alignq128_mask ((__v2di) __A, + (__v2di) __B, __imm, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_alignr_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B, const int __imm) +{ + return (__m128i) __builtin_ia32_alignq128_mask ((__v2di) __A, + (__v2di) __B, __imm, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_alignr_epi64 (__mmask8 __U, __m128i __A, __m128i __B, + const int __imm) +{ + return (__m128i) __builtin_ia32_alignq128_mask ((__v2di) __A, + (__v2di) __B, __imm, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_alignr_epi32 (__m256i __A, __m256i __B, const int __imm) +{ + return (__m256i) __builtin_ia32_alignd256_mask ((__v8si) __A, + (__v8si) __B, __imm, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_alignr_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B, const int __imm) +{ + return (__m256i) __builtin_ia32_alignd256_mask ((__v8si) __A, + (__v8si) __B, __imm, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_alignr_epi32 (__mmask8 __U, __m256i __A, __m256i __B, + const int __imm) +{ + return (__m256i) __builtin_ia32_alignd256_mask ((__v8si) __A, + (__v8si) __B, __imm, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_alignr_epi64 (__m256i __A, __m256i __B, const int __imm) +{ + return (__m256i) __builtin_ia32_alignq256_mask ((__v4di) __A, + (__v4di) __B, __imm, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_alignr_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B, const int __imm) +{ + return (__m256i) __builtin_ia32_alignq256_mask ((__v4di) __A, + (__v4di) __B, __imm, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_alignr_epi64 (__mmask8 __U, __m256i __A, __m256i __B, + const int __imm) +{ + return (__m256i) __builtin_ia32_alignq256_mask ((__v4di) __A, + (__v4di) __B, __imm, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtps_ph (__m128i __W, __mmask8 __U, __m128 __A, + const int __I) +{ + return (__m128i) __builtin_ia32_vcvtps2ph_mask ((__v4sf) __A, __I, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtps_ph (__mmask8 __U, __m128 __A, const int __I) +{ + return (__m128i) __builtin_ia32_vcvtps2ph_mask ((__v4sf) __A, __I, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtps_ph (__m128i __W, __mmask8 __U, __m256 __A, + const int __I) +{ + return (__m128i) __builtin_ia32_vcvtps2ph256_mask ((__v8sf) __A, __I, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtps_ph (__mmask8 __U, __m256 __A, const int __I) +{ + return (__m128i) __builtin_ia32_vcvtps2ph256_mask ((__v8sf) __A, __I, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_srai_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + const int __imm) +{ + return (__m256i) __builtin_ia32_psradi256_mask ((__v8si) __A, __imm, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_srai_epi32 (__mmask8 __U, __m256i __A, const int __imm) +{ + return (__m256i) __builtin_ia32_psradi256_mask ((__v8si) __A, __imm, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_srai_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + const int __imm) +{ + return (__m128i) __builtin_ia32_psradi128_mask ((__v4si) __A, __imm, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_srai_epi32 (__mmask8 __U, __m128i __A, const int __imm) +{ + return (__m128i) __builtin_ia32_psradi128_mask ((__v4si) __A, __imm, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_srai_epi64 (__m256i __A, const int __imm) +{ + return (__m256i) __builtin_ia32_psraqi256_mask ((__v4di) __A, __imm, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_srai_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + const int __imm) +{ + return (__m256i) __builtin_ia32_psraqi256_mask ((__v4di) __A, __imm, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_srai_epi64 (__mmask8 __U, __m256i __A, const int __imm) +{ + return (__m256i) __builtin_ia32_psraqi256_mask ((__v4di) __A, __imm, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srai_epi64 (__m128i __A, const int __imm) +{ + return (__m128i) __builtin_ia32_psraqi128_mask ((__v2di) __A, __imm, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_srai_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + const int __imm) +{ + return (__m128i) __builtin_ia32_psraqi128_mask ((__v2di) __A, __imm, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_srai_epi64 (__mmask8 __U, __m128i __A, const int __imm) +{ + return (__m128i) __builtin_ia32_psraqi128_mask ((__v2di) __A, __imm, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_slli_epi32 (__m128i __W, __mmask8 __U, __m128i __A, int __B) +{ + return (__m128i) __builtin_ia32_pslldi128_mask ((__v4si) __A, __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_slli_epi32 (__mmask8 __U, __m128i __A, int __B) +{ + return (__m128i) __builtin_ia32_pslldi128_mask ((__v4si) __A, __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_slli_epi64 (__m128i __W, __mmask8 __U, __m128i __A, int __B) +{ + return (__m128i) __builtin_ia32_psllqi128_mask ((__v2di) __A, __B, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_slli_epi64 (__mmask8 __U, __m128i __A, int __B) +{ + return (__m128i) __builtin_ia32_psllqi128_mask ((__v2di) __A, __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_slli_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + int __B) +{ + return (__m256i) __builtin_ia32_pslldi256_mask ((__v8si) __A, __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_slli_epi32 (__mmask8 __U, __m256i __A, int __B) +{ + return (__m256i) __builtin_ia32_pslldi256_mask ((__v8si) __A, __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_slli_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + int __B) +{ + return (__m256i) __builtin_ia32_psllqi256_mask ((__v4di) __A, __B, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_slli_epi64 (__mmask8 __U, __m256i __A, int __B) +{ + return (__m256i) __builtin_ia32_psllqi256_mask ((__v4di) __A, __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permutex_pd (__m256d __W, __mmask8 __U, __m256d __X, + const int __imm) +{ + return (__m256d) __builtin_ia32_permdf256_mask ((__v4df) __X, __imm, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permutex_pd (__mmask8 __U, __m256d __X, const int __imm) +{ + return (__m256d) __builtin_ia32_permdf256_mask ((__v4df) __X, __imm, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permute_pd (__m256d __W, __mmask8 __U, __m256d __X, + const int __C) +{ + return (__m256d) __builtin_ia32_vpermilpd256_mask ((__v4df) __X, __C, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permute_pd (__mmask8 __U, __m256d __X, const int __C) +{ + return (__m256d) __builtin_ia32_vpermilpd256_mask ((__v4df) __X, __C, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_permute_pd (__m128d __W, __mmask8 __U, __m128d __X, + const int __C) +{ + return (__m128d) __builtin_ia32_vpermilpd_mask ((__v2df) __X, __C, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_permute_pd (__mmask8 __U, __m128d __X, const int __C) +{ + return (__m128d) __builtin_ia32_vpermilpd_mask ((__v2df) __X, __C, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permute_ps (__m256 __W, __mmask8 __U, __m256 __X, + const int __C) +{ + return (__m256) __builtin_ia32_vpermilps256_mask ((__v8sf) __X, __C, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permute_ps (__mmask8 __U, __m256 __X, const int __C) +{ + return (__m256) __builtin_ia32_vpermilps256_mask ((__v8sf) __X, __C, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_permute_ps (__m128 __W, __mmask8 __U, __m128 __X, + const int __C) +{ + return (__m128) __builtin_ia32_vpermilps_mask ((__v4sf) __X, __C, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_permute_ps (__mmask8 __U, __m128 __X, const int __C) +{ + return (__m128) __builtin_ia32_vpermilps_mask ((__v4sf) __X, __C, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_blend_pd (__mmask8 __U, __m256d __A, __m256d __W) +{ + return (__m256d) __builtin_ia32_blendmpd_256_mask ((__v4df) __A, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_blend_ps (__mmask8 __U, __m256 __A, __m256 __W) +{ + return (__m256) __builtin_ia32_blendmps_256_mask ((__v8sf) __A, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_blend_epi64 (__mmask8 __U, __m256i __A, __m256i __W) +{ + return (__m256i) __builtin_ia32_blendmq_256_mask ((__v4di) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_blend_epi32 (__mmask8 __U, __m256i __A, __m256i __W) +{ + return (__m256i) __builtin_ia32_blendmd_256_mask ((__v8si) __A, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_blend_pd (__mmask8 __U, __m128d __A, __m128d __W) +{ + return (__m128d) __builtin_ia32_blendmpd_128_mask ((__v2df) __A, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_blend_ps (__mmask8 __U, __m128 __A, __m128 __W) +{ + return (__m128) __builtin_ia32_blendmps_128_mask ((__v4sf) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_blend_epi64 (__mmask8 __U, __m128i __A, __m128i __W) +{ + return (__m128i) __builtin_ia32_blendmq_128_mask ((__v2di) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_blend_epi32 (__mmask8 __U, __m128i __A, __m128i __W) +{ + return (__m128i) __builtin_ia32_blendmd_128_mask ((__v4si) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmp_epi64_mask (__m256i __X, __m256i __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_cmpq256_mask ((__v4di) __X, + (__v4di) __Y, __P, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmp_epi32_mask (__m256i __X, __m256i __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_cmpd256_mask ((__v8si) __X, + (__v8si) __Y, __P, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmp_epu64_mask (__m256i __X, __m256i __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_ucmpq256_mask ((__v4di) __X, + (__v4di) __Y, __P, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmp_epu32_mask (__m256i __X, __m256i __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_ucmpd256_mask ((__v8si) __X, + (__v8si) __Y, __P, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmp_pd_mask (__m256d __X, __m256d __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_cmppd256_mask ((__v4df) __X, + (__v4df) __Y, __P, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmp_ps_mask (__m256 __X, __m256 __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_cmpps256_mask ((__v8sf) __X, + (__v8sf) __Y, __P, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmp_epi64_mask (__mmask8 __U, __m256i __X, __m256i __Y, + const int __P) +{ + return (__mmask8) __builtin_ia32_cmpq256_mask ((__v4di) __X, + (__v4di) __Y, __P, + (__mmask8) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmp_epi32_mask (__mmask8 __U, __m256i __X, __m256i __Y, + const int __P) +{ + return (__mmask8) __builtin_ia32_cmpd256_mask ((__v8si) __X, + (__v8si) __Y, __P, + (__mmask8) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmp_epu64_mask (__mmask8 __U, __m256i __X, __m256i __Y, + const int __P) +{ + return (__mmask8) __builtin_ia32_ucmpq256_mask ((__v4di) __X, + (__v4di) __Y, __P, + (__mmask8) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmp_epu32_mask (__mmask8 __U, __m256i __X, __m256i __Y, + const int __P) +{ + return (__mmask8) __builtin_ia32_ucmpd256_mask ((__v8si) __X, + (__v8si) __Y, __P, + (__mmask8) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmp_pd_mask (__mmask8 __U, __m256d __X, __m256d __Y, + const int __P) +{ + return (__mmask8) __builtin_ia32_cmppd256_mask ((__v4df) __X, + (__v4df) __Y, __P, + (__mmask8) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmp_ps_mask (__mmask8 __U, __m256 __X, __m256 __Y, + const int __P) +{ + return (__mmask8) __builtin_ia32_cmpps256_mask ((__v8sf) __X, + (__v8sf) __Y, __P, + (__mmask8) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_epi64_mask (__m128i __X, __m128i __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_cmpq128_mask ((__v2di) __X, + (__v2di) __Y, __P, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_epi32_mask (__m128i __X, __m128i __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_cmpd128_mask ((__v4si) __X, + (__v4si) __Y, __P, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_epu64_mask (__m128i __X, __m128i __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_ucmpq128_mask ((__v2di) __X, + (__v2di) __Y, __P, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_epu32_mask (__m128i __X, __m128i __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_ucmpd128_mask ((__v4si) __X, + (__v4si) __Y, __P, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_pd_mask (__m128d __X, __m128d __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_cmppd128_mask ((__v2df) __X, + (__v2df) __Y, __P, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_ps_mask (__m128 __X, __m128 __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_cmpps128_mask ((__v4sf) __X, + (__v4sf) __Y, __P, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmp_epi64_mask (__mmask8 __U, __m128i __X, __m128i __Y, + const int __P) +{ + return (__mmask8) __builtin_ia32_cmpq128_mask ((__v2di) __X, + (__v2di) __Y, __P, + (__mmask8) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmp_epi32_mask (__mmask8 __U, __m128i __X, __m128i __Y, + const int __P) +{ + return (__mmask8) __builtin_ia32_cmpd128_mask ((__v4si) __X, + (__v4si) __Y, __P, + (__mmask8) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmp_epu64_mask (__mmask8 __U, __m128i __X, __m128i __Y, + const int __P) +{ + return (__mmask8) __builtin_ia32_ucmpq128_mask ((__v2di) __X, + (__v2di) __Y, __P, + (__mmask8) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmp_epu32_mask (__mmask8 __U, __m128i __X, __m128i __Y, + const int __P) +{ + return (__mmask8) __builtin_ia32_ucmpd128_mask ((__v4si) __X, + (__v4si) __Y, __P, + (__mmask8) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmp_pd_mask (__mmask8 __U, __m128d __X, __m128d __Y, + const int __P) +{ + return (__mmask8) __builtin_ia32_cmppd128_mask ((__v2df) __X, + (__v2df) __Y, __P, + (__mmask8) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmp_ps_mask (__mmask8 __U, __m128 __X, __m128 __Y, + const int __P) +{ + return (__mmask8) __builtin_ia32_cmpps128_mask ((__v4sf) __X, + (__v4sf) __Y, __P, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permutex_pd (__m256d __X, const int __M) +{ + return (__m256d) __builtin_ia32_permdf256_mask ((__v4df) __X, __M, + (__v4df) + _mm256_undefined_pd (), + (__mmask8) -1); +} + +#else +#define _mm256_permutex_pd(X, M) \ + ((__m256d) __builtin_ia32_permdf256_mask ((__v4df)(__m256d)(X), (int)(M), \ + (__v4df)(__m256d) \ + _mm256_undefined_pd (), \ + (__mmask8)-1)) + +#define _mm256_maskz_permutex_epi64(M, X, I) \ + ((__m256i) __builtin_ia32_permdi256_mask ((__v4di)(__m256i)(X), \ + (int)(I), \ + (__v4di)(__m256i) \ + (_mm256_setzero_si256 ()),\ + (__mmask8)(M))) + +#define _mm256_mask_permutex_epi64(W, M, X, I) \ + ((__m256i) __builtin_ia32_permdi256_mask ((__v4di)(__m256i)(X), \ + (int)(I), \ + (__v4di)(__m256i)(W), \ + (__mmask8)(M))) + +#define _mm256_insertf32x4(X, Y, C) \ + ((__m256) __builtin_ia32_insertf32x4_256_mask ((__v8sf)(__m256) (X), \ + (__v4sf)(__m128) (Y), (int) (C), \ + (__v8sf)(__m256)_mm256_setzero_ps (), \ + (__mmask8)-1)) + +#define _mm256_mask_insertf32x4(W, U, X, Y, C) \ + ((__m256) __builtin_ia32_insertf32x4_256_mask ((__v8sf)(__m256) (X), \ + (__v4sf)(__m128) (Y), (int) (C), \ + (__v8sf)(__m256)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_insertf32x4(U, X, Y, C) \ + ((__m256) __builtin_ia32_insertf32x4_256_mask ((__v8sf)(__m256) (X), \ + (__v4sf)(__m128) (Y), (int) (C), \ + (__v8sf)(__m256)_mm256_setzero_ps (), \ + (__mmask8)(U))) + +#define _mm256_inserti32x4(X, Y, C) \ + ((__m256i) __builtin_ia32_inserti32x4_256_mask ((__v8si)(__m256i) (X),\ + (__v4si)(__m128i) (Y), (int) (C), \ + (__v8si)(__m256i)_mm256_setzero_si256 (), \ + (__mmask8)-1)) + +#define _mm256_mask_inserti32x4(W, U, X, Y, C) \ + ((__m256i) __builtin_ia32_inserti32x4_256_mask ((__v8si)(__m256i) (X),\ + (__v4si)(__m128i) (Y), (int) (C), \ + (__v8si)(__m256i)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_inserti32x4(U, X, Y, C) \ + ((__m256i) __builtin_ia32_inserti32x4_256_mask ((__v8si)(__m256i) (X),\ + (__v4si)(__m128i) (Y), (int) (C), \ + (__v8si)(__m256i)_mm256_setzero_si256 (), \ + (__mmask8)(U))) + +#define _mm256_extractf32x4_ps(X, C) \ + ((__m128) __builtin_ia32_extractf32x4_256_mask ((__v8sf)(__m256) (X), \ + (int) (C), \ + (__v4sf)(__m128)_mm_setzero_ps (), \ + (__mmask8)-1)) + +#define _mm256_mask_extractf32x4_ps(W, U, X, C) \ + ((__m128) __builtin_ia32_extractf32x4_256_mask ((__v8sf)(__m256) (X), \ + (int) (C), \ + (__v4sf)(__m128)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_extractf32x4_ps(U, X, C) \ + ((__m128) __builtin_ia32_extractf32x4_256_mask ((__v8sf)(__m256) (X), \ + (int) (C), \ + (__v4sf)(__m128)_mm_setzero_ps (), \ + (__mmask8)(U))) + +#define _mm256_extracti32x4_epi32(X, C) \ + ((__m128i) __builtin_ia32_extracti32x4_256_mask ((__v8si)(__m256i) (X),\ + (int) (C), (__v4si)(__m128i)_mm_setzero_si128 (), (__mmask8)-1)) + +#define _mm256_mask_extracti32x4_epi32(W, U, X, C) \ + ((__m128i) __builtin_ia32_extracti32x4_256_mask ((__v8si)(__m256i) (X),\ + (int) (C), (__v4si)(__m128i)(W), (__mmask8)(U))) + +#define _mm256_maskz_extracti32x4_epi32(U, X, C) \ + ((__m128i) __builtin_ia32_extracti32x4_256_mask ((__v8si)(__m256i) (X),\ + (int) (C), (__v4si)(__m128i)_mm_setzero_si128 (), (__mmask8)(U))) + +#define _mm256_shuffle_i64x2(X, Y, C) \ + ((__m256i) __builtin_ia32_shuf_i64x2_256_mask ((__v4di)(__m256i)(X), \ + (__v4di)(__m256i)(Y), (int)(C), \ + (__v4di)(__m256i)_mm256_setzero_si256 (), \ + (__mmask8)-1)) + +#define _mm256_mask_shuffle_i64x2(W, U, X, Y, C) \ + ((__m256i) __builtin_ia32_shuf_i64x2_256_mask ((__v4di)(__m256i)(X), \ + (__v4di)(__m256i)(Y), (int)(C), \ + (__v4di)(__m256i)(W),\ + (__mmask8)(U))) + +#define _mm256_maskz_shuffle_i64x2(U, X, Y, C) \ + ((__m256i) __builtin_ia32_shuf_i64x2_256_mask ((__v4di)(__m256i)(X), \ + (__v4di)(__m256i)(Y), (int)(C), \ + (__v4di)(__m256i)_mm256_setzero_si256 (), \ + (__mmask8)(U))) + +#define _mm256_shuffle_i32x4(X, Y, C) \ + ((__m256i) __builtin_ia32_shuf_i32x4_256_mask ((__v8si)(__m256i)(X), \ + (__v8si)(__m256i)(Y), (int)(C), \ + (__v8si)(__m256i) \ + _mm256_setzero_si256 (), \ + (__mmask8)-1)) + +#define _mm256_mask_shuffle_i32x4(W, U, X, Y, C) \ + ((__m256i) __builtin_ia32_shuf_i32x4_256_mask ((__v8si)(__m256i)(X), \ + (__v8si)(__m256i)(Y), (int)(C), \ + (__v8si)(__m256i)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_shuffle_i32x4(U, X, Y, C) \ + ((__m256i) __builtin_ia32_shuf_i32x4_256_mask ((__v8si)(__m256i)(X), \ + (__v8si)(__m256i)(Y), (int)(C), \ + (__v8si)(__m256i) \ + _mm256_setzero_si256 (), \ + (__mmask8)(U))) + +#define _mm256_shuffle_f64x2(X, Y, C) \ + ((__m256d) __builtin_ia32_shuf_f64x2_256_mask ((__v4df)(__m256d)(X), \ + (__v4df)(__m256d)(Y), (int)(C), \ + (__v4df)(__m256d)_mm256_setzero_pd (),\ + (__mmask8)-1)) + +#define _mm256_mask_shuffle_f64x2(W, U, X, Y, C) \ + ((__m256d) __builtin_ia32_shuf_f64x2_256_mask ((__v4df)(__m256d)(X), \ + (__v4df)(__m256d)(Y), (int)(C), \ + (__v4df)(__m256d)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_shuffle_f64x2(U, X, Y, C) \ + ((__m256d) __builtin_ia32_shuf_f64x2_256_mask ((__v4df)(__m256d)(X), \ + (__v4df)(__m256d)(Y), (int)(C), \ + (__v4df)(__m256d)_mm256_setzero_pd( ),\ + (__mmask8)(U))) + +#define _mm256_shuffle_f32x4(X, Y, C) \ + ((__m256) __builtin_ia32_shuf_f32x4_256_mask ((__v8sf)(__m256)(X), \ + (__v8sf)(__m256)(Y), (int)(C), \ + (__v8sf)(__m256)_mm256_setzero_ps (), \ + (__mmask8)-1)) + +#define _mm256_mask_shuffle_f32x4(W, U, X, Y, C) \ + ((__m256) __builtin_ia32_shuf_f32x4_256_mask ((__v8sf)(__m256)(X), \ + (__v8sf)(__m256)(Y), (int)(C), \ + (__v8sf)(__m256)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_shuffle_f32x4(U, X, Y, C) \ + ((__m256) __builtin_ia32_shuf_f32x4_256_mask ((__v8sf)(__m256)(X), \ + (__v8sf)(__m256)(Y), (int)(C), \ + (__v8sf)(__m256)_mm256_setzero_ps (), \ + (__mmask8)(U))) + +#define _mm256_mask_shuffle_pd(W, U, A, B, C) \ + ((__m256d)__builtin_ia32_shufpd256_mask ((__v4df)(__m256d)(A), \ + (__v4df)(__m256d)(B), (int)(C), \ + (__v4df)(__m256d)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_shuffle_pd(U, A, B, C) \ + ((__m256d)__builtin_ia32_shufpd256_mask ((__v4df)(__m256d)(A), \ + (__v4df)(__m256d)(B), (int)(C), \ + (__v4df)(__m256d) \ + _mm256_setzero_pd (), \ + (__mmask8)(U))) + +#define _mm_mask_shuffle_pd(W, U, A, B, C) \ + ((__m128d)__builtin_ia32_shufpd128_mask ((__v2df)(__m128d)(A), \ + (__v2df)(__m128d)(B), (int)(C), \ + (__v2df)(__m128d)(W), \ + (__mmask8)(U))) + +#define _mm_maskz_shuffle_pd(U, A, B, C) \ + ((__m128d)__builtin_ia32_shufpd128_mask ((__v2df)(__m128d)(A), \ + (__v2df)(__m128d)(B), (int)(C), \ + (__v2df)(__m128d)_mm_setzero_pd (), \ + (__mmask8)(U))) + +#define _mm256_mask_shuffle_ps(W, U, A, B, C) \ + ((__m256) __builtin_ia32_shufps256_mask ((__v8sf)(__m256)(A), \ + (__v8sf)(__m256)(B), (int)(C), \ + (__v8sf)(__m256)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_shuffle_ps(U, A, B, C) \ + ((__m256) __builtin_ia32_shufps256_mask ((__v8sf)(__m256)(A), \ + (__v8sf)(__m256)(B), (int)(C), \ + (__v8sf)(__m256)_mm256_setzero_ps (),\ + (__mmask8)(U))) + +#define _mm_mask_shuffle_ps(W, U, A, B, C) \ + ((__m128) __builtin_ia32_shufps128_mask ((__v4sf)(__m128)(A), \ + (__v4sf)(__m128)(B), (int)(C), \ + (__v4sf)(__m128)(W), \ + (__mmask8)(U))) + +#define _mm_maskz_shuffle_ps(U, A, B, C) \ + ((__m128) __builtin_ia32_shufps128_mask ((__v4sf)(__m128)(A), \ + (__v4sf)(__m128)(B), (int)(C), \ + (__v4sf)(__m128)_mm_setzero_ps (), \ + (__mmask8)(U))) + +#define _mm256_fixupimm_pd(X, Y, Z, C) \ + ((__m256d)__builtin_ia32_fixupimmpd256_mask ((__v4df)(__m256d)(X), \ + (__v4df)(__m256d)(Y), \ + (__v4di)(__m256i)(Z), (int)(C), \ + (__mmask8)(-1))) + +#define _mm256_mask_fixupimm_pd(X, U, Y, Z, C) \ + ((__m256d)__builtin_ia32_fixupimmpd256_mask ((__v4df)(__m256d)(X), \ + (__v4df)(__m256d)(Y), \ + (__v4di)(__m256i)(Z), (int)(C), \ + (__mmask8)(U))) + +#define _mm256_maskz_fixupimm_pd(U, X, Y, Z, C) \ + ((__m256d)__builtin_ia32_fixupimmpd256_maskz ((__v4df)(__m256d)(X), \ + (__v4df)(__m256d)(Y), \ + (__v4di)(__m256i)(Z), (int)(C),\ + (__mmask8)(U))) + +#define _mm256_fixupimm_ps(X, Y, Z, C) \ + ((__m256)__builtin_ia32_fixupimmps256_mask ((__v8sf)(__m256)(X), \ + (__v8sf)(__m256)(Y), \ + (__v8si)(__m256i)(Z), (int)(C), \ + (__mmask8)(-1))) + + +#define _mm256_mask_fixupimm_ps(X, U, Y, Z, C) \ + ((__m256)__builtin_ia32_fixupimmps256_mask ((__v8sf)(__m256)(X), \ + (__v8sf)(__m256)(Y), \ + (__v8si)(__m256i)(Z), (int)(C), \ + (__mmask8)(U))) + +#define _mm256_maskz_fixupimm_ps(U, X, Y, Z, C) \ + ((__m256)__builtin_ia32_fixupimmps256_maskz ((__v8sf)(__m256)(X), \ + (__v8sf)(__m256)(Y), \ + (__v8si)(__m256i)(Z), (int)(C),\ + (__mmask8)(U))) + +#define _mm_fixupimm_pd(X, Y, Z, C) \ + ((__m128d)__builtin_ia32_fixupimmpd128_mask ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), \ + (__v2di)(__m128i)(Z), (int)(C), \ + (__mmask8)(-1))) + + +#define _mm_mask_fixupimm_pd(X, U, Y, Z, C) \ + ((__m128d)__builtin_ia32_fixupimmpd128_mask ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), \ + (__v2di)(__m128i)(Z), (int)(C), \ + (__mmask8)(U))) + +#define _mm_maskz_fixupimm_pd(U, X, Y, Z, C) \ + ((__m128d)__builtin_ia32_fixupimmpd128_maskz ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), \ + (__v2di)(__m128i)(Z), (int)(C),\ + (__mmask8)(U))) + +#define _mm_fixupimm_ps(X, Y, Z, C) \ + ((__m128)__builtin_ia32_fixupimmps128_mask ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), \ + (__v4si)(__m128i)(Z), (int)(C), \ + (__mmask8)(-1))) + +#define _mm_mask_fixupimm_ps(X, U, Y, Z, C) \ + ((__m128)__builtin_ia32_fixupimmps128_mask ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), \ + (__v4si)(__m128i)(Z), (int)(C),\ + (__mmask8)(U))) + +#define _mm_maskz_fixupimm_ps(U, X, Y, Z, C) \ + ((__m128)__builtin_ia32_fixupimmps128_maskz ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), \ + (__v4si)(__m128i)(Z), (int)(C),\ + (__mmask8)(U))) + +#define _mm256_mask_srli_epi32(W, U, A, B) \ + ((__m256i) __builtin_ia32_psrldi256_mask ((__v8si)(__m256i)(A), \ + (int)(B), (__v8si)(__m256i)(W), (__mmask8)(U))) + +#define _mm256_maskz_srli_epi32(U, A, B) \ + ((__m256i) __builtin_ia32_psrldi256_mask ((__v8si)(__m256i)(A), \ + (int)(B), (__v8si)_mm256_setzero_si256 (), (__mmask8)(U))) + +#define _mm_mask_srli_epi32(W, U, A, B) \ + ((__m128i) __builtin_ia32_psrldi128_mask ((__v4si)(__m128i)(A), \ + (int)(B), (__v4si)(__m128i)(W), (__mmask8)(U))) + +#define _mm_maskz_srli_epi32(U, A, B) \ + ((__m128i) __builtin_ia32_psrldi128_mask ((__v4si)(__m128i)(A), \ + (int)(B), (__v4si)_mm_setzero_si128 (), (__mmask8)(U))) + +#define _mm256_mask_srli_epi64(W, U, A, B) \ + ((__m256i) __builtin_ia32_psrlqi256_mask ((__v4di)(__m256i)(A), \ + (int)(B), (__v4di)(__m256i)(W), (__mmask8)(U))) + +#define _mm256_maskz_srli_epi64(U, A, B) \ + ((__m256i) __builtin_ia32_psrlqi256_mask ((__v4di)(__m256i)(A), \ + (int)(B), (__v4di)_mm256_setzero_si256 (), (__mmask8)(U))) + +#define _mm_mask_srli_epi64(W, U, A, B) \ + ((__m128i) __builtin_ia32_psrlqi128_mask ((__v2di)(__m128i)(A), \ + (int)(B), (__v2di)(__m128i)(W), (__mmask8)(U))) + +#define _mm_maskz_srli_epi64(U, A, B) \ + ((__m128i) __builtin_ia32_psrlqi128_mask ((__v2di)(__m128i)(A), \ + (int)(B), (__v2di)_mm_setzero_si128 (), (__mmask8)(U))) + +#define _mm256_mask_slli_epi32(W, U, X, C) \ + ((__m256i)__builtin_ia32_pslldi256_mask ((__v8si)(__m256i)(X), (int)(C),\ + (__v8si)(__m256i)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_slli_epi32(U, X, C) \ + ((__m256i)__builtin_ia32_pslldi256_mask ((__v8si)(__m256i)(X), (int)(C),\ + (__v8si)(__m256i)_mm256_setzero_si256 (), \ + (__mmask8)(U))) + +#define _mm256_mask_slli_epi64(W, U, X, C) \ + ((__m256i)__builtin_ia32_psllqi256_mask ((__v4di)(__m256i)(X), (int)(C),\ + (__v4di)(__m256i)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_slli_epi64(U, X, C) \ + ((__m256i)__builtin_ia32_psllqi256_mask ((__v4di)(__m256i)(X), (int)(C),\ + (__v4di)(__m256i)_mm256_setzero_si256 (), \ + (__mmask8)(U))) + +#define _mm_mask_slli_epi32(W, U, X, C) \ + ((__m128i)__builtin_ia32_pslldi128_mask ((__v4si)(__m128i)(X), (int)(C),\ + (__v4si)(__m128i)(W),\ + (__mmask8)(U))) + +#define _mm_maskz_slli_epi32(U, X, C) \ + ((__m128i)__builtin_ia32_pslldi128_mask ((__v4si)(__m128i)(X), (int)(C),\ + (__v4si)(__m128i)_mm_setzero_si128 (),\ + (__mmask8)(U))) + +#define _mm_mask_slli_epi64(W, U, X, C) \ + ((__m128i)__builtin_ia32_psllqi128_mask ((__v2di)(__m128i)(X), (int)(C),\ + (__v2di)(__m128i)(W),\ + (__mmask8)(U))) + +#define _mm_maskz_slli_epi64(U, X, C) \ + ((__m128i)__builtin_ia32_psllqi128_mask ((__v2di)(__m128i)(X), (int)(C),\ + (__v2di)(__m128i)_mm_setzero_si128 (),\ + (__mmask8)(U))) + +#define _mm256_ternarylogic_epi64(A, B, C, I) \ + ((__m256i) __builtin_ia32_pternlogq256_mask ((__v4di)(__m256i)(A), \ + (__v4di)(__m256i)(B), (__v4di)(__m256i)(C), (int)(I), (__mmask8)-1)) + +#define _mm256_mask_ternarylogic_epi64(A, U, B, C, I) \ + ((__m256i) __builtin_ia32_pternlogq256_mask ((__v4di)(__m256i)(A), \ + (__v4di)(__m256i)(B), (__v4di)(__m256i)(C), (int)(I), (__mmask8)(U))) + +#define _mm256_maskz_ternarylogic_epi64(U, A, B, C, I) \ + ((__m256i) __builtin_ia32_pternlogq256_maskz ((__v4di)(__m256i)(A), \ + (__v4di)(__m256i)(B), (__v4di)(__m256i)(C), (int)(I), (__mmask8)(U))) + +#define _mm256_ternarylogic_epi32(A, B, C, I) \ + ((__m256i) __builtin_ia32_pternlogd256_mask ((__v8si)(__m256i)(A), \ + (__v8si)(__m256i)(B), (__v8si)(__m256i)(C), (int)(I), (__mmask8)-1)) + +#define _mm256_mask_ternarylogic_epi32(A, U, B, C, I) \ + ((__m256i) __builtin_ia32_pternlogd256_mask ((__v8si)(__m256i)(A), \ + (__v8si)(__m256i)(B), (__v8si)(__m256i)(C), (int)(I), (__mmask8)(U))) + +#define _mm256_maskz_ternarylogic_epi32(U, A, B, C, I) \ + ((__m256i) __builtin_ia32_pternlogd256_maskz ((__v8si)(__m256i)(A), \ + (__v8si)(__m256i)(B), (__v8si)(__m256i)(C), (int)(I), (__mmask8)(U))) + +#define _mm_ternarylogic_epi64(A, B, C, I) \ + ((__m128i) __builtin_ia32_pternlogq128_mask ((__v2di)(__m128i)(A), \ + (__v2di)(__m128i)(B), (__v2di)(__m128i)(C), (int)(I), (__mmask8)-1)) + +#define _mm_mask_ternarylogic_epi64(A, U, B, C, I) \ + ((__m128i) __builtin_ia32_pternlogq128_mask ((__v2di)(__m128i)(A), \ + (__v2di)(__m128i)(B), (__v2di)(__m128i)(C), (int)(I), (__mmask8)(U))) + +#define _mm_maskz_ternarylogic_epi64(U, A, B, C, I) \ + ((__m128i) __builtin_ia32_pternlogq128_maskz ((__v2di)(__m128i)(A), \ + (__v2di)(__m128i)(B), (__v2di)(__m128i)(C), (int)(I), (__mmask8)(U))) + +#define _mm_ternarylogic_epi32(A, B, C, I) \ + ((__m128i) __builtin_ia32_pternlogd128_mask ((__v4si)(__m128i)(A), \ + (__v4si)(__m128i)(B), (__v4si)(__m128i)(C), (int)(I), (__mmask8)-1)) + +#define _mm_mask_ternarylogic_epi32(A, U, B, C, I) \ + ((__m128i) __builtin_ia32_pternlogd128_mask ((__v4si)(__m128i)(A), \ + (__v4si)(__m128i)(B), (__v4si)(__m128i)(C), (int)(I), (__mmask8)(U))) + +#define _mm_maskz_ternarylogic_epi32(U, A, B, C, I) \ + ((__m128i) __builtin_ia32_pternlogd128_maskz ((__v4si)(__m128i)(A), \ + (__v4si)(__m128i)(B), (__v4si)(__m128i)(C), (int)(I), (__mmask8)(U))) + +#define _mm256_roundscale_ps(A, B) \ + ((__m256) __builtin_ia32_rndscaleps_256_mask ((__v8sf)(__m256)(A), \ + (int)(B), (__v8sf)(__m256)_mm256_setzero_ps (), (__mmask8)-1)) + +#define _mm256_mask_roundscale_ps(W, U, A, B) \ + ((__m256) __builtin_ia32_rndscaleps_256_mask ((__v8sf)(__m256)(A), \ + (int)(B), (__v8sf)(__m256)(W), (__mmask8)(U))) + +#define _mm256_maskz_roundscale_ps(U, A, B) \ + ((__m256) __builtin_ia32_rndscaleps_256_mask ((__v8sf)(__m256)(A), \ + (int)(B), (__v8sf)(__m256)_mm256_setzero_ps (), (__mmask8)(U))) + +#define _mm256_roundscale_pd(A, B) \ + ((__m256d) __builtin_ia32_rndscalepd_256_mask ((__v4df)(__m256d)(A), \ + (int)(B), (__v4df)(__m256d)_mm256_setzero_pd (), (__mmask8)-1)) + +#define _mm256_mask_roundscale_pd(W, U, A, B) \ + ((__m256d) __builtin_ia32_rndscalepd_256_mask ((__v4df)(__m256d)(A), \ + (int)(B), (__v4df)(__m256d)(W), (__mmask8)(U))) + +#define _mm256_maskz_roundscale_pd(U, A, B) \ + ((__m256d) __builtin_ia32_rndscalepd_256_mask ((__v4df)(__m256d)(A), \ + (int)(B), (__v4df)(__m256d)_mm256_setzero_pd (), (__mmask8)(U))) + +#define _mm_roundscale_ps(A, B) \ + ((__m128) __builtin_ia32_rndscaleps_128_mask ((__v4sf)(__m128)(A), \ + (int)(B), (__v4sf)(__m128)_mm_setzero_ps (), (__mmask8)-1)) + +#define _mm_mask_roundscale_ps(W, U, A, B) \ + ((__m128) __builtin_ia32_rndscaleps_128_mask ((__v4sf)(__m128)(A), \ + (int)(B), (__v4sf)(__m128)(W), (__mmask8)(U))) + +#define _mm_maskz_roundscale_ps(U, A, B) \ + ((__m128) __builtin_ia32_rndscaleps_128_mask ((__v4sf)(__m128)(A), \ + (int)(B), (__v4sf)(__m128)_mm_setzero_ps (), (__mmask8)(U))) + +#define _mm_roundscale_pd(A, B) \ + ((__m128d) __builtin_ia32_rndscalepd_128_mask ((__v2df)(__m128d)(A), \ + (int)(B), (__v2df)(__m128d)_mm_setzero_pd (), (__mmask8)-1)) + +#define _mm_mask_roundscale_pd(W, U, A, B) \ + ((__m128d) __builtin_ia32_rndscalepd_128_mask ((__v2df)(__m128d)(A), \ + (int)(B), (__v2df)(__m128d)(W), (__mmask8)(U))) + +#define _mm_maskz_roundscale_pd(U, A, B) \ + ((__m128d) __builtin_ia32_rndscalepd_128_mask ((__v2df)(__m128d)(A), \ + (int)(B), (__v2df)(__m128d)_mm_setzero_pd (), (__mmask8)(U))) + +#define _mm256_getmant_ps(X, B, C) \ + ((__m256) __builtin_ia32_getmantps256_mask ((__v8sf)(__m256) (X), \ + (int)(((C)<<2) | (B)), \ + (__v8sf)(__m256)_mm256_setzero_ps (), \ + (__mmask8)-1)) + +#define _mm256_mask_getmant_ps(W, U, X, B, C) \ + ((__m256) __builtin_ia32_getmantps256_mask ((__v8sf)(__m256) (X), \ + (int)(((C)<<2) | (B)), \ + (__v8sf)(__m256)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_getmant_ps(U, X, B, C) \ + ((__m256) __builtin_ia32_getmantps256_mask ((__v8sf)(__m256) (X), \ + (int)(((C)<<2) | (B)), \ + (__v8sf)(__m256)_mm256_setzero_ps (), \ + (__mmask8)(U))) + +#define _mm_getmant_ps(X, B, C) \ + ((__m128) __builtin_ia32_getmantps128_mask ((__v4sf)(__m128) (X), \ + (int)(((C)<<2) | (B)), \ + (__v4sf)(__m128)_mm_setzero_ps (), \ + (__mmask8)-1)) + +#define _mm_mask_getmant_ps(W, U, X, B, C) \ + ((__m128) __builtin_ia32_getmantps128_mask ((__v4sf)(__m128) (X), \ + (int)(((C)<<2) | (B)), \ + (__v4sf)(__m128)(W), \ + (__mmask8)(U))) + +#define _mm_maskz_getmant_ps(U, X, B, C) \ + ((__m128) __builtin_ia32_getmantps128_mask ((__v4sf)(__m128) (X), \ + (int)(((C)<<2) | (B)), \ + (__v4sf)(__m128)_mm_setzero_ps (), \ + (__mmask8)(U))) + +#define _mm256_getmant_pd(X, B, C) \ + ((__m256d) __builtin_ia32_getmantpd256_mask ((__v4df)(__m256d) (X), \ + (int)(((C)<<2) | (B)), \ + (__v4df)(__m256d)_mm256_setzero_pd (),\ + (__mmask8)-1)) + +#define _mm256_mask_getmant_pd(W, U, X, B, C) \ + ((__m256d) __builtin_ia32_getmantpd256_mask ((__v4df)(__m256d) (X), \ + (int)(((C)<<2) | (B)), \ + (__v4df)(__m256d)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_getmant_pd(U, X, B, C) \ + ((__m256d) __builtin_ia32_getmantpd256_mask ((__v4df)(__m256d) (X), \ + (int)(((C)<<2) | (B)), \ + (__v4df)(__m256d)_mm256_setzero_pd (),\ + (__mmask8)(U))) + +#define _mm_getmant_pd(X, B, C) \ + ((__m128d) __builtin_ia32_getmantpd128_mask ((__v2df)(__m128d) (X), \ + (int)(((C)<<2) | (B)), \ + (__v2df)(__m128d)_mm_setzero_pd (), \ + (__mmask8)-1)) + +#define _mm_mask_getmant_pd(W, U, X, B, C) \ + ((__m128d) __builtin_ia32_getmantpd128_mask ((__v2df)(__m128d) (X), \ + (int)(((C)<<2) | (B)), \ + (__v2df)(__m128d)(W), \ + (__mmask8)(U))) + +#define _mm_maskz_getmant_pd(U, X, B, C) \ + ((__m128d) __builtin_ia32_getmantpd128_mask ((__v2df)(__m128d) (X), \ + (int)(((C)<<2) | (B)), \ + (__v2df)(__m128d)_mm_setzero_pd (), \ + (__mmask8)(U))) + +#define _mm256_mmask_i32gather_ps(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m256) __builtin_ia32_gather3siv8sf ((__v8sf)(__m256)V1OLD, \ + (void const *)ADDR, \ + (__v8si)(__m256i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm_mmask_i32gather_ps(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m128) __builtin_ia32_gather3siv4sf ((__v4sf)(__m128)V1OLD, \ + (void const *)ADDR, \ + (__v4si)(__m128i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm256_mmask_i32gather_pd(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m256d) __builtin_ia32_gather3siv4df ((__v4df)(__m256d)V1OLD, \ + (void const *)ADDR, \ + (__v4si)(__m128i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm_mmask_i32gather_pd(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m128d) __builtin_ia32_gather3siv2df ((__v2df)(__m128d)V1OLD, \ + (void const *)ADDR, \ + (__v4si)(__m128i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm256_mmask_i64gather_ps(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m128) __builtin_ia32_gather3div8sf ((__v4sf)(__m128)V1OLD, \ + (void const *)ADDR, \ + (__v4di)(__m256i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm_mmask_i64gather_ps(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m128) __builtin_ia32_gather3div4sf ((__v4sf)(__m128)V1OLD, \ + (void const *)ADDR, \ + (__v2di)(__m128i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm256_mmask_i64gather_pd(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m256d) __builtin_ia32_gather3div4df ((__v4df)(__m256d)V1OLD, \ + (void const *)ADDR, \ + (__v4di)(__m256i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm_mmask_i64gather_pd(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m128d) __builtin_ia32_gather3div2df ((__v2df)(__m128d)V1OLD, \ + (void const *)ADDR, \ + (__v2di)(__m128i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm256_mmask_i32gather_epi32(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m256i) __builtin_ia32_gather3siv8si ((__v8si)(__m256i)V1OLD, \ + (void const *)ADDR, \ + (__v8si)(__m256i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm_mmask_i32gather_epi32(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m128i) __builtin_ia32_gather3siv4si ((__v4si)(__m128i)V1OLD, \ + (void const *)ADDR, \ + (__v4si)(__m128i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm256_mmask_i32gather_epi64(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m256i) __builtin_ia32_gather3siv4di ((__v4di)(__m256i)V1OLD, \ + (void const *)ADDR, \ + (__v4si)(__m128i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm_mmask_i32gather_epi64(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m128i) __builtin_ia32_gather3siv2di ((__v2di)(__m128i)V1OLD, \ + (void const *)ADDR, \ + (__v4si)(__m128i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm256_mmask_i64gather_epi32(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m128i) __builtin_ia32_gather3div8si ((__v4si)(__m128i)V1OLD, \ + (void const *)ADDR, \ + (__v4di)(__m256i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm_mmask_i64gather_epi32(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m128i) __builtin_ia32_gather3div4si ((__v4si)(__m128i)V1OLD, \ + (void const *)ADDR, \ + (__v2di)(__m128i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm256_mmask_i64gather_epi64(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m256i) __builtin_ia32_gather3div4di ((__v4di)(__m256i)V1OLD, \ + (void const *)ADDR, \ + (__v4di)(__m256i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm_mmask_i64gather_epi64(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m128i) __builtin_ia32_gather3div2di ((__v2di)(__m128i)V1OLD, \ + (void const *)ADDR, \ + (__v2di)(__m128i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm256_i32scatter_ps(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv8sf ((void *)ADDR, (__mmask8)0xFF, \ + (__v8si)(__m256i)INDEX, \ + (__v8sf)(__m256)V1, (int)SCALE) + +#define _mm256_mask_i32scatter_ps(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv8sf ((void *)ADDR, (__mmask8)MASK, \ + (__v8si)(__m256i)INDEX, \ + (__v8sf)(__m256)V1, (int)SCALE) + +#define _mm_i32scatter_ps(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv4sf ((void *)ADDR, (__mmask8)0xFF, \ + (__v4si)(__m128i)INDEX, \ + (__v4sf)(__m128)V1, (int)SCALE) + +#define _mm_mask_i32scatter_ps(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv4sf ((void *)ADDR, (__mmask8)MASK, \ + (__v4si)(__m128i)INDEX, \ + (__v4sf)(__m128)V1, (int)SCALE) + +#define _mm256_i32scatter_pd(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv4df ((void *)ADDR, (__mmask8)0xFF, \ + (__v4si)(__m128i)INDEX, \ + (__v4df)(__m256d)V1, (int)SCALE) + +#define _mm256_mask_i32scatter_pd(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv4df ((void *)ADDR, (__mmask8)MASK, \ + (__v4si)(__m128i)INDEX, \ + (__v4df)(__m256d)V1, (int)SCALE) + +#define _mm_i32scatter_pd(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv2df ((void *)ADDR, (__mmask8)0xFF, \ + (__v4si)(__m128i)INDEX, \ + (__v2df)(__m128d)V1, (int)SCALE) + +#define _mm_mask_i32scatter_pd(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv2df ((void *)ADDR, (__mmask8)MASK, \ + (__v4si)(__m128i)INDEX, \ + (__v2df)(__m128d)V1, (int)SCALE) + +#define _mm256_i64scatter_ps(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv8sf ((void *)ADDR, (__mmask8)0xFF, \ + (__v4di)(__m256i)INDEX, \ + (__v4sf)(__m128)V1, (int)SCALE) + +#define _mm256_mask_i64scatter_ps(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv8sf ((void *)ADDR, (__mmask8)MASK, \ + (__v4di)(__m256i)INDEX, \ + (__v4sf)(__m128)V1, (int)SCALE) + +#define _mm_i64scatter_ps(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv4sf ((void *)ADDR, (__mmask8)0xFF, \ + (__v2di)(__m128i)INDEX, \ + (__v4sf)(__m128)V1, (int)SCALE) + +#define _mm_mask_i64scatter_ps(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv4sf ((void *)ADDR, (__mmask8)MASK, \ + (__v2di)(__m128i)INDEX, \ + (__v4sf)(__m128)V1, (int)SCALE) + +#define _mm256_i64scatter_pd(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv4df ((void *)ADDR, (__mmask8)0xFF, \ + (__v4di)(__m256i)INDEX, \ + (__v4df)(__m256d)V1, (int)SCALE) + +#define _mm256_mask_i64scatter_pd(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv4df ((void *)ADDR, (__mmask8)MASK, \ + (__v4di)(__m256i)INDEX, \ + (__v4df)(__m256d)V1, (int)SCALE) + +#define _mm_i64scatter_pd(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv2df ((void *)ADDR, (__mmask8)0xFF, \ + (__v2di)(__m128i)INDEX, \ + (__v2df)(__m128d)V1, (int)SCALE) + +#define _mm_mask_i64scatter_pd(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv2df ((void *)ADDR, (__mmask8)MASK, \ + (__v2di)(__m128i)INDEX, \ + (__v2df)(__m128d)V1, (int)SCALE) + +#define _mm256_i32scatter_epi32(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv8si ((void *)ADDR, (__mmask8)0xFF, \ + (__v8si)(__m256i)INDEX, \ + (__v8si)(__m256i)V1, (int)SCALE) + +#define _mm256_mask_i32scatter_epi32(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv8si ((void *)ADDR, (__mmask8)MASK, \ + (__v8si)(__m256i)INDEX, \ + (__v8si)(__m256i)V1, (int)SCALE) + +#define _mm_i32scatter_epi32(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv4si ((void *)ADDR, (__mmask8)0xFF, \ + (__v4si)(__m128i)INDEX, \ + (__v4si)(__m128i)V1, (int)SCALE) + +#define _mm_mask_i32scatter_epi32(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv4si ((void *)ADDR, (__mmask8)MASK, \ + (__v4si)(__m128i)INDEX, \ + (__v4si)(__m128i)V1, (int)SCALE) + +#define _mm256_i32scatter_epi64(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv4di ((void *)ADDR, (__mmask8)0xFF, \ + (__v4si)(__m128i)INDEX, \ + (__v4di)(__m256i)V1, (int)SCALE) + +#define _mm256_mask_i32scatter_epi64(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv4di ((void *)ADDR, (__mmask8)MASK, \ + (__v4si)(__m128i)INDEX, \ + (__v4di)(__m256i)V1, (int)SCALE) + +#define _mm_i32scatter_epi64(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv2di ((void *)ADDR, (__mmask8)0xFF, \ + (__v4si)(__m128i)INDEX, \ + (__v2di)(__m128i)V1, (int)SCALE) + +#define _mm_mask_i32scatter_epi64(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv2di ((void *)ADDR, (__mmask8)MASK, \ + (__v4si)(__m128i)INDEX, \ + (__v2di)(__m128i)V1, (int)SCALE) + +#define _mm256_i64scatter_epi32(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv8si ((void *)ADDR, (__mmask8)0xFF, \ + (__v4di)(__m256i)INDEX, \ + (__v4si)(__m128i)V1, (int)SCALE) + +#define _mm256_mask_i64scatter_epi32(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv8si ((void *)ADDR, (__mmask8)MASK, \ + (__v4di)(__m256i)INDEX, \ + (__v4si)(__m128i)V1, (int)SCALE) + +#define _mm_i64scatter_epi32(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv4si ((void *)ADDR, (__mmask8)0xFF, \ + (__v2di)(__m128i)INDEX, \ + (__v4si)(__m128i)V1, (int)SCALE) + +#define _mm_mask_i64scatter_epi32(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv4si ((void *)ADDR, (__mmask8)MASK, \ + (__v2di)(__m128i)INDEX, \ + (__v4si)(__m128i)V1, (int)SCALE) + +#define _mm256_i64scatter_epi64(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv4di ((void *)ADDR, (__mmask8)0xFF, \ + (__v4di)(__m256i)INDEX, \ + (__v4di)(__m256i)V1, (int)SCALE) + +#define _mm256_mask_i64scatter_epi64(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv4di ((void *)ADDR, (__mmask8)MASK, \ + (__v4di)(__m256i)INDEX, \ + (__v4di)(__m256i)V1, (int)SCALE) + +#define _mm_i64scatter_epi64(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv2di ((void *)ADDR, (__mmask8)0xFF, \ + (__v2di)(__m128i)INDEX, \ + (__v2di)(__m128i)V1, (int)SCALE) + +#define _mm_mask_i64scatter_epi64(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv2di ((void *)ADDR, (__mmask8)MASK, \ + (__v2di)(__m128i)INDEX, \ + (__v2di)(__m128i)V1, (int)SCALE) + +#define _mm256_mask_shuffle_epi32(W, U, X, C) \ + ((__m256i) __builtin_ia32_pshufd256_mask ((__v8si)(__m256i)(X), (int)(C), \ + (__v8si)(__m256i)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_shuffle_epi32(U, X, C) \ + ((__m256i) __builtin_ia32_pshufd256_mask ((__v8si)(__m256i)(X), (int)(C), \ + (__v8si)(__m256i) \ + _mm256_setzero_si256 (), \ + (__mmask8)(U))) + +#define _mm_mask_shuffle_epi32(W, U, X, C) \ + ((__m128i) __builtin_ia32_pshufd128_mask ((__v4si)(__m128i)(X), (int)(C), \ + (__v4si)(__m128i)(W), \ + (__mmask8)(U))) + +#define _mm_maskz_shuffle_epi32(U, X, C) \ + ((__m128i) __builtin_ia32_pshufd128_mask ((__v4si)(__m128i)(X), (int)(C), \ + (__v4si)(__m128i)_mm_setzero_si128 (), \ + (__mmask8)(U))) + +#define _mm256_rol_epi64(A, B) \ + ((__m256i)__builtin_ia32_prolq256_mask ((__v4di)(__m256i)(A), (int)(B), \ + (__v4di)(__m256i)_mm256_setzero_si256 (),\ + (__mmask8)-1)) + +#define _mm256_mask_rol_epi64(W, U, A, B) \ + ((__m256i)__builtin_ia32_prolq256_mask ((__v4di)(__m256i)(A), (int)(B), \ + (__v4di)(__m256i)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_rol_epi64(U, A, B) \ + ((__m256i)__builtin_ia32_prolq256_mask ((__v4di)(__m256i)(A), (int)(B), \ + (__v4di)(__m256i)_mm256_setzero_si256 (),\ + (__mmask8)(U))) + +#define _mm_rol_epi64(A, B) \ + ((__m128i)__builtin_ia32_prolq128_mask ((__v2di)(__m128i)(A), (int)(B), \ + (__v2di)(__m128i)_mm_setzero_si128 (),\ + (__mmask8)-1)) + +#define _mm_mask_rol_epi64(W, U, A, B) \ + ((__m128i)__builtin_ia32_prolq128_mask ((__v2di)(__m128i)(A), (int)(B), \ + (__v2di)(__m128i)(W), \ + (__mmask8)(U))) + +#define _mm_maskz_rol_epi64(U, A, B) \ + ((__m128i)__builtin_ia32_prolq128_mask ((__v2di)(__m128i)(A), (int)(B), \ + (__v2di)(__m128i)_mm_setzero_si128 (),\ + (__mmask8)(U))) + +#define _mm256_ror_epi64(A, B) \ + ((__m256i)__builtin_ia32_prorq256_mask ((__v4di)(__m256i)(A), (int)(B), \ + (__v4di)(__m256i)_mm256_setzero_si256 (),\ + (__mmask8)-1)) + +#define _mm256_mask_ror_epi64(W, U, A, B) \ + ((__m256i)__builtin_ia32_prorq256_mask ((__v4di)(__m256i)(A), (int)(B), \ + (__v4di)(__m256i)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_ror_epi64(U, A, B) \ + ((__m256i)__builtin_ia32_prorq256_mask ((__v4di)(__m256i)(A), (int)(B), \ + (__v4di)(__m256i)_mm256_setzero_si256 (),\ + (__mmask8)(U))) + +#define _mm_ror_epi64(A, B) \ + ((__m128i)__builtin_ia32_prorq128_mask ((__v2di)(__m128i)(A), (int)(B), \ + (__v2di)(__m128i)_mm_setzero_si128 (),\ + (__mmask8)-1)) + +#define _mm_mask_ror_epi64(W, U, A, B) \ + ((__m128i)__builtin_ia32_prorq128_mask ((__v2di)(__m128i)(A), (int)(B), \ + (__v2di)(__m128i)(W), \ + (__mmask8)(U))) + +#define _mm_maskz_ror_epi64(U, A, B) \ + ((__m128i)__builtin_ia32_prorq128_mask ((__v2di)(__m128i)(A), (int)(B), \ + (__v2di)(__m128i)_mm_setzero_si128 (),\ + (__mmask8)(U))) + +#define _mm256_rol_epi32(A, B) \ + ((__m256i)__builtin_ia32_prold256_mask ((__v8si)(__m256i)(A), (int)(B), \ + (__v8si)(__m256i)_mm256_setzero_si256 (),\ + (__mmask8)-1)) + +#define _mm256_mask_rol_epi32(W, U, A, B) \ + ((__m256i)__builtin_ia32_prold256_mask ((__v8si)(__m256i)(A), (int)(B), \ + (__v8si)(__m256i)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_rol_epi32(U, A, B) \ + ((__m256i)__builtin_ia32_prold256_mask ((__v8si)(__m256i)(A), (int)(B), \ + (__v8si)(__m256i)_mm256_setzero_si256 (),\ + (__mmask8)(U))) + +#define _mm_rol_epi32(A, B) \ + ((__m128i)__builtin_ia32_prold128_mask ((__v4si)(__m128i)(A), (int)(B), \ + (__v4si)(__m128i)_mm_setzero_si128 (),\ + (__mmask8)-1)) + +#define _mm_mask_rol_epi32(W, U, A, B) \ + ((__m128i)__builtin_ia32_prold128_mask ((__v4si)(__m128i)(A), (int)(B), \ + (__v4si)(__m128i)(W), \ + (__mmask8)(U))) + +#define _mm_maskz_rol_epi32(U, A, B) \ + ((__m128i)__builtin_ia32_prold128_mask ((__v4si)(__m128i)(A), (int)(B), \ + (__v4si)(__m128i)_mm_setzero_si128 (),\ + (__mmask8)(U))) + +#define _mm256_ror_epi32(A, B) \ + ((__m256i)__builtin_ia32_prord256_mask ((__v8si)(__m256i)(A), (int)(B), \ + (__v8si)(__m256i)_mm256_setzero_si256 (),\ + (__mmask8)-1)) + +#define _mm256_mask_ror_epi32(W, U, A, B) \ + ((__m256i)__builtin_ia32_prord256_mask ((__v8si)(__m256i)(A), (int)(B), \ + (__v8si)(__m256i)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_ror_epi32(U, A, B) \ + ((__m256i)__builtin_ia32_prord256_mask ((__v8si)(__m256i)(A), (int)(B), \ + (__v8si)(__m256i) \ + _mm256_setzero_si256 (), \ + (__mmask8)(U))) + +#define _mm_ror_epi32(A, B) \ + ((__m128i)__builtin_ia32_prord128_mask ((__v4si)(__m128i)(A), (int)(B), \ + (__v4si)(__m128i)_mm_setzero_si128 (),\ + (__mmask8)-1)) + +#define _mm_mask_ror_epi32(W, U, A, B) \ + ((__m128i)__builtin_ia32_prord128_mask ((__v4si)(__m128i)(A), (int)(B), \ + (__v4si)(__m128i)(W), \ + (__mmask8)(U))) + +#define _mm_maskz_ror_epi32(U, A, B) \ + ((__m128i)__builtin_ia32_prord128_mask ((__v4si)(__m128i)(A), (int)(B), \ + (__v4si)(__m128i)_mm_setzero_si128 (),\ + (__mmask8)(U))) + +#define _mm256_alignr_epi32(X, Y, C) \ + ((__m256i)__builtin_ia32_alignd256_mask ((__v8si)(__m256i)(X), \ + (__v8si)(__m256i)(Y), (int)(C), (__v8si)(__m256i)(X), (__mmask8)-1)) + +#define _mm256_mask_alignr_epi32(W, U, X, Y, C) \ + ((__m256i)__builtin_ia32_alignd256_mask ((__v8si)(__m256i)(X), \ + (__v8si)(__m256i)(Y), (int)(C), (__v8si)(__m256i)(W), (__mmask8)(U))) + +#define _mm256_maskz_alignr_epi32(U, X, Y, C) \ + ((__m256i)__builtin_ia32_alignd256_mask ((__v8si)(__m256i)(X), \ + (__v8si)(__m256i)(Y), (int)(C), (__v8si)(__m256i)_mm256_setzero_si256 (),\ + (__mmask8)(U))) + +#define _mm256_alignr_epi64(X, Y, C) \ + ((__m256i)__builtin_ia32_alignq256_mask ((__v4di)(__m256i)(X), \ + (__v4di)(__m256i)(Y), (int)(C), (__v4di)(__m256i)(X), (__mmask8)-1)) + +#define _mm256_mask_alignr_epi64(W, U, X, Y, C) \ + ((__m256i)__builtin_ia32_alignq256_mask ((__v4di)(__m256i)(X), \ + (__v4di)(__m256i)(Y), (int)(C), (__v4di)(__m256i)(W), (__mmask8)(U))) + +#define _mm256_maskz_alignr_epi64(U, X, Y, C) \ + ((__m256i)__builtin_ia32_alignq256_mask ((__v4di)(__m256i)(X), \ + (__v4di)(__m256i)(Y), (int)(C), (__v4di)(__m256i)_mm256_setzero_si256 (),\ + (__mmask8)(U))) + +#define _mm_alignr_epi32(X, Y, C) \ + ((__m128i)__builtin_ia32_alignd128_mask ((__v4si)(__m128i)(X), \ + (__v4si)(__m128i)(Y), (int)(C), (__v4si)(__m128i)(X), (__mmask8)-1)) + +#define _mm_mask_alignr_epi32(W, U, X, Y, C) \ + ((__m128i)__builtin_ia32_alignd128_mask ((__v4si)(__m128i)(X), \ + (__v4si)(__m128i)(Y), (int)(C), (__v4si)(__m128i)(W), (__mmask8)(U))) + +#define _mm_maskz_alignr_epi32(U, X, Y, C) \ + ((__m128i)__builtin_ia32_alignd128_mask ((__v4si)(__m128i)(X), \ + (__v4si)(__m128i)(Y), (int)(C), (__v4si)(__m128i)_mm_setzero_si128 (),\ + (__mmask8)(U))) + +#define _mm_alignr_epi64(X, Y, C) \ + ((__m128i)__builtin_ia32_alignq128_mask ((__v2di)(__m128i)(X), \ + (__v2di)(__m128i)(Y), (int)(C), (__v2di)(__m128i)(X), (__mmask8)-1)) + +#define _mm_mask_alignr_epi64(W, U, X, Y, C) \ + ((__m128i)__builtin_ia32_alignq128_mask ((__v2di)(__m128i)(X), \ + (__v2di)(__m128i)(Y), (int)(C), (__v2di)(__m128i)(X), (__mmask8)-1)) + +#define _mm_maskz_alignr_epi64(U, X, Y, C) \ + ((__m128i)__builtin_ia32_alignq128_mask ((__v2di)(__m128i)(X), \ + (__v2di)(__m128i)(Y), (int)(C), (__v2di)(__m128i)_mm_setzero_si128 (),\ + (__mmask8)(U))) + +#define _mm_mask_cvtps_ph(W, U, A, I) \ + ((__m128i) __builtin_ia32_vcvtps2ph_mask ((__v4sf)(__m128) A, (int) (I), \ + (__v8hi)(__m128i) (W), (__mmask8) (U))) + +#define _mm_maskz_cvtps_ph(U, A, I) \ + ((__m128i) __builtin_ia32_vcvtps2ph_mask ((__v4sf)(__m128) A, (int) (I), \ + (__v8hi)(__m128i) _mm_setzero_si128 (), (__mmask8) (U))) + +#define _mm256_mask_cvtps_ph(W, U, A, I) \ + ((__m128i) __builtin_ia32_vcvtps2ph256_mask ((__v8sf)(__m256) A, (int) (I), \ + (__v8hi)(__m128i) (W), (__mmask8) (U))) + +#define _mm256_maskz_cvtps_ph(U, A, I) \ + ((__m128i) __builtin_ia32_vcvtps2ph256_mask ((__v8sf)(__m256) A, (int) (I), \ + (__v8hi)(__m128i) _mm_setzero_si128 (), (__mmask8) (U))) + +#define _mm256_mask_srai_epi32(W, U, A, B) \ + ((__m256i) __builtin_ia32_psradi256_mask ((__v8si)(__m256i)(A), \ + (int)(B), (__v8si)(__m256i)(W), (__mmask8)(U))) + +#define _mm256_maskz_srai_epi32(U, A, B) \ + ((__m256i) __builtin_ia32_psradi256_mask ((__v8si)(__m256i)(A), \ + (int)(B), (__v8si)_mm256_setzero_si256 (), (__mmask8)(U))) + +#define _mm_mask_srai_epi32(W, U, A, B) \ + ((__m128i) __builtin_ia32_psradi128_mask ((__v4si)(__m128i)(A), \ + (int)(B), (__v4si)(__m128i)(W), (__mmask8)(U))) + +#define _mm_maskz_srai_epi32(U, A, B) \ + ((__m128i) __builtin_ia32_psradi128_mask ((__v4si)(__m128i)(A), \ + (int)(B), (__v4si)_mm_setzero_si128 (), (__mmask8)(U))) + +#define _mm256_srai_epi64(A, B) \ + ((__m256i) __builtin_ia32_psraqi256_mask ((__v4di)(__m256i)(A), \ + (int)(B), (__v4di)_mm256_setzero_si256 (), (__mmask8)-1)) + +#define _mm256_mask_srai_epi64(W, U, A, B) \ + ((__m256i) __builtin_ia32_psraqi256_mask ((__v4di)(__m256i)(A), \ + (int)(B), (__v4di)(__m256i)(W), (__mmask8)(U))) + +#define _mm256_maskz_srai_epi64(U, A, B) \ + ((__m256i) __builtin_ia32_psraqi256_mask ((__v4di)(__m256i)(A), \ + (int)(B), (__v4di)_mm256_setzero_si256 (), (__mmask8)(U))) + +#define _mm_srai_epi64(A, B) \ + ((__m128i) __builtin_ia32_psraqi128_mask ((__v2di)(__m128i)(A), \ + (int)(B), (__v2di)_mm_setzero_si128 (), (__mmask8)-1)) + +#define _mm_mask_srai_epi64(W, U, A, B) \ + ((__m128i) __builtin_ia32_psraqi128_mask ((__v2di)(__m128i)(A), \ + (int)(B), (__v2di)(__m128i)(W), (__mmask8)(U))) + +#define _mm_maskz_srai_epi64(U, A, B) \ + ((__m128i) __builtin_ia32_psraqi128_mask ((__v2di)(__m128i)(A), \ + (int)(B), (__v2di)_mm_setzero_si128 (), (__mmask8)(U))) + +#define _mm256_mask_permutex_pd(W, U, A, B) \ + ((__m256d) __builtin_ia32_permdf256_mask ((__v4df)(__m256d)(A), \ + (int)(B), (__v4df)(__m256d)(W), (__mmask8)(U))) + +#define _mm256_maskz_permutex_pd(U, A, B) \ + ((__m256d) __builtin_ia32_permdf256_mask ((__v4df)(__m256d)(A), \ + (int)(B), (__v4df)(__m256d)_mm256_setzero_pd (), (__mmask8)(U))) + +#define _mm256_mask_permute_pd(W, U, X, C) \ + ((__m256d) __builtin_ia32_vpermilpd256_mask ((__v4df)(__m256d)(X), (int)(C), \ + (__v4df)(__m256d)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_permute_pd(U, X, C) \ + ((__m256d) __builtin_ia32_vpermilpd256_mask ((__v4df)(__m256d)(X), (int)(C), \ + (__v4df)(__m256d)_mm256_setzero_pd (),\ + (__mmask8)(U))) + +#define _mm256_mask_permute_ps(W, U, X, C) \ + ((__m256) __builtin_ia32_vpermilps256_mask ((__v8sf)(__m256)(X), (int)(C), \ + (__v8sf)(__m256)(W), (__mmask8)(U))) + +#define _mm256_maskz_permute_ps(U, X, C) \ + ((__m256) __builtin_ia32_vpermilps256_mask ((__v8sf)(__m256)(X), (int)(C), \ + (__v8sf)(__m256)_mm256_setzero_ps (), \ + (__mmask8)(U))) + +#define _mm_mask_permute_pd(W, U, X, C) \ + ((__m128d) __builtin_ia32_vpermilpd_mask ((__v2df)(__m128d)(X), (int)(C), \ + (__v2df)(__m128d)(W), (__mmask8)(U))) + +#define _mm_maskz_permute_pd(U, X, C) \ + ((__m128d) __builtin_ia32_vpermilpd_mask ((__v2df)(__m128d)(X), (int)(C), \ + (__v2df)(__m128d)_mm_setzero_pd (), \ + (__mmask8)(U))) + +#define _mm_mask_permute_ps(W, U, X, C) \ + ((__m128) __builtin_ia32_vpermilps_mask ((__v4sf)(__m128)(X), (int)(C), \ + (__v4sf)(__m128)(W), (__mmask8)(U))) + +#define _mm_maskz_permute_ps(U, X, C) \ + ((__m128) __builtin_ia32_vpermilps_mask ((__v4sf)(__m128)(X), (int)(C), \ + (__v4sf)(__m128)_mm_setzero_ps (), \ + (__mmask8)(U))) + +#define _mm256_mask_blend_pd(__U, __A, __W) \ + ((__m256d) __builtin_ia32_blendmpd_256_mask ((__v4df) (__A), \ + (__v4df) (__W), \ + (__mmask8) (__U))) + +#define _mm256_mask_blend_ps(__U, __A, __W) \ + ((__m256) __builtin_ia32_blendmps_256_mask ((__v8sf) (__A), \ + (__v8sf) (__W), \ + (__mmask8) (__U))) + +#define _mm256_mask_blend_epi64(__U, __A, __W) \ + ((__m256i) __builtin_ia32_blendmq_256_mask ((__v4di) (__A), \ + (__v4di) (__W), \ + (__mmask8) (__U))) + +#define _mm256_mask_blend_epi32(__U, __A, __W) \ + ((__m256i) __builtin_ia32_blendmd_256_mask ((__v8si) (__A), \ + (__v8si) (__W), \ + (__mmask8) (__U))) + +#define _mm_mask_blend_pd(__U, __A, __W) \ + ((__m128d) __builtin_ia32_blendmpd_128_mask ((__v2df) (__A), \ + (__v2df) (__W), \ + (__mmask8) (__U))) + +#define _mm_mask_blend_ps(__U, __A, __W) \ + ((__m128) __builtin_ia32_blendmps_128_mask ((__v4sf) (__A), \ + (__v4sf) (__W), \ + (__mmask8) (__U))) + +#define _mm_mask_blend_epi64(__U, __A, __W) \ + ((__m128i) __builtin_ia32_blendmq_128_mask ((__v2di) (__A), \ + (__v2di) (__W), \ + (__mmask8) (__U))) + +#define _mm_mask_blend_epi32(__U, __A, __W) \ + ((__m128i) __builtin_ia32_blendmd_128_mask ((__v4si) (__A), \ + (__v4si) (__W), \ + (__mmask8) (__U))) + +#define _mm256_cmp_epu32_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_ucmpd256_mask ((__v8si)(__m256i)(X), \ + (__v8si)(__m256i)(Y), (int)(P),\ + (__mmask8)-1)) + +#define _mm256_cmp_epi64_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpq256_mask ((__v4di)(__m256i)(X), \ + (__v4di)(__m256i)(Y), (int)(P),\ + (__mmask8)-1)) + +#define _mm256_cmp_epi32_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpd256_mask ((__v8si)(__m256i)(X), \ + (__v8si)(__m256i)(Y), (int)(P),\ + (__mmask8)-1)) + +#define _mm256_cmp_epu64_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_ucmpq256_mask ((__v4di)(__m256i)(X), \ + (__v4di)(__m256i)(Y), (int)(P),\ + (__mmask8)-1)) + +#define _mm256_cmp_pd_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_cmppd256_mask ((__v4df)(__m256d)(X), \ + (__v4df)(__m256d)(Y), (int)(P),\ + (__mmask8)-1)) + +#define _mm256_cmp_ps_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpps256_mask ((__v8sf)(__m256)(X), \ + (__v8sf)(__m256)(Y), (int)(P),\ + (__mmask8)-1)) + +#define _mm256_mask_cmp_epi64_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpq256_mask ((__v4di)(__m256i)(X), \ + (__v4di)(__m256i)(Y), (int)(P),\ + (__mmask8)(M))) + +#define _mm256_mask_cmp_epi32_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpd256_mask ((__v8si)(__m256i)(X), \ + (__v8si)(__m256i)(Y), (int)(P),\ + (__mmask8)(M))) + +#define _mm256_mask_cmp_epu64_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_ucmpq256_mask ((__v4di)(__m256i)(X), \ + (__v4di)(__m256i)(Y), (int)(P),\ + (__mmask8)(M))) + +#define _mm256_mask_cmp_epu32_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_ucmpd256_mask ((__v8si)(__m256i)(X), \ + (__v8si)(__m256i)(Y), (int)(P),\ + (__mmask8)(M))) + +#define _mm256_mask_cmp_pd_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_cmppd256_mask ((__v4df)(__m256d)(X), \ + (__v4df)(__m256d)(Y), (int)(P),\ + (__mmask8)(M))) + +#define _mm256_mask_cmp_ps_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpps256_mask ((__v8sf)(__m256)(X), \ + (__v8sf)(__m256)(Y), (int)(P),\ + (__mmask8)(M))) + +#define _mm_cmp_epi64_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpq128_mask ((__v2di)(__m128i)(X), \ + (__v2di)(__m128i)(Y), (int)(P),\ + (__mmask8)-1)) + +#define _mm_cmp_epi32_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpd128_mask ((__v4si)(__m128i)(X), \ + (__v4si)(__m128i)(Y), (int)(P),\ + (__mmask8)-1)) + +#define _mm_cmp_epu64_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_ucmpq128_mask ((__v2di)(__m128i)(X), \ + (__v2di)(__m128i)(Y), (int)(P),\ + (__mmask8)-1)) + +#define _mm_cmp_epu32_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_ucmpd128_mask ((__v4si)(__m128i)(X), \ + (__v4si)(__m128i)(Y), (int)(P),\ + (__mmask8)-1)) + +#define _mm_cmp_pd_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_cmppd128_mask ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (int)(P),\ + (__mmask8)-1)) + +#define _mm_cmp_ps_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpps128_mask ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (int)(P),\ + (__mmask8)-1)) + +#define _mm_mask_cmp_epi64_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpq128_mask ((__v2di)(__m128i)(X), \ + (__v2di)(__m128i)(Y), (int)(P),\ + (__mmask8)(M))) + +#define _mm_mask_cmp_epi32_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpd128_mask ((__v4si)(__m128i)(X), \ + (__v4si)(__m128i)(Y), (int)(P),\ + (__mmask8)(M))) + +#define _mm_mask_cmp_epu64_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_ucmpq128_mask ((__v2di)(__m128i)(X), \ + (__v2di)(__m128i)(Y), (int)(P),\ + (__mmask8)(M))) + +#define _mm_mask_cmp_epu32_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_ucmpd128_mask ((__v4si)(__m128i)(X), \ + (__v4si)(__m128i)(Y), (int)(P),\ + (__mmask8)(M))) + +#define _mm_mask_cmp_pd_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_cmppd128_mask ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (int)(P),\ + (__mmask8)(M))) + +#define _mm_mask_cmp_ps_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpps128_mask ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (int)(P),\ + (__mmask8)(M))) + +#endif + +#define _mm256_permutexvar_ps(A, B) _mm256_permutevar8x32_ps ((B), (A)) + +#ifdef __DISABLE_AVX512VL__ +#undef __DISABLE_AVX512VL__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX512VL__ */ + +#endif /* _AVX512VLINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512vpopcntdqintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512vpopcntdqintrin.h new file mode 100644 index 0000000000..969a23785c --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avx512vpopcntdqintrin.h @@ -0,0 +1,94 @@ +/* Copyright (C) 2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#if !defined _IMMINTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _AVX512VPOPCNTDQINTRIN_H_INCLUDED +#define _AVX512VPOPCNTDQINTRIN_H_INCLUDED + +#ifndef __AVX512VPOPCNTDQ__ +#pragma GCC push_options +//#pragma GCC target("avx512vpopcntdq") +#define __DISABLE_AVX512VPOPCNTDQ__ +#endif /* __AVX512VPOPCNTDQ__ */ + +extern __inline __m512i +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_popcnt_epi32 (__m512i __A) +{ + return (__m512i) __builtin_ia32_vpopcountd_v16si ((__v16si) __A); +} + +extern __inline __m512i +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_popcnt_epi32 (__m512i __A, __mmask16 __U, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpopcountd_v16si_mask ((__v16si) __A, + (__v16si) __B, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_popcnt_epi32 (__mmask16 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_vpopcountd_v16si_mask ((__v16si) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_popcnt_epi64 (__m512i __A) +{ + return (__m512i) __builtin_ia32_vpopcountq_v8di ((__v8di) __A); +} + +extern __inline __m512i +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_popcnt_epi64 (__m512i __A, __mmask8 __U, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpopcountq_v8di_mask ((__v8di) __A, + (__v8di) __B, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_popcnt_epi64 (__mmask8 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_vpopcountq_v8di_mask ((__v8di) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +#ifdef __DISABLE_AVX512VPOPCNTDQ__ +#undef __DISABLE_AVX512VPOPCNTDQ__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX512VPOPCNTDQ__ */ + +#endif /* _AVX512VPOPCNTDQINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avxintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avxintrin.h new file mode 100644 index 0000000000..20c5a078a9 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/avxintrin.h @@ -0,0 +1,1492 @@ +/* Copyright (C) 2008-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +/* Implemented from the specification included in the Intel C++ Compiler + User Guide and Reference, version 11.0. */ + +#ifndef _IMMINTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _AVXINTRIN_H_INCLUDED +#define _AVXINTRIN_H_INCLUDED + +#ifndef __AVX__ +#pragma GCC push_options +#pragma GCC target("avx") +#define __DISABLE_AVX__ +#endif /* __AVX__ */ + +/* Internal data types for implementing the intrinsics. */ +typedef double __v4df __attribute__ ((__vector_size__ (32))); +typedef float __v8sf __attribute__ ((__vector_size__ (32))); +typedef long long __v4di __attribute__ ((__vector_size__ (32))); +typedef unsigned long long __v4du __attribute__ ((__vector_size__ (32))); +typedef int __v8si __attribute__ ((__vector_size__ (32))); +typedef unsigned int __v8su __attribute__ ((__vector_size__ (32))); +typedef short __v16hi __attribute__ ((__vector_size__ (32))); +typedef unsigned short __v16hu __attribute__ ((__vector_size__ (32))); +typedef char __v32qi __attribute__ ((__vector_size__ (32))); +typedef unsigned char __v32qu __attribute__ ((__vector_size__ (32))); + +/* The Intel API is flexible enough that we must allow aliasing with other + vector types, and their scalar components. */ +typedef float __m256 __attribute__ ((__vector_size__ (32), + __may_alias__)); +typedef long long __m256i __attribute__ ((__vector_size__ (32), + __may_alias__)); +typedef double __m256d __attribute__ ((__vector_size__ (32), + __may_alias__)); + +/* Unaligned version of the same types. */ +typedef float __m256_u __attribute__ ((__vector_size__ (32), + __may_alias__, + __aligned__ (1))); +typedef long long __m256i_u __attribute__ ((__vector_size__ (32), + __may_alias__, + __aligned__ (1))); +typedef double __m256d_u __attribute__ ((__vector_size__ (32), + __may_alias__, + __aligned__ (1))); + +/* Compare predicates for scalar and packed compare intrinsics. */ + +/* Equal (ordered, non-signaling) */ +#define _CMP_EQ_OQ 0x00 +/* Less-than (ordered, signaling) */ +#define _CMP_LT_OS 0x01 +/* Less-than-or-equal (ordered, signaling) */ +#define _CMP_LE_OS 0x02 +/* Unordered (non-signaling) */ +#define _CMP_UNORD_Q 0x03 +/* Not-equal (unordered, non-signaling) */ +#define _CMP_NEQ_UQ 0x04 +/* Not-less-than (unordered, signaling) */ +#define _CMP_NLT_US 0x05 +/* Not-less-than-or-equal (unordered, signaling) */ +#define _CMP_NLE_US 0x06 +/* Ordered (nonsignaling) */ +#define _CMP_ORD_Q 0x07 +/* Equal (unordered, non-signaling) */ +#define _CMP_EQ_UQ 0x08 +/* Not-greater-than-or-equal (unordered, signaling) */ +#define _CMP_NGE_US 0x09 +/* Not-greater-than (unordered, signaling) */ +#define _CMP_NGT_US 0x0a +/* False (ordered, non-signaling) */ +#define _CMP_FALSE_OQ 0x0b +/* Not-equal (ordered, non-signaling) */ +#define _CMP_NEQ_OQ 0x0c +/* Greater-than-or-equal (ordered, signaling) */ +#define _CMP_GE_OS 0x0d +/* Greater-than (ordered, signaling) */ +#define _CMP_GT_OS 0x0e +/* True (unordered, non-signaling) */ +#define _CMP_TRUE_UQ 0x0f +/* Equal (ordered, signaling) */ +#define _CMP_EQ_OS 0x10 +/* Less-than (ordered, non-signaling) */ +#define _CMP_LT_OQ 0x11 +/* Less-than-or-equal (ordered, non-signaling) */ +#define _CMP_LE_OQ 0x12 +/* Unordered (signaling) */ +#define _CMP_UNORD_S 0x13 +/* Not-equal (unordered, signaling) */ +#define _CMP_NEQ_US 0x14 +/* Not-less-than (unordered, non-signaling) */ +#define _CMP_NLT_UQ 0x15 +/* Not-less-than-or-equal (unordered, non-signaling) */ +#define _CMP_NLE_UQ 0x16 +/* Ordered (signaling) */ +#define _CMP_ORD_S 0x17 +/* Equal (unordered, signaling) */ +#define _CMP_EQ_US 0x18 +/* Not-greater-than-or-equal (unordered, non-signaling) */ +#define _CMP_NGE_UQ 0x19 +/* Not-greater-than (unordered, non-signaling) */ +#define _CMP_NGT_UQ 0x1a +/* False (ordered, signaling) */ +#define _CMP_FALSE_OS 0x1b +/* Not-equal (ordered, signaling) */ +#define _CMP_NEQ_OS 0x1c +/* Greater-than-or-equal (ordered, non-signaling) */ +#define _CMP_GE_OQ 0x1d +/* Greater-than (ordered, non-signaling) */ +#define _CMP_GT_OQ 0x1e +/* True (unordered, signaling) */ +#define _CMP_TRUE_US 0x1f + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_add_pd (__m256d __A, __m256d __B) +{ + return (__m256d) ((__v4df)__A + (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_add_ps (__m256 __A, __m256 __B) +{ + return (__m256) ((__v8sf)__A + (__v8sf)__B); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_addsub_pd (__m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_addsubpd256 ((__v4df)__A, (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_addsub_ps (__m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_addsubps256 ((__v8sf)__A, (__v8sf)__B); +} + + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_and_pd (__m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_andpd256 ((__v4df)__A, (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_and_ps (__m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_andps256 ((__v8sf)__A, (__v8sf)__B); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_andnot_pd (__m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_andnpd256 ((__v4df)__A, (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_andnot_ps (__m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_andnps256 ((__v8sf)__A, (__v8sf)__B); +} + +/* Double/single precision floating point blend instructions - select + data from 2 sources using constant/variable mask. */ + +#ifdef __OPTIMIZE__ +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_blend_pd (__m256d __X, __m256d __Y, const int __M) +{ + return (__m256d) __builtin_ia32_blendpd256 ((__v4df)__X, + (__v4df)__Y, + __M); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_blend_ps (__m256 __X, __m256 __Y, const int __M) +{ + return (__m256) __builtin_ia32_blendps256 ((__v8sf)__X, + (__v8sf)__Y, + __M); +} +#else +#define _mm256_blend_pd(X, Y, M) \ + ((__m256d) __builtin_ia32_blendpd256 ((__v4df)(__m256d)(X), \ + (__v4df)(__m256d)(Y), (int)(M))) + +#define _mm256_blend_ps(X, Y, M) \ + ((__m256) __builtin_ia32_blendps256 ((__v8sf)(__m256)(X), \ + (__v8sf)(__m256)(Y), (int)(M))) +#endif + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_blendv_pd (__m256d __X, __m256d __Y, __m256d __M) +{ + return (__m256d) __builtin_ia32_blendvpd256 ((__v4df)__X, + (__v4df)__Y, + (__v4df)__M); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_blendv_ps (__m256 __X, __m256 __Y, __m256 __M) +{ + return (__m256) __builtin_ia32_blendvps256 ((__v8sf)__X, + (__v8sf)__Y, + (__v8sf)__M); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_div_pd (__m256d __A, __m256d __B) +{ + return (__m256d) ((__v4df)__A / (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_div_ps (__m256 __A, __m256 __B) +{ + return (__m256) ((__v8sf)__A / (__v8sf)__B); +} + +/* Dot product instructions with mask-defined summing and zeroing parts + of result. */ + +#ifdef __OPTIMIZE__ +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_dp_ps (__m256 __X, __m256 __Y, const int __M) +{ + return (__m256) __builtin_ia32_dpps256 ((__v8sf)__X, + (__v8sf)__Y, + __M); +} +#else +#define _mm256_dp_ps(X, Y, M) \ + ((__m256) __builtin_ia32_dpps256 ((__v8sf)(__m256)(X), \ + (__v8sf)(__m256)(Y), (int)(M))) +#endif + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_hadd_pd (__m256d __X, __m256d __Y) +{ + return (__m256d) __builtin_ia32_haddpd256 ((__v4df)__X, (__v4df)__Y); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_hadd_ps (__m256 __X, __m256 __Y) +{ + return (__m256) __builtin_ia32_haddps256 ((__v8sf)__X, (__v8sf)__Y); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_hsub_pd (__m256d __X, __m256d __Y) +{ + return (__m256d) __builtin_ia32_hsubpd256 ((__v4df)__X, (__v4df)__Y); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_hsub_ps (__m256 __X, __m256 __Y) +{ + return (__m256) __builtin_ia32_hsubps256 ((__v8sf)__X, (__v8sf)__Y); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_max_pd (__m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_maxpd256 ((__v4df)__A, (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_max_ps (__m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_maxps256 ((__v8sf)__A, (__v8sf)__B); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_min_pd (__m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_minpd256 ((__v4df)__A, (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_min_ps (__m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_minps256 ((__v8sf)__A, (__v8sf)__B); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mul_pd (__m256d __A, __m256d __B) +{ + return (__m256d) ((__v4df)__A * (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mul_ps (__m256 __A, __m256 __B) +{ + return (__m256) ((__v8sf)__A * (__v8sf)__B); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_or_pd (__m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_orpd256 ((__v4df)__A, (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_or_ps (__m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_orps256 ((__v8sf)__A, (__v8sf)__B); +} + +#ifdef __OPTIMIZE__ +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_shuffle_pd (__m256d __A, __m256d __B, const int __mask) +{ + return (__m256d) __builtin_ia32_shufpd256 ((__v4df)__A, (__v4df)__B, + __mask); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_shuffle_ps (__m256 __A, __m256 __B, const int __mask) +{ + return (__m256) __builtin_ia32_shufps256 ((__v8sf)__A, (__v8sf)__B, + __mask); +} +#else +#define _mm256_shuffle_pd(A, B, N) \ + ((__m256d)__builtin_ia32_shufpd256 ((__v4df)(__m256d)(A), \ + (__v4df)(__m256d)(B), (int)(N))) + +#define _mm256_shuffle_ps(A, B, N) \ + ((__m256) __builtin_ia32_shufps256 ((__v8sf)(__m256)(A), \ + (__v8sf)(__m256)(B), (int)(N))) +#endif + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sub_pd (__m256d __A, __m256d __B) +{ + return (__m256d) ((__v4df)__A - (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sub_ps (__m256 __A, __m256 __B) +{ + return (__m256) ((__v8sf)__A - (__v8sf)__B); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_xor_pd (__m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_xorpd256 ((__v4df)__A, (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_xor_ps (__m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_xorps256 ((__v8sf)__A, (__v8sf)__B); +} + +#ifdef __OPTIMIZE__ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_pd (__m128d __X, __m128d __Y, const int __P) +{ + return (__m128d) __builtin_ia32_cmppd ((__v2df)__X, (__v2df)__Y, __P); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_ps (__m128 __X, __m128 __Y, const int __P) +{ + return (__m128) __builtin_ia32_cmpps ((__v4sf)__X, (__v4sf)__Y, __P); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmp_pd (__m256d __X, __m256d __Y, const int __P) +{ + return (__m256d) __builtin_ia32_cmppd256 ((__v4df)__X, (__v4df)__Y, + __P); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmp_ps (__m256 __X, __m256 __Y, const int __P) +{ + return (__m256) __builtin_ia32_cmpps256 ((__v8sf)__X, (__v8sf)__Y, + __P); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_sd (__m128d __X, __m128d __Y, const int __P) +{ + return (__m128d) __builtin_ia32_cmpsd ((__v2df)__X, (__v2df)__Y, __P); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_ss (__m128 __X, __m128 __Y, const int __P) +{ + return (__m128) __builtin_ia32_cmpss ((__v4sf)__X, (__v4sf)__Y, __P); +} +#else +#define _mm_cmp_pd(X, Y, P) \ + ((__m128d) __builtin_ia32_cmppd ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (int)(P))) + +#define _mm_cmp_ps(X, Y, P) \ + ((__m128) __builtin_ia32_cmpps ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (int)(P))) + +#define _mm256_cmp_pd(X, Y, P) \ + ((__m256d) __builtin_ia32_cmppd256 ((__v4df)(__m256d)(X), \ + (__v4df)(__m256d)(Y), (int)(P))) + +#define _mm256_cmp_ps(X, Y, P) \ + ((__m256) __builtin_ia32_cmpps256 ((__v8sf)(__m256)(X), \ + (__v8sf)(__m256)(Y), (int)(P))) + +#define _mm_cmp_sd(X, Y, P) \ + ((__m128d) __builtin_ia32_cmpsd ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (int)(P))) + +#define _mm_cmp_ss(X, Y, P) \ + ((__m128) __builtin_ia32_cmpss ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (int)(P))) +#endif + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi32_pd (__m128i __A) +{ + return (__m256d)__builtin_ia32_cvtdq2pd256 ((__v4si) __A); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi32_ps (__m256i __A) +{ + return (__m256)__builtin_ia32_cvtdq2ps256 ((__v8si) __A); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtpd_ps (__m256d __A) +{ + return (__m128)__builtin_ia32_cvtpd2ps256 ((__v4df) __A); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtps_epi32 (__m256 __A) +{ + return (__m256i)__builtin_ia32_cvtps2dq256 ((__v8sf) __A); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtps_pd (__m128 __A) +{ + return (__m256d)__builtin_ia32_cvtps2pd256 ((__v4sf) __A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvttpd_epi32 (__m256d __A) +{ + return (__m128i)__builtin_ia32_cvttpd2dq256 ((__v4df) __A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtpd_epi32 (__m256d __A) +{ + return (__m128i)__builtin_ia32_cvtpd2dq256 ((__v4df) __A); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvttps_epi32 (__m256 __A) +{ + return (__m256i)__builtin_ia32_cvttps2dq256 ((__v8sf) __A); +} + +extern __inline double +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtsd_f64 (__m256d __A) +{ + return __A[0]; +} + +extern __inline float +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtss_f32 (__m256 __A) +{ + return __A[0]; +} + +#ifdef __OPTIMIZE__ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_extractf128_pd (__m256d __X, const int __N) +{ + return (__m128d) __builtin_ia32_vextractf128_pd256 ((__v4df)__X, __N); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_extractf128_ps (__m256 __X, const int __N) +{ + return (__m128) __builtin_ia32_vextractf128_ps256 ((__v8sf)__X, __N); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_extractf128_si256 (__m256i __X, const int __N) +{ + return (__m128i) __builtin_ia32_vextractf128_si256 ((__v8si)__X, __N); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_extract_epi32 (__m256i __X, int const __N) +{ + __m128i __Y = _mm256_extractf128_si256 (__X, __N >> 2); + return _mm_extract_epi32 (__Y, __N % 4); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_extract_epi16 (__m256i __X, int const __N) +{ + __m128i __Y = _mm256_extractf128_si256 (__X, __N >> 3); + return _mm_extract_epi16 (__Y, __N % 8); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_extract_epi8 (__m256i __X, int const __N) +{ + __m128i __Y = _mm256_extractf128_si256 (__X, __N >> 4); + return _mm_extract_epi8 (__Y, __N % 16); +} + +#ifdef __x86_64__ +extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_extract_epi64 (__m256i __X, const int __N) +{ + __m128i __Y = _mm256_extractf128_si256 (__X, __N >> 1); + return _mm_extract_epi64 (__Y, __N % 2); +} +#endif +#else +#define _mm256_extractf128_pd(X, N) \ + ((__m128d) __builtin_ia32_vextractf128_pd256 ((__v4df)(__m256d)(X), \ + (int)(N))) + +#define _mm256_extractf128_ps(X, N) \ + ((__m128) __builtin_ia32_vextractf128_ps256 ((__v8sf)(__m256)(X), \ + (int)(N))) + +#define _mm256_extractf128_si256(X, N) \ + ((__m128i) __builtin_ia32_vextractf128_si256 ((__v8si)(__m256i)(X), \ + (int)(N))) + +#define _mm256_extract_epi32(X, N) \ + (__extension__ \ + ({ \ + __m128i __Y = _mm256_extractf128_si256 ((X), (N) >> 2); \ + _mm_extract_epi32 (__Y, (N) % 4); \ + })) + +#define _mm256_extract_epi16(X, N) \ + (__extension__ \ + ({ \ + __m128i __Y = _mm256_extractf128_si256 ((X), (N) >> 3); \ + _mm_extract_epi16 (__Y, (N) % 8); \ + })) + +#define _mm256_extract_epi8(X, N) \ + (__extension__ \ + ({ \ + __m128i __Y = _mm256_extractf128_si256 ((X), (N) >> 4); \ + _mm_extract_epi8 (__Y, (N) % 16); \ + })) + +#ifdef __x86_64__ +#define _mm256_extract_epi64(X, N) \ + (__extension__ \ + ({ \ + __m128i __Y = _mm256_extractf128_si256 ((X), (N) >> 1); \ + _mm_extract_epi64 (__Y, (N) % 2); \ + })) +#endif +#endif + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_zeroall (void) +{ + __builtin_ia32_vzeroall (); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_zeroupper (void) +{ + __builtin_ia32_vzeroupper (); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permutevar_pd (__m128d __A, __m128i __C) +{ + return (__m128d) __builtin_ia32_vpermilvarpd ((__v2df)__A, + (__v2di)__C); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permutevar_pd (__m256d __A, __m256i __C) +{ + return (__m256d) __builtin_ia32_vpermilvarpd256 ((__v4df)__A, + (__v4di)__C); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permutevar_ps (__m128 __A, __m128i __C) +{ + return (__m128) __builtin_ia32_vpermilvarps ((__v4sf)__A, + (__v4si)__C); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permutevar_ps (__m256 __A, __m256i __C) +{ + return (__m256) __builtin_ia32_vpermilvarps256 ((__v8sf)__A, + (__v8si)__C); +} + +#ifdef __OPTIMIZE__ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permute_pd (__m128d __X, const int __C) +{ + return (__m128d) __builtin_ia32_vpermilpd ((__v2df)__X, __C); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permute_pd (__m256d __X, const int __C) +{ + return (__m256d) __builtin_ia32_vpermilpd256 ((__v4df)__X, __C); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permute_ps (__m128 __X, const int __C) +{ + return (__m128) __builtin_ia32_vpermilps ((__v4sf)__X, __C); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permute_ps (__m256 __X, const int __C) +{ + return (__m256) __builtin_ia32_vpermilps256 ((__v8sf)__X, __C); +} +#else +#define _mm_permute_pd(X, C) \ + ((__m128d) __builtin_ia32_vpermilpd ((__v2df)(__m128d)(X), (int)(C))) + +#define _mm256_permute_pd(X, C) \ + ((__m256d) __builtin_ia32_vpermilpd256 ((__v4df)(__m256d)(X), (int)(C))) + +#define _mm_permute_ps(X, C) \ + ((__m128) __builtin_ia32_vpermilps ((__v4sf)(__m128)(X), (int)(C))) + +#define _mm256_permute_ps(X, C) \ + ((__m256) __builtin_ia32_vpermilps256 ((__v8sf)(__m256)(X), (int)(C))) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permute2f128_pd (__m256d __X, __m256d __Y, const int __C) +{ + return (__m256d) __builtin_ia32_vperm2f128_pd256 ((__v4df)__X, + (__v4df)__Y, + __C); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permute2f128_ps (__m256 __X, __m256 __Y, const int __C) +{ + return (__m256) __builtin_ia32_vperm2f128_ps256 ((__v8sf)__X, + (__v8sf)__Y, + __C); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permute2f128_si256 (__m256i __X, __m256i __Y, const int __C) +{ + return (__m256i) __builtin_ia32_vperm2f128_si256 ((__v8si)__X, + (__v8si)__Y, + __C); +} +#else +#define _mm256_permute2f128_pd(X, Y, C) \ + ((__m256d) __builtin_ia32_vperm2f128_pd256 ((__v4df)(__m256d)(X), \ + (__v4df)(__m256d)(Y), \ + (int)(C))) + +#define _mm256_permute2f128_ps(X, Y, C) \ + ((__m256) __builtin_ia32_vperm2f128_ps256 ((__v8sf)(__m256)(X), \ + (__v8sf)(__m256)(Y), \ + (int)(C))) + +#define _mm256_permute2f128_si256(X, Y, C) \ + ((__m256i) __builtin_ia32_vperm2f128_si256 ((__v8si)(__m256i)(X), \ + (__v8si)(__m256i)(Y), \ + (int)(C))) +#endif + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_broadcast_ss (float const *__X) +{ + return (__m128) __builtin_ia32_vbroadcastss (__X); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcast_sd (double const *__X) +{ + return (__m256d) __builtin_ia32_vbroadcastsd256 (__X); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcast_ss (float const *__X) +{ + return (__m256) __builtin_ia32_vbroadcastss256 (__X); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcast_pd (__m128d const *__X) +{ + return (__m256d) __builtin_ia32_vbroadcastf128_pd256 (__X); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcast_ps (__m128 const *__X) +{ + return (__m256) __builtin_ia32_vbroadcastf128_ps256 (__X); +} + +#ifdef __OPTIMIZE__ +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_insertf128_pd (__m256d __X, __m128d __Y, const int __O) +{ + return (__m256d) __builtin_ia32_vinsertf128_pd256 ((__v4df)__X, + (__v2df)__Y, + __O); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_insertf128_ps (__m256 __X, __m128 __Y, const int __O) +{ + return (__m256) __builtin_ia32_vinsertf128_ps256 ((__v8sf)__X, + (__v4sf)__Y, + __O); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_insertf128_si256 (__m256i __X, __m128i __Y, const int __O) +{ + return (__m256i) __builtin_ia32_vinsertf128_si256 ((__v8si)__X, + (__v4si)__Y, + __O); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_insert_epi32 (__m256i __X, int __D, int const __N) +{ + __m128i __Y = _mm256_extractf128_si256 (__X, __N >> 2); + __Y = _mm_insert_epi32 (__Y, __D, __N % 4); + return _mm256_insertf128_si256 (__X, __Y, __N >> 2); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_insert_epi16 (__m256i __X, int __D, int const __N) +{ + __m128i __Y = _mm256_extractf128_si256 (__X, __N >> 3); + __Y = _mm_insert_epi16 (__Y, __D, __N % 8); + return _mm256_insertf128_si256 (__X, __Y, __N >> 3); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_insert_epi8 (__m256i __X, int __D, int const __N) +{ + __m128i __Y = _mm256_extractf128_si256 (__X, __N >> 4); + __Y = _mm_insert_epi8 (__Y, __D, __N % 16); + return _mm256_insertf128_si256 (__X, __Y, __N >> 4); +} + +#ifdef __x86_64__ +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_insert_epi64 (__m256i __X, long long __D, int const __N) +{ + __m128i __Y = _mm256_extractf128_si256 (__X, __N >> 1); + __Y = _mm_insert_epi64 (__Y, __D, __N % 2); + return _mm256_insertf128_si256 (__X, __Y, __N >> 1); +} +#endif +#else +#define _mm256_insertf128_pd(X, Y, O) \ + ((__m256d) __builtin_ia32_vinsertf128_pd256 ((__v4df)(__m256d)(X), \ + (__v2df)(__m128d)(Y), \ + (int)(O))) + +#define _mm256_insertf128_ps(X, Y, O) \ + ((__m256) __builtin_ia32_vinsertf128_ps256 ((__v8sf)(__m256)(X), \ + (__v4sf)(__m128)(Y), \ + (int)(O))) + +#define _mm256_insertf128_si256(X, Y, O) \ + ((__m256i) __builtin_ia32_vinsertf128_si256 ((__v8si)(__m256i)(X), \ + (__v4si)(__m128i)(Y), \ + (int)(O))) + +#define _mm256_insert_epi32(X, D, N) \ + (__extension__ \ + ({ \ + __m128i __Y = _mm256_extractf128_si256 ((X), (N) >> 2); \ + __Y = _mm_insert_epi32 (__Y, (D), (N) % 4); \ + _mm256_insertf128_si256 ((X), __Y, (N) >> 2); \ + })) + +#define _mm256_insert_epi16(X, D, N) \ + (__extension__ \ + ({ \ + __m128i __Y = _mm256_extractf128_si256 ((X), (N) >> 3); \ + __Y = _mm_insert_epi16 (__Y, (D), (N) % 8); \ + _mm256_insertf128_si256 ((X), __Y, (N) >> 3); \ + })) + +#define _mm256_insert_epi8(X, D, N) \ + (__extension__ \ + ({ \ + __m128i __Y = _mm256_extractf128_si256 ((X), (N) >> 4); \ + __Y = _mm_insert_epi8 (__Y, (D), (N) % 16); \ + _mm256_insertf128_si256 ((X), __Y, (N) >> 4); \ + })) + +#ifdef __x86_64__ +#define _mm256_insert_epi64(X, D, N) \ + (__extension__ \ + ({ \ + __m128i __Y = _mm256_extractf128_si256 ((X), (N) >> 1); \ + __Y = _mm_insert_epi64 (__Y, (D), (N) % 2); \ + _mm256_insertf128_si256 ((X), __Y, (N) >> 1); \ + })) +#endif +#endif + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_load_pd (double const *__P) +{ + return *(__m256d *)__P; +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_store_pd (double *__P, __m256d __A) +{ + *(__m256d *)__P = __A; +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_load_ps (float const *__P) +{ + return *(__m256 *)__P; +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_store_ps (float *__P, __m256 __A) +{ + *(__m256 *)__P = __A; +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_loadu_pd (double const *__P) +{ + return *(__m256d_u *)__P; +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_storeu_pd (double *__P, __m256d __A) +{ + *(__m256d_u *)__P = __A; +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_loadu_ps (float const *__P) +{ + return *(__m256_u *)__P; +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_storeu_ps (float *__P, __m256 __A) +{ + *(__m256_u *)__P = __A; +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_load_si256 (__m256i const *__P) +{ + return *__P; +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_store_si256 (__m256i *__P, __m256i __A) +{ + *__P = __A; +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_loadu_si256 (__m256i_u const *__P) +{ + return *__P; +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_storeu_si256 (__m256i_u *__P, __m256i __A) +{ + *__P = __A; +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskload_pd (double const *__P, __m128i __M) +{ + return (__m128d) __builtin_ia32_maskloadpd ((const __v2df *)__P, + (__v2di)__M); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskstore_pd (double *__P, __m128i __M, __m128d __A) +{ + __builtin_ia32_maskstorepd ((__v2df *)__P, (__v2di)__M, (__v2df)__A); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskload_pd (double const *__P, __m256i __M) +{ + return (__m256d) __builtin_ia32_maskloadpd256 ((const __v4df *)__P, + (__v4di)__M); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskstore_pd (double *__P, __m256i __M, __m256d __A) +{ + __builtin_ia32_maskstorepd256 ((__v4df *)__P, (__v4di)__M, (__v4df)__A); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskload_ps (float const *__P, __m128i __M) +{ + return (__m128) __builtin_ia32_maskloadps ((const __v4sf *)__P, + (__v4si)__M); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskstore_ps (float *__P, __m128i __M, __m128 __A) +{ + __builtin_ia32_maskstoreps ((__v4sf *)__P, (__v4si)__M, (__v4sf)__A); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskload_ps (float const *__P, __m256i __M) +{ + return (__m256) __builtin_ia32_maskloadps256 ((const __v8sf *)__P, + (__v8si)__M); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskstore_ps (float *__P, __m256i __M, __m256 __A) +{ + __builtin_ia32_maskstoreps256 ((__v8sf *)__P, (__v8si)__M, (__v8sf)__A); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_movehdup_ps (__m256 __X) +{ + return (__m256) __builtin_ia32_movshdup256 ((__v8sf)__X); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_moveldup_ps (__m256 __X) +{ + return (__m256) __builtin_ia32_movsldup256 ((__v8sf)__X); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_movedup_pd (__m256d __X) +{ + return (__m256d) __builtin_ia32_movddup256 ((__v4df)__X); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_lddqu_si256 (__m256i const *__P) +{ + return (__m256i) __builtin_ia32_lddqu256 ((char const *)__P); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_stream_si256 (__m256i *__A, __m256i __B) +{ + __builtin_ia32_movntdq256 ((__v4di *)__A, (__v4di)__B); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_stream_pd (double *__A, __m256d __B) +{ + __builtin_ia32_movntpd256 (__A, (__v4df)__B); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_stream_ps (float *__P, __m256 __A) +{ + __builtin_ia32_movntps256 (__P, (__v8sf)__A); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_rcp_ps (__m256 __A) +{ + return (__m256) __builtin_ia32_rcpps256 ((__v8sf)__A); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_rsqrt_ps (__m256 __A) +{ + return (__m256) __builtin_ia32_rsqrtps256 ((__v8sf)__A); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sqrt_pd (__m256d __A) +{ + return (__m256d) __builtin_ia32_sqrtpd256 ((__v4df)__A); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sqrt_ps (__m256 __A) +{ + return (__m256) __builtin_ia32_sqrtps256 ((__v8sf)__A); +} + +#ifdef __OPTIMIZE__ +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_round_pd (__m256d __V, const int __M) +{ + return (__m256d) __builtin_ia32_roundpd256 ((__v4df)__V, __M); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_round_ps (__m256 __V, const int __M) +{ + return (__m256) __builtin_ia32_roundps256 ((__v8sf)__V, __M); +} +#else +#define _mm256_round_pd(V, M) \ + ((__m256d) __builtin_ia32_roundpd256 ((__v4df)(__m256d)(V), (int)(M))) + +#define _mm256_round_ps(V, M) \ + ((__m256) __builtin_ia32_roundps256 ((__v8sf)(__m256)(V), (int)(M))) +#endif + +#define _mm256_ceil_pd(V) _mm256_round_pd ((V), _MM_FROUND_CEIL) +#define _mm256_floor_pd(V) _mm256_round_pd ((V), _MM_FROUND_FLOOR) +#define _mm256_ceil_ps(V) _mm256_round_ps ((V), _MM_FROUND_CEIL) +#define _mm256_floor_ps(V) _mm256_round_ps ((V), _MM_FROUND_FLOOR) + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_unpackhi_pd (__m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_unpckhpd256 ((__v4df)__A, (__v4df)__B); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_unpacklo_pd (__m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_unpcklpd256 ((__v4df)__A, (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_unpackhi_ps (__m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_unpckhps256 ((__v8sf)__A, (__v8sf)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_unpacklo_ps (__m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_unpcklps256 ((__v8sf)__A, (__v8sf)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_testz_pd (__m128d __M, __m128d __V) +{ + return __builtin_ia32_vtestzpd ((__v2df)__M, (__v2df)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_testc_pd (__m128d __M, __m128d __V) +{ + return __builtin_ia32_vtestcpd ((__v2df)__M, (__v2df)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_testnzc_pd (__m128d __M, __m128d __V) +{ + return __builtin_ia32_vtestnzcpd ((__v2df)__M, (__v2df)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_testz_ps (__m128 __M, __m128 __V) +{ + return __builtin_ia32_vtestzps ((__v4sf)__M, (__v4sf)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_testc_ps (__m128 __M, __m128 __V) +{ + return __builtin_ia32_vtestcps ((__v4sf)__M, (__v4sf)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_testnzc_ps (__m128 __M, __m128 __V) +{ + return __builtin_ia32_vtestnzcps ((__v4sf)__M, (__v4sf)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testz_pd (__m256d __M, __m256d __V) +{ + return __builtin_ia32_vtestzpd256 ((__v4df)__M, (__v4df)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testc_pd (__m256d __M, __m256d __V) +{ + return __builtin_ia32_vtestcpd256 ((__v4df)__M, (__v4df)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testnzc_pd (__m256d __M, __m256d __V) +{ + return __builtin_ia32_vtestnzcpd256 ((__v4df)__M, (__v4df)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testz_ps (__m256 __M, __m256 __V) +{ + return __builtin_ia32_vtestzps256 ((__v8sf)__M, (__v8sf)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testc_ps (__m256 __M, __m256 __V) +{ + return __builtin_ia32_vtestcps256 ((__v8sf)__M, (__v8sf)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testnzc_ps (__m256 __M, __m256 __V) +{ + return __builtin_ia32_vtestnzcps256 ((__v8sf)__M, (__v8sf)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testz_si256 (__m256i __M, __m256i __V) +{ + return __builtin_ia32_ptestz256 ((__v4di)__M, (__v4di)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testc_si256 (__m256i __M, __m256i __V) +{ + return __builtin_ia32_ptestc256 ((__v4di)__M, (__v4di)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testnzc_si256 (__m256i __M, __m256i __V) +{ + return __builtin_ia32_ptestnzc256 ((__v4di)__M, (__v4di)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_movemask_pd (__m256d __A) +{ + return __builtin_ia32_movmskpd256 ((__v4df)__A); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_movemask_ps (__m256 __A) +{ + return __builtin_ia32_movmskps256 ((__v8sf)__A); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_undefined_pd (void) +{ + __m256d __Y = __Y; + return __Y; +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_undefined_ps (void) +{ + __m256 __Y = __Y; + return __Y; +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_undefined_si256 (void) +{ + __m256i __Y = __Y; + return __Y; +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_setzero_pd (void) +{ + return __extension__ (__m256d){ 0.0, 0.0, 0.0, 0.0 }; +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_setzero_ps (void) +{ + return __extension__ (__m256){ 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0 }; +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_setzero_si256 (void) +{ + return __extension__ (__m256i)(__v4di){ 0, 0, 0, 0 }; +} + +/* Create the vector [A B C D]. */ +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set_pd (double __A, double __B, double __C, double __D) +{ + return __extension__ (__m256d){ __D, __C, __B, __A }; +} + +/* Create the vector [A B C D E F G H]. */ +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set_ps (float __A, float __B, float __C, float __D, + float __E, float __F, float __G, float __H) +{ + return __extension__ (__m256){ __H, __G, __F, __E, + __D, __C, __B, __A }; +} + +/* Create the vector [A B C D E F G H]. */ +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set_epi32 (int __A, int __B, int __C, int __D, + int __E, int __F, int __G, int __H) +{ + return __extension__ (__m256i)(__v8si){ __H, __G, __F, __E, + __D, __C, __B, __A }; +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set_epi16 (short __q15, short __q14, short __q13, short __q12, + short __q11, short __q10, short __q09, short __q08, + short __q07, short __q06, short __q05, short __q04, + short __q03, short __q02, short __q01, short __q00) +{ + return __extension__ (__m256i)(__v16hi){ + __q00, __q01, __q02, __q03, __q04, __q05, __q06, __q07, + __q08, __q09, __q10, __q11, __q12, __q13, __q14, __q15 + }; +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set_epi8 (char __q31, char __q30, char __q29, char __q28, + char __q27, char __q26, char __q25, char __q24, + char __q23, char __q22, char __q21, char __q20, + char __q19, char __q18, char __q17, char __q16, + char __q15, char __q14, char __q13, char __q12, + char __q11, char __q10, char __q09, char __q08, + char __q07, char __q06, char __q05, char __q04, + char __q03, char __q02, char __q01, char __q00) +{ + return __extension__ (__m256i)(__v32qi){ + __q00, __q01, __q02, __q03, __q04, __q05, __q06, __q07, + __q08, __q09, __q10, __q11, __q12, __q13, __q14, __q15, + __q16, __q17, __q18, __q19, __q20, __q21, __q22, __q23, + __q24, __q25, __q26, __q27, __q28, __q29, __q30, __q31 + }; +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set_epi64x (long long __A, long long __B, long long __C, + long long __D) +{ + return __extension__ (__m256i)(__v4di){ __D, __C, __B, __A }; +} + +/* Create a vector with all elements equal to A. */ +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set1_pd (double __A) +{ + return __extension__ (__m256d){ __A, __A, __A, __A }; +} + +/* Create a vector with all elements equal to A. */ +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set1_ps (float __A) +{ + return __extension__ (__m256){ __A, __A, __A, __A, + __A, __A, __A, __A }; +} + +/* Create a vector with all elements equal to A. */ +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set1_epi32 (int __A) +{ + return __extension__ (__m256i)(__v8si){ __A, __A, __A, __A, + __A, __A, __A, __A }; +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set1_epi16 (short __A) +{ + return _mm256_set_epi16 (__A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set1_epi8 (char __A) +{ + return _mm256_set_epi8 (__A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set1_epi64x (long long __A) +{ + return __extension__ (__m256i)(__v4di){ __A, __A, __A, __A }; +} + +/* Create vectors of elements in the reversed order from the + _mm256_set_XXX functions. */ + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_setr_pd (double __A, double __B, double __C, double __D) +{ + return _mm256_set_pd (__D, __C, __B, __A); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_setr_ps (float __A, float __B, float __C, float __D, + float __E, float __F, float __G, float __H) +{ + return _mm256_set_ps (__H, __G, __F, __E, __D, __C, __B, __A); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_setr_epi32 (int __A, int __B, int __C, int __D, + int __E, int __F, int __G, int __H) +{ + return _mm256_set_epi32 (__H, __G, __F, __E, __D, __C, __B, __A); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_setr_epi16 (short __q15, short __q14, short __q13, short __q12, + short __q11, short __q10, short __q09, short __q08, + short __q07, short __q06, short __q05, short __q04, + short __q03, short __q02, short __q01, short __q00) +{ + return _mm256_set_epi16 (__q00, __q01, __q02, __q03, + __q04, __q05, __q06, __q07, + __q08, __q09, __q10, __q11, + __q12, __q13, __q14, __q15); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_setr_epi8 (char __q31, char __q30, char __q29, char __q28, + char __q27, char __q26, char __q25, char __q24, + char __q23, char __q22, char __q21, char __q20, + char __q19, char __q18, char __q17, char __q16, + char __q15, char __q14, char __q13, char __q12, + char __q11, char __q10, char __q09, char __q08, + char __q07, char __q06, char __q05, char __q04, + char __q03, char __q02, char __q01, char __q00) +{ + return _mm256_set_epi8 (__q00, __q01, __q02, __q03, + __q04, __q05, __q06, __q07, + __q08, __q09, __q10, __q11, + __q12, __q13, __q14, __q15, + __q16, __q17, __q18, __q19, + __q20, __q21, __q22, __q23, + __q24, __q25, __q26, __q27, + __q28, __q29, __q30, __q31); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_setr_epi64x (long long __A, long long __B, long long __C, + long long __D) +{ + return _mm256_set_epi64x (__D, __C, __B, __A); +} + +/* Casts between various SP, DP, INT vector types. Note that these do no + conversion of values, they just change the type. */ +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castpd_ps (__m256d __A) +{ + return (__m256) __A; +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castpd_si256 (__m256d __A) +{ + return (__m256i) __A; +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castps_pd (__m256 __A) +{ + return (__m256d) __A; +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castps_si256(__m256 __A) +{ + return (__m256i) __A; +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castsi256_ps (__m256i __A) +{ + return (__m256) __A; +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castsi256_pd (__m256i __A) +{ + return (__m256d) __A; +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castpd256_pd128 (__m256d __A) +{ + return (__m128d) __builtin_ia32_pd_pd256 ((__v4df)__A); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castps256_ps128 (__m256 __A) +{ + return (__m128) __builtin_ia32_ps_ps256 ((__v8sf)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castsi256_si128 (__m256i __A) +{ + return (__m128i) __builtin_ia32_si_si256 ((__v8si)__A); +} + +/* When cast is done from a 128 to 256-bit type, the low 128 bits of + the 256-bit result contain source parameter value and the upper 128 + bits of the result are undefined. Those intrinsics shouldn't + generate any extra moves. */ + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castpd128_pd256 (__m128d __A) +{ + return (__m256d) __builtin_ia32_pd256_pd ((__v2df)__A); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castps128_ps256 (__m128 __A) +{ + return (__m256) __builtin_ia32_ps256_ps ((__v4sf)__A); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castsi128_si256 (__m128i __A) +{ + return (__m256i) __builtin_ia32_si256_si ((__v4si)__A); +} + +#ifdef __DISABLE_AVX__ +#undef __DISABLE_AVX__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX__ */ + +#endif /* _AVXINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/bmi2intrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/bmi2intrin.h new file mode 100644 index 0000000000..29569ae8de --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/bmi2intrin.h @@ -0,0 +1,109 @@ +/* Copyright (C) 2011-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#if !defined _X86INTRIN_H_INCLUDED && !defined _IMMINTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _BMI2INTRIN_H_INCLUDED +#define _BMI2INTRIN_H_INCLUDED + +#ifndef __BMI2__ +#pragma GCC push_options +#pragma GCC target("bmi2") +#define __DISABLE_BMI2__ +#endif /* __BMI2__ */ + +extern __inline unsigned int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_bzhi_u32 (unsigned int __X, unsigned int __Y) +{ + return __builtin_ia32_bzhi_si (__X, __Y); +} + +extern __inline unsigned int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_pdep_u32 (unsigned int __X, unsigned int __Y) +{ + return __builtin_ia32_pdep_si (__X, __Y); +} + +extern __inline unsigned int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_pext_u32 (unsigned int __X, unsigned int __Y) +{ + return __builtin_ia32_pext_si (__X, __Y); +} + +#ifdef __x86_64__ + +extern __inline unsigned long long +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_bzhi_u64 (unsigned long long __X, unsigned long long __Y) +{ + return __builtin_ia32_bzhi_di (__X, __Y); +} + +extern __inline unsigned long long +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_pdep_u64 (unsigned long long __X, unsigned long long __Y) +{ + return __builtin_ia32_pdep_di (__X, __Y); +} + +extern __inline unsigned long long +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_pext_u64 (unsigned long long __X, unsigned long long __Y) +{ + return __builtin_ia32_pext_di (__X, __Y); +} + +extern __inline unsigned long long +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mulx_u64 (unsigned long long __X, unsigned long long __Y, + unsigned long long *__P) +{ + unsigned __int128 __res = (unsigned __int128) __X * __Y; + *__P = (unsigned long long) (__res >> 64); + return (unsigned long long) __res; +} + +#else /* !__x86_64__ */ + +extern __inline unsigned int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mulx_u32 (unsigned int __X, unsigned int __Y, unsigned int *__P) +{ + unsigned long long __res = (unsigned long long) __X * __Y; + *__P = (unsigned int) (__res >> 32); + return (unsigned int) __res; +} + +#endif /* !__x86_64__ */ + +#ifdef __DISABLE_BMI2__ +#undef __DISABLE_BMI2__ +#pragma GCC pop_options +#endif /* __DISABLE_BMI2__ */ + +#endif /* _BMI2INTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/bmiintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/bmiintrin.h new file mode 100644 index 0000000000..55c6219e97 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/bmiintrin.h @@ -0,0 +1,184 @@ +/* Copyright (C) 2010-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#if !defined _X86INTRIN_H_INCLUDED && !defined _IMMINTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _BMIINTRIN_H_INCLUDED +#define _BMIINTRIN_H_INCLUDED + +#ifndef __BMI__ +#pragma GCC push_options +#pragma GCC target("bmi") +#define __DISABLE_BMI__ +#endif /* __BMI__ */ + +extern __inline unsigned short __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__tzcnt_u16 (unsigned short __X) +{ + return __builtin_ia32_tzcnt_u16 (__X); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__andn_u32 (unsigned int __X, unsigned int __Y) +{ + return ~__X & __Y; +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__bextr_u32 (unsigned int __X, unsigned int __Y) +{ + return __builtin_ia32_bextr_u32 (__X, __Y); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_bextr_u32 (unsigned int __X, unsigned int __Y, unsigned __Z) +{ + return __builtin_ia32_bextr_u32 (__X, ((__Y & 0xff) | ((__Z & 0xff) << 8))); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blsi_u32 (unsigned int __X) +{ + return __X & -__X; +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_blsi_u32 (unsigned int __X) +{ + return __blsi_u32 (__X); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blsmsk_u32 (unsigned int __X) +{ + return __X ^ (__X - 1); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_blsmsk_u32 (unsigned int __X) +{ + return __blsmsk_u32 (__X); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blsr_u32 (unsigned int __X) +{ + return __X & (__X - 1); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_blsr_u32 (unsigned int __X) +{ + return __blsr_u32 (__X); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__tzcnt_u32 (unsigned int __X) +{ + return __builtin_ia32_tzcnt_u32 (__X); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_tzcnt_u32 (unsigned int __X) +{ + return __builtin_ia32_tzcnt_u32 (__X); +} + + +#ifdef __x86_64__ +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__andn_u64 (unsigned long long __X, unsigned long long __Y) +{ + return ~__X & __Y; +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__bextr_u64 (unsigned long long __X, unsigned long long __Y) +{ + return __builtin_ia32_bextr_u64 (__X, __Y); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_bextr_u64 (unsigned long long __X, unsigned int __Y, unsigned int __Z) +{ + return __builtin_ia32_bextr_u64 (__X, ((__Y & 0xff) | ((__Z & 0xff) << 8))); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blsi_u64 (unsigned long long __X) +{ + return __X & -__X; +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_blsi_u64 (unsigned long long __X) +{ + return __blsi_u64 (__X); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blsmsk_u64 (unsigned long long __X) +{ + return __X ^ (__X - 1); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_blsmsk_u64 (unsigned long long __X) +{ + return __blsmsk_u64 (__X); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blsr_u64 (unsigned long long __X) +{ + return __X & (__X - 1); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_blsr_u64 (unsigned long long __X) +{ + return __blsr_u64 (__X); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__tzcnt_u64 (unsigned long long __X) +{ + return __builtin_ia32_tzcnt_u64 (__X); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_tzcnt_u64 (unsigned long long __X) +{ + return __builtin_ia32_tzcnt_u64 (__X); +} + +#endif /* __x86_64__ */ + +#ifdef __DISABLE_BMI__ +#undef __DISABLE_BMI__ +#pragma GCC pop_options +#endif /* __DISABLE_BMI__ */ + +#endif /* _BMIINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/bmmintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/bmmintrin.h new file mode 100644 index 0000000000..f460393508 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/bmmintrin.h @@ -0,0 +1,29 @@ +/* Copyright (C) 2007-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _BMMINTRIN_H_INCLUDED +#define _BMMINTRIN_H_INCLUDED + +# error "SSE5 instruction set removed from compiler" + +#endif /* _BMMINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/cilk.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/cilk.h new file mode 100644 index 0000000000..86038ac1ad --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/cilk.h @@ -0,0 +1,82 @@ +/* cilk.h -*-C++-*- + * + * Copyright (C) 2010-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + */ + +/** @file cilk.h + * + * @brief Provides convenient aliases for Intel(R) Cilk(TM) language keywords. + * + * @details + * Since Intel Cilk Plus is a nonstandard extension to both C and C++, the Intel + * Cilk language keywords all begin with "`_Cilk_`", which guarantees that they + * will not conflict with user-defined identifiers in properly written + * programs. This way, a Cilk-enabled C or C++ compiler can safely compile + * "standard" C and C++ programs. + * + * However, this means that the keywords _look_ like something grafted on to + * the base language. Therefore, you can include this header: + * + * #include "cilk/cilk.h" + * + * and then write the Intel Cilk keywords with a "`cilk_`" prefix instead of + * "`_Cilk_`". + * + * @ingroup language + */ + + +/** @defgroup language Language Keywords + * Definitions for the Intel Cilk language. + * @{ + */ + +#ifndef cilk_spawn +# define cilk_spawn _Cilk_spawn ///< Spawn a task that can execute in parallel. +# define cilk_sync _Cilk_sync ///< Wait for spawned tasks to complete. +# define cilk_for _Cilk_for ///< Execute iterations of a `for` loop in parallel. +#endif + +/// @} diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/cilk_api.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/cilk_api.h new file mode 100644 index 0000000000..6cc62c994b --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/cilk_api.h @@ -0,0 +1,436 @@ +/* cilk_api.h + * + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + */ + +/** @file cilk_api.h + * + * @brief Defines the Intel(R) Cilk(TM) Plus API for use by applications. + * + * @ingroup api + */ + +#ifndef INCLUDED_CILK_API_H +#define INCLUDED_CILK_API_H + + +/** @defgroup api Runtime API +* API to interact with the Intel Cilk Plus runtime. +* @{ +*/ + +#ifndef CILK_STUB /* Real (non-stub) definitions */ + +#if ! defined(__cilk) && ! defined(USE_CILK_API) +# ifdef _WIN32 +# error Cilk API is being used with non-Cilk compiler (or Cilk is disabled) +# else +# warning Cilk API is being used with non-Cilk compiler (or Cilk is disabled) +# endif +#endif + +#include + +#ifdef __cplusplus +# include /* Defines size_t */ +#else +# include /* Defines size_t */ +#endif + +#ifdef _WIN32 +# ifndef IN_CILK_RUNTIME +/* Ensure the library is brought if any of these functions are being called. */ +# pragma comment(lib, "cilkrts") +# endif + +# ifndef __cplusplus +# include +# endif +#endif /* _WIN32 */ + +__CILKRTS_BEGIN_EXTERN_C + +/** Return values from `__cilkrts_set_param()` and `__cilkrts_set_param_w()` + */ +enum __cilkrts_set_param_status { + __CILKRTS_SET_PARAM_SUCCESS = 0, /**< Success - parameter set */ + __CILKRTS_SET_PARAM_UNIMP = 1, /**< Unimplemented parameter */ + __CILKRTS_SET_PARAM_XRANGE = 2, /**< Parameter value out of range */ + __CILKRTS_SET_PARAM_INVALID = 3, /**< Invalid parameter value */ + __CILKRTS_SET_PARAM_LATE = 4 /**< Too late to change parameter value */ +}; + +/** Sets user controllable runtime parameters + * + * Call this function to set runtime parameters that control the behavior + * of the Intel Cilk Plus scheduler. + * + * @param param A string specifying the parameter to be set. One of: + * - `"nworkers"` + * - `"force reduce"` + * @param value A string specifying the parameter value. + * @returns A value from the @ref __cilkrts_set_param_status + * enumeration indicating the result of the operation. + * + * @par The "nworkers" parameter + * + * This parameter specifies the number of worker threads to be created by the + * Intel Cilk Plus runtime. @a Value must be a string of digits to be parsed by + * `strtol()` as a decimal number. + * + * The number of worker threads is: + * 1. the value set with `__cilkrts_set_param("nworkers")`, if it is + * positive; otherwise, + * 2. the value of the CILK_NWORKERS environment variable, if it is + * defined; otherwise + * 3. the number of cores available, as reported by the operating system. + * + * @note + * Technically, Intel Cilk Plus distinguishes between the _user thread_ (the thread + * that the user code was executing on when the Intel Cilk Plus runtime started), + * and _worker threads_ (new threads created by the Intel Cilk Plus runtime to + * support Intel Cilk Plus parallelism). `nworkers` actually includes both the user + * thread and the worker threads; that is, it is one greater than the number of + * true "worker threads". + * + * @note + * Setting `nworkers = 1` produces serial behavior. Intel Cilk Plus spawns and syncs + * will be executed, but with only one worker, continuations will never be + * stolen, so all code will execute in serial. + * + * @warning + * The number of worker threads can only be set *before* the runtime has + * started. Attempting to set it when the runtime is running will have no + * effect, and will return an error code. You can call __cilkrts_end_cilk() + * to shut down the runtime to change the number of workers. + * + * @warning + * The default Intel Cilk scheduler behavior is usually pretty good. The + * ability to override `nworkers` can be useful for experimentation, but it + * won't usually be necessary for getting good performance. + * + * @par The "force reduce" parameter + * + * This parameter controls whether the runtime should allocate a new view + * for a reducer for every parallel strand that it is accessed on. (See + * @ref pagereducers.) @a Value must be `"1"` or `"true"` to enable the + * "force reduce" behavior, or `"0"` or `"false"` to disable it. + * + * "Force reduce" behavior will also be enabled if + * `__cilkrts_set_param("force reduce")` is not called, but the + * `CILK_FORCE_REDUCE` environment variable is defined. + * + * @warning + * When this option is enabled, `nworkers` should be set to `1`. Using "force + * reduce" with more than one worker may result in runtime errors. + * + * @warning + * Enabling this option can significantly reduce performance. Use it + * _only_ as a debugging tool. + */ +CILK_API(int) __cilkrts_set_param(const char *param, const char *value); + +#ifdef _WIN32 +/** + * Sets user controllable parameters using wide strings + * + * @note This variant of __cilkrts_set_param() is only available + * on Windows. + * + * @copydetails __cilkrts_set_param + */ +CILK_API(int) __cilkrts_set_param_w(const wchar_t *param, const wchar_t *value); +#endif + +/** Shuts down and deallocates all Intel Cilk Plus states. If Intel Cilk Plus is still in + * use by the calling thread, the runtime aborts the application. Otherwise, the + * runtime waits for all other threads using Intel Cilk Plus to exit. + */ +CILK_API(void) __cilkrts_end_cilk(void); + +/** Initializes Intel Cilk Plus data structures and start the runtime. + */ +CILK_API(void) __cilkrts_init(void); + +/** Returns the runtime `nworkers` parameter. (See the discussion of `nworkers` + * in the documentation for __cilkrts_set_param().) + */ +CILK_API(int) __cilkrts_get_nworkers(void); + +/** Returns the number of thread data structures. + * + * This function returns the number of data structures that have been allocated + * by the runtime to hold information about user and worker threads. + * + * If you don't already know what this is good for, then you probably don't + * need it. :) + */ +CILK_API(int) __cilkrts_get_total_workers(void); + +/** Returns a small integer identifying the current thread. + * + * What thread is the function running on? Each worker thread + * started by the Intel Cilk Plus runtime library has a unique worker number in the + * range `1 .. nworkers - 1`. + * + * All _user_ threads (threads started by the user, or by other libraries) are + * identified as worker number 0. Therefore, the worker number is not unique + * across multiple user threads. + */ +CILK_API(int) __cilkrts_get_worker_number(void); + +/** Tests whether "force reduce" behavior is enabled. + * + * @return Non-zero if force-reduce mode is on, zero if it is off. + */ +CILK_API(int) __cilkrts_get_force_reduce(void); + +/** Interacts with tools + */ +CILK_API(void) + __cilkrts_metacall(unsigned int tool, unsigned int code, void *data); + +#ifdef _WIN32 +/// Windows exception description record. +typedef struct _EXCEPTION_RECORD _EXCEPTION_RECORD; + +/** Function signature for Windows exception notification callbacks. + */ +typedef void (*__cilkrts_pfn_seh_callback)(const _EXCEPTION_RECORD *exception); + +/** Specifies a function to call when a non-C++ exception is caught. + * + * Intel Cilk Plus parallelism plays nicely with C++ exception handling, but + * the Intel Cilk Plus runtime has no way to unwind the stack across a strand + * boundary for Microsoft SEH ("Structured Exception Handling") exceptions. + * Therefore, when the runtime catches such an exception, it must abort the + * application. + * + * If an SEH callback has been set, the runtime will call it before aborting. + * + * @param pfn A pointer to a callback function to be called before the + * runtime aborts the program because of an SEH exception. + */ +CILK_API(int) __cilkrts_set_seh_callback(__cilkrts_pfn_seh_callback pfn); +#endif /* _WIN32 */ + +#if __CILKRTS_ABI_VERSION >= 1 +/* Pedigree API is available only for compilers that use ABI version >= 1. */ + + +/** @name Pedigrees + */ +//@{ + +// @cond internal + +/** Support for __cilkrts_get_pedigree. + */ +CILK_API(__cilkrts_pedigree) +__cilkrts_get_pedigree_internal(__cilkrts_worker *w); + +/** Support for __cilkrts_bump_worker_rank. + */ +CILK_API(int) +__cilkrts_bump_worker_rank_internal(__cilkrts_worker* w); + +/// @endcond + + +/** Gets the current pedigree in a linked list representation. + * + * This routine returns a copy of the last node in the pedigree list. + * For example, if the current pedigree (in order) is <1, 2, 3, 4>, + * then this method returns a node with rank == 4, and whose parent + * field points to the node with rank of 3. In summary, following the + * nodes in the chain visits the terms of the pedigree in reverse. + * + * The returned node is guaranteed to be valid only until the caller + * of this routine has returned. + */ +__CILKRTS_INLINE +__cilkrts_pedigree __cilkrts_get_pedigree(void) +{ + return __cilkrts_get_pedigree_internal(__cilkrts_get_tls_worker()); +} + +/** Context used by __cilkrts_get_pedigree_info. + * + * @deprecated + * This data structure is only used by the deprecated + * __cilkrts_get_pedigree_info function. + * + * Callers should initialize the `data` array to NULL and set the `size` + * field to `sizeof(__cilkrts_pedigree_context_t)` before the first call + * to `__cilkrts_get_pedigree_info()`. Also, callers should not examine or + * modify `data` thereafter. + */ +typedef struct +{ + __STDNS size_t size; /**< Size of the struct in bytes */ + void *data[3]; /**< Opaque context data */ +} __cilkrts_pedigree_context_t; + +/** Gets pedigree information. + * + * @deprecated + * Use __cilkrts_get_pedigree() instead. + * + * This routine allows code to walk up the stack of Intel Cilk Plus frames to gather + * the pedigree. + * + * Initialize the pedigree walk by filling the pedigree context with NULLs + * and setting the size field to `sizeof(__cilkrts_pedigree_context)`. + * Other than initialization to NULL to start the walk, user coder should + * consider the pedigree context data opaque and should not examine or + * modify it. + * + * @returns 0 - Success - birthrank is valid + * @returns >0 - End of pedigree walk + * @returns -1 - Failure - No worker bound to thread + * @returns -2 - Failure - Sanity check failed, + * @returns -3 - Failure - Invalid context size + * @returns -4 - Failure - Internal error - walked off end of chain of frames + */ +CILK_API(int) +__cilkrts_get_pedigree_info(/* In/Out */ __cilkrts_pedigree_context_t *context, + /* Out */ uint64_t *sf_birthrank); + +/** Gets the rank of the currently executing worker. + * + * @deprecated + * Use `__cilkrts_get_pedigree().rank` instead. + * + * @returns 0 - Success - *rank is valid + * @returns <0 - Failure - *rank is not changed + */ +CILK_EXPORT_AND_INLINE +int __cilkrts_get_worker_rank(uint64_t *rank) +{ + *rank = __cilkrts_get_pedigree().rank; + return 0; +} + +/** Increments the pedigree rank of the currently executing worker. + * + * @returns 0 - Success - rank was incremented + * @returns -1 - Failure + */ +CILK_EXPORT_AND_INLINE +int __cilkrts_bump_worker_rank(void) +{ + return __cilkrts_bump_worker_rank_internal(__cilkrts_get_tls_worker()); +} + +/** Increments the pedigree rank for a `cilk_for` loop. + * Obsolete. + * + * @deprecated + * This function was provided to allow the user to manipulate the pedigree + * rank of a `cilk_for` loop. The compiler now generates code to do that + * manipulation automatically, so this function is now unnecessary. It may + * be called, but will have no effect. + */ +CILK_EXPORT_AND_INLINE +int __cilkrts_bump_loop_rank(void) +{ + return 0; +} + +//@} + +#endif /* __CILKRTS_ABI_VERSION >= 1 */ + +__CILKRTS_END_EXTERN_C + +#else /* CILK_STUB */ + +// Programs compiled with CILK_STUB are not linked with the Intel Cilk Plus runtime +// library, so they should not have external references to runtime functions. +// Therefore, the functions are replaced with stubs. + +#ifdef _WIN32 +#define __cilkrts_set_param_w(name,value) ((value), 0) +#define __cilkrts_set_seh_callback(pfn) (0) +#endif +#define __cilkrts_set_param(name,value) ((value), 0) +#define __cilkrts_end_cilk() ((void) 0) +#define __cilkrts_init() ((void) 0) +#define __cilkrts_get_nworkers() (1) +#define __cilkrts_get_total_workers() (1) +#define __cilkrts_get_worker_number() (0) +#define __cilkrts_get_force_reduce() (0) +#define __cilkrts_metacall(tool,code,data) ((tool), (code), (data), 0) + +#if __CILKRTS_ABI_VERSION >= 1 +/* Pedigree stubs */ +#define __cilkrts_get_pedigree_info(context, sf_birthrank) (-1) +#define __cilkrts_get_worker_rank(rank) (*(rank) = 0) +#define __cilkrts_bump_worker_rank() (-1) +#define __cilkrts_bump_loop_rank() (-1) + +/* + * A stub method for __cilkrts_get_pedigree. + * Returns an empty __cilkrts_pedigree. + */ +__CILKRTS_INLINE +__cilkrts_pedigree __cilkrts_get_pedigree_stub(void) +{ + __cilkrts_pedigree ans; + ans.rank = 0; + ans.parent = NULL; + return ans; +} + +/* Renamed to an actual stub method. */ +#define __cilkrts_get_pedigree() __cilkrts_get_pedigree_stub() + +#endif /* __CILKRTS_ABI_VERSION >= 1 */ + +#endif /* CILK_STUB */ + +//@} + +#endif /* INCLUDED_CILK_API_H */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/cilk_api_linux.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/cilk_api_linux.h new file mode 100644 index 0000000000..0ebd57cba2 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/cilk_api_linux.h @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + * + */ + +/* THIS FILE IS DEPRECATED. USE cilk_api.h INSTEAD. */ +#include diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/cilk_stub.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/cilk_stub.h new file mode 100644 index 0000000000..b4a54f37c9 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/cilk_stub.h @@ -0,0 +1,71 @@ +/* cilk_stub.h -*-C++-*- + * + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + * + */ + +#ifndef INCLUDED_CILK_STUB_DOT_H +#define INCLUDED_CILK_STUB_DOT_H + +/* Definitions for creating a serialization from an Intel(R) Cilk(TM) Plus program. + * These definitions are suitable for use by a compiler that is not + * Cilk-enabled. + */ + +/* Pretend we are a non-Cilk compiler */ +#undef __cilk +#define CILK_STUB + +/* Replace Intel Cilk keywords with serial equivalents */ +#define _Cilk_spawn +#define _Cilk_sync +#define _Cilk_for for + +/* Replace simd-loop keywords with serial equivalents */ +#define _Simd +#define _Safelen(...) +#define _Reduction(...) + +#endif /* ! defined(INCLUDED_CILK_STUB_DOT_H) */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/cilk_undocumented.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/cilk_undocumented.h new file mode 100644 index 0000000000..71a51ec52d --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/cilk_undocumented.h @@ -0,0 +1,139 @@ +/* + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + * + ****************************************************************************** + * + * cilk_undocumented.h + * + * This file defines exported functions that are not included in the standard + * documentation. + */ + +#ifndef INCLUDED_CILK_UNDOCUMENTED_H +#define INCLUDED_CILK_UNDOCUMENTED_H + +#include + +#ifndef CILK_STUB + +__CILKRTS_BEGIN_EXTERN_C + +/* + * __cilkrts_synched + * + * Allows an application to determine if there are any outstanding children at + * this instant. This function will examine the current full frame to + * determine this. This function will return a valid result only when called + * within a spawn continuation, within the stack frame of the continuation + * itself. + */ + +CILK_EXPORT __CILKRTS_NOTHROW +int __cilkrts_synched(void); + +/* + * __cilkrts_cilkscreen_puts + * + * Allows an application to write a string to the Cilkscreen log. + * The standard error stream will be flushed after the write. + */ + +CILK_EXPORT __CILKRTS_NOTHROW +void __cilkrts_cilkscreen_puts(const char *); + +/* + * __cilkrts_get_sf + * + * A debugging aid that allows an application to get the __cilkrts_stack_frame + * for the current function. Only compiled into the DLL in debug builds. + */ + +CILK_EXPORT __CILKRTS_NOTHROW +void *__cilkrts_get_sf(void); + +/** + * Returns the size of stacks created by Intel(R) Cilk(TM) Plus. + */ +CILK_EXPORT __CILKRTS_NOTHROW +size_t __cilkrts_get_stack_size(void); + +/** + * Dumps runtime statistics to stderr. + * Undocumented API for debugging. + */ +CILK_EXPORT __CILKRTS_NOTHROW +void __cilkrts_dump_stats(void); + +struct __cilk_tbb_unwatch_thunk; +struct __cilk_tbb_stack_op_thunk; + +CILK_EXPORT __CILKRTS_NOTHROW +int __cilkrts_watch_stack(struct __cilk_tbb_unwatch_thunk *u, + struct __cilk_tbb_stack_op_thunk o); + +#ifndef IN_CILK_RUNTIME +#ifdef _WIN32 +/* Do not use CILK_API because __cilkrts_worker_stub must be __stdcall */ +CILK_EXPORT unsigned __CILKRTS_NOTHROW __stdcall +__cilkrts_worker_stub(void *arg); +#else +/* Do not use CILK_API because __cilkrts_worker_stub have default visibility */ +CILK_EXPORT void* __CILKRTS_NOTHROW +__cilkrts_worker_stub(void *arg); +#endif /* _WIN32 */ +#endif /* IN_CILK_RUNTIME */ + +__CILKRTS_END_EXTERN_C + +#else /* CILK_STUB */ + +/* Stubs for the api functions */ + +#define __cilkrts_get_stack_size() (0) +#define __cilkrts_synched() (1) + +#endif /* CILK_STUB */ + +#endif /* INCLUDED_CILK_UNDOCUMENTED_H */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/common.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/common.h new file mode 100644 index 0000000000..91b2928e7e --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/common.h @@ -0,0 +1,385 @@ +/** common.h + * + * Copyright (C) 2010-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + */ + +/** @file common.h + * + * @brief Defines common macros and structures used by the Intel(R) Cilk(TM) Plus runtime. + * + * @ingroup common + */ + +/** @defgroup common Common Definitions + * Definitions for runtime macros, structures, and classes. + * @{ + */ + +#ifndef INCLUDED_CILK_COMMON +#define INCLUDED_CILK_COMMON + +#ifdef __cplusplus +/** Namespace for all Intel Cilk Plus definitions that can be included in user code. + */ +namespace cilk { + + /** Namespace for definitions re-used in other Intel Cilk Plus definitions. + */ + namespace internal {} +} +#endif + +/** Intel Cilk Plus library version = 1.02 + */ +#define CILK_LIBRARY_VERSION 102 + +#ifdef __cplusplus +# include +#else +# include +#endif + +/** + * Prefix standard library function and type names with __STDNS to + * get correct lookup in both C and C++. + */ +#ifdef __cplusplus +# define __STDNS std:: +#else +# define __STDNS +#endif + +/** + * @def CILK_EXPORT + * Define export of runtime functions from shared library. + * Should be exported only from cilkrts*.dll/cilkrts*.so + * @def CILK_EXPORT_DATA + * Define export of runtime data from shared library. + */ +#ifdef _WIN32 +# ifdef IN_CILK_RUNTIME +# define CILK_EXPORT __declspec(dllexport) +# define CILK_EXPORT_DATA __declspec(dllexport) +# else +# define CILK_EXPORT __declspec(dllimport) +# define CILK_EXPORT_DATA __declspec(dllimport) +# endif /* IN_CILK_RUNTIME */ +#elif defined(__CYGWIN__) || defined(__APPLE__) || defined(_DARWIN_C_SOURCE) +# define CILK_EXPORT /* nothing */ +# define CILK_EXPORT_DATA /* nothing */ +#else /* Unix/gcc */ +# if defined(IN_CILK_RUNTIME) && defined(HAVE_ATTRIBUTE_VISIBILITY) +# define CILK_EXPORT __attribute__((visibility("protected"))) +# define CILK_EXPORT_DATA __attribute__((visibility("protected"))) +# else +# define CILK_EXPORT /* nothing */ +# define CILK_EXPORT_DATA /* nothing */ +# endif /* IN_CILK_RUNTIME */ +#endif /* Unix/gcc */ + +/** + * @def __CILKRTS_BEGIN_EXTERN_C + * Macro to denote the start of a section in which all names have "C" linkage. + * That is, none of the names are to be mangled. + * @see __CILKRTS_END_EXTERN_C + * @see __CILKRTS_EXTERN_C + * + * @def __CILKRTS_END_EXTERN_C + * Macro to denote the end of a section in which all names have "C" linkage. + * That is, none of the names are to be mangled. + * @see __CILKRTS_BEGIN_EXTERN_C + * @see __CILKRTS_EXTERN_C + * + * @def __CILKRTS_EXTERN_C + * Macro to prefix a single definition which has "C" linkage. + * That is, the defined name is not to be mangled. + * @see __CILKRTS_BEGIN_EXTERN_C + * @see __CILKRTS_END_EXTERN_C + */ +#ifdef __cplusplus +# define __CILKRTS_BEGIN_EXTERN_C extern "C" { +# define __CILKRTS_END_EXTERN_C } +# define __CILKRTS_EXTERN_C extern "C" +#else +# define __CILKRTS_BEGIN_EXTERN_C +# define __CILKRTS_END_EXTERN_C +# define __CILKRTS_EXTERN_C +#endif + +/** + * OS-independent macro to specify a function which is known to not throw + * an exception. + */ +#ifdef __cplusplus +# ifdef _WIN32 +# define __CILKRTS_NOTHROW __declspec(nothrow) +# else /* Unix/gcc */ +# define __CILKRTS_NOTHROW __attribute__((nothrow)) +# endif /* Unix/gcc */ +#else +# define __CILKRTS_NOTHROW /* nothing */ +#endif /* __cplusplus */ + +/** Cache alignment. (Good enough for most architectures.) + */ +#define __CILKRTS_CACHE_LINE__ 64 + +/** + * Macro to specify alignment of a data member in a structure. + * Because of the way that gcc's alignment attribute is defined, @a n must + * be a numeric literal, not just a compile-time constant expression. + */ +#ifdef _WIN32 +# define CILK_ALIGNAS(n) __declspec(align(n)) +#else /* Unix/gcc */ +# define CILK_ALIGNAS(n) __attribute__((__aligned__(n))) +#endif + +/** + * Macro to specify cache-line alignment of a data member in a structure. + */ +#define __CILKRTS_CACHE_ALIGN CILK_ALIGNAS(__CILKRTS_CACHE_LINE__) + +/** + * Macro to specify a class as being at least as strictly aligned as some + * type on Windows. gcc does not provide a way of doing this, so on Unix, + * this just specifies the largest natural type alignment. Put the macro + * between the `class` keyword and the class name: + * + * class CILK_ALIGNAS_TYPE(foo) bar { ... }; + */ +#ifdef _WIN32 +# define CILK_ALIGNAS_TYPE(t) __declspec(align(__alignof(t))) +#else /* Unix/gcc */ +# define CILK_ALIGNAS_TYPE(t) __attribute__((__aligned__)) +#endif + +/** + * @def CILK_API(RET_TYPE) + * A function called explicitly by the programmer. + * @def CILK_ABI(RET_TYPE) + * A function called by compiler-generated code. + * @def CILK_ABI_THROWS(RET_TYPE) + * An ABI function that may throw an exception + * + * Even when these are the same definitions, they should be separate macros so + * that they can be easily found in the code. + */ + +#ifdef _WIN32 +# define CILK_API(RET_TYPE) CILK_EXPORT RET_TYPE __CILKRTS_NOTHROW __cdecl +# define CILK_ABI(RET_TYPE) CILK_EXPORT RET_TYPE __CILKRTS_NOTHROW __cdecl +# define CILK_ABI_THROWS(RET_TYPE) CILK_EXPORT RET_TYPE __cdecl +#else +# define CILK_API(RET_TYPE) CILK_EXPORT RET_TYPE __CILKRTS_NOTHROW +# define CILK_ABI(RET_TYPE) CILK_EXPORT RET_TYPE __CILKRTS_NOTHROW +# define CILK_ABI_THROWS(RET_TYPE) CILK_EXPORT RET_TYPE +#endif + +/** + * __CILKRTS_ASSERT should be defined for debugging only, otherwise it + * interferes with vectorization. Since NDEBUG is not reliable (it must be + * set by the user), we must use a platform-specific detection of debug mode. + */ +#if defined(_WIN32) && defined(_DEBUG) + /* Windows debug */ +# define __CILKRTS_ASSERT(e) assert(e) +#elif (! defined(_WIN32)) && ! defined(__OPTIMIZE__) + /* Unix non-optimized */ +# define __CILKRTS_ASSERT(e) assert(e) +#elif defined __cplusplus + /* C++ non-debug */ +# define __CILKRTS_ASSERT(e) static_cast(0) +#else + /* C non-debug */ +# define __CILKRTS_ASSERT(e) ((void) 0) +#endif + +/** + * OS-independent macro to specify a function that should be inlined + */ +#ifdef __cplusplus + // C++ +# define __CILKRTS_INLINE inline +#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + // C99 +# define __CILKRTS_INLINE static inline +#elif defined(_MSC_VER) + // C89 on Windows +# define __CILKRTS_INLINE __inline +#else + // C89 on GCC-compatible systems +# define __CILKRTS_INLINE extern __inline__ +#endif + +/** + * Functions marked as CILK_EXPORT_AND_INLINE have both + * inline versions defined in the Intel Cilk Plus API, as well as + * non-inlined versions that are exported (for + * compatibility with previous versions that did not + * inline the functions). + */ +#ifdef COMPILING_CILK_API_FUNCTIONS +# define CILK_EXPORT_AND_INLINE CILK_EXPORT +#else +# define CILK_EXPORT_AND_INLINE __CILKRTS_INLINE +#endif + +/** + * Try to determine if compiler supports rvalue references. + */ +#if defined(__cplusplus) && !defined(__CILKRTS_RVALUE_REFERENCES) +# if __cplusplus >= 201103L // C++11 +# define __CILKRTS_RVALUE_REFERENCES 1 +# elif defined(__GXX_EXPERIMENTAL_CXX0X__) +# define __CILKRTS_RVALUE_REFERENCES 1 +# elif __cplusplus >= 199711L && __cplusplus < 201103L + // Compiler recognizes a language version prior to C++11 +# elif __INTEL_COMPILER == 1200 && defined(__STDC_HOSTED__) + // Intel compiler version 12.0 + // __cplusplus has a non-standard definition. In the absence of a + // proper definition, look for the C++0x macro, __STDC_HOSTED__. +# define __CILKRTS_RVALUE_REFERENCES 1 +# elif __INTEL_COMPILER > 1200 && defined(CHAR16T) + // Intel compiler version >= 12.1 + // __cplusplus has a non-standard definition. In the absence of a + // proper definition, look for the Intel macro, CHAR16T +# define __CILKRTS_RVALUE_REFERENCES 1 +# endif +#endif + +/* + * Include stdint.h to define the standard integer types. + * + * Unfortunately Microsoft doesn't provide stdint.h until Visual Studio 2010, + * so use our own definitions until those are available + */ + +#if ! defined(_MSC_VER) || (_MSC_VER >= 1600) +# include +#else +# ifndef __MS_STDINT_TYPES_DEFINED__ +# define __MS_STDINT_TYPES_DEFINED__ + typedef signed char int8_t; + typedef short int16_t; + typedef int int32_t; + typedef __int64 int64_t; + + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; + typedef unsigned __int64 uint64_t; +# endif /* __MS_STDINT_TYPES_DEFINED__ */ +#endif /* ! defined(_MSC_VER) || (_MSC_VER >= 1600) */ + +/** + * @brief Application Binary Interface (ABI) version of the Intel Cilk Plus runtime + * library. + * + * The compiler determines the ABI version used for compilation. Object files + * compiled with higher ABI versions are not compatible with libraries compiled + * with lower ABI versions. However, an object file compiled with a lower ABI + * version can be used with a library compiled with a higher ABI version + * (unless otherwise stated.) + */ +#ifndef __CILKRTS_ABI_VERSION +# ifdef IN_CILK_RUNTIME +# define __CILKRTS_ABI_VERSION 1 +# elif defined(__INTEL_COMPILER) && (__INTEL_COMPILER <= 1200) + // Intel compilers prior to version 12.1 support only ABI 0 +# define __CILKRTS_ABI_VERSION 0 +# else + // Non-Intel compiler or Intel compiler after version 12.0. +# define __CILKRTS_ABI_VERSION 1 +# endif +#endif + +// These structs are exported because the inlining of +// the internal version of API methods require a worker +// structure as parameter. +__CILKRTS_BEGIN_EXTERN_C + /// Worker struct, exported for inlined API methods + /// @ingroup api + struct __cilkrts_worker; + + /// Worker struct, exported for inlined API methods + /// @ingroup api + typedef struct __cilkrts_worker __cilkrts_worker; + + /// Worker struct pointer, exported for inlined API methods + /// @ingroup api + typedef struct __cilkrts_worker *__cilkrts_worker_ptr; + + + /// Fetch the worker out of TLS. + CILK_ABI(__cilkrts_worker_ptr) __cilkrts_get_tls_worker(void); + + /// void *, defined to work around complaints from the compiler + /// about using __declspec(nothrow) after the "void *" return type + typedef void * __cilkrts_void_ptr; + +__CILKRTS_END_EXTERN_C + + +#if __CILKRTS_ABI_VERSION >= 1 +// Pedigree API is available only for compilers that use ABI version >= 1. + +/** Pedigree information kept in the worker and stack frame. + * @ingroup api + */ +typedef struct __cilkrts_pedigree +{ + /** Rank at start of spawn helper. Saved rank for spawning functions */ + uint64_t rank; + + /** Link to next in chain */ + const struct __cilkrts_pedigree *parent; +} __cilkrts_pedigree; + +#endif // __CILKRTS_ABI_VERSION >= 1 + +/// @} + +#endif /* INCLUDED_CILK_COMMON */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/holder.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/holder.h new file mode 100644 index 0000000000..66899a25bc --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/holder.h @@ -0,0 +1,1011 @@ +/* + * Copyright (C) 2011-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + * + */ + +/* + * holder.h + * + * Purpose: hyperobject to provide different views of an object to each + * parallel strand. + */ + +#ifndef HOLDER_H_INCLUDED +#define HOLDER_H_INCLUDED + +#include +#include +#include + +#ifdef __cplusplus + +/* C++ Interface + * + * Classes: holder + * + * Description: + * ============ + * This component provides a hyperobject that isolates a parallel uses of a + * common variable where it is not necessary to preserve changes from + * different parallel strands. In effect, a holder acts a bit like + * thread-local storage, but has qualities that work better with the + * fork-join structure of Intel(R) Cilk(TM) Plus. In particular, a holder has the + * following qualities: + * + * - The view of a holder before the first spawn within a function is the same + * as the view after each sync (as in the case of a reducer). + * - The view of a holder within the first spawned child of a function (or the + * first child spawned after a sync) is the same as the view on entry to the + * function. + * - The view of a holder before entering a _Cilk_for loop is the same as the + * view during the first iteration of the loop and the view at the end of + * the loop. + * - The view of a holder in the continuation of a spawn or in an arbitrary + * iteration of a _Cilk_for loop is *non-deterministic*. It is generally + * recommended that the holder be explicitly put into a known state in these + * situations. + * + * A holder can be used as an alternative to parameter-passing. They are most + * useful for replacing non-local variables without massive refactoring. A + * holder takes advantage of the fact that, most of the time, a holder view + * does not change after a spawn or from one iteration of a parallel for loop + * to the next (i.e., stealing is the exception, not the rule). When the + * holder view is a large object that is expensive to construct, this + * optimization can save significant time versus creating a separate local + * object for each view. In addition, a holder using the "keep last" policy + * will have the same value after a sync as the serialization of the same + * program. The last quality will often allow the program to avoid + * recomputing a value. + * + * Usage Example: + * ============== + * Function 'compute()' is a complex function that computes a value using a + * memoized algorithm, storing intermediate results in a hash table. Compute + * calls several other functions, each of which calls several other functions, + * all of which share a global hash table. In all, there are over a dozen + * functions with a total of about 60 references to the hash table. + *.. + * hash_table memos; + * + * void h(const X& x); // Uses memos + * + * double compute(const X& x) + * { + * memos.clear(); + * // ... + * memos[i] = x; + * ... + * g(i); // Uses memos + * // ... + * std::for_each(c.begin(), c.end(), h); // Call h for each element of c + * } + * + * int main() + * { + * const std::size_t ARRAY_SIZE = 1000000; + * extern X myArray[ARRAY_SIZE]; + * + * for (std::size_t i = 0; i < ARRAY_SIZE; ++i) + * { + * compute(myArray[i]); + * } + * } + *.. + * We would like to replace the 'for' loop in 'main' with a 'cilk_for'. + * Although the hash table is cleared on entry to each call to 'compute()', + * and although the values stored in the hash table are no longer used after + * 'compute()' returns, the use of the hash table as a global variable + * prevents 'compute()' from being called safely in parallel. One way to do + * this would be to make 'memos' a private variable within the cilk_for loop + * and pass it down to the actual computation, so that each loop iteration has + * its own private copy: + *.. + * cilk_for (std::size_t i = 0; i < ARRAY_SIZE; ++i) + * { + * hash_table memos; + * compute(myArray[i], memos); + * } + *.. + * The problem with this approach is that it requires changing the signature + * of 'compute', 'h', 'g', and every one of the dozen or so functions that + * reference 'memos' as well as any function that calls those functions. This + * may break the abstraction of 'compute' and other functions, exposing an + * implementation detail that was not part of the interface. In addition, the + * function 'h' is called through a templated algorithm, 'for_each', which + * requires a fixed interface. Finally, there is constructor and destructor + * overhead for 'hash_table' each time through the loop. + * + * The alternative approach is to replace 'memos' with a holder. The holder + * would be available to all of the functions involved, but would not cause a + * race between parallel loop iterations. In order to make this work, each + * use of the 'memos' variable must be (mechanically) replaced by a use of the + * holder: + *.. + * cilk::holder > memos_h; + * + * void h(const X& x); // Uses memos_h + * + * double compute(const X& x) + * { + * memos_h().clear(); // operator() used to "dereference" the holder + * // ... + * memos_h()[i] = x; // operator() used to "dereference" the holder + * ... + * g(i); // Uses memos_h + * // ... + * std::for_each(c.begin(), c.end(), h); // Call h for each element of c + * } + *.. + * Note that each reference to the holder must be modified with an empty pair + * of parenthesis. This syntax is needed because there is no facility in C++ + * for a "smart reference" that would allow 'memos_h' to be a perfect + * replacement for 'memos'. One way that a user can avoid this syntax change + * is to wrap the holder in a class that has the same inteface as + * 'hash_table' but redirects all calls to the holder: + *.. + * template + * class hash_table_holder + * { + * private: + * cilk::holder > m_holder; + * public: + * void clear() { m_holder().clear(); } + * V& operator[](const K& x) { return m_holder()[x]; } + * std::size_t size() const { return m_holder().size(); } + * // etc. ... + * }; + *.. + * Using the above wrapper, the original code can be left unchanged except for + * replacing 'hash_table' with 'hash_table_holder' and replacing 'for' with + * 'cilk_for': + *.. + * hash_table_holder memos; + * + * void h(const X& x); // Uses memos + * + * double compute(const X& x) + * { + * memos.clear(); // Calls hash_table_holder::clear(). + * // ... + * } + *.. + * The above changes have no benefit over the use of thread-local storage. + * What if one of the functions has a 'cilk_spawn', however? + *.. + * void h(const X& x) + * { + * Y y = x.nested(); + * double d, w; + * if (y) + * { + * w = cilk_spawn compute_width(y); // May use 'memos' + * d = compute_depth(y); // Does not use 'memos' + * cilk_sync; + * compute(y); // recursive call. Uses 'memos'. + * } + * } + *.. + * In the above example, the view of the holder within 'compute_width' is the + * same as the view on entry to 'h'. More importantly, the view of the holder + * within the recursive call to 'compute' is the same as the view on entry to + * 'h', even if a different worker is executing the recursive call. Thus, the + * holder view within a Intel Cilk Plus program has useful qualities not found in + * thread-local storage. + */ + +namespace cilk { + + /** + * After a sync, the value stored in a holder matches the most recent + * value stored into the holder by one of the starnds entering the sync. + * The holder policy used to instantiate the holder determines which of + * the entering strands determines the final value of the holder. A policy + * of 'holder_keep_indeterminate' (the default) is the most efficient, and + * results in an indeterminate value depending on the runtime schedule + * (see below for more specifics). An indeterminate value after a sync is + * often acceptable, especially if the value of the holder is not reused + * after the sync. All of the remaining policies retain the value of the + * last strand that would be executed in the serialization of the program. + * They differ in the mechanism used to move the value from one view to + * another. A policy of 'holder_keep_last_copy' moves values by + * copy-assignment. A policy of 'holder_keep_last_swap' moves values by + * calling 'swap'. A policy of 'holder_keep_last_move' is available only + * for compilers that support C++0x rvalue references and moves values by + * move-assignment. A policy of 'holder_keep_last' attempts to choose the + * most efficient mechanism: member-function 'swap' if the view type + * supports it, otherwise move-assignment if supported, otherwise + * copy-assignment. (The swap member function for a class that provides + * one is almost always as fast or faster than move-assignment or + * copy-assignment.) + * + * The behavior of 'holder_keep_indeterminate', while indeterminate, is + * not random and can be used for advanced programming or debugging. With + * a policy of 'holder_keep_intermediate', values are never copied or + * moved between views. The value of the view after a sync is the same as + * the value set in the last spawned child before a steal occurs or the + * last value set in the continuation if no steal occurs. Using this + * knowledge, a programmer can use a holder to detect the earliest steal + * in a piece of code. An indeterminate holder is also useful for keeping + * cached data similar to the way some applications might use thread-local + * storage. + */ + enum holder_policy { + holder_keep_indeterminate, + holder_keep_last, + holder_keep_last_copy, + holder_keep_last_swap, +#ifdef __CILKRTS_RVALUE_REFERENCES + holder_keep_last_move +#endif + }; + + namespace internal { + + // Private special-case holder policy using the swap member-function + const holder_policy holder_keep_last_member_swap = + (holder_policy) (holder_keep_last_swap | 0x10); + + /* The constant, 'has_member_swap::value', will be 'true' if 'T' + * has a non-static member function with prototype 'void swap(T&)'. + * The mechanism used to detect 'swap' is the most portable among + * present-day compilers, but is not the most robust. Specifically, + * the prototype for 'swap' must exactly match 'void swap(T&)'. + * Near-matches like a 'swap' function that returns 'int' instead of + * 'void' will not be detected. Detection will also fail if 'T' + * inherits 'swap' from a base class. + */ + template + class has_member_swap + { + // This technique for detecting member functions was described by + // Rani Sharoni in comp.lang.c++.moderated: + // http://groups.google.com/group/comp.lang.c++.moderated/msg/2b06b2432fddfb60 + + // sizeof(notchar) is guaranteed larger than 1 + struct notchar { char x[2]; }; + + // Instantiationg Q will fail unless U contains a + // non-static member with prototype 'void swap(U&)'. + template struct Q { }; + + // First 'test' is preferred overload if U::swap exists with the + // correct prototype. Second 'test' is preferred overload + // otherwise. + template static char test(Q*); + template static notchar test(...); + + public: + /// 'value' will be true if T has a non-static member function + /// with prototype 'void swap(T&)'. + static const bool value = (1 == sizeof(test(0))); + }; + + template const bool has_member_swap::value; + + /** + * @brief Utility class for exception safety. + * + * The constuctor for this class takes a pointer and an allocator and + * holds on to them. The destructor deallocates the pointed-to + * object, without calling its destructor, typically to recover memory + * in case an exception is thrown. The release member clears the + * pointer so that the deallocation is prevented, i.e., when the + * exception danger has passed. The behavior of this class is similar + * to auto_ptr and unique_ptr. + */ + template > + class auto_deallocator + { + Allocator m_alloc; + Type* m_ptr; + + // Non-copiable + auto_deallocator(const auto_deallocator&); + auto_deallocator& operator=(const auto_deallocator&); + + public: + /// Constructor + explicit auto_deallocator(Type* p, const Allocator& a = Allocator()) + : m_alloc(a), m_ptr(p) { } + + /// Destructor - free allocated resources + ~auto_deallocator() { if (m_ptr) m_alloc.deallocate(m_ptr, 1); } + + /// Remove reference to resource + void release() { m_ptr = 0; } + }; + + /** + * Pure-abstract base class to initialize holder views + */ + template + class init_base + { + public: + virtual ~init_base() { } + virtual init_base* clone_self(Allocator& a) const = 0; + virtual void delete_self(Allocator& a) = 0; + virtual void construct_view(Type* p, Allocator& a) const = 0; + }; + + /** + * Class to default-initialize a holder view + */ + template + class default_init : public init_base + { + typedef init_base base; + + /// Private constructor (called from static make() function). + default_init() { } + + // Non-copiable + default_init(const default_init&); + default_init& operator=(const default_init&); + + public: + // Static factory function + static default_init* make(Allocator& a); + + // Virtual function overrides + virtual ~default_init(); + virtual base* clone_self(Allocator& a) const; + virtual void delete_self(Allocator& a); + virtual void construct_view(Type* p, Allocator& a) const; + }; + + template + default_init* + default_init::make(Allocator&) + { + // Return a pointer to a singleton. All instances of this class + // are identical, so we need only one. + static default_init self; + return &self; + } + + template + default_init::~default_init() + { + } + + template + init_base* + default_init::clone_self(Allocator& a) const + { + return make(a); + } + + template + void default_init::delete_self(Allocator&) + { + // Since make() returned a shared singleton, there is nothing to + // delete here. + } + + template + void + default_init::construct_view(Type* p, + Allocator&) const + { + ::new((void*) p) Type(); + // TBD: In a C++0x library, this should be rewritten + // std::allocator_traits::construct(a, p); + } + + /** + * Class to copy-construct a view from a stored exemplar. + */ + template + class exemplar_init : public init_base + { + typedef init_base base; + + Type* m_exemplar; + + // Private constructors (called from make() functions). + exemplar_init(const Type& val, Allocator& a); +#ifdef __CILKRTS_RVALUE_REFERENCES + exemplar_init(Type&& val, Allocator& a); +#endif + + // Non-copyiable + exemplar_init(const exemplar_init&); + exemplar_init& operator=(const exemplar_init&); + + public: + // Static factory functions + static exemplar_init* make(const Type& val, + Allocator& a = Allocator()); +#ifdef __CILKRTS_RVALUE_REFERENCES + static exemplar_init* make(Type&& val, + Allocator& a = Allocator()); +#endif + + // Virtual function overrides + virtual ~exemplar_init(); + virtual base* clone_self(Allocator& a) const; + virtual void delete_self(Allocator& a); + virtual void construct_view(Type* p, Allocator& a) const; + }; + + template + exemplar_init::exemplar_init(const Type& val, + Allocator& a) + { + m_exemplar = a.allocate(1); + auto_deallocator guard(m_exemplar, a); + a.construct(m_exemplar, val); + guard.release(); + } + +#ifdef __CILKRTS_RVALUE_REFERENCES + template + exemplar_init::exemplar_init(Type&& val, + Allocator& a) + { + m_exemplar = a.allocate(1); + auto_deallocator guard(m_exemplar, a); + a.construct(m_exemplar, std::forward(val)); + guard.release(); + } +#endif + + template + exemplar_init* + exemplar_init::make(const Type& val, + Allocator& a) + { + typedef typename Allocator::template rebind::other + self_alloc_t; + self_alloc_t alloc(a); + + exemplar_init *self = alloc.allocate(1); + auto_deallocator guard(self, alloc); + + // Don't use allocator to construct self. Allocator should be + // used only on elements of type 'Type'. + ::new((void*) self) exemplar_init(val, a); + + guard.release(); + + return self; + } + +#ifdef __CILKRTS_RVALUE_REFERENCES + template + exemplar_init* + exemplar_init::make(Type&& val, + Allocator& a) + { + typedef typename Allocator::template rebind::other + self_alloc_t; + self_alloc_t alloc(a); + + exemplar_init *self = alloc.allocate(1); + auto_deallocator guard(self, alloc); + + // Don't use allocator to construct self. Allocator should be + // used only on elements of type 'Type'. + ::new((void*) self) exemplar_init(std::forward(val), a); + + guard.release(); + + return self; + } +#endif + + template + exemplar_init::~exemplar_init() + { + // Called only by delete_self, which deleted the exemplar using an + // allocator. + __CILKRTS_ASSERT(0 == m_exemplar); + } + + template + init_base* + exemplar_init::clone_self(Allocator& a) const + { + return make(*m_exemplar, a); + } + + template + void exemplar_init::delete_self(Allocator& a) + { + typename Allocator::template rebind::other alloc(a); + + a.destroy(m_exemplar); + a.deallocate(m_exemplar, 1); + m_exemplar = 0; + + this->~exemplar_init(); + alloc.deallocate(this, 1); + } + + template + void + exemplar_init::construct_view(Type* p, + Allocator& a) const + { + a.construct(p, *m_exemplar); + // TBD: In a C++0x library, this should be rewritten + // std::allocator_traits::construct(a, p, *m_exemplar); + } + + /** + * Class to construct a view using a stored functor. The functor, + * 'f', must be be invokable using the expression 'Type x = f()'. + */ + template + class functor_init : + public init_base + { + typedef typename Allocator::value_type value_type; + typedef init_base base; + typedef typename Allocator::template rebind::other f_alloc; + + Func *m_functor; + + /// Private constructors (called from make() functions + functor_init(const Func& f, Allocator& a); +#ifdef __CILKRTS_RVALUE_REFERENCES + functor_init(Func&& f, Allocator& a); +#endif + + // Non-copiable + functor_init(const functor_init&); + functor_init& operator=(const functor_init&); + + public: + // Static factory functions + static functor_init* make(const Func& val, + Allocator& a = Allocator()); +#ifdef __CILKRTS_RVALUE_REFERENCES + static functor_init* make(Func&& val, + Allocator& a = Allocator()); +#endif + + // Virtual function overrides + virtual ~functor_init(); + virtual base* clone_self(Allocator& a) const; + virtual void delete_self(Allocator& a); + virtual void + construct_view(value_type* p, Allocator& a) const; + }; + + /// Specialization to strip off reference from 'Func&'. + template + struct functor_init + : functor_init { }; + + /// Specialization to strip off reference and cvq from 'const Func&'. + template + struct functor_init + : functor_init { }; + + template + functor_init::functor_init(const Func& f, + Allocator& a) + { + f_alloc alloc(a); + + m_functor = alloc.allocate(1); + auto_deallocator guard(m_functor, alloc); + alloc.construct(m_functor, f); + guard.release(); + } + +#ifdef __CILKRTS_RVALUE_REFERENCES + template + functor_init::functor_init(Func&& f, + Allocator& a) + { + f_alloc alloc(a); + + m_functor = alloc.allocate(1); + auto_deallocator guard(m_functor, alloc); + alloc.construct(m_functor, std::forward(f)); + guard.release(); + } +#endif + + template + functor_init* + functor_init::make(const Func& f, Allocator& a) + { + typedef typename Allocator::template rebind::other + self_alloc_t; + self_alloc_t alloc(a); + + functor_init *self = alloc.allocate(1); + auto_deallocator guard(self, alloc); + + // Don't use allocator to construct self. Allocator should be + // used only on elements of type 'Func'. + ::new((void*) self) functor_init(f, a); + + guard.release(); + + return self; + } + +#ifdef __CILKRTS_RVALUE_REFERENCES + template + functor_init* + functor_init::make(Func&& f, Allocator& a) + { + typedef typename Allocator::template rebind::other + self_alloc_t; + self_alloc_t alloc(a); + + functor_init *self = alloc.allocate(1); + auto_deallocator guard(self, alloc); + + // Don't use allocator to construct self. Allocator should be + // used only on elements of type 'Func'. + ::new((void*) self) functor_init(std::forward(f), a); + + guard.release(); + + return self; + } +#endif + + template + functor_init::~functor_init() + { + // Called only by delete_self, which deleted the functor using an + // allocator. + __CILKRTS_ASSERT(0 == m_functor); + } + + template + init_base* + functor_init::clone_self(Allocator& a) const + { + return make(*m_functor, a); + } + + template + inline + void functor_init::delete_self(Allocator& a) + { + typename Allocator::template rebind::other alloc(a); + f_alloc fa(a); + + fa.destroy(m_functor); + fa.deallocate(m_functor, 1); + m_functor = 0; + + this->~functor_init(); + alloc.deallocate(this, 1); + } + + template + void functor_init::construct_view(value_type* p, + Allocator& a) const + { + a.construct(p, (*m_functor)()); + // In C++0x, the above should be written + // std::allocator_traits::construct(a, p, m_functor()); + } + + /** + * Functor called to reduce a holder + */ + template + struct holder_reduce_functor; + + /** + * Specialization to keep the left (first) value. + */ + template + struct holder_reduce_functor + { + void operator()(Type* left, Type* right) const { } + }; + + /** + * Specialization to copy-assign from the right (last) value. + */ + template + struct holder_reduce_functor + { + void operator()(Type* left, Type* right) const { + *left = *right; + } + }; + + /* + * Specialization to keep the right (last) value via swap. + */ + template + struct holder_reduce_functor + { + void operator()(Type* left, Type* right) const { + using std::swap; + swap(*left, *right); + } + }; + +#ifdef __CILKRTS_RVALUE_REFERENCES + /* + * Specialization to move-assign from the right (last) value. + */ + template + struct holder_reduce_functor + { + void operator()(Type* left, Type* right) const { + *left = std::move(*right); + } + }; +#endif + + /* + * Specialization to keep the right (last) value via the swap member + * function. + */ + template + struct holder_reduce_functor + { + void operator()(Type* left, Type* right) const { + left->swap(*right); + } + }; + + /* + * Specialization to keep the right (last) value by the most efficient + * means detectable. + */ + template + struct holder_reduce_functor : + holder_reduce_functor::value ? + holder_keep_last_member_swap : +#ifdef __CILKRTS_RVALUE_REFERENCES + holder_keep_last_move +#else + holder_keep_last_copy +#endif + )> + { + }; + } // end namespace internal + + /** + * Monoid for holders. + * Allocator type is required to be thread-safe. + */ + template > + class holder_monoid : public monoid_base + { + // Allocator is mutable because the copy of the monoid inside the + // reducer is const (to avoid races on the shared state). However, + // the allocator is required to be thread-safe, so it is ok (and + // necessary) to modify. + mutable Allocator m_allocator; + internal::init_base *m_initializer; + + public: + /// This constructor uses default-initialization for both the leftmost + /// view and each identity view. + holder_monoid(const Allocator& a = Allocator()) + : m_allocator(a) + , m_initializer( + internal::default_init::make(m_allocator)) + { } + + /// These constructors use 'val' as an exemplar to copy-construct both + /// the leftmost view and each identity view. + holder_monoid(const Type& val, const Allocator& a = Allocator()) + : m_allocator(a) + , m_initializer(internal::exemplar_init::make( + val, m_allocator)) { } + /// This constructor uses 'f' as a functor to construct both + /// the leftmost view and each identity view. + template + holder_monoid(const Func& f, const Allocator& a = Allocator()) + : m_allocator(a) + , m_initializer( + internal::functor_init::make(f,m_allocator)) + { } + + /// Copy constructor + holder_monoid(const holder_monoid& rhs) + : m_allocator(rhs.m_allocator) + , m_initializer(rhs.m_initializer->clone_self(m_allocator)) { } + + /// "Extended" copy constructor with allocator + holder_monoid(const holder_monoid& rhs, const Allocator& a) + : m_allocator(a) + , m_initializer(rhs.m_initializer->clone_self(m_allocator)) { } + +#ifdef __CILKRTS_RVALUE_REFERENCES + /// Move constructor + holder_monoid(holder_monoid&& rhs) + : m_allocator(rhs.m_allocator) + , m_initializer(rhs.m_initializer) { + rhs.m_initializer = + internal::default_init::make(m_allocator); + } + + /// "Extended" move constructor with allocator + holder_monoid(holder_monoid&& rhs, const Allocator& a) + : m_allocator(a) + , m_initializer(0) { + if (a != rhs.m_allocator) + m_initializer = rhs.m_initializer->clone_self(a); + else { + m_initializer = rhs.m_initializer; + rhs.m_initializer = + internal::default_init::make(m_allocator); + } + } +#endif + /// Destructor + ~holder_monoid() { m_initializer->delete_self(m_allocator); } + + holder_monoid& operator=(const holder_monoid& rhs) { + if (this == &rhs) return *this; + m_initializer->delete_self(m_allocator); + m_initializer = rhs.m_initializer->clone_self(m_allocator); + } + +#ifdef __CILKRTS_RVALUE_REFERENCES + holder_monoid& operator=(holder_monoid&& rhs) { + if (m_allocator != rhs.m_allocator) + // Delegate to copy-assignment on unequal allocators + return operator=(static_cast(rhs)); + std::swap(m_initializer, rhs.m_initializer); + return *this; + } +#endif + + /// Constructs IDENTITY value into the uninitilized '*p' + void identity(Type* p) const + { m_initializer->construct_view(p, m_allocator); } + + /// Calls the destructor on the object pointed-to by 'p' + void destroy(Type* p) const + { m_allocator.destroy(p); } + + /// Return a pointer to size bytes of raw memory + void* allocate(std::size_t s) const { + __CILKRTS_ASSERT(sizeof(Type) == s); + return m_allocator.allocate(1); + } + + /// Deallocate the raw memory at p + void deallocate(void* p) const { + m_allocator.deallocate(static_cast(p), sizeof(Type)); + } + + void reduce(Type* left, Type* right) const { + internal::holder_reduce_functor()(left, right); + } + + void swap(holder_monoid& other) { + __CILKRTS_ASSERT(m_allocator == other.m_allocator); + std::swap(m_initializer, other.m_initializer); + } + + Allocator get_allocator() const { + return m_allocator; + } + }; + + // Namespace-scope swap + template + inline void swap(holder_monoid& a, + holder_monoid& b) + { + a.swap(b); + } + + /** + * Hyperobject to provide different views of an object to each + * parallel strand. + */ + template > + class holder : public reducer > + { + typedef holder_monoid monoid_type; + typedef reducer imp; + + // Return a value of Type constructed using the functor Func. + template + Type make_value(const Func& f) const { + struct obj { + union { + char buf[sizeof(Type)]; + void* align1; + double align2; + }; + + obj(const Func& f) { f(static_cast(buf)); } + ~obj() { static_cast(buf)->~Type(); } + + operator Type&() { return *static_cast(buf); } + }; + + return obj(f); + } + + public: + /// Default constructor uses default-initialization for both the + /// leftmost view and each identity view. + holder(const Allocator& alloc = Allocator()) + : imp(monoid_type(alloc)) { } + + /// Construct from an exemplar that is used to initialize both the + /// leftmost view and each identity view. + holder(const Type& v, const Allocator& alloc = Allocator()) + // Alas, cannot use an rvalue reference for 'v' because it is used + // twice in the same expression for initializing imp. + : imp(monoid_type(v, alloc), v) { } + + /// Construct from a functor that is used to initialize both the + /// leftmost view and each identity view. The functor, 'f', must be be + /// invokable using the expression 'Type x = f()'. + template + holder(const Func& f, const Allocator& alloc = Allocator()) + // Alas, cannot use an rvalue for 'f' because it is used twice in + // the same expression for initializing imp. + : imp(monoid_type(f, alloc), make_value(f)) { } + }; + +} // end namespace cilk + +#else /* C */ +# error Holders are currently available only for C++ +#endif /* __cplusplus */ + +#endif /* HOLDER_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/hyperobject_base.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/hyperobject_base.h new file mode 100644 index 0000000000..dd7ccfd902 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/hyperobject_base.h @@ -0,0 +1,183 @@ +/* + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + * + */ + +#ifndef INCLUDED_CILK_HYPEROBJECT_BASE +#define INCLUDED_CILK_HYPEROBJECT_BASE + +#ifdef __cplusplus +# include +# include +#else +# include +# include +#endif + +#include + +#if defined _WIN32 || defined _WIN64 +# if !defined CILK_STUB && !defined IN_CILK_RUNTIME + /* bring in the Intel(R) Cilk(TM) Plus library, which has definitions for some of these + * functions. */ +# pragma comment(lib, "cilkrts") +# endif +#endif + +/* The __CILKRTS_STRAND_PURE attribute tells the compiler that the value + * returned by 'func' for a given argument to 'func' will remain valid until + * the next strand boundary (spawn or sync) or until the next call to a + * function with the __CILKRTS_STRAND_STALE attribute using the same function + * argument. + */ +#if 0 && defined __cilk && (defined __GNUC__ && !defined _WIN32) && defined __cilkartsrev +# define __CILKRTS_STRAND_PURE(func) \ + func __attribute__((__cilk_hyper__("lookup"))) +# define __CILKRTS_STRAND_STALE(func) \ + func __attribute__((__cilk_hyper__("flush"))) +#else +# define __CILKRTS_STRAND_PURE(func) func +# define __CILKRTS_STRAND_STALE(func) func +#endif + +/***************************************************************************** + * C runtime interface to the hyperobject subsystem + *****************************************************************************/ + +__CILKRTS_BEGIN_EXTERN_C + +/* Callback function signatures. The 'r' argument always points to the + * reducer itself and is commonly ignored. */ +typedef void (*cilk_c_reducer_reduce_fn_t)(void* r, void* lhs, void* rhs); +typedef void (*cilk_c_reducer_identity_fn_t)(void* r, void* view); +typedef void (*cilk_c_reducer_destroy_fn_t)(void* r, void* view); +typedef void* (*cilk_c_reducer_allocate_fn_t)(void* r, __STDNS size_t bytes); +typedef void (*cilk_c_reducer_deallocate_fn_t)(void* r, void* view); + +/** Representation of the monoid */ +typedef struct cilk_c_monoid { + cilk_c_reducer_reduce_fn_t reduce_fn; + cilk_c_reducer_identity_fn_t identity_fn; + cilk_c_reducer_destroy_fn_t destroy_fn; + cilk_c_reducer_allocate_fn_t allocate_fn; + cilk_c_reducer_deallocate_fn_t deallocate_fn; +} cilk_c_monoid; + +/** Base of the hyperobject */ +typedef struct __cilkrts_hyperobject_base +{ + cilk_c_monoid __c_monoid; + unsigned long long __flags; + __STDNS ptrdiff_t __view_offset; /* offset (in bytes) to leftmost view */ + __STDNS size_t __view_size; /* Size of each view */ +} __cilkrts_hyperobject_base; + + +#ifndef CILK_STUB + +/* Library functions. */ +CILK_EXPORT + void __cilkrts_hyper_create(__cilkrts_hyperobject_base *key); +CILK_EXPORT void __CILKRTS_STRAND_STALE( + __cilkrts_hyper_destroy(__cilkrts_hyperobject_base *key)); +CILK_EXPORT void* __CILKRTS_STRAND_PURE( + __cilkrts_hyper_lookup(__cilkrts_hyperobject_base *key)); + +CILK_EXPORT + void* __cilkrts_hyperobject_alloc(void* ignore, __STDNS size_t bytes); +CILK_EXPORT + void __cilkrts_hyperobject_dealloc(void* ignore, void* view); + +/* No-op destroy function */ +CILK_EXPORT + void __cilkrts_hyperobject_noop_destroy(void* ignore, void* ignore2); + + +#else // CILK_STUB + +// Programs compiled with CILK_STUB are not linked with the Intel Cilk Plus runtime +// library, so they should not have external references to cilkrts functions. +// Furthermore, they don't need the hyperobject functionality, so the +// functions can be stubbed. + +#define __cilkrts_hyperobject_create __cilkrts_hyperobject_create__stub +__CILKRTS_INLINE + void __cilkrts_hyper_create(__cilkrts_hyperobject_base *key) + {} + +#define __cilkrts_hyperobject_destroy __cilkrts_hyperobject_destroy__stub +__CILKRTS_INLINE + void __cilkrts_hyper_destroy(__cilkrts_hyperobject_base *key) + {} + +#define __cilkrts_hyperobject_lookup __cilkrts_hyperobject_lookup__stub +__CILKRTS_INLINE + void* __cilkrts_hyper_lookup(__cilkrts_hyperobject_base *key) + { return (char*)(key) + key->__view_offset; } + +// Pointers to these functions are stored into monoids, so real functions +// are needed. + +#define __cilkrts_hyperobject_alloc __cilkrts_hyperobject_alloc__stub +__CILKRTS_INLINE + void* __cilkrts_hyperobject_alloc(void* ignore, __STDNS size_t bytes) + { assert(0); return __STDNS malloc(bytes); } + +#define __cilkrts_hyperobject_dealloc __cilkrts_hyperobject_dealloc__stub +__CILKRTS_INLINE + void __cilkrts_hyperobject_dealloc(void* ignore, void* view) + { assert(0); __STDNS free(view); } + +#define __cilkrts_hyperobject_noop_destroy \ + __cilkrts_hyperobject_noop_destroy__stub +__CILKRTS_INLINE + void __cilkrts_hyperobject_noop_destroy(void* ignore, void* ignore2) + {} + +#endif + +__CILKRTS_END_EXTERN_C + +#endif /* INCLUDED_CILK_HYPEROBJECT_BASE */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/metaprogramming.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/metaprogramming.h new file mode 100644 index 0000000000..2df7cf6467 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/metaprogramming.h @@ -0,0 +1,621 @@ +/* metaprogramming.h -*- C++ -*- + * + * Copyright (C) 2012-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + */ + +/** @file metaprogramming.h + * + * @brief Defines metaprogramming utility classes used in the Intel(R) Cilk(TM) Plus library. + * + * @ingroup common + */ + +#ifndef METAPROGRAMMING_H_INCLUDED +#define METAPROGRAMMING_H_INCLUDED + +#ifdef __cplusplus + +#include +#include +#include +#ifdef _WIN32 +#include +#endif +#include + +namespace cilk { + +namespace internal { + +/** Test if a class is empty. + * + * If @a Class is an empty (and therefore necessarily stateless) class, then + * the "empty base-class optimization" guarantees that + * `sizeof(check_for_empty_class) == sizeof(char)`. Conversely, if + * `sizeof(check_for_empty_class) > sizeof(char)`, then @a Class is not + * empty, and we must discriminate distinct instances of @a Class. + * + * Typical usage: + * + * // General definition of A for non-empty B: + * template ::value> > + * class A { ... }; + * + * // Specialized definition of A for empty B: + * template + * class A { ... }; + * + * @tparam Class The class to be tested for emptiness. + * + * @result The `value` member will be `true` if @a Class is empty, + * `false` otherwise. + * + * @ingroup common + */ +template +class class_is_empty { + class check_for_empty_class : public Class + { + char m_data; + public: + // Declared but not defined + check_for_empty_class(); + check_for_empty_class(const check_for_empty_class&); + check_for_empty_class& operator=(const check_for_empty_class&); + ~check_for_empty_class(); + }; +public: + + /** Constant is true if and only if @a Class is empty. + */ + static const bool value = (sizeof(check_for_empty_class) == sizeof(char)); +}; + + +/** Get the alignment of a type. + * + * For example: + * + * align_of::value == 8 + * + * @tparam Tp The type whose alignment is to be computed. + * + * @result The `value` member of an instantiation of this class template + * will hold the integral alignment requirement of @a Tp. + * + * @pre @a Tp shall be a complete type. + * + * @ingroup common + */ +template +struct align_of +{ +private: + struct imp { + char m_padding; + Tp m_val; + + // The following declarations exist to suppress compiler-generated + // definitions, in case @a Tp does not have a public default + // constructor, copy constructor, or destructor. + imp(const imp&); // Declared but not defined + ~imp(); // Declared but not defined + }; + +public: + /// The integral alignment requirement of @a Tp. + static const std::size_t value = (sizeof(imp) - sizeof(Tp)); +}; + + +/** A class containing raw bytes with a specified alignment and size. + * + * An object of type `aligned_storage` will have alignment `A` and + * size at least `S`. Its contents will be uninitialized bytes. + * + * @tparam Size The required minimum size of the resulting class. + * @tparam Alignment The required alignment of the resulting class. + * + * @pre @a Alignment shall be a power of 2 no greater than 64. + * + * @note This is implemented using the `CILK_ALIGNAS` macro, which uses + * the non-standard, implementation-specific features + * `__declspec(align(N))` on Windows, and + * `__attribute__((__aligned__(N)))` on Unix. The `gcc` implementation + * of `__attribute__((__aligned__(N)))` requires a numeric literal `N` + * (_not_ an arbitrary compile-time constant expression). Therefore, + * this class is implemented using specialization on the required + * alignment. + * + * @note The template class is specialized only for the supported + * alignments. An attempt to instantiate it for an unsupported + * alignment will result in a compilation error. + */ +template +struct aligned_storage; + +/// @cond +template class aligned_storage + { CILK_ALIGNAS( 1) char m_bytes[Size]; }; +template class aligned_storage + { CILK_ALIGNAS( 2) char m_bytes[Size]; }; +template class aligned_storage + { CILK_ALIGNAS( 4) char m_bytes[Size]; }; +template class aligned_storage + { CILK_ALIGNAS( 8) char m_bytes[Size]; }; +template class aligned_storage + { CILK_ALIGNAS(16) char m_bytes[Size]; }; +template class aligned_storage + { CILK_ALIGNAS(32) char m_bytes[Size]; }; +template class aligned_storage + { CILK_ALIGNAS(64) char m_bytes[Size]; }; +/// @endcond + +/** A buffer of uninitialized bytes with the same size and alignment as a + * specified type. + * + * The class `storage_for_object` will have the same size and alignment + * properties as `Type`, but it will contain only raw (uninitialized) bytes. + * This allows the definition of a data member which can contain a `Type` + * object which is initialized explicitly under program control, rather + * than implicitly as part of the initialization of the containing class. + * For example: + * + * class C { + * storage_for_object _member; + * public: + * C() ... // Does NOT initialize _member + * void initialize(args) + * { new (_member.pointer()) MemberClass(args); } + * const MemberClass& member() const { return _member.object(); } + * MemberClass& member() { return _member.object(); } + * + * @tparam Type The type whose size and alignment are to be reflected + * by this class. + */ +template +class storage_for_object : + aligned_storage< sizeof(Type), align_of::value > +{ +public: + /// Return a typed reference to the buffer. + const Type& object() const { return *reinterpret_cast(this); } + /// Return a typed reference to the buffer. + Type& object() { return *reinterpret_cast(this); } +}; + + +/** Get the functor class corresponding to a binary function type. + * + * The `binary_functor` template class can be instantiated with a binary + * functor class or with a real binary function, and will yield an equivalent + * binary functor class in either case. + * + * @tparam F A binary functor class, a binary function type, or a pointer to + * binary function type. + * + * @result `binary_functor::%type` will be the same as @a F if @a F is + * a class. It will be a `std::pointer_to_binary_function` wrapper + * if @a F is a binary function or binary function pointer type. + * (It will _not_ necessarily be an `Adaptable Binary Function` + * class, since @a F might be a non-adaptable binary functor + * class.) + * + * @ingroup common + */ +template +struct binary_functor { + /// The binary functor class equivalent to @a F. + typedef F type; +}; + +/// @copydoc binary_functor +/// Specialization for binary function. +template +struct binary_functor { + /// The binary functor class equivalent to @a F. + typedef std::pointer_to_binary_function type; +}; + +/// @copydoc binary_functor +/// Specialization for pointer to binary function. +template +struct binary_functor { + /// The binary functor class equivalent to @a F. + typedef std::pointer_to_binary_function type; +}; + + +/** Indirect binary function class with specified types. + * + * `typed_indirect_binary_function` is an `Adaptable Binary Function` class + * based on an existing binary functor class or binary function type @a F. If + * @a F is a stateless class, then this class will be empty, and its + * `operator()` will invoke @a F's `operator()`. Otherwise, an object of this + * class will hold a pointer to an object of type @a F, and will refer its + * `operator()` calls to the pointed-to @a F object. + * + * That is, suppose that we have the declarations: + * + * F *p; + * typed_indirect_binary_function ibf(p); + * + * Then: + * + * - `ibf(x, y) == (*p)(x, y)`. + * - `ibf(x, y)` will not do a pointer dereference if `F` is an empty class. + * + * @note Just to repeat: if `F` is an empty class, then + * `typed_indirect_binary_function\' is also an empty class. + * This is critical for its use in the + * @ref cilk::cilk_lib_1_1::min_max_internal::view_base + * "min/max reducer view classes", where it allows the view to + * call a comparison functor in the monoid without actually + * having to allocate a pointer in the view class when the + * comparison class is empty. + * + * @note If you have an `Adaptable Binary Function` class or a binary + * function type, then you can use the + * @ref indirect_binary_function class, which derives the + * argument and result types parameter type instead of requiring + * you to specify them as template arguments. + * + * @tparam F A binary functor class, a binary function type, or a pointer to + * binary function type. + * @param A1 The first argument type. + * @param A2 The second argument type. + * @param R The result type. + * + * @see min_max::comparator_base + * @see indirect_binary_function + * + * @ingroup common + */ +template < typename F + , typename A1 + , typename A2 + , typename R + , typename Functor = typename binary_functor::type + , bool FunctorIsEmpty = class_is_empty::value + > +class typed_indirect_binary_function : std::binary_function +{ + const F* f; +public: + /// Constructor captures a pointer to the wrapped function. + typed_indirect_binary_function(const F* f) : f(f) {} + + /// Return the comparator pointer, or `NULL` if the comparator is stateless. + const F* pointer() const { return f; } + + /// Apply the pointed-to functor to the arguments. + R operator()(const A1& a1, const A2& a2) const { return (*f)(a1, a2); } +}; + + +/// @copydoc typed_indirect_binary_function +/// Specialization for an empty functor class. (This is only possible if @a F +/// itself is an empty class. If @a F is a function or pointer-to-function +/// type, then the functor will contain a pointer.) +template +class typed_indirect_binary_function : + std::binary_function +{ +public: + /// Return `NULL` for the comparator pointer of a stateless comparator. + const F* pointer() const { return 0; } + + /// Constructor discards the pointer to a stateless functor class. + typed_indirect_binary_function(const F* f) {} + + /// Create an instance of the stateless functor class and apply it to the arguments. + R operator()(const A1& a1, const A2& a2) const { return F()(a1, a2); } +}; + + +/** Indirect binary function class with inferred types. + * + * This is identical to @ref cilk::internal::typed_indirect_binary_function, + * except that it derives the binary function argument and result types from + * the parameter type @a F instead of taking them as additional template + * parameters. If @a F is a class type, then it must be an `Adaptable Binary + * Function`. + * + * @see typed_indirect_binary_function + * + * @ingroup common + */ +template ::type> +class indirect_binary_function : + typed_indirect_binary_function< F + , typename Functor::first_argument_type + , typename Functor::second_argument_type + , typename Functor::result_type + > +{ + typedef typed_indirect_binary_function< F + , typename Functor::first_argument_type + , typename Functor::second_argument_type + , typename Functor::result_type + > + base; +public: + indirect_binary_function(const F* f) : base(f) {} ///< Constructor +}; + + +/** Choose a type based on a boolean constant. + * + * This metafunction is identical to C++11's condition metafunction. + * It needs to be here until we can reasonably assume that users will be + * compiling with C++11. + * + * @tparam Cond A boolean constant. + * @tparam IfTrue A type. + * @tparam IfFalse A type. + * @result The `type` member will be a typedef of @a IfTrue if @a Cond + * is true, and a typedef of @a IfFalse if @a Cond is false. + * + * @ingroup common + */ +template +struct condition +{ + typedef IfTrue type; ///< The type selected by the condition. +}; + +/// @copydoc condition +/// Specialization for @a Cond == `false`. +template +struct condition +{ + typedef IfFalse type; ///< The type selected by the condition. +}; + + +/** @def __CILKRTS_STATIC_ASSERT + * + * @brief Compile-time assertion. + * + * Causes a compilation error if a compile-time constant expression is false. + * + * @par Usage example. + * This assertion is used in reducer_min_max.h to avoid defining + * legacy reducer classes that would not be binary-compatible with the + * same classes compiled with earlier versions of the reducer library. + * + * __CILKRTS_STATIC_ASSERT( + * internal::class_is_empty< internal::binary_functor >::value, + * "cilk::reducer_max only works with an empty Compare class"); + * + * @note In a C++11 compiler, this is just the language predefined + * `static_assert` macro. + * + * @note In a non-C++11 compiler, the @a Msg string is not directly included + * in the compiler error message, but it may appear if the compiler + * prints the source line that the error occurred on. + * + * @param Cond The expression to test. + * @param Msg A string explaining the failure. + * + * @ingroup common + */ +#if defined(__INTEL_CXX11_MODE__) || defined(__GXX_EXPERIMENTAL_CXX0X__) +# define __CILKRTS_STATIC_ASSERT(Cond, Msg) static_assert(Cond, Msg) +#else +# define __CILKRTS_STATIC_ASSERT(Cond, Msg) \ + typedef int __CILKRTS_STATIC_ASSERT_DUMMY_TYPE \ + [::cilk::internal::static_assert_failure<(Cond)>::Success] + +/// @cond internal + template struct static_assert_failure { }; + template <> struct static_assert_failure { enum { Success = 1 }; }; + +# define __CILKRTS_STATIC_ASSERT_DUMMY_TYPE \ + __CILKRTS_STATIC_ASSERT_DUMMY_TYPE1(__cilkrts_static_assert_, __LINE__) +# define __CILKRTS_STATIC_ASSERT_DUMMY_TYPE1(a, b) \ + __CILKRTS_STATIC_ASSERT_DUMMY_TYPE2(a, b) +# define __CILKRTS_STATIC_ASSERT_DUMMY_TYPE2(a, b) a ## b +/// @endcond + +#endif + +/// @cond internal + +/** @name Aligned heap management. + */ +//@{ + +/** Implementation-specific aligned memory allocation function. + * + * @param size The minimum number of bytes to allocate. + * @param alignment The required alignment (must be a power of 2). + * @return The address of a block of memory of at least @a size + * bytes. The address will be a multiple of @a alignment. + * `NULL` if the allocation fails. + * + * @see deallocate_aligned() + */ +inline void* allocate_aligned(std::size_t size, std::size_t alignment) +{ +#ifdef _WIN32 + return _aligned_malloc(size, alignment); +#else +#if defined(__ANDROID__) || defined(__VXWORKS__) + return memalign(std::max(alignment, sizeof(void*)), size); +#else + void* ptr; + return (posix_memalign(&ptr, std::max(alignment, sizeof(void*)), size) == 0) ? ptr : 0; +#endif +#endif +} + +/** Implementation-specific aligned memory deallocation function. + * + * @param ptr A pointer which was returned by a call to alloc_aligned(). + */ +inline void deallocate_aligned(void* ptr) +{ +#ifdef _WIN32 + _aligned_free(ptr); +#else + std::free(ptr); +#endif +} + +/** Class to allocate and guard an aligned pointer. + * + * A new_aligned_pointer object allocates aligned heap-allocated memory when + * it is created, and automatically deallocates it when it is destroyed + * unless its `ok()` function is called. + * + * @tparam T The type of the object to allocate on the heap. The allocated + * will have the size and alignment of an object of type T. + */ +template +class new_aligned_pointer { + void* m_ptr; +public: + /// Constructor allocates the pointer. + new_aligned_pointer() : + m_ptr(allocate_aligned(sizeof(T), internal::align_of::value)) {} + /// Destructor deallocates the pointer. + ~new_aligned_pointer() { if (m_ptr) deallocate_aligned(m_ptr); } + /// Get the pointer. + operator void*() { return m_ptr; } + /// Return the pointer and release the guard. + T* ok() { + T* ptr = static_cast(m_ptr); + m_ptr = 0; + return ptr; + } +}; + +//@} + +/// @endcond + +} // namespace internal + +//@{ + +/** Allocate an aligned data structure on the heap. + * + * `cilk::aligned_new([args])` is equivalent to `new T([args])`, except + * that it guarantees that the returned pointer will be at least as aligned + * as the alignment requirements of type `T`. + * + * @ingroup common + */ +template +T* aligned_new() +{ + internal::new_aligned_pointer ptr; + new (ptr) T(); + return ptr.ok(); +} + +template +T* aligned_new(const T1& x1) +{ + internal::new_aligned_pointer ptr; + new (ptr) T(x1); + return ptr.ok(); +} + +template +T* aligned_new(const T1& x1, const T2& x2) +{ + internal::new_aligned_pointer ptr; + new (ptr) T(x1, x2); + return ptr.ok(); +} + +template +T* aligned_new(const T1& x1, const T2& x2, const T3& x3) +{ + internal::new_aligned_pointer ptr; + new (ptr) T(x1, x2, x3); + return ptr.ok(); +} + +template +T* aligned_new(const T1& x1, const T2& x2, const T3& x3, const T4& x4) +{ + internal::new_aligned_pointer ptr; + new (ptr) T(x1, x2, x3, x4); + return ptr.ok(); +} + +template +T* aligned_new(const T1& x1, const T2& x2, const T3& x3, const T4& x4, const T5& x5) +{ + internal::new_aligned_pointer ptr; + new (ptr) T(x1, x2, x3, x4, x5); + return ptr.ok(); +} + +//@} + + +/** Deallocate an aligned data structure on the heap. + * + * `cilk::aligned_delete(ptr)` is equivalent to `delete ptr`, except that it + * operates on a pointer that was allocated by aligned_new(). + * + * @ingroup common + */ +template +void aligned_delete(const T* ptr) +{ + ptr->~T(); + internal::deallocate_aligned((void*)ptr); +} + +} // namespace cilk + +#endif // __cplusplus + +#endif // METAPROGRAMMING_H_INCLUDED diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer.h new file mode 100644 index 0000000000..09c2e19690 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer.h @@ -0,0 +1,2002 @@ +/* reducer.h -*- C++ -*- + * + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + */ + +/** @file reducer.h + * + * @brief Defines foundation classes for creating Intel(R) Cilk(TM) Plus reducers. + * + * @ingroup Reducers + * + * @see @ref pagereducers + * + * @defgroup Reducers Reducers + */ + +#ifndef REDUCER_H_INCLUDED +#define REDUCER_H_INCLUDED + +#include "cilk/hyperobject_base.h" +#include "cilk/metaprogramming.h" + +#ifdef __cplusplus + +//===================== C++ interfaces =================================== + +#include + +namespace cilk { + +/** Class for provisionally constructed objects. + * + * The monoid_base::construct() functions manually construct both a + * monoid and a view. If one of these is constructed successfully, and the + * construction of the other (or some other initialization) fails, then the + * first one must be destroyed to avoid a memory leak. Because the + * construction is explicit, the destruction must be explicit, too. + * + * A provisional_guard object wraps a pointer to a newly constructed + * object. A call to its confirm() function confirms that the object is + * really going to be used. If the guard is destroyed without being + * confirmed, then the pointed-to object is destroyed (but not + * deallocated). + * + * Expected usage: + * + * provisional_guard x1_provisional( new (x1) T1 ); + * … more initialization … + * x1_provisional.confirm(); + * + * or + * + * provisional_guard x1_provisional( new (x1) T1 ); + * x1_provisional.confirm_if( new (x2) T2 ); + * + * If an exception is thrown in the "more initialization" code in the + * first example, or in the `T2` constructor in the second example, then + * `x1_provisional` will not be confirmed, so when its destructor is + * called during exception unwinding, the `T1` object that was constructed + * in `x1` will be destroyed. + * + * **NOTE**: Do *not* be tempted to chain a `provisional_guard` + * constructor with `confirm_if` as in this example: + * + * // BAD IDEA + * provisional_guard( new (x1) T1 ).confirm_if( new (x2) T2 ); + * + * The code above is problematic because the evaluation of the T2 + * constructor is unsequenced with respect to the call to the + * `provisional_guard` constructor (and with respect the T1 constructor). + * Thus, the compiler may choose to evaluate `new (x2) T2` before + * constructing the guard and leak the T1 object if the `T2` constructor + * throws. + * + * @tparam Type The type of the provisionally constructed object. + */ +template +class provisional_guard { + Type* m_ptr; + +public: + + /** Constructor. Creates a guard for a provisionally constructed object. + * + * @param ptr A pointer to the provisionally constructed object. + */ + provisional_guard(Type* ptr) : m_ptr(ptr) {} + + /** Destructor. Destroy the object pointed to by the contained pointer + * if it has not been confirmed. + */ + ~provisional_guard() { if (m_ptr) m_ptr->~Type(); } + + /** Confirm the provisional construction. Do *not* delete the contained + * pointer when the guard is destroyed. + */ + void confirm() { m_ptr = 0; } + + /** Confirm provisional construction if argument is non-null. Note that + * if an exception is thrown during evaluation of the argument + * expression, then this function will not be called, and the + * provisional object will not be confirmed. This allows the usage: + * + * x1_provisional.confirm_if( new (x2) T2() ); + * + * @param cond An arbitrary pointer. The provisional object will be + * confirmed if @a cond is not null. + * + * @returns The value of the @a cond argument. + */ + template + Cond* confirm_if(Cond* cond) { if (cond) m_ptr = 0; return cond; } +}; + +/** Base class for defining monoids. + * + * The monoid_base class template is useful for creating classes that model + * the monoid concept. It provides the core type and memory management + * functionality. A subclass of monoid_base need only declare and implement + * the `identity` and `reduce` functions. + * + * The monoid_base class also manages the integration between the monoid, the + * reducer class that is based on it, and an optional view class which wraps + * value objects and restricts access to their operations. + * + * @tparam Value The value type for the monoid. + * @tparam View An optional view class that serves as a proxy for the value + * type. + * + * @see monoid_with_view + */ +template +class monoid_base +{ + +public: + + /** Value type of the monoid. + */ + typedef Value value_type; + + /** View type of the monoid. Defaults to be the same as the value type. + * @see monoid_with_view + */ + typedef View view_type; + + enum { + /** Should reducers created with this monoid be aligned? + * + * @details + * "Aligned" means that the view is allocated at a cache-line aligned + * offset in the reducer, and the reducer must be cache-line aligned. + * "Unaligned" means that the reducer as a whole is just naturally + * aligned, but it contains a large enough block of uninitialized + * storage for a cache-line aligned view to be allocated in it at + * reducer construction time. + * + * Since the standard heap allocator (new reducer) does not allocate + * cache-line aligned storage, only unaligned reducers can be safely + * allocated on the heap. + * + * Default is false (unaligned) unless overridden in a subclass. + * + * @since 1.02 + * (In Intel Cilk Plus library versions 1.0 and 1.01, the default was true. + * In Intel Cilk Plus library versions prior to 1.0, reducers were always + * aligned, and this data member did not exist.) + */ + align_reducer = false + }; + + /** Destroys a view. Destroys (without deallocating) the @a View object + * pointed to by @a p. + * + * @param p The address of the @a View object to be destroyed. + */ + void destroy(view_type* p) const { p->~view_type(); } + + /** Allocates raw memory. Allocate @a s bytes of memory with no + * initialization. + * + * @param s The number of bytes of memory to allocate. + * @return An untyped pointer to the allocated memory. + */ + void* allocate(size_t s) const { return operator new(s); } + + /** Deallocates raw memory pointed to by @a p + * without doing any destruction. + * + * @param p Pointer to the memory to be deallocated. + * + * @pre @a p points to a block of memory that was allocated by a + * call to allocate(). + */ + void deallocate(void* p) const { operator delete(p); } + + /** Creates the identity value. Constructs (without allocating) a @a View + * object representing the default value of the @a Value type. + * + * @param p A pointer to a block of raw memory large enough to hold a + * @a View object. + * + * @post The memory pointed to by @a p contains a @a View object that + * represents the default value of the @a View type. + * + * @deprecated This function constructs the @a View object with its default + * constructor, which will often, but not always, yield the + * appropriate identity value. Monoid classes should declare + * their identity function explicitly, rather than relying on + * this default definition. + */ + void identity(View* p) const { new ((void*) p) View(); } + + + /** @name Constructs the monoid and the view with arbitrary arguments. + * + * A @ref reducer object contains monoid and view data members, which are + * declared as raw storage (byte arrays), so that they are not implicitly + * constructed when the reducer is constructed. Instead, a reducer + * constructor calls one of the monoid class's static construct() + * functions with the addresses of the monoid and the view, and the + * construct() function uses placement `new` to construct them. + * This allows the monoid to determine the order in which the monoid and + * view are constructed, and to make one of them dependent on the other. + * + * Any arguments to the reducer constructor are just passed on as + * additional arguments to the construct() function (after the monoid + * and view addresses are set). + * + * A monoid whose needs are satisfied by the suite of construct() + * functions below, such as @ref monoid_with_view, can just inherit them + * from monoid_base. Other monoids will need to provide their own versions + * to override the monoid_base functions. + */ + //@{ + + /** Default-constructs the monoid, identity-constructs the view. + * + * @param monoid Address of uninitialized monoid object. + * @param view Address of uninitialized initial view object. + */ + //@{ + template + static void construct(Monoid* monoid, View* view) + { + provisional_guard guard( new((void*) monoid) Monoid() ); + monoid->identity(view); + guard.confirm(); + } + //@} + + /** Default-constructs the monoid, and passes one to five const reference + * arguments to the view constructor. + */ + //@{ + + template + static void construct(Monoid* monoid, View* view, const T1& x1) + { + provisional_guard guard( new((void*) monoid) Monoid() ); + guard.confirm_if( new((void*) view) View(x1) ); + } + + template + static void construct(Monoid* monoid, View* view, + const T1& x1, const T2& x2) + { + provisional_guard guard( new((void*) monoid) Monoid() ); + guard.confirm_if( new((void*) view) View(x1, x2) ); + } + + template + static void construct(Monoid* monoid, View* view, + const T1& x1, const T2& x2, const T3& x3) + { + provisional_guard guard( new((void*) monoid) Monoid() ); + guard.confirm_if( new((void*) view) View(x1, x2, x3) ); + } + + template + static void construct(Monoid* monoid, View* view, + const T1& x1, const T2& x2, const T3& x3, + const T4& x4) + { + provisional_guard guard( new((void*) monoid) Monoid() ); + guard.confirm_if( new((void*) view) View(x1, x2, x3, x4) ); + } + + template + static void construct(Monoid* monoid, View* view, + const T1& x1, const T2& x2, const T3& x3, + const T4& x4, const T5& x5) + { + provisional_guard guard( new((void*) monoid) Monoid() ); + guard.confirm_if( new((void*) view) View(x1, x2, x3, x4, x5) ); + } + + //@} + + /** Default-constructs the monoid, and passes one non-const reference + * argument to the view constructor. + */ + //@{ + template + static void construct(Monoid* monoid, View* view, T1& x1) + { + provisional_guard guard( new((void*) monoid) Monoid() ); + guard.confirm_if( new((void*) view) View(x1) ); + } + //@} + + /** Copy-constructs the monoid, and identity-constructs the view + * constructor. + * + * @param monoid Address of uninitialized monoid object. + * @param view Address of uninitialized initial view object. + * @param m Object to be copied into `*monoid` + */ + //@{ + template + static void construct(Monoid* monoid, View* view, const Monoid& m) + { + provisional_guard guard( new((void*) monoid) Monoid(m) ); + monoid->identity(view); + guard.confirm(); + } + //@} + + /** Copy-constructs the monoid, and passes one to four const reference + * arguments to the view constructor. + */ + //@{ + + template + static void construct(Monoid* monoid, View* view, const Monoid& m, + const T1& x1) + { + provisional_guard guard( new((void*) monoid) Monoid(m) ); + guard.confirm_if( new((void*) view) View(x1) ); + } + + template + static void construct(Monoid* monoid, View* view, const Monoid& m, + const T1& x1, const T2& x2) + { + provisional_guard guard( new((void*) monoid) Monoid(m) ); + guard.confirm_if( new((void*) view) View(x1, x2) ); + } + + template + static void construct(Monoid* monoid, View* view, const Monoid& m, + const T1& x1, const T2& x2, const T3& x3) + { + provisional_guard guard( new((void*) monoid) Monoid(m) ); + guard.confirm_if( new((void*) view) View(x1, x2, x3) ); + } + + template + static void construct(Monoid* monoid, View* view, const Monoid& m, + const T1& x1, const T2& x2, const T3& x3, + const T4& x4) + { + provisional_guard guard( new((void*) monoid) Monoid(m) ); + guard.confirm_if( new((void*) view) View(x1, x2, x3, x4) ); + } + + //@} + + //@} +}; + + +/** Monoid class that gets its value type and identity and reduce operations + * from its view. + * + * A simple implementation of the monoid-view-reducer architecture would + * distribute knowledge about the type and operations for the reduction + * between the monoid and the view - the identity and reduction operations are + * specified in the monoid, the reduction operations are implemented in the + * view, and the value type is specified in both the monoid and the view. + * This is inelegant. + * + * monoid_with_view is a subclass of @ref monoid_base that gets its value type + * and its identity and reduction operations from its view class. No + * customization of the monoid_with_view class itself is needed beyond + * instantiating it with an appropriate view class. (Customized subclasses of + * monoid_with_view may be needed for other reasons, such as to keep some + * state for the reducer.) All of the Intel Cilk Plus predefined reducers use + * monoid_with_view or one of its subclasses. + * + * The view class `View` of a monoid_with_view must provide the following + * public definitions: + * + * Definition | Meaning + * ---------------------------------|-------- + * `value_type` | a typedef of the value type for the reduction + * `View()` | a default constructor which constructs the identity value for the reduction + * `void reduce(const View* other)` | a member function which applies the reduction operation to the values of `this` view and the `other` view, leaving the result as the value of `this` view, and leaving the value of the `other` view undefined (but valid) + * + * @tparam View The view class for the monoid. + * @tparam Align If true, reducers instantiated on this monoid will be + * cache-aligned. By default, library reducers (unlike legacy + * library reducer _wrappers_) are aligned only as required by + * contents. + */ +template +class monoid_with_view : public monoid_base +{ +public: + /** Should reducers created with this monoid be aligned? + */ + enum { align_reducer = Align }; + + /** Create the identity value. + * + * Implements the monoid `identity` operation by using the @a View class's + * default constructor. + * + * @param p A pointer to a block of raw memory large enough to hold a + * @p View object. + */ + void identity(View* p) const { new((void*) p) View(); } + + /** Reduce the values of two views. + * + * Implements the monoid `reduce` operation by calling the left view's + * `%reduce()` function with the right view as an operand. + * + * @param left The left operand of the reduce operation. + * @param right The right operand of the reduce operation. + * @post The left view contains the result of the reduce + * operation, and the right view is undefined. + */ + void reduce(View* left, View* right) const { left->reduce(right); } +}; + + +/** Base class for simple views with (usually) scalar values. + * + * The scalar_view class is intended as a base class which provides about half + * of the required definitions for simple views. It defines the `value_type` + * required by a @ref monoid_with_view (but not the identity constructor and + * reduce operation, which are inherently specific to a particular kind of + * reduction). It also defines the value access functions which will be called + * by the corresponding @ref reducer functions. (It uses copy semantics for + * the view_move_in() and view_move_out() functions, which is appropriate + * for simple scalar types, but not necessarily for more complex types like + * STL containers. + * + * @tparam Type The type of value wrapped by the view. + */ +template +class scalar_view +{ +protected: + Type m_value; ///< The wrapped accumulator variable. + +public: + /** Value type definition required by @ref monoid_with_view. + */ + typedef Type value_type; + + /** Default constructor. + */ + scalar_view() : m_value() {} + + /** Value constructor. + */ + scalar_view(const Type& v) : m_value(v) {} + + /** @name Value functions required by the reducer class. + * + * Note that the move in/out functions use simple assignment semantics. + */ + //@{ + + /** Set the value of the view. + */ + void view_move_in(Type& v) { m_value = v; } + + /** Get the value of the view. + */ + void view_move_out(Type& v) { v = m_value; } + + /** Set the value of the view. + */ + void view_set_value(const Type& v) { m_value = v; } + + /** Get the value of the view. + */ + Type const& view_get_value() const { return m_value; } + + /** Type returned by view_get_value. + */ + typedef Type const& return_type_for_get_value; + + /** Get a reference to the value contained in the view. For legacy + * reducer support only. + */ + Type & view_get_reference() { return m_value; } + + /** Get a reference to the value contained in the view. For legacy + * reducer support only. + */ + Type const& view_get_reference() const { return m_value; } + //@} +}; + + +/** Wrapper class for move-in construction. + * + * Some types allow their values to be _moved_ as an alternative to copying. + * Moving a value may be much faster than copying it, but may leave the value + * of the move's source undefined. Consider the `swap` operation provided by + * many STL container classes: + * + * list x, y; + * x = y; // Copy + * x.swap(y); // Move + * + * The assignment _copies_ the value of `y` into `x` in time linear in the + * size of `y`, leaving `y` unchanged. The `swap` _moves_ the value of `y` + * into `x` in constant time, but it also moves the value of `x` into `y`, + * potentially leaving `y` undefined. + * + * A move_in_wrapper simply wraps a pointer to an object. It is created by a + * call to cilk::move_in(). Passing a move_in_wrapper to a view constructor + * (actually, passing it to a reducer constructor, which passes it to the + * monoid `construct()` function, which passes it to the view constructor) + * allows, but does not require, the value pointed to by the wrapper to be + * moved into the view instead of copied. + * + * A view class exercises this option by defining a _move-in constructor_, + * i.e., a constructor with a move_in_wrapper parameter. The constructor calls + * the wrapper's `value()` function to get a reference to its pointed-to + * value, and can then use that reference in a move operation. + * + * A move_in_wrapper also has an implicit conversion to its pointed-to value, + * so if a view class does not define a move-in constructor, its ordinary + * value constructor will be called with the wrapped value. For example, an + * @ref ReducersAdd "op_add" view does not have a move-in constructor, so + * + * int x; + * reducer< op_add > xr(move_in(x)); + * + * will simply call the `op_add_view(const int &)` constructor. But an + * @ref ReducersList "op_list_append" view does have a move-in constructor, + * so + * + * list x; + * reducer< op_list_append > xr(move_in(x)); + * + * will call the `op_list_append_view(move_in_wrapper)` constructor, + * which can `swap` the value of `x` into the view. + * + * @note Remember that passing the value of a variable to a reducer + * constructor using a move_in_wrapper leaves the variable undefined. + * You cannot assume that the constructor either will or will not copy + * or move the value. + * + * @tparam Type The type of the wrapped value. + * + * @see cilk::move_in() + */ +template +class move_in_wrapper +{ + Type *m_pointer; +public: + + /** Constructor that captures the address of its argument. This is almost + * always called from the @ref move_in function. + */ + explicit move_in_wrapper(Type& ref) : m_pointer(&ref) { } + + /** Implicit conversion to the wrapped value. This allows a move_in_wrapper + * to be used where a value of the wrapped type is expected, in which case + * the wrapper is completely transparent. + */ + operator Type&() const { return *m_pointer; } + + /** Get a reference to the pointed-to value. This has the same effect as + * the implicit conversion, but makes the intent clearer in a move-in + * constructor. + */ + Type& value() const { return *m_pointer; } +}; + +/** Function to create a move_in_wrapper for a value. + * + * @tparam Type The type of the argument, which will be the `type` of the + * created wrapper. + * + * @see move_in_wrapper + */ +template +inline +move_in_wrapper move_in(Type& ref) + { return move_in_wrapper(ref); } + + +/** @copydoc move_in(Type&) + * + * @note Applying a function that is explicitly specified as modifying its + * argument to a const argument is obviously an irrational thing to + * do. This move_in() variant is just provided to allow calling a + * move-in constructor with a function return value, which the + * language treats as a const. Using it for any other purpose will + * probably end in tears. + */ +template +inline +move_in_wrapper move_in(const Type& ref) + { return move_in_wrapper(ref); } + + +/** Wrapper class to allow implicit downcasts to reducer subclasses. + * + * The Intel Cilk Plus library contains a collection of reducer wrapper classes which + * were created before the `cilk::reducer` style was developed. For + * example, `cilk::reducer_opadd` provided essentially the same + * functionality that is now provided by + * `cilk::reducer< cilk::op_add >`. These legacy reducer classes are + * deprecated, but still supported, and they have been reimplemented as + * subclasses of the corresponding `cilk::reducer` classes. For example: + * + * template + * reducer_opadd : public reducer< op_add > { ... }; + * + * This reimplementation allows transparent conversion between legacy and + * new reducers. That is, a `reducer*` or `reducer&` can be + * used anywhere that a `reducer_opadd*` or `reducer_opadd&` is expected, + * and vice versa. + * + * The conversion from the legacy reducer to the new reducer is just an + * up-cast, which is provided for free by C++. The conversion from the new + * reducer to the legacy reducer is a down-cast, though, which requires an + * explicit conversion member function in the `reducer` class. The challenge + * is to define a function in the reducer template class which will convert + * each cilk::reducer specialization to the corresponding legacy reducer, + * if there is one. + * + * The trick is in the legacy_reducer_downcast template class, which provides + * a mapping from `cilk::reducer` specializations to legacy reducer classes. + * `reducer` has a conversion function to convert itself to + * `legacy_reducer_downcast< reducer >::%type`. By default, + * `legacy_reducer_downcast::%type` is just a trivial subclass of + * `Reducer`, which is uninteresting, but a reducer with a legacy counterpart + * will have a specialization of `legacy_reducer_downcast` whose `type` is + * the corresponding legacy reducer. For example: + * + * template + * struct legacy_reducer_downcast< reducer< op_add > > + * { + * typedef reducer_opadd type; + * }; + * + * + * @tparam Reducer The new-style reducer class whose corresponding legacy + * reducer class is `type`, if there is such a legacy reducer + * class. + */ +template +struct legacy_reducer_downcast +{ + /** The related legacy reducer class. + * + * By default, this is just a trivial subclass of Reducer, but it can be + * overridden in the specialization of legacy_reducer_downcast for + * a reducer that has a corresponding legacy reducers. + */ + struct type : Reducer { }; +}; + + +namespace internal { +/// @cond internal + +template +struct reducer_set_get +{ + // sizeof(notchar) != sizeof(char) + struct notchar { char x[2]; }; + + // `does_view_define_return_type_for_get_value(View*)` returns `char` if + // `View` defines `return_type_for_get_value`, and `notchar` if it doesn't. + + template + struct using_type {}; + + template + static char does_view_define_return_type_for_get_value( + using_type*); + + template + static notchar does_view_define_return_type_for_get_value(...); + + // `VIEW_DOES_DEFINE_RETURN_TYPE_FOR_GET_VALUE` is true if `View` defines + // `return_type_for_get_value`. + + enum { VIEW_DOES_DEFINE_RETURN_TYPE_FOR_GET_VALUE = + sizeof( does_view_define_return_type_for_get_value(0) ) + == sizeof(char) } ; + + // `return_type_for_get_value` is `View::return_type_for_get_value` + // if it is defined, and just `Value` otherwise. + + template + struct return_type_for_view_get_value { + typedef Value type; + }; + + template + struct return_type_for_view_get_value { + typedef typename InnerView::return_type_for_get_value type; + }; + +public: + + typedef + typename + return_type_for_view_get_value< + View, + VIEW_DOES_DEFINE_RETURN_TYPE_FOR_GET_VALUE + >::type + return_type_for_get_value; + + static void move_in(View& view, Value& v) { view.view_move_in(v); } + static void move_out(View& view, Value& v) { view.view_move_out(v); } + + static void set_value(View& view, const Value& v) + { view.view_set_value(v); } + + static return_type_for_get_value get_value(const View& view) + { return view.view_get_value(); } +}; + +template +struct reducer_set_get +{ + typedef const Value& return_type_for_get_value; + + static void move_in(Value& view, Value& v) { view = v; } + static void move_out(Value& view, Value& v) { v = view; } + + static void set_value(Value& view, const Value& v) + { view = v; } + + static return_type_for_get_value get_value(const Value& view) + { return view; } +}; + +/// @endcond + + +/** Base class defining the data layout that is common to all reducers. + */ +template +class reducer_base { + typedef typename Monoid::view_type view_type; + + // This makes the reducer a hyper-object. (Partially initialized in + // the derived reducer_content class.) + // + __cilkrts_hyperobject_base m_base; + + // The monoid is allocated here as raw bytes, and is constructed explicitly + // by a call to the monoid_type::construct() function in the constructor of + // the `reducer` subclass. + // + storage_for_object m_monoid; + + // Used for sanity checking at destruction. + // + void* m_initialThis; + + // The leftmost view comes next. It is defined in the derived + // reducer_content class. + + /** @name C-callable wrappers for the C++-coded monoid dispatch functions. + */ + //@{ + + static void reduce_wrapper(void* r, void* lhs, void* rhs); + static void identity_wrapper(void* r, void* view); + static void destroy_wrapper(void* r, void* view); + static void* allocate_wrapper(void* r, __STDNS size_t bytes); + static void deallocate_wrapper(void* r, void* view); + + //@} + +protected: + + /** Constructor. + * + * @param leftmost The address of the leftmost view in the reducer. + */ + reducer_base(char* leftmost) + { + static const cilk_c_monoid c_monoid_initializer = { + (cilk_c_reducer_reduce_fn_t) &reduce_wrapper, + (cilk_c_reducer_identity_fn_t) &identity_wrapper, + (cilk_c_reducer_destroy_fn_t) &destroy_wrapper, + (cilk_c_reducer_allocate_fn_t) &allocate_wrapper, + (cilk_c_reducer_deallocate_fn_t) &deallocate_wrapper + }; + + m_base.__c_monoid = c_monoid_initializer; + m_base.__flags = 0; + m_base.__view_offset = (char*)leftmost - (char*)this; + m_base.__view_size = sizeof(view_type); + m_initialThis = this; + + __cilkrts_hyper_create(&m_base); + } + + /** Destructor. + */ + __CILKRTS_STRAND_STALE(~reducer_base()) + { + // Make sure we haven't been memcopy'd or corrupted + __CILKRTS_ASSERT( + this == m_initialThis || + // Allow for a layout bug that may put the initialThis field one + // word later in 1.0 reducers than in 0.9 and 1.1 reducers. + this == *(&m_initialThis + 1) + ); + __cilkrts_hyper_destroy(&m_base); + } + + /** Monoid data member. + * + * @return A pointer to the reducer's monoid data member. + */ + Monoid* monoid_ptr() { return &m_monoid.object(); } + + /** Leftmost view data member. + * + * @return A pointer to the reducer's leftmost view data member. + * + * @note This function returns the address of the *leftmost* view, + * which is unique for the lifetime of the reducer. It is + * intended to be used in constructors and destructors. + * Use the reducer::view() function to access the per-strand + * view instance. + */ + view_type* leftmost_ptr() + { + char* view_addr = (char*)this + m_base.__view_offset; + return reinterpret_cast(view_addr); + } + +public: + + /** @name Access the current view. + * + * These functions return a reference to the instance of the reducer's + * view that was created for the current strand of a parallel computation + * (and create it if it doesn't already exist). Note the difference from + * the (private) leftmost_ptr() function, which returns a pointer to the + * _leftmost_ view, which is the same in all strands. + */ + //@{ + + /** Per-strand view instance. + * + * @return A reference to the per-strand view instance. + */ + view_type& view() + { + return *static_cast(__cilkrts_hyper_lookup(&m_base)); + } + + /** @copydoc view() + */ + const view_type& view() const + { + return const_cast(this)->view(); + } + + //@} + + /** Initial view pointer field. + * + * @internal + * + * @return a reference to the m_initialThis field. + * + * @note This function is provided for "white-box" testing of the + * reducer layout code. There is never any reason for user code + * to call it. + */ + const void* const & initial_this() const { return m_initialThis; } +}; + +template +void reducer_base::reduce_wrapper(void* r, void* lhs, void* rhs) +{ + Monoid* monoid = static_cast(r)->monoid_ptr(); + monoid->reduce(static_cast(lhs), + static_cast(rhs)); +} + +template +void reducer_base::identity_wrapper(void* r, void* view) +{ + Monoid* monoid = static_cast(r)->monoid_ptr(); + monoid->identity(static_cast(view)); +} + +template +void reducer_base::destroy_wrapper(void* r, void* view) +{ + Monoid* monoid = static_cast(r)->monoid_ptr(); + monoid->destroy(static_cast(view)); +} + +template +void* reducer_base::allocate_wrapper(void* r, __STDNS size_t bytes) +{ + Monoid* monoid = static_cast(r)->monoid_ptr(); + return monoid->allocate(bytes); +} + +template +void reducer_base::deallocate_wrapper(void* r, void* view) +{ + Monoid* monoid = static_cast(r)->monoid_ptr(); + monoid->deallocate(static_cast(view)); +} + + +/** Base class defining the data members of a reducer. + * + * @tparam Aligned The `m_view` data member, and therefore the entire + * structure, are cache-line aligned if this parameter + * is `true'. + */ +template +class reducer_content; + +/** Base class defining the data members of an aligned reducer. + */ +template +class reducer_content : public reducer_base +{ + typedef typename Monoid::view_type view_type; + + // The leftmost view is defined as raw bytes. It will be constructed + // by the monoid `construct` function. It is cache-aligned, which + // will push it into a new cache line. Furthermore, its alignment causes + // the reducer as a whole to be cache-aligned, which makes the reducer + // size a multiple of a cache line. Since there is nothing in the reducer + // after the view, all this means that the leftmost view gets one or more + // cache lines all to itself, which prevents false sharing. + // + __CILKRTS_CACHE_ALIGN + char m_leftmost[sizeof(view_type)]; + + /** Test if the reducer is cache-line-aligned. + * + * Used in assertions. + */ + bool reducer_is_cache_aligned() const + { return 0 == ((std::size_t) this & (__CILKRTS_CACHE_LINE__ - 1)); } + +protected: + + /** Constructor. + */ + reducer_content() : reducer_base((char*)&m_leftmost) + { +#ifndef CILK_IGNORE_REDUCER_ALIGNMENT + assert(reducer_is_cache_aligned() && + "Reducer should be cache aligned. Please see comments following " + "this assertion for explanation and fixes."); +#endif + /* "REDUCER SHOULD BE CACHE ALIGNED" ASSERTION. + * + * This Reducer class instantiation specifies cache-line alignment of the + * leftmost view field (and, implicitly, of the reducer itself). You got + * this assertion because a reducer with this class was allocated at a + * non-cache-aligned address, probably because it was allocated on the + * heap with `new`. This can be a problem for two reasons: + * + * 1. If the leftmost view is not on a cache line by itself, there might + * be a slowdown resulting from accesses to the same cache line from + * different threads. + * + * 2. The compiler thinks that reducer is cache-line aligned, but it + * really isn't. If the reducer is contained in a structure, then the + * compiler will believe that the containing structure, and other + * fields contained in it, are also more aligned than they really + * are. In particular, if the structure contains a numeric array that + * is used in a vectorizable loop, then the compiler might generate + * invalid vector instructions, resulting in a runtime error. + * + * The compiler will always allocate reducer variables, and structure + * variables containing reducers, with their required alignment. + * Reducers, and structures containing a reducer, which are allocated + * on the heap with `new` will _not_ be properly aligned. + * + * There are three ways that you can fix this assertion failure. + * + * A. Rewrite your code to use the new-style `reducer< op_XXX >` + * instead of the legacy `reducer_XXX`. The new-style reducers + * are not declared to be cache-aligned, and will work properly if + * they are not cache-aligned. + * + * B. If you must allocate an old-style reducer or a structure containing + * a reducer on the heap, figure out how to align it correctly. The + * suggested fix is to use `cilk::aligned_new()` and + * `cilk::aligned_delete()` instead of `new` and `delete`, as follows: + * + * Type* ptr = cilk::aligned_new(constructor-arguments); + * cilk::aligned_delete(ptr); + * + * C. Define the macro CILK_IGNORE_REDUCER_ALIGNMENT, which will suppress + * the assertion check. Do this only if you are comfortable that + * problem (2) above will not occur. + */ + } +}; + +/** Base class defining the data members of an unaligned reducer. + */ +template +class reducer_content : public reducer_base +{ + typedef typename Monoid::view_type view_type; ///< The view type. + + // Reserve space for the leftmost view. The view will be allocated at an + // aligned offset in this space at runtime, to guarantee that the view + // will get one or more cache lines all to itself, to prevent false + // sharing. + // + // The number of bytes to reserve is determined as follows: + // * Start with the view size. + // * Round up to a multiple of the cache line size, to get the total size + // of the cache lines that will be dedicated to the view. + // * Add (cache line size - 1) filler bytes to guarantee that the reserved + // area will contain a cache-aligned block of the required cache lines, + // no matter where the reserved area starts. + // + char m_leftmost[ + // View size rounded up to multiple cache lines + ( (sizeof(view_type) + __CILKRTS_CACHE_LINE__ - 1) + & ~ (__CILKRTS_CACHE_LINE__ - 1) + ) + // plus filler to allow alignment. + + __CILKRTS_CACHE_LINE__ - 1 + ]; + +protected: + + /** Constructor. Find the first cache-aligned position in the reserved + * area, and pass it to the base constructor as the leftmost view + * address. + */ + reducer_content() : + reducer_base( + (char*)( ((std::size_t)&m_leftmost + __CILKRTS_CACHE_LINE__ - 1) + & ~ (__CILKRTS_CACHE_LINE__ - 1) ) ) + {} +}; + + +} // namespace internal + + +// The __cilkrts_hyperobject_ functions are defined differently depending on +// whether a file is compiled with or without the CILK_STUB option. Therefore, +// reducers compiled in the two modes should be link-time incompatible, so that +// object files compiled with stubbed reducers won't be linked into an +// unstubbed program, or vice versa. We achieve this by putting the reducer +// class definition into the cilk::stub namespace in a stubbed compilation. + +#ifdef CILK_STUB +namespace stub { +#endif + +/** Reducer class. + * + * A reducer is instantiated on a Monoid. The Monoid provides the value + * type, associative reduce function, and identity for the reducer. + * + * @tparam Monoid The monoid class that the reducer is instantiated on. It + * must model the @ref reducers_monoid_concept "monoid + * concept". + * + * @see @ref pagereducers + */ +template +class reducer : public internal::reducer_content +{ + typedef internal::reducer_content base; + using base::monoid_ptr; + using base::leftmost_ptr; + public: + typedef Monoid monoid_type; ///< The monoid type. + typedef typename Monoid::value_type value_type; ///< The value type. + typedef typename Monoid::view_type view_type; ///< The view type. + + private: + typedef internal::reducer_set_get set_get; + + reducer(const reducer&); ///< Disallow copying. + reducer& operator=(const reducer&); ///< Disallow assignment. + + public: + + /** @name Constructors + * + * All reducer constructors call the static `construct()` function of the + * monoid class to construct the reducer's monoid and leftmost view. + * + * The reducer constructor arguments are simply passed through to the + * construct() function. Thus, the constructor parameters accepted by a + * particular reducer class are determined by its monoid class. + */ + //@{ + + /** 0 – 6 const reference parameters. + */ + //@{ + + reducer() + { + monoid_type::construct(monoid_ptr(), leftmost_ptr()); + } + + template + reducer(const T1& x1) + { + monoid_type::construct(monoid_ptr(), leftmost_ptr(), x1); + } + + template + reducer(const T1& x1, const T2& x2) + { + monoid_type::construct(monoid_ptr(), leftmost_ptr(), x1, x2); + } + + template + reducer(const T1& x1, const T2& x2, const T3& x3) + { + monoid_type::construct(monoid_ptr(), leftmost_ptr(), x1, x2, x3); + } + + template + reducer(const T1& x1, const T2& x2, const T3& x3, const T4& x4) + { + monoid_type::construct(monoid_ptr(), leftmost_ptr(), x1, x2, x3, x4); + } + + template + reducer(const T1& x1, const T2& x2, const T3& x3, const T4& x4, + const T5& x5) + { + monoid_type::construct(monoid_ptr(), leftmost_ptr(), + x1, x2, x3, x4, x5); + } + + template + reducer(const T1& x1, const T2& x2, const T3& x3, const T4& x4, + const T5& x5, const T6& x6) + { + monoid_type::construct(monoid_ptr(), leftmost_ptr(), + x1, x2, x3, x4, x5, x6); + } + + //@} + + /** 1 non-const reference parameter. + */ + //@{ + + template + reducer(T1& x1) + { + monoid_type::construct(monoid_ptr(), leftmost_ptr(), x1); + } + + //@} + + /** Destructor. + */ + __CILKRTS_STRAND_STALE(~reducer()) + { + leftmost_ptr()->~view_type(); + monoid_ptr()->~monoid_type(); + } + + //@{ + /** Get the monoid. + * + * @return A reference to the monoid object belonging to this reducer. + */ + Monoid& monoid() { return *monoid_ptr(); } + + const Monoid& monoid() const + { return const_cast(this)->monoid(); } + //@} + + //@{ + /** Access the current view. + * + * Return a reference to the instance of the reducer's view that was + * created for the current strand of a parallel computation (and create + * it if it doesn't already exist). + */ + view_type& view() { return base::view(); } + const view_type& view() const { return base::view(); } + //@} + + + /** @name Dereference the reducer to get the view. + * + * "Dereferencing" a reducer yields the view for the current strand. The + * view, in turn, acts as a proxy for its contained value, exposing only + * those operations which are consistent with the reducer's monoid. Thus, + * all modifications of the reducer's accumulator variable are written as + * + * *reducer OP ... + * + * or + * + * reducer->func(...) + * + * (The permitted operations on a reducer's accumulator are listed in the + * documentation for that particular kind of reducer.) + * + * @note `*r` is a synonym for `r.view()`. Recommended style is to use + * `*r` (or `r->`) in the common case where code is simply + * updating the accumulator variable wrapped in the view, and to + * use `r.view()` in the unusual case where it is desirable to + * call attention to the view itself. + */ + //@{ + + //@{ + /** Dereference operator. + * + * @return A reference to the per-strand view instance. + */ + view_type& operator*() { return view(); } + view_type const& operator*() const { return view(); } + //@} + + //@{ + /** Pointer operator. + * + * @return A pointer to the per-strand view instance. + */ + view_type* operator->() { return &view(); } + view_type const* operator->() const { return &view(); } + //@} + + //@{ + /** Deprecated view access. + * + * `r()` is a synonym for `*r` which was used with early versions of + * Intel Cilk Plus reducers. `*r` is now the preferred usage. + * + * @deprecated Use operator*() instead of operator()(). + * + * @return A reference to the per-strand view instance. + */ + view_type& operator()() { return view(); } + view_type const& operator()() const { return view(); } + //@} + + //@} + + /** @name Set and get the value. + * + * These functions are used to set an initial value for the reducer before + * starting the reduction, or to get the final value after the reduction + * is complete. + * + * @note These functions are completely different from the view + * operations that are made available via operator*() and + * operator->(), which are used to _modify_ the reducer's value + * _during_ the reduction. + * + * @warning These functions _can_ be called at any time, and in + * general, they will refer to the value contained in the view + * for the current strand. However, using them other than to + * set the reduction's initial value or get its final value + * will almost always result in undefined behavior. + */ + //@{ + + /** Move a value into the reducer. + * + * This function is used to set the initial value of the reducer's + * accumulator variable by either copying or _moving_ the value of @a obj + * into it. Moving a value can often be performed in constant time, even + * for large container objects, but has the side effect of leaving the + * value of @a obj undefined. (See the description of the + * @ref move_in_wrapper class for a discussion of moving values.) + * + * @par Usage + * A move_in() call to initialize a reducer is often paired with a + * move_out() call to get its final value: + * + * reducer xr; + * xr.move_in(x); + * … do the reduction … + * xr.move_out(x); + * + * @par Assumptions + * - You cannot assume either that this will function will copy its + * value or that it will move it. + * - You must assume that the value of @a obj will be undefined + * after the call to move_in(). + * - You can assume that move_in() will be at least as efficient as + * set_value(), and you should therefore prefer move_in() unless + * you need the value of @a obj to be unchanged after the call. + * (But you should usually prefer the move-in constructor over a + * move_in() call - see the note below.) + * + * @note The behavior of a default constructor followed by move-in + * initialization: + * + * reducer xr; + * xr.move_in(x); + * + * @note is not necessarily the same as a move-in constructor: + * + * reducer xr(move_in(x)); + * + * @note In particular, when @a Type is a container type with a + * non-empty allocator, the move-in constructor will create the + * accumulator variable with the same allocator as the input + * argument @a x, while the default constructor will create the + * accumulator variable with a default allocator. The mismatch of + * allocators in the latter case means that the input argument + * @a x may have to be copied in linear time instead of being + * moved in constant time. + * + * @note Best practice is to prefer the move-in constructor over the + * move-in function unless the move-in function is required for + * some specific reason. + * + * @warning Calling this function other than to set the initial value + * for a reduction will almost always result in undefined + * behavior. + * + * @param obj The object containing the value that will be moved into the + * reducer. + * + * @post The reducer contains the value that was initially in @a obj. + * @post The value of @a obj is undefined. + * + * @see set_value() + */ + void move_in(value_type& obj) { set_get::move_in(view(), obj);} + + /** Move the value out of the reducer. + * + * This function is used to retrieve the final value of the reducer's + * accumulator variable by either copying or _moving_ the value of @a obj + * into it. Moving a value can often be performed in constant time, even + * for large container objects, but has the side effect of leaving the + * value of the reducer's accumulator variable undefined. (See the + * description of the @ref move_in_wrapper class for a discussion of + * moving values.) + * + * @par Usage + * A move_in() call to initialize a reducer is often paired with a + * move_out() call to get its final value: + * + * reducer xr; + * xr.move_in(x); + * … do the reduction … + * xr.move_out(x); + * + * @par Assumptions + * - You cannot assume either that this will function will copy its + * value or that it will move it. + * - You must assume that the value of the reducer's accumulator + * variable will be undefined after the call to move_out(). + * - You can assume that move_out() will be at least as efficient as + * get_value(), and you should therefore prefer move_out() unless + * you need the accumulator variable to be preserved after the + * call. + * + * @warning Calling this function other than to retrieve the final + * value of a reduction will almost always result in undefined + * behavior. + * + * @param obj The object that the value of the reducer will be moved into. + * + * @post @a obj contains the value that was initially in the reducer. + * @post The value of the reducer is undefined. + * + * @see get_value() + */ + void move_out(value_type& obj) { set_get::move_out(view(), obj); } + + /** Set the value of the reducer. + * + * This function sets the initial value of the reducer's accumulator + * variable to the value of @a obj. + * + * @note The behavior of a default constructor followed by + * initialization: + * + * reducer xr; + * xr.set_value(x); + * + * @note is not necessarily the same as a value constructor: + * + * reducer xr(x); + * + * @note In particular, when @a Type is a container type with a + * non-empty allocator, the value constructor will create the + * accumulator variable with the same allocator as the input + * argument @a x, while the default constructor will create the + * accumulator variable with a default allocator. + * + * @warning Calling this function other than to set the initial value + * for a reduction will almost always result in undefined + * behavior. + * + * @param obj The object containing the value that will be copied into + * the reducer. + * + * @post The reducer contains a copy of the value in @a obj. + * + * @see move_in() + */ + void set_value(const value_type& obj) { set_get::set_value(view(), obj); } + + /** Get the value of the reducer. + * + * This function gets the final value of the reducer's accumulator + * variable. + * + * @warning Calling this function other than to retrieve the final + * value of a reduction will almost always result in undefined + * behavior. + * + * @return A reference to the value contained in the reducer. + * + * @see move_out() + */ + typename set_get::return_type_for_get_value get_value() const + { return set_get::get_value(view()); } + + //@} + + /** Implicit downcast to legacy reducer wrapper, if any. + * + * @see legacy_reducer_downcast + */ + operator typename legacy_reducer_downcast::type& () + { + typedef typename legacy_reducer_downcast::type downcast_type; + return *reinterpret_cast(this); + } + + + /** Implicit downcast to legacy reducer wrapper, if any. + * + * @see legacy_reducer_downcast + */ + operator const typename legacy_reducer_downcast::type& () const + { + typedef typename legacy_reducer_downcast::type downcast_type; + return *reinterpret_cast(this); + } +}; + +#ifdef CILK_STUB +} // namespace stub +using stub::reducer; +#endif + +} // end namespace cilk + +#endif /* __cplusplus */ + +/** @page page_reducers_in_c Creating and Using Reducers in C + * + * @tableofcontents + * + * The Intel Cilk Plus runtime supports reducers written in C as well as in C++. The + * basic logic is the same, but the implementation details are very + * different. The C++ reducer implementation uses templates heavily to create + * very generic components. The C reducer implementation uses macros, which + * are a much blunter instrument. The most immediate consequence is that the + * monoid/view/reducer architecture is mostly implicit rather than explicit + * in C reducers. + * + * @section reducers_c_overview Overview of Using Reducers in C + * + * The basic usage pattern for C reducers is: + * + * 1. Create and initialize a reducer object. + * 2. Tell the Intel Cilk Plus runtime about the reducer. + * 3. Update the value contained in the reducer in a parallel computation. + * 4. Tell the Intel Cilk Plus runtime that you are done with the reducer. + * 5. Retrieve the value from the reducer. + * + * @subsection reducers_c_creation Creating and Initializing a C Reducer + * + * The basic pattern for creating and initializing a reducer object in C is + * + * CILK_C_DECLARE_REDUCER(value-type) reducer-name = + * CILK_C_INIT_REDUCER(value-type, + * reduce-function, + * identity-function, + * destroy-function, + * initial-value); + * + * This is simply an initialized definition of a variable named + * _reducer-name_. The @ref CILK_C_DECLARE_REDUCER macro expands to an + * anonymous `struct` declaration for a reducer object containing a view of + * type _value-type_, and the @ref CILK_C_INIT_REDUCER macro expands to a + * struct initializer. + * + * @subsection reducers_c_reduce_func Reduce Functions + * + * The reduce function for a reducer is called when a parallel execution + * strand terminates, to combine the values computed by the terminating + * strand and the strand to its left. It takes three arguments: + * + * - `void* reducer` - the address of the reducer. + * - `void* left` - the address of the value for the left strand. + * - `void* right` - the address of the value for the right (terminating) + * strand. + * + * It must apply the reducer's reduction operation to the `left` and `right` + * values, leaving the result in the `left` value. The `right` value is + * undefined after the reduce function call. + * + * @subsection reducers_c_identity_func Identity Functions + * + * The identity function for a reducer is called when a parallel execution + * strand begins, to initialize its value to the reducer's identity value. It + * takes two arguments: + * + * - `void* reducer` - the address of the reducer. + * - `void* v` - the address of a freshly allocated block of memory of size + * `sizeof(value-type)`. + * + * It must initialize the memory pointed to by `v` so that it contains the + * reducer's identity value. + * + * @subsection reducers_c_destroy_func Destroy Functions + * + * The destroy function for a reducer is called when a parallel execution + * strand terminates, to do any necessary cleanup before its value is + * deallocated. It takes two arguments: + * + * - `void* reducer` - the address of the reducer. + * - `void* p` - the address of the value for the terminating strand. + * + * It must release any resources belonging to the value pointed to by `p`, to + * avoid a resource leak when the memory containing the value is deallocated. + * + * The runtime function `__cilkrts_hyperobject_noop_destroy` can be used for + * the destructor function if the reducer's values do not need any cleanup. + * + * @subsection reducers_c_register Tell the Intel Cilk Plus Runtime About the + * Reducer + * + * Call the @ref CILK_C_REGISTER_REDUCER macro to register the reducer with + * the Intel Cilk Plus runtime: + * + * CILK_C_REGISTER_REDUCER(reducer-name); + * + * The runtime will manage reducer values for all registered reducers when + * parallel execution strands begin and end. + * + * @subsection reducers_c_update Update the Value Contained in the Reducer + * + * The @ref REDUCER_VIEW macro returns a reference to the reducer's value for + * the current parallel strand: + * + * REDUCER_VIEW(reducer-name) = REDUCER_VIEW(reducer-name) OP x; + * + * C++ reducer views restrict access to the wrapped value so that it can only + * be modified in ways consistent with the reducer's operation. No such + * protection is provided for C reducers. It is entirely the responsibility + * of the user to avoid modifying the value in any inappropriate way. + * + * @subsection c_reducers_unregister Tell the Intel Cilk Plus Runtime That You Are + * Done with the Reducer + * + * When the parallel computation is complete, call the @ref + * CILK_C_UNREGISTER_REDUCER macro to unregister the reducer with the + * Intel Cilk Plus runtime: + * + * CILK_C_UNREGISTER_REDUCER(reducer-name); + * + * The runtime will stop managing reducer values for the reducer. + * + * @subsection c_reducers_retrieve Retrieve the Value from the Reducer + * + * When the parallel computation is complete, use the @ref REDUCER_VIEW macro + * to retrieve the final value computed by the reducer. + * + * @subsection reducers_c_example_custom Example - Creating and Using a + * Custom C Reducer + * + * The `IntList` type represents a simple list of integers. + * + * struct _intListNode { + * int value; + * _intListNode* next; + * } IntListNode; + * typedef struct { IntListNode* head; IntListNode* tail; } IntList; + * + * // Initialize a list to be empty + * void IntList_init(IntList* list) { list->head = list->tail = 0; } + * + * // Append an integer to the list + * void IntList_append(IntList* list, int x) + * { + * IntListNode* node = (IntListNode*) malloc(sizeof(IntListNode)); + * if (list->tail) list->tail->next = node; else list->head = node; + * list->tail = node; + * } + * + * // Append the right list to the left list, and leave the right list + * // empty + * void IntList_concat(IntList* left, IntList* right) + * { + * if (left->head) { + * left->tail->next = right->head; + * if (right->tail) left->tail = right->tail; + * } + * else { + * *left = *right; + * } + * IntList_init(*right); + * } + * + * This code creates a reducer that supports creating an `IntList` by + * appending values to it. + * + * void identity_IntList(void* reducer, void* list) + * { + * IntList_init((IntList*)list); + * } + * + * void reduce_IntList(void* reducer, void* left, void* right) + * { + * IntList_concat((IntList*)left, (IntList*)right); + * } + * + * CILK_C_DECLARE_REDUCER(IntList) my_list_int_reducer = + * CILK_C_INIT_REDUCER(IntList, + * reduce_int_list, + * identity_int_list, + * __cilkrts_hyperobject_noop_destroy); + * // Initial value omitted // + * ListInt_init(&REDUCER_VIEW(my_int_list_reducer)); + * + * CILK_C_REGISTER_REDUCER(my_int_list_reducer); + * cilk_for (int i = 0; i != n; ++i) { + * IntList_append(&REDUCER_VIEW(my_int_list_reducer), a[i]); + * } + * CILK_C_UNREGISTER_REDUCER(my_int_list_reducer); + * + * IntList result = REDUCER_VIEW(my_int_list_reducer); + * + * @section reducers_c_predefined Predefined C Reducers + * + * Some of the predefined reducer classes in the Intel Cilk Plus library come with + * a set of predefined macros to provide the same capabilities in C. + * In general, two macros are provided for each predefined reducer family: + * + * - `CILK_C_REDUCER_operation(reducer-name, type-name, initial-value)` - + * Declares a reducer object named _reducer-name_ with initial value + * _initial-value_ to perform a reduction using the _operation_ on values + * of the type specified by _type-name_. This is the equivalent of the + * general code described in @ref reducers_c_creation : + * + * CILK_C_DECLARE_REDUCER(type) reducer-name = + * CILK_C_INIT_REDUCER(type, ..., initial-value); + * + * where _type_ is the C type corresponding to _type_name_. See @ref + * reducers_c_type_names below for the _type-names_ that you can use. + * + * - `CILK_C_REDUCER_operation_TYPE(type-name)` - Expands to the `typedef` + * name for the type of the reducer object declared by + * `CILK_C_REDUCER_operation(reducer-name, type-name, initial-value)`. + * + * See @ref reducers_c_example_predefined. + * + * The predefined C reducers are: + * + * | Operation | Name | Documentation | + * |-------------------|---------------|-------------------------------| + * | addition | `OPADD` | @ref ReducersAdd | + * | bitwise AND | `OPAND` | @ref ReducersAnd | + * | bitwise OR | `OPOR` | @ref ReducersOr | + * | bitwise XOR | `OPXOR` | @ref ReducersXor | + * | multiplication | `OPMUL` | @ref ReducersMul | + * | minimum | `MIN` | @ref ReducersMinMax | + * | minimum & index | `MIN_INDEX` | @ref ReducersMinMax | + * | maximum | `MAX` | @ref ReducersMinMax | + * | maximum & index | `MAX_INDEX` | @ref ReducersMinMax | + * + * @subsection reducers_c_type_names Numeric Type Names + * + * The type and function names created by the C reducer definition macros + * incorporate both the reducer kind (`opadd`, `opxor`, etc.) and the value + * type of the reducer (`int`, `double`, etc.). The value type is represented + * by a _numeric type name_ string. The types supported in C reducers, and + * their corresponding numeric type names, are given in the following table: + * + * | Type | Numeric Type Name | + * |-----------------------|-------------------------------| + * | `char` | `char` | + * | `unsigned char` | `uchar` | + * | `signed char` | `schar` | + * | `wchar_t` | `wchar_t` | + * | `short` | `short` | + * | `unsigned short` | `ushort` | + * | `int` | `int` | + * | `unsigned int` | `uint` | + * | `unsigned int` | `unsigned` (alternate name) | + * | `long` | `long` | + * | `unsigned long` | `ulong` | + * | `long long` | `longlong` | + * | `unsigned long long` | `ulonglong` | + * | `float` | `float` | + * | `double` | `double` | + * | `long double` | `longdouble` | + * + * @subsection reducers_c_example_predefined Example - Using a Predefined C + * Reducer + * + * To compute the sum of all the values in an array of `unsigned int`: + * + * CILK_C_REDUCER_OPADD(sum, uint, 0); + * CILK_C_REGISTER_REDUCER(sum); + * cilk_for(int i = 0; i != n; ++i) { + * REDUCER_VIEW(sum) += a[i]; + * } + * CILK_C_UNREGISTER_REDUCER(sum); + * printf("The sum is %u\n", REDUCER_VIEW(sum)); + */ + + + /** @name C language reducer macros + * + * These macros are used to declare and work with reducers in C code. + * + * @see @ref page_reducers_in_c + */ + //@{ + +/// @cond internal + +/** @name Compound identifier macros. + * + * These macros are used to construct an identifier by concatenating two or + * three identifiers. + */ +//@{ + +/** Expand to an identifier formed by concatenating two identifiers. + */ +#define __CILKRTS_MKIDENT(a,b) __CILKRTS_MKIDENT_IMP(a,b,) + +/** Expand to an identifier formed by concatenating three identifiers. + */ +#define __CILKRTS_MKIDENT3(a,b,c) __CILKRTS_MKIDENT_IMP(a,b,c) + +/** Helper macro to do the concatenation. + */ +#define __CILKRTS_MKIDENT_IMP(a,b,c) a ## b ## c + +//@} + +/** Compiler-specific keyword for the "type of" operator. + */ +#if defined(__GNUC__) && !defined(__INTEL_COMPILER) +# define _Typeof __typeof__ +#endif + +/** @name Predefined reducer function declaration macros. + * + * These macros are used to create the function headers for the identity, + * reduction, and destructor functions for a builtin reducer family. The + * macro can be followed by a semicolon to create a declaration, or by a + * brace-enclosed body to create a definition. + */ +//@{ + +/** Create an identity function header. + * + * @note The name of the function's value pointer parameter will always be `v`. + * + * @param name The reducer family name. + * @param tn The type name. + */ +#define __CILKRTS_DECLARE_REDUCER_IDENTITY(name,tn) CILK_EXPORT \ + void __CILKRTS_MKIDENT3(name,_identity_,tn)(void* key, void* v) + +/** Create a reduction function header. + * + * @param name The reducer family name. + * @param tn The type name. + * @param l The name to use for the function's left value pointer parameter. + * @param r The name to use for the function's right value pointer + * parameter. + */ +#define __CILKRTS_DECLARE_REDUCER_REDUCE(name,tn,l,r) CILK_EXPORT \ + void __CILKRTS_MKIDENT3(name,_reduce_,tn)(void* key, void* l, void* r) + +/** Create a destructor function header. + * + * @param name The reducer family name. + * @param tn The type name. + * @param p The name to use for the function's value pointer parameter. + */ +#define __CILKRTS_DECLARE_REDUCER_DESTROY(name,tn,p) CILK_EXPORT \ + void __CILKRTS_MKIDENT3(name,_destroy_,tn)(void* key, void* p) + +//@} + +/// @endcond + + +/*************************************************************************** + * Real implementation + ***************************************************************************/ + +/** Declaration of a C reducer structure type. + * + * This macro expands into an anonymous structure declaration for a C reducer + * structure which contains a @a Type value. For example: + * + * CILK_C_DECLARE_REDUCER(int) my_add_int_reducer = + * CILK_C_INIT_REDUCER(int, …); + * + * @param Type The type of the value contained in the reducer object. + * + * @see @ref reducers_c_creation + */ +#define CILK_C_DECLARE_REDUCER(Type) struct { \ + __cilkrts_hyperobject_base __cilkrts_hyperbase; \ + __CILKRTS_CACHE_ALIGN Type value; \ + } + +/** Initializer for a C reducer structure. + * + * This macro expands into a brace-enclosed structure initializer for a C + * reducer structure that was declared with + * `CILK_C_DECLARE_REDUCER(Type)`. For example: + * + * CILK_C_DECLARE_REDUCER(int) my_add_int_reducer = + * CILK_C_INIT_REDUCER(int, + * add_int_reduce, + * add_int_identity, + * __cilkrts_hyperobject_noop_destroy, + * 0); + * + * @param Type The type of the value contained in the reducer object. Must + * be the same as the @a Type argument of the + * CILK_C_DECLARE_REDUCER macro call that created the + * reducer. + * @param Reduce The address of the @ref reducers_c_reduce_func + * "reduce function" for the reducer. + * @param Identity The address of the @ref reducers_c_identity_func + * "identity function" for the reducer. + * @param Destroy The address of the @ref reducers_c_destroy_func + * "destroy function" for the reducer. + * @param ... The initial value for the reducer. (A single expression if + * @a Type is a scalar type; a list of values if @a Type is a + * struct or array type.) + * + * @see @ref reducers_c_creation + */ + +#define CILK_C_INIT_REDUCER(Type, Reduce, Identity, Destroy, ...) \ + { { { Reduce \ + , Identity \ + , Destroy \ + , __cilkrts_hyperobject_alloc \ + , __cilkrts_hyperobject_dealloc \ + } \ + , 0 \ + , __CILKRTS_CACHE_LINE__ \ + , sizeof(Type) \ + } \ + , __VA_ARGS__ \ + } + +/** Register a reducer with the Intel Cilk Plus runtime. + * + * The runtime will manage reducer values for all registered reducers when + * parallel execution strands begin and end. For example: + * + * CILK_C_REGISTER_REDUCER(my_add_int_reducer); + * cilk_for (int i = 0; i != n; ++i) { + * … + * } + * + * @param Expr The reducer to be registered. + * + * @see @ref page_reducers_in_c + */ +#define CILK_C_REGISTER_REDUCER(Expr) \ + __cilkrts_hyper_create(&(Expr).__cilkrts_hyperbase) + +/** Unregister a reducer with the Intel Cilk Plus runtime. + * + * The runtime will stop managing reducer values for a reducer after it is + * unregistered. For example: + * + * cilk_for (int i = 0; i != n; ++i) { + * … + * } + * CILK_C_UNREGISTER_REDUCER(my_add_int_reducer); + * + * @param Expr The reducer to be unregistered. + * + * @see @ref page_reducers_in_c + */ +#define CILK_C_UNREGISTER_REDUCER(Expr) \ + __cilkrts_hyper_destroy(&(Expr).__cilkrts_hyperbase) + +/** Get the current view for a reducer. + * + * The `REDUCER_VIEW(reducer-name)` returns a reference to the reducer's + * value for the current parallel strand. This can be used to initialize the + * value of the reducer before it is used, to modify the value of the reducer + * on the current parallel strand, or to retrieve the final value of the + * reducer at the end of the parallel computation. + * + * REDUCER_VIEW(my_add_int_reducer) = REDUCER_VIEW(my_add_int_reducer) + x; + * + * @note C++ reducer views restrict access to the wrapped value so that it + * can only be modified in ways consistent with the reducer's operation. No + * such protection is provided for C reducers. It is entirely the + * responsibility of the user to refrain from modifying the value in any + * inappropriate way. + * + * @param Expr The reducer whose value is to be returned. + * + * @see @ref page_reducers_in_c + */ +#define REDUCER_VIEW(Expr) (*(_Typeof((Expr).value)*) \ + __cilkrts_hyper_lookup(&(Expr).__cilkrts_hyperbase)) + +//@} C language reducer macros + +#endif // CILK_REDUCER_H_INCLUDED diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_file.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_file.h new file mode 100644 index 0000000000..a372cee7cd --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_file.h @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + * + */ + + diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_list.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_list.h new file mode 100644 index 0000000000..80204af1d9 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_list.h @@ -0,0 +1,1150 @@ +/* reducer_list.h -*- C++ -*- + * + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + */ + +/** @file reducer_list.h + * + * @brief Defines classes for parallel list creation by appending or + * prepending reducers. + * + * @ingroup ReducersList + * + * @see ReducersList + */ + +#ifndef REDUCER_LIST_H_INCLUDED +#define REDUCER_LIST_H_INCLUDED + +#include +#include + +/** @defgroup ReducersList List Reducers + * + * List-append and list-prepend reducers create standard lists by + * concatenating a set of lists or values in parallel. + * + * @ingroup Reducers + * + * You should be familiar with @ref pagereducers "Intel(R) Cilk(TM) Plus reducers" + * (from file `reducers.md`) and particularly with @ref reducers_using, before + * trying to use the information in this file. + * + * @section redlist_usage Usage Example + * + * // Create a list containing the labels of the nodes of a tree in + * // "inorder" (left subtree, root, right subtree). + * + * struct Tree { Tree* left; Tree* right; string label; ... }; + * + * list x; + * cilk::reducer< cilk::op_list_append > xr(cilk::move_in(x)); + * collect_labels(tree, xr); + * xr.move_out(x); + * + * void collect_labels(Tree* node, + * cilk::reducer< cilk::op_list_append >& xr) + * { + * if (node) { + * cilk_spawn collect_labels(node->left, xr); + * xr->push_back(node->label); + * collect_labels(node->right, xr); + * cilk_sync; + * } + * } + * + * @section redlist_monoid The Monoid + * + * @subsection redlist_monoid_values Value Set + * + * The __value set__ of a list reducer is the set of values of the class + * `std::list`, which we refer to as the reducer's _list + * type_. + * + * @subsection redlist_monoid_operator Operator + * + * The operator of a list-append reducer is defined as + * + * x CAT y == (every element of x, followed by every element of y) + * + * The operator of a list-prepend reducer is defined as + * + * x RCAT y == (every element of y, followed by every element of x) + * + * @subsection redlist_monoid_identity Identity + * + * The identity value of a list reducer is the empty list, which is the value + * of the expression `std::list([allocator])`. + * + * @section redlist_operations Operations + * + * In the operation descriptions below, the type name `List` refers to the + * reducer's string type, `std::list`. + * + * @subsection redlist_constructors Constructors + * + * Any argument list which is valid for a `std::list` constructor is valid for + * a list reducer constructor. The usual move-in constructor is also provided: + * + * reducer(move_in(List& variable)) + * + * A list reducer with no constructor arguments, or with only an allocator + * argument, will initially contain the identity value, an empty list. + * + * @subsection redlist_get_set Set and Get + * + * r.set_value(const List& value) + * const List& = r.get_value() const + * r.move_in(List& variable) + * r.move_out(List& variable) + * + * @subsection redlist_view_ops View Operations + * + * The view of a list-append reducer provides the following member functions: + * + * void push_back(const Type& element) + * void insert_back(List::size_type n, const Type& element) + * template void insert_back(Iter first, Iter last) + * void splice_back(List& x) + * void splice_back(List& x, List::iterator i) + * void splice_back(List& x, List::iterator first, List::iterator last) + * + * The view of a list-prepend reducer provides the following member functions: + * + * void push_front(const Type& element) + * void insert_front(List::size_type n, const Type& element) + * template void insert_front(Iter first, Iter last) + * void splice_front(List& x) + * void splice_front(List& x, List::iterator i) + * void splice_front(List& x, List::iterator first, List::iterator last) + * + * The `push_back` and `push_front` functions are the same as the + * corresponding `std::list` functions. The `insert_back`, `splice_back`, + * `insert_front`, and `splice_front` functions are the same as the + * `std::list` `insert` and `splice` functions, with the first parameter + * fixed to the end or beginning of the list, respectively. + * + * @section redlist_performance Performance Considerations + * + * An efficient reducer requires that combining the values of two views (using + * the view `reduce()` function) be a constant-time operations. Two lists can + * be merged in constant time using the `splice()` function if they have the + * same allocator. Therefore, the lists for new views are created (by the view + * identity constructor) using the same allocator as the list that was created + * when the reducer was constructed. + * + * The performance of adding elements to a list reducer depends on the view + * operations that are used: + * + * * The `push` functions add a single element to the list, and therefore + * take constant time. + * * An `insert` function that inserts _N_ elements adds each of them + * individually, and therefore takes _O(N)_ time. + * * A `splice` function that inserts _N_ elements just adjusts a couple of + * pointers, and therefore takes constant time, _if the splice is from a + * list with the same allocator as the reducer_. Otherwise, it is + * equivalent to an `insert`, and takes _O(N)_ time. + * + * This means that for best performance, if you will be adding elements to a + * list reducer in batches, you should `splice` them from a list having the + * same allocator as the reducer. + * + * The reducer `move_in` and `move_out` functions do a constant-time `swap` if + * the variable has the same allocator as the reducer, and a linear-time copy + * otherwise. + * + * Note that the allocator of a list reducer is determined when the reducer is + * constructed. The following two examples may have very different behavior: + * + * list a_list; + * + * reducer< list_append reducer1(move_in(a_list)); + * ... parallel computation ... + * reducer1.move_out(a_list); + * + * reducer< list_append reducer2; + * reducer2.move_in(a_list); + * ... parallel computation ... + * reducer2.move_out(a_list); + * + * * `reducer1` will be constructed with the same allocator as `a_list`, + * because the list was specified in the constructor. The `move_in` + * and `move_out` can therefore be done with a `swap` in constant time. + * * `reducer2` will be constructed with a _default_ allocator, + * "`Allocator()`", which may or may not be the same as the allocator of + * `a_list`. Therefore, the `move_in` and `move_out` may have to be done + * with a copy in _O(N)_ time. + * + * (All instances of an allocator type with no internal state (like + * `std::allocator`) are "the same". You only need to worry about the "same + * allocator" issue when you create list reducers with custom allocator types.) + * + * @section redlist_types Type and Operator Requirements + * + * `std::list` must be a valid type. + */ + + +namespace cilk { + +namespace internal { + +/** @ingroup ReducersList */ +//@{ + +/** Base class for list-append and prepend view classes. + * + * @note This class provides the definitions that are required for a class + * that will be used as the parameter of a @ref list_monoid_base + * specialization. + * + * @tparam Type The list element type (not the list type). + * @tparam Allocator The list's allocator class. + * + * @see ReducersList + * @see list_monoid_base + */ +template +class list_view_base +{ +protected: + /// The type of the contained list. + typedef std::list list_type; + + /// The list accumulator variable. + list_type m_value; + +public: + + /** @name Monoid support. + */ + //@{ + + /// Required by @ref monoid_with_view + typedef list_type value_type; + + /// Required by @ref list_monoid_base + Allocator get_allocator() const + { + return m_value.get_allocator(); + } + + //@} + + + /** @name Constructors. + */ + //@{ + + /// Standard list constructor. + explicit list_view_base(const Allocator& a = Allocator()) : m_value(a) {} + explicit list_view_base( + typename list_type::size_type n, + const Type& value = Type(), + const Allocator& a = Allocator() ) : m_value(n, value, a) {} + template + list_view_base(Iter first, Iter last, const Allocator& a = Allocator()) : + m_value(first, last, a) {} + list_view_base(const list_type& list) : m_value(list) {} + + /// Move-in constructor. + explicit list_view_base(move_in_wrapper w) + : m_value(w.value().get_allocator()) + { + m_value.swap(w.value()); + } + + //@} + + /** @name Reducer support. + */ + //@{ + + /// Required by reducer::move_in() + void view_move_in(value_type& v) + { + if (m_value.get_allocator() == v.get_allocator()) + // Equal allocators. Do a (fast) swap. + m_value.swap(v); + else + // Unequal allocators. Do a (slow) copy. + m_value = v; + v.clear(); + } + + /// Required by reducer::move_out() + void view_move_out(value_type& v) + { + if (m_value.get_allocator() == v.get_allocator()) + // Equal allocators. Do a (fast) swap. + m_value.swap(v); + else + // Unequal allocators. Do a (slow) copy. + v = m_value; + m_value.clear(); + } + + /// Required by reducer::set_value() + void view_set_value(const value_type& v) { m_value = v; } + + /// Required by reducer::get_value() + value_type const& view_get_value() const { return m_value; } + + /// Type returned by view_get_value. + typedef value_type const& return_type_for_get_value; + + // Required by legacy wrapper get_reference() + value_type & view_get_reference() { return m_value; } + value_type const& view_get_reference() const { return m_value; } + + //@} +}; + + +/** Base class for list-append and prepend monoid classes. + * + * The key to efficient reducers is that the `identity` operation, which + * creates a new per-strand view, and the `reduce` operation, which combines + * two per-strand views, must be constant-time operations. Two lists can be + * concatenated in constant time only if they have the same allocator. + * Therefore, all the per-strand list accumulator variables must be created + * with the same allocator as the leftmost view list. + * + * This means that a list reduction monoid must have a copy of the allocator + * of the leftmost view's list, so that it can use it in the `identity` + * operation. This, in turn, requires that list reduction monoids have a + * specialized `construct()` function, which constructs the leftmost view + * before the monoid, and then passes the leftmost view's allocator to the + * monoid constructor. + * + * @tparam View The list-append or prepend view class. + * @tparam Align If `false` (the default), reducers instantiated on this + * monoid will be naturally aligned (the Intel Cilk Plus library 1.0 + * behavior). If `true`, reducers instantiated on this monoid + * will be cache-aligned for binary compatibility with + * reducers in Intel Cilk Plus library version 0.9. + * + * @see ReducersList + * @see list_view_base + */ +template +class list_monoid_base : public monoid_with_view +{ + typedef typename View::value_type list_type; + typedef typename list_type::allocator_type allocator_type; + typedef provisional_guard view_guard; + + allocator_type m_allocator; + +public: + + /** Constructor. + * + * There is no default constructor for list monoids, because the allocator + * must always be specified. + * + * @param allocator The list allocator to be used when + * identity-constructing new views. + */ + list_monoid_base(const allocator_type& allocator = allocator_type()) : + m_allocator(allocator) {} + + /** Creates an identity view. + * + * List view identity constructors take the list allocator as an argument. + * + * @param v The address of the uninitialized memory in which the view + * will be constructed. + */ + void identity(View *v) const { ::new((void*) v) View(m_allocator); } + + /** @name construct functions + * + * All `construct()` functions first construct the leftmost view, using + * the optional @a x1, @a x2, and @a x3 arguments that were passed in from + * the reducer constructor. They then call the view's `get_allocator()` + * function to get the list allocator from its contained list, and pass it + * to the monoid constructor. + */ + //@{ + + template + static void construct(Monoid* monoid, View* view) + { + view_guard vg( new((void*) view) View() ); + vg.confirm_if( new((void*) monoid) Monoid(view->get_allocator()) ); + } + + template + static void construct(Monoid* monoid, View* view, const T1& x1) + { + view_guard vg( new((void*) view) View(x1) ); + vg.confirm_if( new((void*) monoid) Monoid(view->get_allocator()) ); + } + + template + static void construct(Monoid* monoid, View* view, + const T1& x1, const T2& x2) + { + view_guard vg( new((void*) view) View(x1, x2) ); + vg.confirm_if( new((void*) monoid) Monoid(view->get_allocator()) ); + } + + template + static void construct(Monoid* monoid, View* view, + const T1& x1, const T2& x2, const T3& x3) + { + view_guard vg( new((void*) view) View(x1, x2, x3) ); + vg.confirm_if( new((void*) monoid) Monoid(view->get_allocator()) ); + } + + //@} +}; + +//@} + +} // namespace internal + + +/** @ingroup ReducersList */ +//@{ + +/** The list-append reducer view class. + * + * This is the view class for reducers created with + * `cilk::reducer< cilk::op_list_append >`. It holds the + * accumulator variable for the reduction, and allows only append operations + * to be performed on it. + * + * @note The reducer "dereference" operation (`reducer::operator *()`) + * yields a reference to the view. Thus, for example, the view class's + * `push_back` operation would be used in an expression like + * `r->push_back(a)`, where `r` is a list-append reducer variable. + * + * @tparam Type The list element type (not the list type). + * @tparam Allocator The list allocator type. + * + * @see ReducersList + * @see op_list_append + */ +template ::allocator_type> +class op_list_append_view : public internal::list_view_base +{ + typedef internal::list_view_base base; + typedef std::list list_type; + typedef typename list_type::iterator iterator; + + iterator end() { return this->m_value.end(); } + +public: + + /** @name Constructors. + * + * All op_list_append_view constructors simply pass their arguments on to + * the @ref internal::list_view_base base class constructor. + * + * @ref internal::list_view_base supports all the std::list constructor + * forms, as well as the reducer move_in constructor form. + */ + //@{ + + op_list_append_view() : base() {} + + template + op_list_append_view(const T1& x1) : base(x1) {} + + template + op_list_append_view(const T1& x1, const T2& x2) : base(x1, x2) {} + + template + op_list_append_view(const T1& x1, const T2& x2, const T3& x3) : + base(x1, x2, x3) {} + + //@} + + /** @name View modifier operations. + */ + //@{ + + /** Adds an element at the end of the list. + * + * This is equivalent to `list.push_back(element)` + */ + void push_back(const Type& element) + { this->m_value.push_back(element); } + + /** Inserts elements at the end of the list. + * + * This is equivalent to `list.insert(list.end(), n, element)` + */ + void insert_back(typename list_type::size_type n, const Type& element) + { this->m_value.insert(end(), n, element); } + + /** Inserts elements at the end of the list. + * + * This is equivalent to `list.insert(list.end(), first, last)` + */ + template + void insert_back(Iter first, Iter last) + { this->m_value.insert(end(), first, last); } + + /** Splices elements at the end of the list. + * + * This is equivalent to `list.splice(list.end(), x)` + */ + void splice_back(list_type& x) { + if (x.get_allocator() == this->m_value.get_allocator()) + this->m_value.splice(end(), x); + else { + insert_back(x.begin(), x.end()); + x.clear(); + } + } + + /** Splices elements at the end of the list. + * + * This is equivalent to `list.splice(list.end(), x, i)` + */ + void splice_back(list_type& x, iterator i) { + if (x.get_allocator() == this->m_value.get_allocator()) + this->m_value.splice(end(), x, i); + else { + push_back(*i); + x.erase(i); + } + } + + /** Splices elements at the end of the list. + * + * This is equivalent to `list.splice(list.end(), x, first, last)` + */ + void splice_back(list_type& x, iterator first, iterator last) { + if (x.get_allocator() == this->m_value.get_allocator()) + this->m_value.splice(end(), x, first, last); + else { + insert_back(first, last); + x.erase(first, last); + } + } + + //@} + + /** Reduces the views of two strands. + * + * This function is invoked by the @ref op_list_append monoid to combine + * the views of two strands when the right strand merges with the left + * one. It appends the value contained in the right-strand view to the + * value contained in the left-strand view, and leaves the value in the + * right-strand view undefined. + * + * @param right A pointer to the right-strand view. (`this` points to + * the left-strand view.) + * + * @note Used only by the @ref op_list_append monoid to implement the + * monoid reduce operation. + */ + void reduce(op_list_append_view* right) + { + __CILKRTS_ASSERT( + this->m_value.get_allocator() == right->m_value.get_allocator()); + this->m_value.splice(end(), right->m_value); + } +}; + + +/** The list-prepend reducer view class. + * + * This is the view class for reducers created with + * `cilk::reducer< cilk::op_list_prepend >`. It holds the + * accumulator variable for the reduction, and allows only prepend operations + * to be performed on it. + * + * @note The reducer "dereference" operation (`reducer::operator *()`) + * yields a reference to the view. Thus, for example, the view class's + * `push_front` operation would be used in an expression like + * `r->push_front(a)`, where `r` is a list-prepend reducer variable. + * + * @tparam Type The list element type (not the list type). + * @tparam Allocator The list allocator type. + * + * @see ReducersList + * @see op_list_prepend + */ +template ::allocator_type> +class op_list_prepend_view : public internal::list_view_base +{ + typedef internal::list_view_base base; + typedef std::list list_type; + typedef typename list_type::iterator iterator; + + iterator begin() { return this->m_value.begin(); } + +public: + + /** @name Constructors. + * + * All op_list_prepend_view constructors simply pass their arguments on to + * the @ref internal::list_view_base base class constructor. + * + * @ref internal::list_view_base supports all the std::list constructor + * forms, as well as the reducer move_in constructor form. + * + */ + //@{ + + op_list_prepend_view() : base() {} + + template + op_list_prepend_view(const T1& x1) : base(x1) {} + + template + op_list_prepend_view(const T1& x1, const T2& x2) : base(x1, x2) {} + + template + op_list_prepend_view(const T1& x1, const T2& x2, const T3& x3) : + base(x1, x2, x3) {} + + //@} + + /** @name View modifier operations. + */ + //@{ + + /** Adds an element at the beginning of the list. + * + * This is equivalent to `list.push_front(element)` + */ + void push_front(const Type& element) + { this->m_value.push_front(element); } + + /** Inserts elements at the beginning of the list. + * + * This is equivalent to `list.insert(list.begin(), n, element)` + */ + void insert_front(typename list_type::size_type n, const Type& element) + { this->m_value.insert(begin(), n, element); } + + /** Inserts elements at the beginning of the list. + * + * This is equivalent to `list.insert(list.begin(), first, last)` + */ + template + void insert_front(Iter first, Iter last) + { this->m_value.insert(begin(), first, last); } + + /** Splices elements at the beginning of the list. + * + * This is equivalent to `list.splice(list.begin(), x)` + */ + void splice_front(list_type& x) { + if (x.get_allocator() == this->m_value.get_allocator()) + this->m_value.splice(begin(), x); + else { + insert_front(x.begin(), x.begin()); + x.clear(); + } + } + + /** Splices elements at the beginning of the list. + * + * This is equivalent to `list.splice(list.begin(), x, i)` + */ + void splice_front(list_type& x, iterator i) { + if (x.get_allocator() == this->m_value.get_allocator()) + this->m_value.splice(begin(), x, i); + else { + push_front(*i); + x.erase(i); + } + } + + /** Splices elements at the beginning of the list. + * + * This is equivalent to `list.splice(list.begin(), x, first, last)` + */ + void splice_front(list_type& x, iterator first, iterator last) { + if (x.get_allocator() == this->m_value.get_allocator()) + this->m_value.splice(begin(), x, first, last); + else { + insert_front(first, last); + x.erase(first, last); + } + } + + //@} + + /** Reduces the views of two strands. + * + * This function is invoked by the @ref op_list_prepend monoid to combine + * the views of two strands when the right strand merges with the left + * one. It prepends the value contained in the right-strand view to the + * value contained in the left-strand view, and leaves the value in the + * right-strand view undefined. + * + * @param right A pointer to the right-strand view. (`this` points to + * the left-strand view.) + * + * @note Used only by the @ref op_list_prepend monoid to implement the + * monoid reduce operation. + */ + /** Reduce operation. + * + * Required by @ref monoid_base. + */ + void reduce(op_list_prepend_view* right) + { + __CILKRTS_ASSERT( + this->m_value.get_allocator() == right->m_value.get_allocator()); + this->m_value.splice(begin(), right->m_value); + } +}; + + + +/** Monoid class for list-append reductions. Instantiate the cilk::reducer + * template class with a op_list_append monoid to create a list-append reducer + * class. For example, to create a list of strings: + * + * cilk::reducer< cilk::op_list_append > r; + * + * @tparam Type The list element type (not the list type). + * @tparam Alloc The list allocator type. + * @tparam Align If `false` (the default), reducers instantiated on this + * monoid will be naturally aligned (the Intel Cilk Plus library 1.0 + * behavior). If `true`, reducers instantiated on this monoid + * will be cache-aligned for binary compatibility with + * reducers in Intel Cilk Plus library version 0.9. + * + * @see ReducersList + * @see op_list_append_view + */ +template ::allocator_type, + bool Align = false> +struct op_list_append : + public internal::list_monoid_base, Align> +{ + /// Construct with default allocator. + op_list_append() {} + /// Construct with specified allocator. + op_list_append(const Allocator& alloc) : + internal::list_monoid_base, Align>(alloc) {} +}; + +/** Monoid class for list-prepend reductions. Instantiate the cilk::reducer + * template class with a op_list_prepend monoid to create a list-prepend + * reducer class. For example, to create a list of strings: + * + * cilk::reducer< cilk::op_list_prepend > r; + * + * @tparam Type The list element type (not the list type). + * @tparam Alloc The list allocator type. + * @tparam Align If `false` (the default), reducers instantiated on this + * monoid will be naturally aligned (the Intel Cilk Plus library 1.0 + * behavior). If `true`, reducers instantiated on this monoid + * will be cache-aligned for binary compatibility with + * reducers in Intel Cilk Plus library version 0.9. + * + * @see ReducersList + * @see op_list_prepend_view + */ +template ::allocator_type, + bool Align = false> +struct op_list_prepend : + public internal::list_monoid_base, Align> +{ + /// Construct with default allocator. + op_list_prepend() {} + /// Construct with specified allocator. + op_list_prepend(const Allocator& alloc) : + internal::list_monoid_base, Align>(alloc) {} +}; + + +/** Deprecated list-append reducer wrapper class. + * + * reducer_list_append is the same as + * @ref reducer<@ref op_list_append>, except that reducer_list_append is a + * proxy for the contained view, so that accumulator variable update + * operations can be applied directly to the reducer. For example, an element + * is appended to a `reducer<%op_list_append>` with `r->push_back(a)`, but an + * element can be appended to a `%reducer_list_append` with `r.push_back(a)`. + * + * @deprecated Users are strongly encouraged to use `reducer` + * reducers rather than the old wrappers like reducer_list_append. + * The `reducer` reducers show the reducer/monoid/view + * architecture more clearly, are more consistent in their + * implementation, and present a simpler model for new + * user-implemented reducers. + * + * @note Implicit conversions are provided between `%reducer_list_append` + * and `reducer<%op_list_append>`. This allows incremental code + * conversion: old code that used `%reducer_list_append` can pass a + * `%reducer_list_append` to a converted function that now expects a + * pointer or reference to a `reducer<%op_list_append>`, and vice + * versa. + * + * @tparam Type The value type of the list. + * @tparam Allocator The allocator type of the list. + * + * @see op_list_append + * @see reducer + * @see ReducersList + */ +template > +class reducer_list_append : + public reducer > +{ + typedef reducer > base; + using base::view; +public: + + /// The reducer's list type. + typedef typename base::value_type list_type; + + /// The list's element type. + typedef Type list_value_type; + + /// The reducer's primitive component type. + typedef Type basic_value_type; + + /// The monoid type. + typedef typename base::monoid_type Monoid; + + /** @name Constructors + */ + //@{ + + /** Constructs a reducer with an empty list. + */ + reducer_list_append() {} + + /** Constructs a reducer with a specified initial list value. + */ + reducer_list_append(const std::list &initial_value) : + base(initial_value) {} + + //@} + + + /** @name Forwarded functions + * @details Functions that update the contained accumulator variable are + * simply forwarded to the contained @ref op_and_view. */ + //@{ + + /// @copydoc op_list_append_view::push_back(const Type&) + void push_back(const Type& element) { view().push_back(element); } + + //@} + + /** Allows mutable access to the list within the current view. + * + * @warning If this method is called before the parallel calculation is + * complete, the list returned by this method will be a partial + * result. + * + * @returns A mutable reference to the list within the current view. + */ + list_type &get_reference() { return view().view_get_reference(); } + + /** Allows read-only access to the list within the current view. + * + * @warning If this method is called before the parallel calculation is + * complete, the list returned by this method will be a partial + * result. + * + * @returns A const reference to the list within the current view. + */ + list_type const &get_reference() const { return view().view_get_reference(); } + + /// @name Dereference + //@{ + /** Dereferencing a wrapper is a no-op. It simply returns the wrapper. + * Combined with the rule that a wrapper forwards view operations to the + * view, this means that view operations can be written the same way on + * reducers and wrappers, which is convenient for incrementally + * converting code using wrappers to code using reducers. That is: + * + * reducer< op_list_append > r; + * r->push_back(a); // *r returns the view + * // push_back is a view member function + * + * reducer_list_append w; + * w->push_back(a); // *w returns the wrapper + * // push_back is a wrapper member function that + * // calls the corresponding view function + */ + //@{ + reducer_list_append& operator*() { return *this; } + reducer_list_append const& operator*() const { return *this; } + + reducer_list_append* operator->() { return this; } + reducer_list_append const* operator->() const { return this; } + //@} + + /** @name Upcast + * @details In Intel Cilk Plus library 0.9, reducers were always cache-aligned. + * In library 1.0, reducer cache alignment is optional. By default, + * reducers are unaligned (i.e., just naturally aligned), but legacy + * wrappers inherit from cache-aligned reducers for binary compatibility. + * + * This means that a wrapper will automatically be upcast to its aligned + * reducer base class. The following conversion operators provide + * pseudo-upcasts to the corresponding unaligned reducer class. + */ + //@{ + operator reducer< op_list_append >& () + { + return *reinterpret_cast< + reducer< op_list_append >* + >(this); + } + operator const reducer< op_list_append >& () const + { + return *reinterpret_cast< + const reducer< op_list_append >* + >(this); + } + //@} + +}; + + +/** Deprecated list-prepend reducer wrapper class. + * + * reducer_list_prepend is the same as + * @ref reducer<@ref op_list_prepend>, except that reducer_list_prepend is a + * proxy for the contained view, so that accumulator variable update operations + * can be applied directly to the reducer. For example, an element is prepended + * to a `reducer` with `r->push_back(a)`, but an element is + * prepended to a `reducer_list_prepend` with `r.push_back(a)`. + * + * @deprecated Users are strongly encouraged to use `reducer` + * reducers rather than the old wrappers like reducer_list_prepend. + * The `reducer` reducers show the reducer/monoid/view + * architecture more clearly, are more consistent in their + * implementation, and present a simpler model for new + * user-implemented reducers. + * + * @note Implicit conversions are provided between `%reducer_list_prepend` + * and `reducer<%op_list_prepend>`. This allows incremental code + * conversion: old code that used `%reducer_list_prepend` can pass a + * `%reducer_list_prepend` to a converted function that now expects a + * pointer or reference to a `reducer<%op_list_prepend>`, and vice + * versa. + * + * @tparam Type The value type of the list. + * @tparam Allocator The allocator type of the list. + * + * @see op_list_prepend + * @see reducer + * @see ReducersList + */ +template > +class reducer_list_prepend : + public reducer > +{ + typedef reducer > base; + using base::view; +public: + + /** The reducer's list type. + */ + typedef typename base::value_type list_type; + + /** The list's element type. + */ + typedef Type list_value_type; + + /** The reducer's primitive component type. + */ + typedef Type basic_value_type; + + /** The monoid type. + */ + typedef typename base::monoid_type Monoid; + + /** @name Constructors + */ + //@{ + + /** Constructs a reducer with an empty list. + */ + reducer_list_prepend() {} + + /** Constructs a reducer with a specified initial list value. + */ + reducer_list_prepend(const std::list &initial_value) : + base(initial_value) {} + + //@} + + /** @name Forwarded functions + * @details Functions that update the contained accumulator variable are + * simply forwarded to the contained @ref op_and_view. + */ + //@{ + + /// @copydoc op_list_prepend_view::push_front(const Type&) + void push_front(const Type& element) { view().push_front(element); } + + //@} + + /** Allows mutable access to the list within the current view. + * + * @warning If this method is called before the parallel calculation is + * complete, the list returned by this method will be a partial + * result. + * + * @returns A mutable reference to the list within the current view. + */ + list_type &get_reference() { return view().view_get_reference(); } + + /** Allows read-only access to the list within the current view. + * + * @warning If this method is called before the parallel calculation is + * complete, the list returned by this method will be a partial + * result. + * + * @returns A const reference to the list within the current view. + */ + list_type const &get_reference() const { return view().view_get_reference(); } + + /// @name Dereference + /** Dereferencing a wrapper is a no-op. It simply returns the wrapper. + * Combined with the rule that a wrapper forwards view operations to the + * view, this means that view operations can be written the same way on + * reducers and wrappers, which is convenient for incrementally + * converting code using wrappers to code using reducers. That is: + * + * reducer< op_list_prepend > r; + * r->push_front(a); // *r returns the view + * // push_front is a view member function + * + * reducer_list_prepend w; + * w->push_front(a); // *w returns the wrapper + * // push_front is a wrapper member function that + * // calls the corresponding view function + */ + //@{ + reducer_list_prepend& operator*() { return *this; } + reducer_list_prepend const& operator*() const { return *this; } + + reducer_list_prepend* operator->() { return this; } + reducer_list_prepend const* operator->() const { return this; } + //@} + + /** @name Upcast + * @details In Intel Cilk Plus library 0.9, reducers were always cache-aligned. + * In library 1.0, reducer cache alignment is optional. By default, + * reducers are unaligned (i.e., just naturally aligned), but legacy + * wrappers inherit from cache-aligned reducers for binary compatibility. + * + * This means that a wrapper will automatically be upcast to its aligned + * reducer base class. The following conversion operators provide + * pseudo-upcasts to the corresponding unaligned reducer class. + */ + //@{ + operator reducer< op_list_prepend >& () + { + return *reinterpret_cast< + reducer< op_list_prepend >* + >(this); + } + operator const reducer< op_list_prepend >& () const + { + return *reinterpret_cast< + const reducer< op_list_prepend >* + >(this); + } + //@} + +}; + +/// @cond internal + +/** Metafunction specialization for reducer conversion. + * + * This specialization of the @ref legacy_reducer_downcast template class + * defined in reducer.h causes the `reducer< op_list_append >` + * class to have an `operator reducer_list_append& ()` + * conversion operator that statically downcasts the `reducer` + * to the corresponding `reducer_list_append` type. (The reverse conversion, + * from `reducer_list_append` to `reducer`, is just an upcast, + * which is provided for free by the language.) + */ +template +struct legacy_reducer_downcast > > +{ + typedef reducer_list_append type; +}; + +/** Metafunction specialization for reducer conversion. + * + * This specialization of the @ref legacy_reducer_downcast template class + * defined in reducer.h causes the + * `reducer< op_list_prepend >` class to have an + * `operator reducer_list_prepend& ()` conversion operator + * that statically downcasts the `reducer` to the + * corresponding `reducer_list_prepend` type. (The reverse conversion, from + * `reducer_list_prepend` to `reducer`, is just an upcast, + * which is provided for free by the language.) + */ +template +struct legacy_reducer_downcast > > +{ + typedef reducer_list_prepend type; +}; + +/// @endcond + +//@} + +} // Close namespace cilk + +#endif // REDUCER_LIST_H_INCLUDED diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_max.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_max.h new file mode 100644 index 0000000000..3982cb11c2 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_max.h @@ -0,0 +1,57 @@ +/* reducer_max.h -*- C++ -*- + * + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + */ + +/** @file reducer_max.h + * + * @brief Defines classes for doing parallel maximum reductions. + * + * @ingroup ReducersMinMax + * + * @see ReducersMinMax + */ + +#include "reducer_min_max.h" diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_min.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_min.h new file mode 100644 index 0000000000..912979d722 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_min.h @@ -0,0 +1,57 @@ +/* reducer_min.h -*- C++ -*- + * + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + */ + +/** @file reducer_min.h + * + * @brief Defines classes for doing parallel minimum reductions. + * + * @ingroup ReducersMinMax + * + * @see ReducersMinMax + */ + +#include "reducer_min_max.h" diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_min_max.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_min_max.h new file mode 100644 index 0000000000..641aa82390 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_min_max.h @@ -0,0 +1,3741 @@ +/* reducer_min_max.h -*- C++ -*- + * + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + */ + +/** @file reducer_min_max.h + * + * @brief Defines classes for doing parallel minimum and maximum reductions. + * + * @ingroup ReducersMinMax + * + * @see ReducersMinMax + */ + +#ifndef REDUCER_MIN_MAX_H_INCLUDED +#define REDUCER_MIN_MAX_H_INCLUDED + +#include + +#ifdef __cplusplus + +#include +#include + +/** @defgroup ReducersMinMax Minimum and Maximum Reducers + * + * Minimum and maximum reducers allow the computation of the minimum or + * maximum of a set of values in parallel. + * + * @ingroup Reducers + * + * You should be familiar with @ref pagereducers "Intel(R) Cilk(TM) Plus reducers", + * described in file `reducers.md`, and particularly with @ref reducers_using, + * before trying to use the information in this file. + * + * @section redminmax_usage Usage Examples + * + * cilk::reducer< cilk::op_max > rm; + * cilk_for (int i = 0; i < ARRAY_SIZE; ++i) + * { + * rm->calc_max(a[i]); // or *rm = cilk::max_of(*max, a[i]) + * } + * std::cout << "maximum value is " << rm.get_value() << std::endl; + * + * and + * + * cilk::reducer< cilk::op_min_index > rmi; + * cilk_for (int i = 0; i < ARRAY_SIZE; ++i) + * { + * rmi->calc_min(i, a[i]) // or *rmi = cilk::min_of(*rmi, i, a[i]); + * } + * std::cout << "minimum value a[" << rmi.get_value().first << "] = " + * << rmi.get_value().second << std::endl; + * + * @section redminmax_monoid The Monoid + * + * @subsection redminmax_monoid_values Value Set + * + * The value set of a minimum or maximum reducer is the set of values of + * `Type`, augmented with a "special identity value" which is not a value of + * `Type`, but which is defined to be greater than (less than) any value of + * `Type`. + * + * @subsection redminmax_monoid_operator Operator + * + * By default, the operator of a minimum reducer is defined as + * + * x MIN y == (x < y) ? x : y + * + * Thus, `a1 MIN a2 MIN … an` is the first `ai` which is not greater than any + * other `ai`. + * + * The operator of a maximum reducer is defined as + * + * x MAX y == (x > y) ? x : y + * + * Thus, `a1 MAX a2 MAX … an` is the first `ai` which is not less than any + * other `ai`. + * + * @subsection redminmax_monoid_comparators Comparators + * + * Min/max reducers are not limited to finding the minimum or maximum value + * determined by the `<` or `>` operator. In fact, all min/max reducers use a + * _comparator_, which is either a function or an object of a function class + * that defines a [strict weak ordering] + * (http://en.wikipedia.org/wiki/Strict_weak_ordering#Strict_weak_orderings) + * on a set of values. (This is exactly the same as the requirement for the + * comparison predicate for STL associative containers and sorting + * algorithms.) + * + * Just as with STL algorithms and containers, the comparator type parameter + * for min/max reducers is optional. If it is omitted, it defaults to + * `std::less`, which gives the behavior described in the previous section. + * Using non-default comparators (anything other than `std::less`) with + * min/max reducers is just like using them with STL containers and + * algorithms. + * + * Taking comparator objects into account, the reduction operation `MIN` for a + * minimum reducer is defined as + * + * x MIN y == compare(x, y) ? x : y + * + * where `compare()` is the reducer's comparator. Similarly, the reduction + * operation MAX for a maximum reducer is defined as + * + * x MAX y == compare(y, x) ? x : y + * + * (If `compare(x, y) == x < y`, then `compare(y, x) == x > y`.) + * + * @subsection redminmax_monoid_identity Identity + * + * The identity value of a min/max reducer is its monoid's + * ["special identity value"](#redminmax_monoid_values), which is not a value + * of the reducer's data type. (See @ref redminmax_initial.) + * + * @section redminmax_index Value and Index Reducers + * + * Min/max reducers come in two families. The _value_ reducers, with the + * `op_min` and `op_max` monoids, simply find the smallest or largest value + * from a set of values. The _index_ reducers, with the `op_min_index` and + * `op_max_index` monoids, also record an index value associated with the + * first occurrence of the smallest or largest value. + * + * In the `%op_min_index` usage example [above](#redminmax_usage), the values + * are taken from an array, and the index of a value is the index of the array + * element it comes from. More generally, though, an index can be any sort of + * key which identifies a particular value in a collection of values. For + * example, if the values were taken from the nodes of a tree, then the + * "index" of a value might be a pointer to the node containing that value. + * + * A min/max index reducer is essentially the same as a min/max value reducer + * whose value type is an (index, value) pair, and whose comparator ignores + * the index part of the pair. (index, value) pairs are represented by + * `std::pair` objects. This has the consequence that wherever + * the interface of a min/max value reducer has a `Type`, the interface of a + * min/max index reducer has a `std::pair`. (There are + * convenience variants of the `reducer(Type)` constructor and the + * `calc_min()`, `calc_max()`, `%min_of()`, and `%max_of()` functions that + * take an index argument and a value argument instead of a single index/value + * pair argument.) + * + * @section redminmax_operations Operations + * + * @subsection redminmax_constructors Constructors + * + * @subsubsection redminmax_constructors_value Min/Max Value Reducers + * + * reducer() // identity + * reducer(const Compare& compare) // identity + * reducer(const Type& value) + * reducer(move_in(Type& variable)) + * reducer(const Type& value, const Compare& compare) + * reducer(move_in(Type& variable), const Compare& compare) + * + * @subsubsection redminmax_constructors_index Min/Max Index Reducers + * + * reducer() // identity + * reducer(const Compare& compare) // identity + * reducer(const std::pair& pair) + * reducer(const Index& index, const Type& value) + * reducer(move_in(std::pair& variable)) + * reducer(const std::pair& pair, const Compare& compare) + * reducer(const Index& index, const Type& value, const Compare& compare) + * reducer(move_in(std::pair& variable), const Compare& compare) + * + * See the explanation of the following two constructors in + * @ref redminmax_index_vector. + * + * reducer(const Index& index) + * reducer(const Index& index, const Compare& compare) + * + * @subsection redminmax_get_set Set and Get + * + * r.set_value(const Type& value) + * Type = r.get_value() const + * r.move_in(Type& variable) + * r.move_out(Type& variable) + * + * Note that for an index reducer, the `Type` in these operations is actually a + * `std::pair`. (See @ref redminmax_index.) There is _not_ a + * `set_value(value, index)` operation. + * + * @subsection redminmax_initial Initial Values and is_set() + * + * The initial value of the leftmost view of a default-initialized min/max + * reducer, or of a non-leftmost view (created for a stolen parallel strand) + * is the special identity value, which is not a value of the reducer's value + * type. + * + * A view will have a real (non-identity) value if: + * + * - it is the leftmost view of a reducer that was constructed with an + * initial value, or + * - it was assigned a value with a call to `reducer.set_value()` or + * `reducer.move_in()`, or + * - it has been updated with a call to `reducer->calc_min()` or + * `reducer->calc_max()`, or + * - it has been updated with an assignment `*reducer = min_of(*reducer, x)` + * or `*reducer = max_of(*reducer, x)`. + * + * Calling `get_value()` or `move_out()` on a reducer whose view has the + * special identity value will yield an undefined result. The `is_set()` + * function can be used to test whether a view has the special identity value + * or a real value. If a reducer's current view has the special identity + * value, then `reducer()->is_set()` will return `false` (and + * `reducer.get_value()` will return an undefined value); if the view has a + * real value, them `reducer->is_set()` will return `true` and + * `reducer.get_value()` will return the value. + * + * @subsubsection redminmax_index_vector Special Issues with Min/Max Index Reducers + * + * The index portion of the computed index/value pair will be wrong in the + * following special case: + * + * - The reducer's value type is a simple numeric type. + * - The reducer uses the default comparator (`std::less`). + * - The reducer is updated at least once with a call to `calc_min()` or + * `calc_max()` or an assignment with `min_of()` or `max_of()`. + * - The value in _every_ update to the reducer is the maximum value of the + * value type (for a min_index reducer) or the minimum value of the value + * type (for a max_index reducer). + * + * In this case, `reducer.get_value().first` should be the index argument from + * the first reducer update, but it will actually be the default value of the + * `Index` type. Now, in the common case where the index type is an integer + * type and the reducer is finding the smallest or largest element in an + * array, the default value of the index type will be zero, which is the + * index of the first element in the array, so everything will work out: + * + * unsigned a[3] = {0, 0, 0}; + * reducer< op_max_index > r; + * for (int i = 0; i < 3; ++i) r->calc_max(i, a[i]); + * // r.get_value() = (0, 0) + * + * However, it doesn't always work out so well: + * + * typedef std::map my_map; + * my_map a; + * a["first"] = 0; + * a["second"] = 0; + * a["third"] = 0; + * reducer< op_max_index > r; + * for (typename my_map::iterator i = a.begin(); i != a.end(); ++i) + * r.calc_max(i->first, i->second); + * // r.get_value() = ("", 0), should be ("first", 0) + * + * If you know that no data value is associated with the default index value, + * then you can treat the default index value as a flag meaning "use the index + * of the first data value." But suppose that you don't know whether there is + * an element in the map with index `""`. Then you won't know what to do when + * `r.get_value().first == ""`. + * + * As a workaround for this conundrum, you can specify an alternative + * "default" index value. Either provide an index argument, _but not a + * value argument_, to the reducer constructor: + * + * reducer< op_max_index > + * r(a.empty() ? std::string() : a.begin()->first); + * + * or specify the default index with the view `set_default_index()` function: + * + * reducer< op_max_index > r; + * if (!a.empty()) r->set_default_index(a.begin()->first); + * + * Note that setting a default index, unlike setting an initial value, does + * not mark the view as having a non-identity value: + * + * reducer< op_min_index > r; + * r->set_default_index(-1); + * // r->is_set() = false + * // r.get_value() is undefined + * + * @subsection redminmax_view_ops View Operations + * + * The basic reduction operation is `x = x MIN a` for a minimum reducer, or + * `x = x MAX a` for a maximum reducer. The basic syntax for these operations + * uses the `calc_min()` and `calc_max()` member functions of the view class. + * An assignment syntax is also provided, using the `%cilk::min_of()` and + * `%cilk::max_of()` global functions: + * + * Class | Modifier | Assignment + * ---------------|---------------------|----------- + * `op_min` | `r->calc_min(x)` | `*r = min_of(*r, x)` or `*r = min_of(x, *r)` + * `op_max` | `r->calc_max(x)` | `*r = max_of(*r, x)` or `*r = max_of(x, *r)` + * `op_min_index` | `r->calc_min(i, x)` | `*r = min_of(*r, i, x)` or `*r = min_of(i, x, *r)` + * `op_max_index` | `r->calc_max(i, x)` | `*r = max_of(*r, i, x)` or `*r = max_of(i, x, *r)` + * + * Wherever an "`i`, `x`" argument pair is shown in the table above, a single + * pair argument may be passed instead. For example: + * + * Index index; + * Type value; + * std::pair ind_val(index, value); + * // The following statements are all equivalent. + * r->calc_min(index, value); + * r->calc_min(ind_val); + * *r = min_of(*r, index, value); + * *r = min_of(*r, ind_val); + * + * The `calc_min()` and `calc_max()` member functions return a reference to + * the view, so they can be chained: + * + * r->calc_max(x).calc_max(y).calc_max(z); + * + * In a `%min_of()` or `%max_of()` assignment, the view on the left-hand side + * of the assignment must be the same as the view argument in the call. + * Otherwise, the behavior is undefined (but an assertion error will occur if + * the code is compiled with debugging enabled). + * + * *r = max_of(*r, x); // OK + * *r1 = max_of(*r2, y); // ERROR + * + * `%min_of()` and `%max_of()` calls can be nested: + * + * *r = max_of(max_of(max_of(*r, x), y), z); + * *r = min_of(i, a[i], min_of(j, a[j], min_of(k, a[k], *r))); + * + * @section redminmax_compatibility Binary Compatibility Issues + * + * Most Intel Cilk Plus library reducers provide binary compatibility between + * `reducer_KIND` reducers compiled with Intel Cilk Plus library version 0.9 + * (distributed with Intel® C++ Composer XE version 13.0 and earlier) and the + * ame reducers compiled with Intel Cilk Plus library version 1.0 and later. + * + * Because of implementation changes that were needed to allow vectorization + * of loops containing min/max reducers, this binary compatibility is _not_ + * generally available for min/max reducers, either between Intel Cilk Plus library + * versions 0.9 and 1.0, or between versions 1.0 and 1.1. (Code compiled with + * different versions can be linked together safely, but min/max reducers in + * different library versions are in different namespaces, so reducer objects + * cannot be shared between them.) + * + * If this is an inconvenience, the simplest solution is just to recompile any + * existing code you may have that uses min/max reducers. If that is + * impossible, you can define the `CILK_LIBRARY_0_9_REDUCER_MINMAX` macro (on + * the compiler command line, or in your source code before including + * `reducer_min_max.h`) when compiling with the new library. This will cause + * it to generate numeric reducers that will be link-time and run-time + * compatible with the 0.9 library. + * + * @subsection redminmax_compatibility_stateful Non-empty Comparators + * + * The representation of min/max reducers with non-empty comparator objects or + * with comparator functions is so different in between the 0.9 and 1.1 + * libraries that there is no way to make them binary compatible, even when + * compiling with `CILK_LIBRARY_0_9_REDUCER_MINMAX`. Therefore, the + * `reducer_{min|max}[_index]` wrapper classes have been coded in the 1.0 and + * later library so that they will not even compile when instantiated with a + * non-empty comparator class. + * + * This is not a problem when using an empty comparator class, such as the + * default `std::less`. + * + * @section redminmax_types Type Requirements + * + * `Type` and `Index` must be `Copy Constructible`, `Default Constructible`, + * and `Assignable`. + * + * `Compare` must be `Copy Constructible` if the reducer is constructed with a + * `compare` argument, and `Default Constructible` otherwise. + * + * The `Compare` function must induce a strict weak ordering on the elements + * of `Type`. + * + * @section redminmax_in_c Minimum and Maximum Reducers in C + * + * These macros can be used to do minimum and maximum reductions in C: + * + * Declaration | Type | Operation + * -----------------------------|-----------------------------------|---------- + * @ref CILK_C_REDUCER_MIN |@ref CILK_C_REDUCER_MIN_TYPE |@ref CILK_C_REDUCER_MIN_CALC + * @ref CILK_C_REDUCER_MAX |@ref CILK_C_REDUCER_MAX_TYPE |@ref CILK_C_REDUCER_MAX_CALC + * @ref CILK_C_REDUCER_MIN_INDEX |@ref CILK_C_REDUCER_MIN_INDEX_TYPE |@ref CILK_C_REDUCER_MIN_INDEX_CALC + * @ref CILK_C_REDUCER_MAX_INDEX |@ref CILK_C_REDUCER_MAX_INDEX_TYPE |@ref CILK_C_REDUCER_MAX_INDEX_CALC + * + * For example: + * + * CILK_C_REDUCER_MIN(r, int, INT_MAX); + * CILK_C_REGISTER_REDUCER(r); + * cilk_for(int i = 0; i != n; ++i) { + * CILK_C_REDUCER_MIN_CALC(r, a[i]); + * } + * CILK_C_UNREGISTER_REDUCER(r); + * printf("The smallest value in a is %d\n", REDUCER_VIEW(r)); + * + * + * CILK_C_REDUCER_MAX_INDEX(r, uint, 0); + * CILK_C_REGISTER_REDUCER(r); + * cilk_for(int i = 0; i != n; ++i) { + * CILK_C_REDUCER_MAX_INDEX_CALC(r, i, a[i]); + * } + * CILK_C_UNREGISTER_REDUCER(r); + * printf("The largest value in a is %u at %d\n", + * REDUCER_VIEW (r).value, REDUCER_VIEW(r).index); + * + * See @ref reducers_c_predefined. + */ + +namespace cilk { + +/** @defgroup ReducersMinMaxBinComp Binary compatibility + * + * If the macro `CILK_LIBRARY_0_9_REDUCER_MINMAX` is defined, then we generate + * reducer code and data structures which are binary-compatible with code that + * was compiled with the old min/max wrapper definitions, so we want the + * mangled names of the legacy min/max reducer wrapper classes to be the + * same as the names produced by the old definitions. + * + * Conversely, if the macro is not defined, then we generate binary- + * incompatible code, so we want different mangled names, to make sure that + * the linker does not allow new and old compiled legacy wrappers to be passed + * to one another. (Global variables are a different, and probably insoluble, + * problem.) + * + * Similarly, min/max classes compiled with and without + * CILK_LIBRARY_0_9_REDUCER_MINMAX are binary-incompatible, and must get + * different mangled names. + * + * The trick is, when compiling in normal (non-compatibility) mode, wrap + * everything in an extra namespace, and then `use` it into the top-level cilk + * namespace. Then + * + * * Classes and functions compiled in normal mode will be in + * different namespaces from the same classes and functions compiled in + * compatibility mode. + * * The legacy wrapper classes and functions will be in the same namespace + * as the same classes and functions compiled with the 0.9 library if and + * only if they are compiled in compatibility mode. + * + * @ingroup ReducersMinMax + */ + +#ifndef CILK_LIBRARY_0_9_REDUCER_MINMAX +/** Namespace to wrap min/max reducer definitions when not compiling in "binary + * compatibility" mode. + * + * By default, all of the min/max reducer definitions are defined in this + * namespace and then imported into namespace ::cilk, so that they do not + * clash with the legacy definitions with the same names. However, if the + * macro `CILK_LIBRARY_0_9_REDUCER_MINMAX` is defined, then the min/max + * definitions go directly into namespace ::cilk, so that, for example, + * cilk::reducer_max defined with the 1.0 library is equivalent (to the + * linker) to cilk::reducer_max defined with the 0.9 library. + * + * @ingroup ReducersMinMaxBinComp + * @ingroup ReducersMinMax + */ +namespace cilk_lib_1_1 { +#endif + +/** Namespace containing internal implementation classes and functions for + * min/max reducers. + * + * @ingroup ReducersMinMax + */ +namespace min_max_internal { + +using ::cilk::internal::binary_functor; +using ::cilk::internal::typed_indirect_binary_function; +using ::cilk::internal::class_is_empty; + +/** @defgroup ReducersMinMaxIsSet The "is_set optimization" + * + * The obvious definition of the identity value for a max or min reducer is as + * the smallest (or largest) value of the value type. However, for an + * arbitrary comparator and/or an arbitrary value type, the largest / smallest + * value may not be known. It may not even be defined - what is the largest + * string? + * + * Therefore, min/max reducers represent their value internally as a pair + * `(value, is_set)`. When `is_set` is true, the pair represents the known + * value `value`; when `is_set` is false, the pair represents the identity + * value. + * + * This is an effective solution, but the most common use of min/max reducers + * is probably with numeric types and the default definition of minimum or + * maximum (using `std::less`), in which case there are well-defined, knowable + * smallest and largest values. Testing `is_set` for every comparison is then + * unnecessary and wasteful. + * + * The "is_set optimization" just means generating code that doesn't use + * `is_set` when it isn't needed. It is implemented using two metaprogramming + * classes: + * + * - do_is_set_optimization tests whether the optimization is applicable. + * - identity_value gets the appropriate identity value for a type. + * + * The is_set optimization is the reason that min/max reducers compiled with + * Intel Cilk Plus library 1.0 are binary-incompatible with the same reducers + * compiled with library 0.9, and therefore the optimization is suppressed when + * compiling in + * ReducersMinMaxBinComp "binary compatibility mode". + * + * @ingroup ReducersMinMax + */ + +/** Tests whether the ReducersMinMaxIsSet "is_set optimization" is + * applicable. + * + * The @ref do_is_set_optimization class is used to test whether the is_set + * optimization should be applied for a particular reducer. It is instantiated + * with a value type and a comparator, and defines a boolean constant, + * `value`. Then `%do_is_set_optimization::%value` can be used as + * a boolean template parameter to control the specialization of another + * class. + * + * In ReducersMinMaxBinComp "binary compatibility mode" (i.e., when the + * `CILK_LIBRARY_0_9_REDUCER_MINMAX` macro is defined), `value` will always + * be false. + * + * @tparam Type The value type for the reducer. + * @tparam Compare The comparator type for the reducer. + * + * @result The `value` data member will be `true` if @a Type is a numeric + * type, @a Compare is `std::less`, and + * `CILK_LIBRARY_0_9_REDUCER_MINMAX` is not defined. + * + * @see ReducersMinMaxIsSet + * @see @ref view_content + * + * @ingroup ReducersMinMaxIsSet + */ +template < typename Type, + typename Compare > +struct do_is_set_optimization +{ + /// `True` if the is_set optimization should be applied to min/max reducers + /// with this value type and comparator; `false` otherwise. + static const bool value = false; +}; + +#ifndef CILK_LIBRARY_0_9_REDUCER_MINMAX +/// @cond +template +struct do_is_set_optimization > +{ + /// True in the special case where optimization is possible. + static const bool value = std::numeric_limits::is_specialized; +}; +/// @endcond +#endif + + +/** Gets the identity value when using the ReducersMinMaxIsSet + * "is_set optimization". + * + * This class defines a function which assigns the appropriate identity value + * to a variable when the is_set optimization is applicable. + * + * @tparam Type The value type for the reducer. + * @tparam Compare The comparator type for the reducer. + * @tparam ForMax `true` to get the identity value for a max reducer (i.e., + * the smallest value of @a Type), `false` to get the identity + * value for a min reducer (i.e., the largest value of + * @a Type). + * + * @result If @a Type and @a Compare qualify for the is_set optimization, the + * `set_identity()' function will set its argument variable to the + * smallest or largest value of @a Type, depending on @a ForMax. + * Otherwise, `set_identity()` will be a no-op. + * + * @see ReducersMinMaxIsSet + * + * @ingroup ReducersMinMaxIsSet + * @see @ref view_content + */ +template < typename Type, + typename Compare, + bool ForMax, + bool = std::numeric_limits::is_specialized, + bool = std::numeric_limits::has_infinity > +struct identity_value { + /// Assign the identity value to the reference parameter. + static void set_identity(Type&) {} +}; + +/// @cond +template +struct identity_value, true, true, true> { + /// Floating max identity is negative infinity. + static void set_identity(Type& id) + { id = -std::numeric_limits::infinity(); } +}; + +template +struct identity_value, true, true, false> { + /// Integer max identity is minimum value of type. + static void set_identity(Type& id) + { id = std::numeric_limits::min(); } +}; + +template +struct identity_value, false, true, true> { + /// Floating min identity is positive infinity. + static void set_identity(Type& id) + { id = std::numeric_limits::infinity(); } +}; + +template +struct identity_value, false, true, false> { + /// Integer min identity is maximum value of type. + static void set_identity(Type& id) + { id = std::numeric_limits::max(); } +}; + +/// @endcond + + +/** Adapter class to reverse the arguments of a predicate. + * + * Observe that: + * + * (x < y) == (y > x) + * max(x, y) == (x < y) ? y : x + * min(x, y) == (y < x) ? y : x == (x > y) ? y : x + * + * More generally, if `c` is a predicate defining a `Strict Weak Ordering`, + * and `c*(x, y) == c(y, x)`, then + * + * max(x, y, c) == c(x, y) ? y : x + * min(x, y, c) == c(y, x) ? y : x == c*(x, y) ? y : x == max(x, y, c*) + * + * For any predicate `C` with argument type `T`, the template class + * `%reverse_predicate` defines a predicate which is identical to `C`, + * except that its arguments are reversed. Thus, for example, we could + * implement `%op_min_view` as + * `%op_max_view >`. + * (Actually, op_min_view and op_max_view are both implemented as subclasses + * of a common base class, view_base.) + * + * @note If `C` is an empty functor class, then `reverse_predicate(C)` will + * also be an empty functor class. + * + * @tparam Predicate The predicate whose arguments are to be reversed. + * @tparam Argument @a Predicate's argument type. + * + * @ingroup ReducersMinMax + */ +template +class reverse_predicate : private binary_functor::type { + typedef typename binary_functor::type base; +public: + /// Default constructor + reverse_predicate() : base() {} + /// Constructor with predicate object + reverse_predicate(const Predicate& p) : base(p) {} + /// The reversed predicate operation + bool operator()(const Argument& x, const Argument& y) const + { return base::operator()(y, x); } +}; + + +/** Class to represent the comparator for a min/max view class. + * + * This class is intended to accomplish two objectives in the implementation + * of min/max views. + * + * 1. To minimize data bloat, when we have a reducer with a non-stateless + * comparator, we want to keep a single instance of the comparator object + * in the monoid, and just call it from the views. + * 2. In ReducersMinMaxBinComp "binary compatibility mode", views for + * reducers with a stateless comparator must have the same content as in + * Intel Cilk Plus library 0.9 - that is, they must contain only `value` and + * `is_set` data members. + * + * To achieve the first objective, we use the + * @ref internal::typed_indirect_binary_function class defined in + * metaprogramming.h to wrap a pointer to the actual comparator. If no + * pointer is needed because the actual comparator is stateless, the + * `typed_indirect_binary_function` class will be empty, too. + * + * To achieve the second objective, we make the + * `typed_indirect_binary_function` class a base class of the view rather than + * a data member, so the "empty base class" rule will ensure no that no + * additional space is allocated in the view unless it is needed. + * + * We could simply use typed_indirect_binary_function as the base class of the + * view, but this would mean writing comparisons as `(*this)(x, y)`, which is + * just weird. So, instead, we comparator_base as a subclass of + * typed_indirect_binary_function which provides function `compare()` + * as a synonym for `operator()`. + * + * @tparam Type The value type of the comparator class. + * @tparam Compare A predicate class. + * + * @see internal::typed_indirect_binary_function + * + * @ingroup ReducersMinMax + */ +template +class comparator_base : private typed_indirect_binary_function +{ + typedef typed_indirect_binary_function base; +protected: + comparator_base(const Compare* f) : base(f) {} ///< Constructor. + + /// Comparison function. + bool compare(const Type& a, const Type& b) const + { + return base::operator()(a, b); + } + + /// Get the comparator pointer. + const Compare* compare_pointer() const { return base::pointer(); } +}; + + +/** @defgroup ReducersMinMaxViewContent Content classes for min/max views + * + * @ingroup ReducersMinMax + * + * Minimum and maximum reducer view classes inherit from a "view content" + * class. The content class defines the actual data members for the view, + * and provides typedefs and member functions for accessing the data members + * as needed to support the view functionality. + * + * There are two content classes, which encapsulate the differences between + * simple min/max reducers and min/max with index reducers: + * + * - view_content + * - index_view_content + * + * @note An obvious, and arguably simpler, encapsulation strategy would be + * to just let the `Type` of a min/max view be an (index, value) pair + * structure for min_index and max_index reducers. Then all views + * would just have a `Type` data member and an `is_set` data member, + * and the comparator for min_index and max_index views could be + * customized to consider only the value component of the (index, + * value) `Type` pair. Unfortunately, this would break binary + * compatibility with reducer_max_index and reducer_min_index in + * Intel Cilk Plus library 0.9, because the memory layout of an + * (index, value) pair followed by a `bool` is different from the + * memory layout of an index data member followed by a value data + * member followed by a `bool` data member. The content class is + * designed to exactly replicate the layout of the views in library 0.9 + * reducers. + * + * A content class `C`, and its objects `c`, must define the following: + * + * Definition | Meaning + * ------------------------------------|-------- + * `C::value_type` | A typedef for `Type` of the view. (A `std::pair` for min_index and max_index views). + * `C::comp_value_type` | A typedef for the type of value compared by the view's `compare()` function. + * `C()` | Constructs the content with the identity value. + * `C(const value_type&)` | Constructs the content with a specified value. + * `c.is_set()` | Returns true if the content has a known value. + * `c.value()` | Returns the content's value. + * `c.set_value(const value_type&)` | Sets the content's value. (The value becomes known.) + * `c.comp_value()` | Returns a const reference to the value or component of the value that is to be compared by the view's comparator. + * `C::comp_value(const value_type&)` | Returns a const reference to a value or component of a value that is to be compared by the view's comparator. + * + * @see view_base + */ + +/** Content class for op_min_view and op_max_view. + * + * @tparam Type The value type of the op_min_view or op_max_view. + * @tparam Compare The comparator class specified for the op_min_view or + * op_max_view. (_Not_ the derived comparator class actually + * used by the view_base. For example, the view_content of an + * `op_min_view` will have `Compare = std::less`, + * but its comparator_base will have + * `Compare = reverse_predicate< std::less >`.) + * @tparam ForMax `true` if this is the content class for an op_max_view, + * `false` if it is for an op_min_view. + * + * @note The general implementation of view_content uses an `is_set` data + * member. There is also a specialization which implements the + * ReducersMinMaxIsSet "is_set optimization". View classes that + * inherit from view_content do not need to know anything about the + * difference, though; the details are abstracted away in the + * view_content interface. + * + * @see ReducersMinMaxViewContent + * + * @ingroup ReducersMinMaxViewContent + * @ingroup ReducersMinMax + */ +template < typename Type + , typename Compare + , bool ForMax + , bool = do_is_set_optimization::value + > +class view_content { +protected: +/// @cond + Type m_value; + bool m_is_set; +/// @endcond +public: + /// The value type of the view. + typedef Type value_type; + + /// The type compared by the view's `compare()` function (which is the same + /// as the value type for view_content). + typedef Type comp_value_type; + + /// Construct with the identity value. + view_content() : m_value(), m_is_set(false) {} + + /// Construct with a defined value. + view_content(const value_type& value) : m_value(value), m_is_set(true) {} + + /// Gets the value. + value_type value() const { return m_value; } + + /// Sets the value. + void set_value(const value_type& value) + { + m_value = value; + } + + /// Sets the is_set flag. + void set_is_set() + { + m_is_set = true; + } + + /// Sets the index part of the value (which is meaningless for non-index + ///reducers, but required for view_base). + void set_default_index(const value_type&) {} + + /// Gets the comparison value (which, for view_content, is the same as the + /// value). + const comp_value_type& comp_value() const { return m_value; } + + /// Given an arbitrary value, gets the corresponding comparison value + /// (which, for view_content, is the same as the value). + static const comp_value_type& comp_value(const value_type& value) + { + return value; + } + + /// Gets a const reference to value part of the value (which is the same as + /// the value for view_content). + const Type& get_reference() const { return m_value; } + + /// Gets a const reference to the index part of the value (which is + /// meaningless for non-index reducers, but required for view_base. + const Type& get_index_reference() const { return m_value; } + + /// Tests if the value is defined. + bool is_set() const { return m_is_set; } + + /// Tests if the view has a comparable value. + bool has_value() const { return is_set(); } +}; + +/// @cond + +/* This is the specialization of the view_content class for cases where + * the is_set optimization is applicable). + */ +template < typename Type + , typename Compare + , bool ForMax + > +class view_content : + public view_content +{ + typedef view_content base; + typedef identity_value Identity; + +public: + typedef typename base::value_type value_type;; + typedef typename base::comp_value_type comp_value_type;; + + view_content() : base() { Identity::set_identity(this->m_value); } + + view_content(const value_type& value) : base(value) {} + + bool has_value() const { return true; } +}; + +/// @endcond + + +/** Content class for op_min_index_view and op_max_index_view. + * + * @tparam Index The index type of the op_min_index_view or + op_max_index_view. + * @tparam Type The value type of the op_min_view or op_max_view. (_Not_ + * the value type of the view, which will be + * `std::pair`.) + * @tparam Compare The comparator class specified for the op_min_index_view or + * op_max_index_view. (_Not_ the derived comparator class + * actually used by the view_base. For example, the + * index_view_content of an `op_min_index_view` will have + * `Compare = std::less`, but its comparator_base will + * have `Compare = reverse_predicate< std::less >`.) + * @tparam ForMax `true` if this is the content class for an + * op_max_index_view, `false` if it is for an + * op_min_index_view. + * + * @see ReducersMinMaxViewContent + * + * @ingroup ReducersMinMaxViewContent + * @ingroup ReducersMinMax + */ +template < typename Index + , typename Type + , typename Compare + , bool ForMax + , bool = do_is_set_optimization::value + > +class index_view_content { +protected: +/// @cond + Index m_index; + Type m_value; + bool m_is_set; +/// @endcond +public: + /// The value type of the view (which is an pair for + /// index_view_content). + typedef std::pair value_type; + + /// The type compared by the view's `compare()` function (which is the data + /// value type for index_view_content). + typedef Type comp_value_type; + + /// Construct with the identity value. + index_view_content() : m_index(), m_value(), m_is_set(false) {} + + /// Construct with an index/value pair. + index_view_content(const value_type& value) : + m_index(value.first), m_value(value.second), m_is_set(true) {} + + /// Construct with an index and a value. + index_view_content(const Index& index, const Type& value) : + m_index(index), m_value(value), m_is_set(true) {} + + /// Construct with just an index. + index_view_content(const Index& index) : + m_index(index), m_value(), m_is_set(false) {} + + /// Gets the value. + value_type value() const { return value_type(m_index, m_value); } + + /// Sets the value. + void set_value(const value_type& value) + { + m_index = value.first; + m_value = value.second; + } + + /// Sets the is_set flag. + void set_is_set() + { + m_is_set = true; + } + + /// Sets the (initial) index, without marking the view as set. + void set_default_index(const Index& index) + { + m_index = index; + } + + /// Gets the comparison value (which, for index_view_content, is the value + /// component of the index/value pair). + const comp_value_type& comp_value() const { return m_value; } + + /// Given an arbitrary value (i.e., index/value pair), gets the + /// corresponding comparison value (which, for index_view_content, is the + /// value component of the index/value pair). + static const comp_value_type& comp_value(const value_type& value) + { return value.second; } + + /// Gets a const reference to the value part of the value. + const Type& get_reference() const { return m_value; } + + /// Gets a const reference to the index part of the value. + const Index& get_index_reference() const { return m_index; } + + /// Tests if the value is defined. + bool is_set() const { return m_is_set; } + + /// Tests if the view has a comparable value. + bool has_value() const { return is_set(); } +}; + + +/// @cond + +/* This is the specialization of the index_view_content class for cases where + * the is_set optimization is applicable). + */ +template < typename Index + , typename Type + , typename Compare + , bool ForMax + > +class index_view_content : + public index_view_content +{ + typedef index_view_content base; + typedef identity_value Identity; +public: + typedef typename base::value_type value_type;; + typedef typename base::comp_value_type comp_value_type;; + + index_view_content() : base() { Identity::set_identity(this->m_value); } + + index_view_content(const value_type& value) : base(value) {} + + index_view_content(const Index& index, const Type& value) : + base(index, value) {} + + index_view_content(const Index& index) : base() { + Identity::set_identity(this->m_value); + this->m_index = index; + } + + /// Test if the view has a comparable value. + bool has_value() const { return true; } +}; + +/// @endcond + + +template class rhs_proxy; + +/** Creates an rhs_proxy. + */ +template +inline rhs_proxy +make_proxy(const typename View::value_type& value, const View& view); + +template class view_base; + + +/** Class to represent the right-hand side of + * `*reducer = {min|max}_of(*reducer, value)`. + * + * The only assignment operator for a min/max view class takes a rhs_proxy as + * its operand. This results in the syntactic restriction that the only + * expressions that can be assigned to a min/max view are ones which generate + * an rhs_proxy - that is, expressions of the form `max_of(view, value)` and + * `min_of(view, value)`. + * + * @warning + * The lhs and rhs views in such an assignment must be the same; otherwise, + * the behavior will be undefined. (I.e., `*r1 = min_of(*r1, x)` is legal; + * `*r1 = min_of(*r2, x)` is illegal.) This condition will be checked with a + * runtime assertion when compiled in debug mode. + * + * @tparam View The view class (op_{min|max}[_index]_view) that this proxy + * was created from. + * + * @see view_base + * + * @ingroup ReducersMinMax + */ +template +class rhs_proxy { + typedef typename View::less_type less_type; + typedef typename View::compare_type compare_type; + typedef typename View::value_type value_type; + typedef typename View::content_type content_type; + typedef typename content_type::comp_value_type comp_value_type; + + friend class view_base; + friend rhs_proxy make_proxy( + const typename View::value_type& value, + const View& view); + + typed_indirect_binary_function< + compare_type, comp_value_type, comp_value_type, bool> + m_comp; + const View* m_view; + value_type m_value; + + rhs_proxy& operator=(const rhs_proxy&); // Disable assignment operator + rhs_proxy(); // Disable default constructor + + // Constructor (called from view_base::make_proxy). + rhs_proxy(const View* view, + const value_type& value, + const compare_type* compare) : + m_view(view), m_value(value), m_comp(compare) {} + + // Checks matching view, then return value (called from view_base::assign). + value_type value(const typename View::base* view) const + { + __CILKRTS_ASSERT(view == m_view); + return m_value; + } + +public: + + /** Supports max_of(max_of(view, value), value) and the like. + */ + rhs_proxy calc(const value_type& x) const + { + return rhs_proxy( + m_view, + m_comp( content_type::comp_value(m_value), + content_type::comp_value(x) + ) ? x : m_value, + m_comp.pointer()); + } +}; + + +template +inline rhs_proxy +make_proxy(const typename View::value_type& value, const View& view) +{ + return rhs_proxy(&view, value, view.compare_pointer()); +} + +//@} + +/** Base class for min and max view classes. + * + * This class accumulates the minimum or maximum of a set of values which have + * occurred as arguments to the `calc()` function, as determined by a + * comparator. The accumulated value will be the first `calc()` argument value + * `x` such that `compare(x, y)` is false for every `calc()` argument value + * `y`. + * + * If the comparator is `std::less`, then the accumulated value is the first + * argument value which is not less than any other argument value, i.e., the + * maximum. Similarly, if the comparator is `reverse_predicate`, + * which is equivalent to `std::greater`, then the accumulated value is the + * first argument value which is not greater than any other argument value, + * i.e., the minimum. + * + * @note This class provides the definitions that are required for a class + * that will be used as the parameter of a + * min_max_internal::monoid_base specialization. + * + * @tparam Content A content class that provides the value types and data + * members for the view. + * @tparam Less A "less than" binary predicate that defines the min or + * max function. + * @tparam Compare A binary predicate to be used to compare the values. + * (The same as @a Less for max reducers; its reversal for + * min reducers.) + * + * @see ReducersMinMaxViewContent + * @see op_max_view + * @see op_min_view + * @see op_max_index_view + * @see op_min_index_view + * @see monoid_base + * + * @ingroup ReducersMinMax + */ +template +class view_base : + // comparator_base comes first to ensure that it will get empty base class + // treatment + private comparator_base, + private Content +{ + typedef comparator_base base; + using base::compare; + using Content::value; + using Content::set_value; + using Content::has_value; + using Content::set_is_set; + using Content::comp_value; + typedef Content content_type; + + template friend class rhs_proxy; + template + friend rhs_proxy make_proxy(const typename View::value_type& value, const View& view); + +public: + + /** @name Monoid support. + */ + //@{ + + /** Value type. Required by @ref monoid_with_view. + */ + typedef typename Content::value_type value_type; + + /** The type of the comparator specified by the user, that defines the + * ordering on @a Type. Required by min_max::monoid_base. + */ + typedef Less less_type; + + /** The type of the comparator actually used by the view. Required by + * min_max::monoid_base. (This is the same as the @ref less_type for a + * max reducer, or `reverse_predicate` for a min reducer.) + */ + typedef Compare compare_type; + + /** Reduces two views. Required by @ref monoid_with_view. + */ + void reduce(view_base* other) + { + if ( other->is_set() && + ( !this->is_set() || + compare(this->comp_value(), other->comp_value()) ) ) + { + this->set_value(other->value()); + this->set_is_set(); + } + } + + //@} + + /** Default constructor. Initializes to identity value. + */ + explicit view_base(const compare_type* compare) : + base(compare), Content() {} + + /** Value constructor. + */ + template + view_base(const T1& x1, const compare_type* compare) : + base(compare), Content(x1) {} + + /** Value constructor. + */ + template + view_base(const T1& x1, const T2& x2, const compare_type* compare) : + base(compare), Content(x1, x2) {} + + + /** Move-in constructor. + */ + explicit view_base(move_in_wrapper w, const compare_type* compare) : + base(compare), Content(w.value()) {} + + /** @name Reducer support. + */ + //@{ + + void view_move_in(value_type& v) + { set_value(v); set_is_set();} + void view_move_out(value_type& v) + { v = value(); } + void view_set_value(const value_type& v) + { set_value(v); set_is_set(); } + value_type view_get_value() const + { return value(); } + // view_get_reference() NOT SUPPORTED + + //@} + + /** Sets the contained index data member, without marking the view as set. + * (Meaningless for non-index reducers.) + */ + using Content::set_default_index; + + /** Is the value defined? + */ + using Content::is_set; + + /** Reference to contained value data member. + * @deprecated For legacy reducers only. + */ + using Content::get_reference; + + /** Reference to contained index data member. + * (Meaningless for non-index reducers.) + * @deprecated For legacy reducers only. + */ + using Content::get_index_reference; + +protected: + + /** Updates the min/max value. + */ + void calc(const value_type& x) + { + if (!has_value() || compare(comp_value(), comp_value(x))) set_value(x); + set_is_set(); + } + + /** Assigns the result of a `{min|max}_of(view, value)` expression to the + * view. + * + * @see rhs_proxy + */ + template + void assign(const rhs_proxy& rhs) + { + calc(rhs.value(this)); + } + +}; + + +/** Base class for min and max monoid classes. + * + * The unique characteristic of minimum and maximum reducers is that they + * incorporate a comparator functor that defines what "minimum" or "maximum" + * means. The monoid for a reducer contains the comparator that will be used + * for the reduction. If the comparator is a function or a class with state, + * then each view will have a pointer to the comparator. + * + * This means that the `construct()` functions first construct the monoid + * (possibly with an explicit comparator argument), and then construct the + * view with a pointer to the monoid's comparator. + * + * @tparam View The view class. + * @tparam Align If true, reducers instantiated on this monoid will be + * aligned. By default, library reducers (unlike legacy + * library reducer _wrappers_) are unaligned. + * + * @see view_base + * + * @ingroup ReducersMinMax + */ +template +class monoid_base : public monoid_with_view +{ + typedef typename View::compare_type compare_type; + typedef typename View::less_type less_type; + + const compare_type m_compare; + + const compare_type* compare_pointer() const { return &m_compare; } + +public: + + /** Default constructor uses default comparator. + */ + monoid_base() : m_compare() {} + + /** Constructor. + * + * @param compare The comparator to use. + */ + monoid_base(const compare_type& compare) : m_compare(compare) {} + + /** Creates an identity view. + * + * List view identity constructors take the list allocator as an argument. + * + * @param v The address of the uninitialized memory in which the view + * will be constructed. + */ + void identity(View *v) const { ::new((void*) v) View(compare_pointer()); } + + /** @name construct functions + * + * Min/max monoid `construct()` functions optionally take one or two value + * arguments, a @ref move_in argument, and/or a comparator argument. + */ + //@{ + + template + static void construct(Monoid* monoid, View* view) + { + provisional_guard mg( new((void*) monoid) Monoid ); + mg.confirm_if( new((void*) view) View(monoid->compare_pointer()) ); + } + + template + static void construct(Monoid* monoid, View* view, const T1& x1) + { + provisional_guard mg( new((void*) monoid) Monoid ); + mg.confirm_if( new((void*) view) View(x1, monoid->compare_pointer()) ); + } + + template + static void construct(Monoid* monoid, View* view, + const T1& x1, const T2& x2) + { + provisional_guard mg( new((void*) monoid) Monoid ); + mg.confirm_if( new((void*) view) View(x1, x2, + monoid->compare_pointer()) ); + } + + template + static void construct(Monoid* monoid, View* view, const less_type& compare) + { + provisional_guard mg( new((void*) monoid) Monoid(compare) ); + mg.confirm_if( new((void*) view) View(monoid->compare_pointer()) ); + } + + template + static void construct(Monoid* monoid, View* view, const T1& x1, + const less_type& compare) + { + provisional_guard mg( new((void*) monoid) Monoid(compare) ); + mg.confirm_if( new((void*) view) View(x1, monoid->compare_pointer()) ); + } + + template + static void construct(Monoid* monoid, View* view, + const T1& x1, const T2& x2, const less_type& compare) + { + provisional_guard mg( new((void*) monoid) Monoid(compare) ); + mg.confirm_if( new((void*) view) View(x1, x2, + monoid->compare_pointer()) ); + } + + //@} +}; + +} //namespace min_max_internal + + +/** @defgroup ReducersMinMaxMaxValue Maximum reducers (value only) + * + * These reducers will find the largest value from a set of values. + * + * @ingroup ReducersMinMax + */ +//@{ + +/** The maximum reducer view class. + * + * This is the view class for reducers created with + * `cilk::reducer< cilk::op_max >`. It accumulates the maximum, + * as determined by a comparator, of a set of values which have occurred as + * arguments to the `calc_max()` function. The accumulated value will be the + * first argument `x` such that `compare(x, y)` is false for every argument + * `y`. + * + * If the comparator is `std::less`, then the accumulated value is the first + * argument value which is not less than any other argument value, i.e., the + * maximum. + * + * @note The reducer "dereference" operation (`reducer::operator *()`) + * yields a reference to the view. Thus, for example, the view class's + * `calc_max()` function would be used in an expression like + * `r->calc_max(a)` where `r` is an op_max reducer variable. + * + * @tparam Type The type of the values compared by the reducer. This will + * be the value type of a monoid_with_view that is + * instantiated with this view. + * @tparam Compare A `Strict Weak Ordering` whose argument type is @a Type. It + * defines the "less than" relation used to compute the + * maximum. + * + * @see ReducersMinMax + * @see op_max + */ +template +class op_max_view : public min_max_internal::view_base< + min_max_internal::view_content, + Compare, + Compare> +{ + typedef min_max_internal::view_base< + min_max_internal::view_content, + Compare, + Compare> base; + using base::calc; + using base::assign; + friend class min_max_internal::rhs_proxy; + +public: + + /** @name Constructors. + * + * All op_max_view constructors simply pass their arguments on to the + * @ref view_base base class. + */ + //@{ + + template + op_max_view(const T1& x1) : base(x1) {} + + template + op_max_view(const T1& x1, const T2& x2) : base(x1, x2) {} + + //@} + + /** @name View modifier operations. + */ + //@{ + + /** Maximizes with a value. + * + * If @a x is greater than the current value of the view (as defined by + * the reducer's comparator), or if the view was created without an + * initial value and its value has never been updated (with `calc_max()` + * or `= max_of()`), then the value of the view is set to @a x. + * + * @param x The value to maximize the view's value with. + * + * @return A reference to the view. (Allows chaining + * `view.comp_max(a).comp_max(b)…`.) + */ + op_max_view& calc_max(const Type& x) { calc(x); return *this; } + + /** Assigns the result of a `max_of(view, value)` expression to the view. + * + * @param rhs An rhs_proxy value created by a `max_of(view, value)` + * expression. + * + * @return A reference to the view. + * + * @see min_max_internal::view_base::rhs_proxy + */ + op_max_view& operator=(const min_max_internal::rhs_proxy& rhs) + { assign(rhs); return *this; } + + //@} +}; + + +/** Computes the maximum of the value in an op_max_view and another value. + * + * The result of this computation can only be assigned back to the original + * view or used in another max_of() call. For example, + * + * *reducer = max_of(*reducer, x); + * *reducer = max_of(x, *reducer); + * + * @see min_max_internal::rhs_proxy + */ +template +inline min_max_internal::rhs_proxy< op_max_view > +max_of(const op_max_view& view, const Type& value) +{ + return min_max_internal::make_proxy(value, view); +} + +/// @copydoc max_of(const op_max_view&, const Type&) +template +inline min_max_internal::rhs_proxy< op_max_view > +max_of(const Type& value, const op_max_view& view) +{ + return min_max_internal::make_proxy(value, view); +} + +/** Computes nested maximum. + * + * Compute the maximum of the result of a max_of() call and another value. + * + * The result of this computation can only be assigned back to the original + * view or wrapper, or used in another max_of() call. For example, + * + * *reducer = max_of(x, max_of(y, *reducer)); + * wrapper = max_of(max_of(wrapper, x), y); + * + * @see min_max_internal::rhs_proxy + */ +template +inline min_max_internal::rhs_proxy< op_max_view > +max_of(const min_max_internal::rhs_proxy< op_max_view >& proxy, + const Type& value) +{ + return proxy.calc(value); +} + +/// @copydoc max_of(const min_max_internal::rhs_proxy< op_max_view >&, const Type&) +template +inline min_max_internal::rhs_proxy< op_max_view > +max_of(const Type& value, + const min_max_internal::rhs_proxy< op_max_view >& proxy) +{ + return proxy.calc(value); +} + + +/** Monoid class for maximum reductions. Instantiate the cilk::reducer template + * class with an op_max monoid to create a maximum reducer class. For example, + * to compute the maximum of a set of `int` values: + * + * cilk::reducer< cilk::op_max > r; + * + * @see ReducersMinMax + * @see op_max_view + */ +template , bool Align = false> +class op_max : + public min_max_internal::monoid_base, Align> +{ + typedef min_max_internal::monoid_base, Align> + base; +public: + /// Construct with default comparator. + op_max() {} + /// Construct with specified comparator. + op_max(const Compare& compare) : base(compare) {} +}; + +//@} + + +/** @defgroup ReducersMinMaxMinValue Minimum reducers (value only) + * + * These reducers will find the smallest value from a set of values. + * + * @ingroup ReducersMinMax + */ +//@{ + +/** The minimum reducer view class. + * + * This is the view class for reducers created with + * `cilk::reducer< cilk::op_min >`. It accumulates the minimum, + * as determined by a comparator, of a set of values which have occurred as + * arguments to the `calc_min()` function. The accumulated value will be the + * first argument `x` such that `compare(y, x)` is false for every argument + * `y`. + * + * If the comparator is `std::less`, then the accumulated value is the first + * argument value which no other argument value is less than, i.e., the + * minimum. + * + * @note The reducer "dereference" operation (`reducer::operator *()`) + * yields a reference to the view. Thus, for example, the view class's + * `calc_min()` function would be used in an expression like + * `r->calc_min(a)` where `r` is an op_min reducer variable. + * + * @tparam Type The type of the values compared by the reducer. This will + * be the value type of a monoid_with_view that is + * instantiated with this view. + * @tparam Compare A `Strict Weak Ordering` whose argument type is @a Type. It + * defines the "less than" relation used to compute the + * minimum. + * + * @see ReducersMinMax + * @see op_min + */ +template +class op_min_view : public min_max_internal::view_base< + min_max_internal::view_content, + Compare, + min_max_internal::reverse_predicate > +{ + typedef min_max_internal::view_base< + min_max_internal::view_content, + Compare, + min_max_internal::reverse_predicate > base; + using base::calc; + using base::assign; + friend class min_max_internal::rhs_proxy; + +public: + /** @name Constructors. + * + * All op_min_view constructors simply pass their arguments on to the + * @ref view_base base class. + */ + //@{ + + template + op_min_view(const T1& x1) : base(x1) {} + + template + op_min_view(const T1& x1, const T2& x2) : base(x1, x2) {} + + //@} + + /** @name View modifier operations. + */ + //@{ + + /** Minimizes with a value. + * + * If @a x is less than the current value of the view (as defined by the + * reducer's comparator), or if the view was created without an initial + * value and its value has never been updated (with `calc_min()` or + * `= min_of()`), then the value of the view is set to @a x. + * + * @param x The value to minimize the view's value with. + * + * @return A reference to the view. (Allows chaining + * `view.comp_min(a).comp_min(b)…`.) + */ + op_min_view& calc_min(const Type& x) { calc(x); return *this; } + + /** Assigns the result of a `min_of(view, value)` expression to the view. + * + * @param rhs An rhs_proxy value created by a `min_of(view, value)` + * expression. + * + * @return A reference to the view. + * + * @see min_max_internal::view_base::rhs_proxy + */ + op_min_view& operator=(const min_max_internal::rhs_proxy& rhs) + { assign(rhs); return *this; } +}; + + +/** Computes the minimum of the value in a view and another value. + * + * The result of this computation can only be assigned back to the original + * view or used in another min_of() call. For example, + * + * *reducer = min_of(*reducer, x); + * *reducer = min_of(x, *reducer); + * + * @see min_max_internal::view_base::rhs_proxy + */ +template +inline min_max_internal::rhs_proxy< op_min_view > +min_of(const op_min_view& view, const Type& value) +{ + return min_max_internal::make_proxy(value, view); +} + +/// @copydoc min_of(const op_min_view&, const Type&) +template +inline min_max_internal::rhs_proxy< op_min_view > +min_of(const Type& value, const op_min_view& view) +{ + return min_max_internal::make_proxy(value, view); +} + +/** Computes nested minimum. + * + * Compute the minimum of the result of a min_of() call and another value. + * + * The result of this computation can only be assigned back to the original + * view or wrapper, or used in another min_of() call. For example, + * + * *reducer = min_of(x, min_of(y, *reducer)); + * wrapper = min_of(min_of(wrapper, x), y); + * + * @see min_max_internal::rhs_proxy + */ +template +inline min_max_internal::rhs_proxy< op_min_view > +min_of(const min_max_internal::rhs_proxy< op_min_view >& proxy, + const Type& value) +{ + return proxy.calc(value); +} + +/// @copydoc min_of(const min_max_internal::rhs_proxy< op_min_view >&, const Type&) +template +inline min_max_internal::rhs_proxy< op_min_view > +min_of(const Type& value, + const min_max_internal::rhs_proxy< op_min_view >& proxy) +{ + return proxy.calc(value); +} + + +/** Monoid class for minimum reductions. Instantiate the cilk::reducer template + * class with an op_min monoid to create a minimum reducer class. For example, + * to compute the minimum of a set of `int` values: + * + * cilk::reducer< cilk::op_min > r; + * + * @see ReducersMinMax + * @see op_min_view + */ +template , bool Align = false> +class op_min : public min_max_internal::monoid_base, Align> { + typedef min_max_internal::monoid_base, Align> base; +public: + /// Construct with default comparator. + op_min() {} + /// Construct with specified comparator. + op_min(const Compare& compare) : base(compare) {} +}; + +//@} + + +/** @defgroup ReducersMinMaxMaxIndex Maximum reducers (value and index) + * + * These reducers will find the largest value from a set of values, and its + * index in the set. + * + * @ingroup ReducersMinMax + */ +//@{ + +/** The maximum index reducer view class. + * + * This is the view class for reducers created with + * `cilk::reducer< cilk::op_max_index >`. It accumulates + * the maximum, as determined by a comparator, of a set of values which have + * occurred as arguments to the `calc_max()` function, and records the index + * of the maximum value. The accumulated value will be the first argument `x` + * such that `compare(x, y)` is false for every argument `y`. + * + * If the comparator is `std::less`, then the accumulated value is the first + * argument value which is not less than any other argument value, i.e., the + * maximum. + * + * @note The reducer "dereference" operation (`reducer::operator *()`) + * yields a reference to the view. Thus, for example, the view class's + * `calc_max()` function would be used in an expression like + * `r->calc_max(i, a)`where `r` is an op_max_index reducer + * variable. + * + * @note The word "index" suggests an integer index into an array, but there + * is no restriction on the index type or how it should be used. In + * general, it may be convenient to use it for any kind of key that + * can be used to locate the maximum value in the collection that it + * came from - for example: + * - An index into an array. + * - A key into an STL map. + * - An iterator into any STL container. + * + * @note A max_index reducer is essentially a max reducer whose value type + * is a `std::pair`. This fact is camouflaged in the view + * `calc_max` function, the global `max_of` functions, and the reducer + * value constructor, which can all take an index argument and a value + * argument as an alternative to a single `std::pair` argument. + * However, the reducer `set_value()`, `get_value()`, `move_in()`, and + * `move_out()` functions work only with pairs, not with individual + * value and/or index arguments. + * + * @tparam Index The type of the indices associated with the values. + * @tparam Type The type of the values compared by the reducer. This will + * be the value type of a monoid_with_view that is + * instantiated with this view. + * @tparam Compare Used to compare the values. It must be a binary predicate. + * If it is omitted, then the view computes the conventional + * arithmetic maximum. + * + * @see ReducersMinMax + * @see op_max_index + */ +template +class op_max_index_view : public min_max_internal::view_base< + min_max_internal::index_view_content, + Compare, + Compare> +{ + typedef min_max_internal::view_base< + min_max_internal::index_view_content, + Compare, + Compare> base; + using base::calc; + using base::assign; + typedef std::pair pair_type; + friend class min_max_internal::rhs_proxy; + +public: + /** @name Constructors. + * + * All op_max_index_view constructors simply pass their arguments on to the + * @ref view_base base class, except for the `(index, value [, compare])` + * constructors, which create a `std::pair` containing the index and value. + */ + //@{ + + op_max_index_view() : base() {} + + template + op_max_index_view(const T1& x1) : base(x1) {} + + template + op_max_index_view(const T1& x1, const T2& x2) : base(x1, x2) {} + + template + op_max_index_view(const T1& x1, const T2& x2, const T3& x3) : base(x1, x2, x3) {} + + op_max_index_view(const Index& i, const Type& v) : base(pair_type(i, v)) {} + + op_max_index_view(const Index& i, const Type& v, const typename base::compare_type* c) : + base(pair_type(i, v), c) {} + + //@} + + /** Maximizes with a value and index. + * + * If @a x is greater than the current value of the view (as defined by + * the reducer's comparator), or if the view was created without an + * initial value and its value has never been updated (with `calc_max()` + * or `= max_of()`), then the value of the view is set to @a x, and the + * index is set to @a i.. + * + * @param i The index of the value @a x. + * @param x The value to maximize the view's value with. + * + * @return A reference to the view. (Allows + * `view.comp_max(i, a).comp_max(j, b)…`.) + */ + op_max_index_view& calc_max(const Index& i, const Type& x) + { calc(pair_type(i, x)); return *this; } + + /** Maximizes with an index/value pair. + * + * If @a pair.second is greater than the current value of the view (as + * defined by the reducer's comparator), or if the view was created + * without an initial value and its value has never been updated (with + * `calc_max()` or `= max_of()`), then the value of the view is set to + * @a pair.second, and the index is set to @a pair.first. + * + * @param pair A pair containing a value to maximize the view's value + * with and its associated index. + * + * @return A reference to the view. (Allows + * `view.comp_max(p1).comp_max(p2)…`.) + */ + op_max_index_view& calc_max(const pair_type& pair) + { calc(pair); return *this; } + + /** Assigns the result of a `max_of(view, index, value)` expression to the + * view. + * + * @param rhs An rhs_proxy value created by a `max_of(view, index, value)` + * expression. + * + * @return A reference to the view. + * + * @see min_max_internal::view_base::rhs_proxy + */ + op_max_index_view& operator=(const min_max_internal::rhs_proxy& rhs) + { assign(rhs); return *this; } +}; + + +/** Computes the maximum of the value in a view and another value. + * + * The result of this computation can only be assigned back to the original + * view or used in another max_of() call. For example, + * + * *reducer = max_of(*reducer, i, x); + * *reducer = max_of(i, x, *reducer); + * + * @see min_max_internal::rhs_proxy + */ +template +inline min_max_internal::rhs_proxy< op_max_index_view > +max_of(const op_max_index_view& view, + const Index& index, const Type& value) +{ + return min_max_internal::make_proxy(std::pair(index, value), view); +} + +/// @copydoc max_of(const op_max_index_view&, const Index&, const Type&) +template +inline min_max_internal::rhs_proxy< op_max_index_view > +max_of(const Index& index, const Type& value, + const op_max_index_view& view) +{ + return min_max_internal::make_proxy(std::pair(index, value), view); +} + +/// @copydoc max_of(const op_max_index_view&, const Index&, const Type&) +template +inline min_max_internal::rhs_proxy< op_max_index_view > +max_of(const op_max_index_view& view, + const std::pair& pair) +{ + return min_max_internal::make_proxy(pair, view); +} + +/// @copydoc max_of(const op_max_index_view&, const Index&, const Type&) +template +inline min_max_internal::rhs_proxy< op_max_index_view > +max_of(const std::pair& pair, + const op_max_index_view& view) +{ + return min_max_internal::make_proxy(pair, view); +} + +/** Computes the nested maximum between the value in a view and other values. + * + * Compute the maximum of the result of a max_of() call and another value. + * + * The result of this computation can only be assigned back to the original + * view or used in another max_of() call. For example, + * + * *reducer = max_of(x, max_of(y, *reducer)); + * *reducer = max_of(max_of(*reducer, x), y); + * + * @see min_max_internal::rhs_proxy + */ +template +inline min_max_internal::rhs_proxy< op_max_index_view > +max_of(const min_max_internal::rhs_proxy< op_max_index_view >& proxy, + const Index& index, const Type& value) +{ + return proxy.calc(std::pair(index, value)); +} + +/// @copydoc max_of(const min_max_internal::rhs_proxy< op_max_index_view >&, const Index&, const Type&) +template +inline min_max_internal::rhs_proxy< op_max_index_view > +max_of(const Index& index, const Type& value, + const min_max_internal::rhs_proxy< op_max_index_view >& proxy) +{ + return proxy.calc(std::pair(index, value)); +} + +/// @copydoc max_of(const min_max_internal::rhs_proxy< op_max_index_view >&, const Index&, const Type&) +template +inline min_max_internal::rhs_proxy< op_max_index_view > +max_of(const min_max_internal::rhs_proxy< op_max_index_view >& proxy, + const std::pair& pair) +{ + return proxy.calc(pair); +} + +/// @copydoc max_of(const min_max_internal::rhs_proxy< op_max_index_view >&, const Index&, const Type&) +template +inline min_max_internal::rhs_proxy< op_max_index_view > +max_of(const std::pair& pair, + const min_max_internal::rhs_proxy< op_max_index_view >& proxy) +{ + return proxy.calc(pair); +} + + +/** Monoid class for maximum reductions with index. Instantiate the + * cilk::reducer template class with an op_max_index monoid to create a + * max_index reducer class. For example, to compute the maximum of an array of + * `double` values and the array index of the max value: + * + * cilk::reducer< cilk::op_max_index > r; + * + * @see ReducersMinMax + * @see op_max_index_view + */ +template < typename Index + , typename Type + , typename Compare=std::less + , bool Align = false + > +class op_max_index : public min_max_internal::monoid_base, Align> +{ + typedef min_max_internal::monoid_base< + op_max_index_view, Align> base; +public: + /// Construct with default comparator. + op_max_index() {} + /// Construct with specified comparator. + op_max_index(const Compare& compare) : base(compare) {} +}; + +//@} + + + +/** @defgroup ReducersMinMaxMinIndex Minimum reducers (value and index) + * + * These reducers will find the smallest value from a set of values, and its + * index in the set. + * + * @ingroup ReducersMinMax + */ +//@{ + +/** The minimum index reducer view class. + * + * This is the view class for reducers created with + * `cilk::reducer >`. It accumulates + * the minimum, as determined by a comparator, of a set of values which have + * occurred as arguments to the `calc_min()` function, and records the index + * of the minimum value. The accumulated value will be the first argument `x` + * such that `compare(y, x)` is false for every argument `y`. + * + * If the comparator is `std::less`, then the accumulated value is the first + * argument value which no other argument value is less than, i.e., the + * minimum. + * + * @note The reducer "dereference" operation (`reducer::operator *()`) + * yields a reference to the view. Thus, for example, the view class's + * `calc_min()` function would be + * used in an expression like `r->calc_min(i, a)`where `r` is an + * op_min_index reducer variable. + * + * @note The word "index" suggests an integer index into an array, but there + * is no restriction on the index type or how it should be used. In + * general, it may be convenient to use it for any kind of key that + * can be used to locate the minimum value in the collection that it + * came from - for example: + * - An index into an array. + * - A key into an STL map. + * - An iterator into any STL container. + * + * @note A min_index reducer is essentially a min reducer whose value type + * is a `std::pair`. This fact is camouflaged in the view + * `calc_min` function, the global `min_of` functions, and the reducer + * value constructor, which can all take an index argument and a value + * argument as an alternative to a single `std::pair` argument. + * However, the reducer `set_value()`, `get_value()`, `move_in()`, and + * `move_out()` functions work only with pairs, not with individual + * value and/or index arguments. + * + * @tparam Index The type of the indices associated with the values. + * @tparam Type The type of the values compared by the reducer. This will + * be the value type of a monoid_with_view that is + * instantiated with this view. + * @tparam Compare Used to compare the values. It must be a binary predicate. + * If it is omitted, then the view computes the conventional + * arithmetic minimum. + * + * @see ReducersMinMax + * @see op_min_index + */ +template +class op_min_index_view : public min_max_internal::view_base< + min_max_internal::index_view_content, + Compare, + min_max_internal::reverse_predicate > +{ + typedef min_max_internal::view_base< + min_max_internal::index_view_content, + Compare, + min_max_internal::reverse_predicate > base; + using base::calc; + using base::assign; + typedef std::pair pair_type; + friend class min_max_internal::rhs_proxy; + +public: + /** @name Constructors. + * + * All op_min_index_view constructors simply pass their arguments on to the + * @ref view_base base class, except for the `(index, value [, compare])` + * constructors, which create a `std::pair` containing the index and value. + */ + //@{ + + op_min_index_view() : base() {} + + template + op_min_index_view(const T1& x1) : base(x1) {} + + template + op_min_index_view(const T1& x1, const T2& x2) : base(x1, x2) {} + + template + op_min_index_view(const T1& x1, const T2& x2, const T3& x3) : base(x1, x2, x3) {} + + op_min_index_view(const Index& i, const Type& v) : base(pair_type(i, v)) {} + + op_min_index_view(const Index& i, const Type& v, const typename base::compare_type* c) : + base(pair_type(i, v), c) {} + + //@} + + /** Minimizes with a value and index. + * + * If @a x is greater than the current value of the view (as defined by + * the reducer's comparator), or if the view was created without an + * initial value and its value has never been updated (with `calc_min()` + * or `= min_of()`), then the value of the view is set to @a x, and the + * index is set to @a i.. + * + * @param i The index of the value @a x. + * @param x The value to minimize the view's value with. + * + * @return A reference to the view. (Allows + * `view.comp_min(i, a).comp_min(j, b)…`.) + */ + op_min_index_view& calc_min(const Index& i, const Type& x) + { calc(pair_type(i, x)); return *this; } + + /** Maximizes with an index/value pair. + * + * If @a pair.second is less than the current value of the view (as + * defined by the reducer's comparator), or if the view was created + * without an initial value and its value has never been updated (with + * `calc_min()` or `= min_of()`), then the value of the view is set to + * @a pair.second, and the index is set to @a pair.first. + * + * @param pair A pair containing a value to minimize the view's value + * with and its associated index. + * + * @return A reference to the view. (Allows + * `view.comp_min(p1).comp_min(p2)…`.) + */ + op_min_index_view& calc_min(const pair_type& pair) + { calc(pair); return *this; } + + /** Assigns the result of a `min_of(view, index, value)` expression to the + * view. + * + * @param rhs An rhs_proxy value created by a `min_of(view, index, value)` + * expression. + * + * @return A reference to the view. + * + * @see min_max_internal::view_base::rhs_proxy + */ + op_min_index_view& operator=(const min_max_internal::rhs_proxy& rhs) + { assign(rhs); return *this; } +}; + + +/** Computes the minimum of the value in a view and another value. + * + * The result of this computation can only be assigned back to the original + * view or used in another min_of() call. For example, + * + * *reducer = min_of(*reducer, i, x); + * *reducer = min_of(i, x, *reducer); + * + * @see min_max_internal::min_min_view_base::rhs_proxy + */ +template +inline min_max_internal::rhs_proxy< op_min_index_view > +min_of(const op_min_index_view& view, + const Index& index, const Type& value) +{ + return min_max_internal::make_proxy(std::pair(index, value), view); +} + +/// @copydoc min_of(const op_min_index_view&, const Index&, const Type&) +template +inline min_max_internal::rhs_proxy< op_min_index_view > +min_of(const Index& index, const Type& value, + const op_min_index_view& view) +{ + return min_max_internal::make_proxy(std::pair(index, value), view); +} + +/// @copydoc min_of(const op_min_index_view&, const Index&, const Type&) +template +inline min_max_internal::rhs_proxy< op_min_index_view > +min_of(const op_min_index_view& view, + const std::pair& pair) +{ + return min_max_internal::make_proxy(pair, view); +} + +/// @copydoc min_of(const op_min_index_view&, const Index&, const Type&) +template +inline min_max_internal::rhs_proxy< op_min_index_view > +min_of(const std::pair& pair, + const op_min_index_view& view) +{ + return min_max_internal::make_proxy(pair, view); +} + +/** Computes nested minimum between the value in a view and other values. + * + * Compute the minimum of the result of a min_of() call and another value. + * + * The result of this computation can only be assigned back to the original + * view or used in another min_of() call. For example, + * + * *reducer = min_of(x, min_of(y, *reducer)); + * *reducer = min_of(min_of(*reducer, x), y); + * + * @see min_max_internal::min_min_view_base::rhs_proxy + */ +template +inline min_max_internal::rhs_proxy< op_min_index_view > +min_of(const min_max_internal::rhs_proxy< op_min_index_view >& proxy, + const Index& index, const Type& value) +{ + return proxy.calc(std::pair(index, value)); +} + +/// @copydoc min_of(const min_max_internal::rhs_proxy< op_min_index_view >&, const Index&, const Type&) +template +inline min_max_internal::rhs_proxy< op_min_index_view > +min_of(const Index& index, const Type& value, + const min_max_internal::rhs_proxy< op_min_index_view >& proxy) +{ + return proxy.calc(std::pair(index, value)); +} + +/// @copydoc min_of(const min_max_internal::rhs_proxy< op_min_index_view >&, const Index&, const Type&) +template +inline min_max_internal::rhs_proxy< op_min_index_view > +min_of(const min_max_internal::rhs_proxy< op_min_index_view >& proxy, + const std::pair& pair) +{ + return proxy.calc(pair); +} + +/// @copydoc min_of(const min_max_internal::rhs_proxy< op_min_index_view >&, const Index&, const Type&) +template +inline min_max_internal::rhs_proxy< op_min_index_view > +min_of(const std::pair& pair, + const min_max_internal::rhs_proxy< op_min_index_view >& proxy) +{ + return proxy.calc(pair); +} + + +/** Monoid class for minimum reductions with index. Instantiate the + * cilk::reducer template class with an op_min_index monoid to create a + * min_index reducer class. For example, to compute the minimum of an array of + * `double` values and the array index of the min value: + * + * cilk::reducer< cilk::op_min_index > r; + * + * @see ReducersMinMax + * @see op_min_index_view + */ +template < typename Index + , typename Type + , typename Compare=std::less + , bool Align = false + > +class op_min_index : public min_max_internal::monoid_base, Align> +{ + typedef min_max_internal::monoid_base< + op_min_index_view, Align> base; +public: + /// Construct with default comparator. + op_min_index() {} + /// Construct with specified comparator. + op_min_index(const Compare& compare) : base(compare) {} +}; + +//@} + + +/** Deprecated maximum reducer wrapper class. + * + * reducer_max is the same as @ref reducer<@ref op_max>, except that + * reducer_max is a proxy for the contained view, so that accumulator + * variable update operations can be applied directly to the reducer. For + * example, a value is maximized with a `reducer<%op_max>` with + * `r->calc_max(a)`, but a value can be maximized with a `%reducer_max` with + * `r.calc_max(a)`. + * + * + * @deprecated Users are strongly encouraged to use `reducer` + * reducers rather than the old wrappers like reducer_max. + * The `reducer` reducers show the reducer/monoid/view + * architecture more clearly, are more consistent in their + * implementation, and present a simpler model for new + * user-implemented reducers. + * + * @note Implicit conversions are provided between `%reducer_max` + * and `reducer<%op_max>`. This allows incremental code + * conversion: old code that used `%reducer_max` can pass a + * `%reducer_max` to a converted function that now expects a + * pointer or reference to a `reducer<%op_max>`, and vice + * versa. **But see @ref redminmax_compatibility.** + * + * @tparam Type The value type of the reducer. + * @tparam Compare The "less than" comparator type for the reducer. + * + * @see op_max + * @see op_max_view + * @see reducer + * @see ReducersMinMax + * @ingroup ReducersMinMaxMaxValue + */ +template > +class reducer_max : public reducer< op_max > +{ + __CILKRTS_STATIC_ASSERT( + ::cilk::internal::class_is_empty< + typename ::cilk::internal::binary_functor::type >::value, + "cilk::reducer_max only works with " + "an empty Compare class"); + typedef reducer< op_max > base; +public: + + /// Type of data in a reducer_max. + typedef Type basic_value_type; + + /// The view type for the reducer. + typedef typename base::view_type view_type; + + /// The view type for the reducer. + typedef typename base::view_type View; + + /// The monoid type for the reducer. + typedef typename base::monoid_type monoid_type; + + /// The monoid type for the reducer. + typedef typename base::monoid_type Monoid; + + /// The view's rhs proxy type. + typedef min_max_internal::rhs_proxy rhs_proxy; + + using base::view; + + /** @name Constructors + */ + //@{ + + /// Constructs the wrapper in its identity state (either `!is_set()`, or + /// `value() == identity value`). + reducer_max() : base() {} + + /// Constructs the wrapper with a specified initial value. + explicit reducer_max(const Type& initial_value) : base(initial_value) {} + + /// Constructs the wrapper in its identity state with a specified + /// comparator. + explicit reducer_max(const Compare& comp) : base(comp) {} + + /// Constructs the wrapper with a specified initial value and a specified + /// comparator. + reducer_max(const Type& initial_value, const Compare& comp) + : base(initial_value, comp) {} + + //@} + + /** @name Forwarded functions + * @details Functions that update the contained accumulator variable are + * simply forwarded to the contained @ref op_max_view. */ + //@{ + + /// @copydoc cilk_lib_1_1::min_max_internal::view_content::is_set() const + bool is_set() const { return view().is_set(); } + + /// @copydoc op_max_view::calc_max(const Type&) + reducer_max& calc_max(const Type& x) + { view().calc_max(x); return *this; } + + /// @copydoc op_max_view::operator=(const min_max_internal::rhs_proxy&) + reducer_max& operator=(const rhs_proxy& rhs) + { view() = rhs; return *this; } + + //@} + + /** Allows read-only access to the value within the current view. + * + * @returns A const reference to the value within the current view. + */ + const Type& get_reference() const { return view().get_reference(); } + + /// @name Dereference + /** Dereferencing a wrapper is a no-op. It simply returns the wrapper. + * Combined with the rule that a wrapper forwards view operations to the + * view, this means that view operations can be written the same way on + * reducers and wrappers, which is convenient for incrementally + * converting code using wrappers to code using reducers. That is: + * + * reducer< op_max > r; + * r->calc_max(a); // *r returns the view + * // calc_max is a view member function + * + * reducer_max w; + * w->calc_max(a); // *w returns the wrapper + * // calc_max is a wrapper member function that + * // calls the corresponding view function + */ + //@{ + reducer_max& operator*() { return *this; } + reducer_max const& operator*() const { return *this; } + + reducer_max* operator->() { return this; } + reducer_max const* operator->() const { return this; } + //@} + + /** @name Upcast + * @details In Intel Cilk Plus library 0.9, reducers were always cache-aligned. + * In library 1.0, reducer cache alignment is optional. By default, + * reducers are unaligned (i.e., just naturally aligned), but legacy + * wrappers inherit from cache-aligned reducers for binary compatibility. + * + * This means that a wrapper will automatically be upcast to its aligned + * reducer base class. The following conversion operators provide + * pseudo-upcasts to the corresponding unaligned reducer class. + */ + //@{ + operator reducer< op_max >& () + { + return *reinterpret_cast< reducer< op_max >* >(this); + } + + operator const reducer< op_max >& () const + { + return *reinterpret_cast< const reducer< op_max >* >(this); + } + //@} +}; + + +/// @cond internal +// The legacy definition of max_of(reducer_max, value) has different +// behavior and a different return type than this definition. We add an +// unused third argument to this version of the function to give it a different +// signature, so that they won't end up sharing a single object file entry. +struct max_of_1_0_t {}; +const max_of_1_0_t max_of_1_0 = {}; +/// @endcond + +/** Computes the maximum of the value in a reducer_max and another value. + * + * @deprecated Because reducer_max is deprecated. + * + * The result of this computation can only be assigned back to the original + * reducer or used in another max_of() call. For example, + * + * reducer = max_of(reducer, x); + * reducer = max_of(x, reducer); + * + * @see min_max_internal::rhs_proxy + * + * @ingroup ReducersMinMaxMaxValue + */ +template +inline min_max_internal::rhs_proxy< op_max_view > +max_of(const reducer_max& r, const Type& value, + const max_of_1_0_t& = max_of_1_0) +{ + return min_max_internal::make_proxy(value, r.view()); +} + +/// @copydoc max_of(const reducer_max&, const Type&, const max_of_1_0_t&) +/// @ingroup ReducersMinMaxMaxValue +template +inline min_max_internal::rhs_proxy< op_max_view > +max_of(const Type& value, const reducer_max& r, + const max_of_1_0_t& = max_of_1_0) +{ + return min_max_internal::make_proxy(value, r.view()); +} + + +/** Deprecated minimum reducer wrapper class. + * + * reducer_min is the same as @ref reducer<@ref op_min>, except that + * reducer_min is a proxy for the contained view, so that accumulator + * variable update operations can be applied directly to the reducer. For + * example, a value is minimized with a `reducer<%op_min>` with + * `r->calc_min(a)`, but a value can be minimized with a `%reducer_min` with + * `r.calc_min(a)`. + * + * + * @deprecated Users are strongly encouraged to use `reducer` + * reducers rather than the old wrappers like reducer_min. + * The `reducer` reducers show the reducer/monoid/view + * architecture more clearly, are more consistent in their + * implementation, and present a simpler model for new + * user-implemented reducers. + * + * @note Implicit conversions are provided between `%reducer_min` + * and `reducer<%op_min>`. This allows incremental code + * conversion: old code that used `%reducer_min` can pass a + * `%reducer_min` to a converted function that now expects a + * pointer or reference to a `reducer<%op_min>`, and vice + * versa. **But see @ref redminmax_compatibility.** + * + * @tparam Type The value type of the reducer. + * @tparam Compare The "less than" comparator type for the reducer. + * + * @see op_min + * @see op_min_view + * @see reducer + * @see ReducersMinMax + * @ingroup ReducersMinMaxMinValue + */ +template > +class reducer_min : public reducer< op_min > +{ + __CILKRTS_STATIC_ASSERT( + ::cilk::internal::class_is_empty< + typename ::cilk::internal::binary_functor::type >::value, + "cilk::reducer_min only works with " + "an empty Compare class"); + typedef reducer< op_min > base; +public: + + /// Type of data in a reducer_min. + typedef Type basic_value_type; + + /// The view type for the reducer. + typedef typename base::view_type view_type; + + /// The view type for the reducer. + typedef typename base::view_type View; + + /// The monoid type for the reducer. + typedef typename base::monoid_type monoid_type; + + /// The monoid type for the reducer. + typedef typename base::monoid_type Monoid; + + /// The view's rhs proxy type. + typedef min_max_internal::rhs_proxy rhs_proxy; + + using base::view; + + /** @name Constructors + */ + //@{ + + /// Constructs the wrapper in its identity state (either `!is_set()`, or + /// `value() == identity value`). + reducer_min() : base() {} + + /// Constructs the wrapper with a specified initial value. + explicit reducer_min(const Type& initial_value) : base(initial_value) {} + + /// Constructs the wrapper in its identity state with a specified + /// comparator. + explicit reducer_min(const Compare& comp) : base(comp) {} + + /// Constructs the wrapper with a specified initial value and a specified + /// comparator. + reducer_min(const Type& initial_value, const Compare& comp) + : base(initial_value, comp) {} + + //@} + + /** @name Forwarded functions + * @details Functions that update the contained accumulator variable are + * simply forwarded to the contained @ref op_min_view. */ + //@{ + + /// @copydoc cilk_lib_1_1::min_max_internal::view_content::is_set() const + bool is_set() const { return view().is_set(); } + + /// @copydoc op_min_view::calc_min(const Type&) + reducer_min& calc_min(const Type& x) + { view().calc_min(x); return *this; } + + /// @copydoc op_min_view::operator=(const min_max_internal::rhs_proxy&) + reducer_min& operator=(const rhs_proxy& rhs) + { view() = rhs; return *this; } + + //@} + + /** Allows read-only access to the value within the current view. + * + * @returns A const reference to the value within the current view. + */ + const Type& get_reference() const { return view().get_reference(); } + + /// @name Dereference + /** Dereferencing a wrapper is a no-op. It simply returns the wrapper. + * Combined with the rule that a wrapper forwards view operations to the + * view, this means that view operations can be written the same way on + * reducers and wrappers, which is convenient for incrementally + * converting code using wrappers to code using reducers. That is: + * + * reducer< op_min > r; + * r->calc_min(a); // *r returns the view + * // calc_min is a view member function + * + * reducer_min w; + * w->calc_min(a); // *w returns the wrapper + * // calc_min is a wrapper member function that + * // calls the corresponding view function + */ + //@{ + reducer_min& operator*() { return *this; } + reducer_min const& operator*() const { return *this; } + + reducer_min* operator->() { return this; } + reducer_min const* operator->() const { return this; } + //@} + + /** @name Upcast + * @details In Intel Cilk Plus library 0.9, reducers were always cache-aligned. + * In library 1.0, reducer cache alignment is optional. By default, + * reducers are unaligned (i.e., just naturally aligned), but legacy + * wrappers inherit from cache-aligned reducers for binary compatibility. + * + * This means that a wrapper will automatically be upcast to its aligned + * reducer base class. The following conversion operators provide + * pseudo-upcasts to the corresponding unaligned reducer class. + */ + //@{ + operator reducer< op_min >& () + { + return *reinterpret_cast< reducer< op_min >* >(this); + } + + operator const reducer< op_min >& () const + { + return *reinterpret_cast< const reducer< op_min >* >(this); + } + //@} +}; + + +/** Computes the minimum of a reducer and a value. + * + * @deprecated Because reducer_min is deprecated. + */ +//@{ +// The legacy definition of min_of(reducer_min, value) has different +// behavior and a different return type than this definition. We add an +// unused third argument to this version of the function to give it a different +// signature, so that they won't end up sharing a single object file entry. +struct min_of_1_0_t {}; +const min_of_1_0_t min_of_1_0 = {}; + +template +inline min_max_internal::rhs_proxy< op_min_view > +min_of(const reducer_min& r, const Type& value, + const min_of_1_0_t& = min_of_1_0) +{ + return min_max_internal::make_proxy(value, r.view()); +} + +template +inline min_max_internal::rhs_proxy< op_min_view > +min_of(const Type& value, const reducer_min& r, + const min_of_1_0_t& = min_of_1_0) +{ + return min_max_internal::make_proxy(value, r.view()); +} +//@} + + +/** Deprecated maximum with index reducer wrapper class. + * + * reducer_max_index is the same as @ref reducer<@ref op_max_index>, except + * that reducer_max_index is a proxy for the contained view, so that + * accumulator variable update operations can be applied directly to the + * reducer. For example, a value is maximized with a `reducer<%op_max_index>` + * with `r->calc_max(i, a)`, but a value can be maximized with a + * `%reducer_max` with `r.calc_max(i, aa)`. + * + * + * @deprecated Users are strongly encouraged to use `reducer` + * reducers rather than the old wrappers like reducer_max. + * The `reducer` reducers show the reducer/monoid/view + * architecture more clearly, are more consistent in their + * implementation, and present a simpler model for new + * user-implemented reducers. + * + * @note Implicit conversions are provided between `%reducer_max_index` + * and `reducer<%op_max_index>`. This allows incremental code + * conversion: old code that used `%reducer_max_index` can pass a + * `%reducer_max_index` to a converted function that now expects a + * pointer or reference to a `reducer<%op_max_index>`, and vice + * versa. **But see @ref redminmax_compatibility.** + * + * @tparam Index The index type of the reducer. + * @tparam Type The value type of the reducer. + * @tparam Compare The "less than" comparator type for the reducer. + * + * @see op_max_index + * @see op_max_index_view + * @see reducer + * @see ReducersMinMax + * @ingroup ReducersMinMaxMaxIndex + */ +template < typename Index + , typename Type + , typename Compare = std::less + > +class reducer_max_index : + public reducer< op_max_index > +{ + __CILKRTS_STATIC_ASSERT( + ::cilk::internal::class_is_empty< + typename ::cilk::internal::binary_functor::type >::value, + "cilk::reducer_max_index only works with " + "an empty Compare class"); + typedef reducer< op_max_index > base; +public: + + /// Type of data in a reducer_max_index. + typedef Type basic_value_type; + + /// The view type for the reducer. + typedef typename base::view_type view_type; + + /// The view type for the reducer. + typedef typename base::view_type View; + + /// The monoid type for the reducer. + typedef typename base::monoid_type monoid_type; + + /// The monoid type for the reducer. + typedef typename base::monoid_type Monoid; + + /// The view's rhs proxy type. + typedef min_max_internal::rhs_proxy rhs_proxy; + + using base::view; + + /** @name Constructors + */ + //@{ + + /// Constructs the wrapper in its identity state (`!is_set()`). + reducer_max_index() : base() {} + + /// Construct with a specified initial index and value. + reducer_max_index(const Index& initial_index, + const Type& initial_value) + : base(initial_index, initial_value) {} + + /// Constructs the wrapper with a specified comparator. + explicit reducer_max_index(const Compare& comp) : base(comp) {} + + /// Constructs the wrapper with a specified initial index, value, + /// and comparator. + reducer_max_index(const Index& initial_index, + const Type& initial_value, + const Compare& comp) + : base(initial_index, initial_value, comp) {} + + //@} + + /** @name Set / Get + */ + //@{ + + /// Sets the index and value of this object. + void set_value(const Index& index, const Type& value) + { base::set_value(std::make_pair(index, value)); } + + /// Returns the maximum value. + const Type& get_value() const + { return view().get_reference(); } + + /// Returns the maximum index. + const Index& get_index() const + { return view().get_index_reference(); } + + /// Returns a const reference to value data member in the view. + const Type& get_reference() const + { return view().get_reference(); } + + /// Returns a const reference to index data member in the view. + const Index& get_index_reference() const + { return view().get_index_reference(); } + + //@} + + /** @name Forwarded functions + * @details Functions that update the contained accumulator variable are + * simply forwarded to the contained @ref op_max_view. */ + //@{ + + /// @copydoc cilk_lib_1_1::min_max_internal::view_content::is_set() const + bool is_set() const { return view().is_set(); } + + /// @copydoc op_max_index_view::calc_max(const Index&, const Type&) + reducer_max_index& calc_max(const Index& i, const Type& x) + { view().calc_max(i, x); return *this; } + + /// @copydoc op_max_view::operator=(const min_max_internal::rhs_proxy&) + reducer_max_index& operator=(const rhs_proxy& rhs) + { view() = rhs; return *this; } + + //@} + + /// @name Dereference + /** Dereferencing a wrapper is a no-op. It simply returns the wrapper. + * Combined with the rule that a wrapper forwards view operations to the + * view, this means that view operations can be written the same way on + * reducers and wrappers, which is convenient for incrementally + * converting code using wrappers to code using reducers. That is: + * + * reducer< op_max_index > r; + * r->calc_max(i, a); // *r returns the view + * // calc_max is a view member function + * + * reducer_max_index w; + * w->calc_max(i, a); // *w returns the wrapper + * // calc_max is a wrapper member function that + * // calls the corresponding view function + */ + //@{ + reducer_max_index& operator*() { return *this; } + reducer_max_index const& operator*() const { return *this; } + + reducer_max_index* operator->() { return this; } + reducer_max_index const* operator->() const { return this; } + //@} + + /** @name Upcast + * @details In Intel Cilk Plus library 0.9, reducers were always cache-aligned. + * In library 1.0, reducer cache alignment is optional. By default, + * reducers are unaligned (i.e., just naturally aligned), but legacy + * wrappers inherit from cache-aligned reducers for binary compatibility. + * + * This means that a wrapper will automatically be upcast to its aligned + * reducer base class. The following conversion operators provide + * pseudo-upcasts to the corresponding unaligned reducer class. + */ + //@{ + operator reducer< op_max_index >& () + { + return *reinterpret_cast< reducer< op_max_index >* >(this); + } + + operator const reducer< op_max_index >& () const + { + return *reinterpret_cast< const reducer< op_max_index >* >(this); + } + //@} + +}; + + +/** Deprecated minimum with index reducer wrapper class. + * + * reducer_min_index is the same as @ref reducer<@ref op_min_index>, except + * that reducer_min_index is a proxy for the contained view, so that + * accumulator variable update operations can be applied directly to the + * reducer. For example, a value is minimized with a `reducer<%op_min_index>` + * with `r->calc_min(i, a)`, but a value can be minimized with a + * `%reducer_min` with `r.calc_min(i, aa)`. + * + * + * @deprecated Users are strongly encouraged to use `reducer` + * reducers rather than the old wrappers like reducer_min. + * The `reducer` reducers show the reducer/monoid/view + * architecture more clearly, are more consistent in their + * implementation, and present a simpler model for new + * user-implemented reducers. + * + * @note Implicit conversions are provided between `%reducer_min_index` + * and `reducer<%op_min_index>`. This allows incremental code + * conversion: old code that used `%reducer_min_index` can pass a + * `%reducer_min_index` to a converted function that now expects a + * pointer or reference to a `reducer<%op_min_index>`, and vice + * versa. **But see @ref redminmax_compatibility.** + * + * @tparam Index The index type of the reducer. + * @tparam Type The value type of the reducer. + * @tparam Compare The "less than" comparator type for the reducer. + * + * @see op_min_index + * @see op_min_index_view + * @see reducer + * @see ReducersMinMax + * @ingroup ReducersMinMaxMinIndex + */ +template < typename Index + , typename Type + , typename Compare = std::less + > +class reducer_min_index : + public reducer< op_min_index > +{ + __CILKRTS_STATIC_ASSERT( + ::cilk::internal::class_is_empty< + typename ::cilk::internal::binary_functor::type >::value, + "cilk::reducer_min_index only works with " + "an empty Compare class"); + typedef reducer< op_min_index > base; +public: + + /// Type of data in a reducer_min_index. + typedef Type basic_value_type; + + /// The view type for the reducer. + typedef typename base::view_type view_type; + + /// The view type for the reducer. + typedef typename base::view_type View; + + /// The monoid type for the reducer. + typedef typename base::monoid_type monoid_type; + + /// The monoid type for the reducer. + typedef typename base::monoid_type Monoid; + + /// The view's rhs proxy type. + typedef min_max_internal::rhs_proxy rhs_proxy; + + using base::view; + + /** @name Constructors + */ + //@{ + + /// Constructs the wrapper in its identity state (`!is_set()`). + reducer_min_index() : base() {} + + /// Construct with a specified initial index and value. + reducer_min_index(const Index& initial_index, + const Type& initial_value) + : base(initial_index, initial_value) {} + + /// Constructs the wrapper with a specified comparator. + explicit reducer_min_index(const Compare& comp) : base(comp) {} + + /// Constructs the wrapper with a specified initial index, value, + /// and comparator. + reducer_min_index(const Index& initial_index, + const Type& initial_value, + const Compare& comp) + : base(initial_index, initial_value, comp) {} + + //@} + + /** @name Set / Get + */ + //@{ + + /// Sets the index and value of this object. + void set_value(const Index& index, const Type& value) + { base::set_value(std::make_pair(index, value)); } + + /// Returns the minimum value. + const Type& get_value() const + { return view().get_reference(); } + + /// Returns the minimum index. + const Index& get_index() const + { return view().get_index_reference(); } + + /// Returns a const reference to value data member in the view. + const Type& get_reference() const + { return view().get_reference(); } + + /// Returns a const reference to index data member in the view. + const Index& get_index_reference() const + { return view().get_index_reference(); } + + //@} + + /** @name Forwarded functions + * @details Functions that update the contained accumulator variable are + * simply forwarded to the contained @ref op_min_view. */ + //@{ + + /// @copydoc cilk_lib_1_1::min_max_internal::view_content::is_set() const + bool is_set() const { return view().is_set(); } + + /// @copydoc op_min_index_view::calc_min(const Index&, const Type&) + reducer_min_index& calc_min(const Index& i, const Type& x) + { view().calc_min(i, x); return *this; } + + /// @copydoc op_min_view::operator=(const min_max_internal::rhs_proxy&) + reducer_min_index& operator=(const rhs_proxy& rhs) + { view() = rhs; return *this; } + + //@} + + /// @name Dereference + /** Dereferencing a wrapper is a no-op. It simply returns the wrapper. + * Combined with the rule that a wrapper forwards view operations to the + * view, this means that view operations can be written the same way on + * reducers and wrappers, which is convenient for incrementally + * converting code using wrappers to code using reducers. That is: + * + * reducer< op_min_index > r; + * r->calc_min(i, a); // *r returns the view + * // calc_min is a view member function + * + * reducer_min_index w; + * w->calc_min(i, a); // *w returns the wrapper + * // calc_min is a wrapper member function that + * // calls the corresponding view function + */ + //@{ + reducer_min_index& operator*() { return *this; } + reducer_min_index const& operator*() const { return *this; } + + reducer_min_index* operator->() { return this; } + reducer_min_index const* operator->() const { return this; } + //@} + + /** @name Upcast + * @details In Intel Cilk Plus library 0.9, reducers were always cache-aligned. + * In library 1.0, reducer cache alignment is optional. By default, + * reducers are unaligned (i.e., just naturally aligned), but legacy + * wrappers inherit from cache-aligned reducers for binary compatibility. + * + * This means that a wrapper will automatically be upcast to its aligned + * reducer base class. The following conversion operators provide + * pseudo-upcasts to the corresponding unaligned reducer class. + */ + //@{ + operator reducer< op_min_index >& () + { + return *reinterpret_cast< reducer< op_min_index >* >(this); + } + + operator const reducer< op_min_index >& () const + { + return *reinterpret_cast< const reducer< op_min_index >* >(this); + } + //@} + +}; + + +#ifndef CILK_LIBRARY_0_9_REDUCER_MINMAX +} // namespace cilk_lib_1_1 +using namespace cilk_lib_1_1; +#endif + + +/// @cond internal +/** Metafunction specialization for reducer conversion. + * + * These specializations of the @ref legacy_reducer_downcast template class + * defined in reducer.h causes each `reducer< op_xxxx >` classes to have + * an `operator reducer_xxxx& ()` conversion operator that statically + * downcasts the `reducer` to the corresponding `reducer_xxxx` type. + * (The reverse conversion, from `reducer_xxxx` to `reducer`, is just + * an upcast, which is provided for free by the language.) + */ +template +struct legacy_reducer_downcast< reducer< op_max > > +{ + typedef reducer_max type; +}; + +template +struct legacy_reducer_downcast< reducer< op_min > > +{ + typedef reducer_min type; +}; + +template +struct legacy_reducer_downcast< reducer< op_max_index > > +{ + typedef reducer_max_index type; +}; + +template +struct legacy_reducer_downcast< reducer< op_min_index > > +{ + typedef reducer_min_index type; +}; +/// @endcond + +} // namespace cilk + +#endif // __cplusplus + + +/** @name C language reducer macros + * + * These macros are used to declare and work with numeric minimum and maximum reducers in C + * code. + * + * @see @ref page_reducers_in_c + */ + //@{ + + +#ifdef CILK_C_DEFINE_REDUCERS + +/* Integer min/max constants */ +#include + +/* Wchar_t min/max constants */ +#if defined(_MSC_VER) || defined(__ANDROID__) +# include +#else +# include +#endif + +/* Floating-point min/max constants */ +#include +#ifndef HUGE_VALF + static const unsigned int __huge_valf[] = {0x7f800000}; +# define HUGE_VALF (*((const float *)__huge_valf)) +#endif + +#ifndef HUGE_VALL + static const unsigned int __huge_vall[] = {0, 0, 0x00007f80, 0}; +# define HUGE_VALL (*((const long double *)__huge_vall)) +#endif + +#endif + +/** Declares max reducer type name. + * + * This macro expands into the identifier which is the name of the max reducer + * type for a specified numeric type. + * + * @param tn The @ref reducers_c_type_names "numeric type name" specifying the type of the + * reducer. + * + * @see @ref reducers_c_predefined + */ +#define CILK_C_REDUCER_MAX_TYPE(tn) \ + __CILKRTS_MKIDENT(cilk_c_reducer_max_,tn) + +/** Declares a max reducer object. + * + * This macro expands into a declaration of a max reducer object for a specified numeric + * type. For example: + * + * CILK_C_REDUCER_MAX(my_reducer, double, -DBL_MAX); + * + * @param obj The variable name to be used for the declared reducer object. + * @param tn The @ref reducers_c_type_names "numeric type name" specifying the type of the + * reducer. + * @param v The initial value for the reducer. (A value which can be assigned to the + * numeric type represented by @a tn.) + * + * @see @ref reducers_c_predefined + */ +#define CILK_C_REDUCER_MAX(obj,tn,v) \ + CILK_C_REDUCER_MAX_TYPE(tn) obj = \ + CILK_C_INIT_REDUCER(_Typeof(obj.value), \ + __CILKRTS_MKIDENT(cilk_c_reducer_max_reduce_,tn), \ + __CILKRTS_MKIDENT(cilk_c_reducer_max_identity_,tn), \ + __cilkrts_hyperobject_noop_destroy, v) + +/** Maximizes with a value. + * + * `CILK_C_REDUCER_MAX_CALC(reducer, v)` sets the current view of the + * reducer to the max of its previous value and a specified new value. + * This is equivalent to + * + * REDUCER_VIEW(reducer) = max(REDUCER_VIEW(reducer), v) + * + * @param reducer The reducer whose contained value is to be updated. + * @param v The value that it is to be maximized with. + */ +#define CILK_C_REDUCER_MAX_CALC(reducer, v) do { \ + _Typeof((reducer).value)* view = &(REDUCER_VIEW(reducer)); \ + _Typeof(v) __value = (v); \ + if (*view < __value) { \ + *view = __value; \ + } } while (0) + +/// @cond internal + +/** Declares the max reducer functions for a numeric type. + * + * This macro expands into external function declarations for functions which implement + * the reducer functionality for the max reducer type for a specified numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer type name, + * function names, etc. + */ +#define CILK_C_REDUCER_MAX_DECLARATION(t,tn,id) \ + typedef CILK_C_DECLARE_REDUCER(t) CILK_C_REDUCER_MAX_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_max,tn,l,r); \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_max,tn); + +/** Defines the max reducer functions for a numeric type. + * + * This macro expands into function definitions for functions which implement the + * reducer functionality for the max reducer type for a specified numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer type name, + * function names, etc. + */ +#define CILK_C_REDUCER_MAX_DEFINITION(t,tn,id) \ + typedef CILK_C_DECLARE_REDUCER(t) CILK_C_REDUCER_MAX_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_max,tn,l,r) \ + { if (*(t*)l < *(t*)r) *(t*)l = *(t*)r; } \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_max,tn) \ + { *(t*)v = id; } + +//@{ +/** @def CILK_C_REDUCER_MAX_INSTANCE + * @brief Declare or define implementation functions for a reducer type. + * + * In the runtime source file c_reducers.c, the macro `CILK_C_DEFINE_REDUCERS` will be defined, and + * this macro will generate reducer implementation functions. Everywhere else, `CILK_C_DEFINE_REDUCERS` + * will be undefined, and this macro will expand into external declarations for the functions. + */ +#ifdef CILK_C_DEFINE_REDUCERS +# define CILK_C_REDUCER_MAX_INSTANCE(t,tn,id) \ + CILK_C_REDUCER_MAX_DEFINITION(t,tn,id) +#else +# define CILK_C_REDUCER_MAX_INSTANCE(t,tn,id) \ + CILK_C_REDUCER_MAX_DECLARATION(t,tn,id) +#endif +//@} + +/* Declare or define an instance of the reducer type and its functions for each + * numeric type. + */ +__CILKRTS_BEGIN_EXTERN_C +CILK_C_REDUCER_MAX_INSTANCE(char, char, CHAR_MIN) +CILK_C_REDUCER_MAX_INSTANCE(unsigned char, uchar, 0) +CILK_C_REDUCER_MAX_INSTANCE(signed char, schar, SCHAR_MIN) +CILK_C_REDUCER_MAX_INSTANCE(wchar_t, wchar_t, WCHAR_MIN) +CILK_C_REDUCER_MAX_INSTANCE(short, short, SHRT_MIN) +CILK_C_REDUCER_MAX_INSTANCE(unsigned short, ushort, 0) +CILK_C_REDUCER_MAX_INSTANCE(int, int, INT_MIN) +CILK_C_REDUCER_MAX_INSTANCE(unsigned int, uint, 0) +CILK_C_REDUCER_MAX_INSTANCE(unsigned int, unsigned, 0) // alternate name +CILK_C_REDUCER_MAX_INSTANCE(long, long, LONG_MIN) +CILK_C_REDUCER_MAX_INSTANCE(unsigned long, ulong, 0) +CILK_C_REDUCER_MAX_INSTANCE(long long, longlong, LLONG_MIN) +CILK_C_REDUCER_MAX_INSTANCE(unsigned long long, ulonglong, 0) +CILK_C_REDUCER_MAX_INSTANCE(float, float, -HUGE_VALF) +CILK_C_REDUCER_MAX_INSTANCE(double, double, -HUGE_VAL) +CILK_C_REDUCER_MAX_INSTANCE(long double, longdouble, -HUGE_VALL) +__CILKRTS_END_EXTERN_C + +/// @endcond + +/** Max_index reducer type name. + * + * This macro expands into the identifier which is the name of the max_index reducer + * type for a specified numeric type. + * + * @param tn The @ref reducers_c_type_names "numeric type name" specifying the type of the + * reducer. + * + * @see @ref reducers_c_predefined + */ +#define CILK_C_REDUCER_MAX_INDEX_TYPE(tn) \ + __CILKRTS_MKIDENT(cilk_c_reducer_max_index_,tn) + +/** Declares an op_max_index reducer object. + * + * This macro expands into a declaration of a max_index reducer object for a specified + * numeric type. For example: + * + * CILK_C_REDUCER_MAX_INDEX(my_reducer, double, -DBL_MAX_INDEX); + * + * @param obj The variable name to be used for the declared reducer object. + * @param tn The @ref reducers_c_type_names "numeric type name" specifying the type of the + * reducer. + * @param v The initial value for the reducer. (A value which can be assigned to the + * numeric type represented by @a tn.) + * + * @see @ref reducers_c_predefined + */ +#define CILK_C_REDUCER_MAX_INDEX(obj,tn,v) \ + CILK_C_REDUCER_MAX_INDEX_TYPE(tn) obj = \ + CILK_C_INIT_REDUCER(_Typeof(obj.value), \ + __CILKRTS_MKIDENT(cilk_c_reducer_max_index_reduce_,tn), \ + __CILKRTS_MKIDENT(cilk_c_reducer_max_index_identity_,tn), \ + __cilkrts_hyperobject_noop_destroy, {0, v}) + +/** Maximizes with a value. + * + * `CILK_C_REDUCER_MAX_INDEX_CALC(reducer, i, v)` sets the current view of the + * reducer to the max of its previous value and a specified new value. + * This is equivalent to + * + * REDUCER_VIEW(reducer) = max_index(REDUCER_VIEW(reducer), v) + * + * If the value of the reducer is changed to @a v, then the index of the reducer is + * changed to @a i. + * + * @param reducer The reducer whose contained value and index are to be updated. + * @param i The index associated with the new value. + * @param v The value that it is to be maximized with. + */ +#define CILK_C_REDUCER_MAX_INDEX_CALC(reducer, i, v) do { \ + _Typeof((reducer).value)* view = &(REDUCER_VIEW(reducer)); \ + _Typeof(v) __value = (v); \ + if (view->value < __value) { \ + view->index = (i); \ + view->value = __value; \ + } } while (0) + +/// @cond internal + +/** Declares the max_index view type. + * + * The view of a max_index reducer is a structure containing both the + * maximum value for the reducer and the index that was associated with + * that value in the sequence of input values. + */ +#define CILK_C_REDUCER_MAX_INDEX_VIEW(t,tn) \ + typedef struct { \ + __STDNS ptrdiff_t index; \ + t value; \ + } __CILKRTS_MKIDENT(cilk_c_reducer_max_index_view_,tn) + +/** Declares the max_index reducer functions for a numeric type. + * + * This macro expands into external function declarations for functions which implement + * the reducer functionality for the max_index reducer type for a specified numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer type name, + * function names, etc. + */ +#define CILK_C_REDUCER_MAX_INDEX_DECLARATION(t,tn,id) \ + CILK_C_REDUCER_MAX_INDEX_VIEW(t,tn); \ + typedef CILK_C_DECLARE_REDUCER( \ + __CILKRTS_MKIDENT(cilk_c_reducer_max_index_view_,tn)) \ + CILK_C_REDUCER_MAX_INDEX_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_max_index,tn,l,r); \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_max_index,tn); + +/** Defines the max_index reducer functions for a numeric type. + * + * This macro expands into function definitions for functions which implement the + * reducer functionality for the max_index reducer type for a specified numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer type name, + * function names, etc. + */ +#define CILK_C_REDUCER_MAX_INDEX_DEFINITION(t,tn,id) \ + CILK_C_REDUCER_MAX_INDEX_VIEW(t,tn); \ + typedef CILK_C_DECLARE_REDUCER( \ + __CILKRTS_MKIDENT(cilk_c_reducer_max_index_view_,tn)) \ + CILK_C_REDUCER_MAX_INDEX_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_max_index,tn,l,r) \ + { typedef __CILKRTS_MKIDENT(cilk_c_reducer_max_index_view_,tn) view_t; \ + if (((view_t*)l)->value < ((view_t*)r)->value) \ + *(view_t*)l = *(view_t*)r; } \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_max_index,tn) \ + { typedef __CILKRTS_MKIDENT(cilk_c_reducer_max_index_view_,tn) view_t; \ + ((view_t*)v)->index = 0; ((view_t*)v)->value = id; } + +//@{ +/** @def CILK_C_REDUCER_MAX_INDEX_INSTANCE + * @brief Declare or define implementation functions for a reducer type. + * + * In the runtime source file c_reducers.c, the macro `CILK_C_DEFINE_REDUCERS` will be defined, and + * this macro will generate reducer implementation functions. Everywhere else, `CILK_C_DEFINE_REDUCERS` + * will be undefined, and this macro will expand into external declarations for the functions. + */ +#ifdef CILK_C_DEFINE_REDUCERS +# define CILK_C_REDUCER_MAX_INDEX_INSTANCE(t,tn,id) \ + CILK_C_REDUCER_MAX_INDEX_DEFINITION(t,tn,id) +#else +# define CILK_C_REDUCER_MAX_INDEX_INSTANCE(t,tn,id) \ + CILK_C_REDUCER_MAX_INDEX_DECLARATION(t,tn,id) +#endif +//@} + +/* Declares or defines an instance of the reducer type and its functions for each + * numeric type. + */ +__CILKRTS_BEGIN_EXTERN_C +CILK_C_REDUCER_MAX_INDEX_INSTANCE(char, char, CHAR_MIN) +CILK_C_REDUCER_MAX_INDEX_INSTANCE(unsigned char, uchar, 0) +CILK_C_REDUCER_MAX_INDEX_INSTANCE(signed char, schar, SCHAR_MIN) +CILK_C_REDUCER_MAX_INDEX_INSTANCE(wchar_t, wchar_t, WCHAR_MIN) +CILK_C_REDUCER_MAX_INDEX_INSTANCE(short, short, SHRT_MIN) +CILK_C_REDUCER_MAX_INDEX_INSTANCE(unsigned short, ushort, 0) +CILK_C_REDUCER_MAX_INDEX_INSTANCE(int, int, INT_MIN) +CILK_C_REDUCER_MAX_INDEX_INSTANCE(unsigned int, uint, 0) +CILK_C_REDUCER_MAX_INDEX_INSTANCE(unsigned int, unsigned, 0) // alternate name +CILK_C_REDUCER_MAX_INDEX_INSTANCE(long, long, LONG_MIN) +CILK_C_REDUCER_MAX_INDEX_INSTANCE(unsigned long, ulong, 0) +CILK_C_REDUCER_MAX_INDEX_INSTANCE(long long, longlong, LLONG_MIN) +CILK_C_REDUCER_MAX_INDEX_INSTANCE(unsigned long long, ulonglong, 0) +CILK_C_REDUCER_MAX_INDEX_INSTANCE(float, float, -HUGE_VALF) +CILK_C_REDUCER_MAX_INDEX_INSTANCE(double, double, -HUGE_VAL) +CILK_C_REDUCER_MAX_INDEX_INSTANCE(long double, longdouble, -HUGE_VALL) +__CILKRTS_END_EXTERN_C + +/// @endcond + +/** Declares min reducer type name. + * + * This macro expands into the identifier which is the name of the min reducer + * type for a specified numeric type. + * + * @param tn The @ref reducers_c_type_names "numeric type name" specifying the type of the + * reducer. + * + * @see @ref reducers_c_predefined + */ +#define CILK_C_REDUCER_MIN_TYPE(tn) \ + __CILKRTS_MKIDENT(cilk_c_reducer_min_,tn) + +/** Declares a min reducer object. + * + * This macro expands into a declaration of a min reducer object for a specified numeric + * type. For example: + * + * CILK_C_REDUCER_MIN(my_reducer, double, DBL_MAX); + * + * @param obj The variable name to be used for the declared reducer object. + * @param tn The @ref reducers_c_type_names "numeric type name" specifying the type of the + * reducer. + * @param v The initial value for the reducer. (A value which can be assigned to the + * numeric type represented by @a tn.) + * + * @see @ref reducers_c_predefined + */ +#define CILK_C_REDUCER_MIN(obj,tn,v) \ + CILK_C_REDUCER_MIN_TYPE(tn) obj = \ + CILK_C_INIT_REDUCER(_Typeof(obj.value), \ + __CILKRTS_MKIDENT(cilk_c_reducer_min_reduce_,tn), \ + __CILKRTS_MKIDENT(cilk_c_reducer_min_identity_,tn), \ + __cilkrts_hyperobject_noop_destroy, v) + +/** Minimizes with a value. + * + * `CILK_C_REDUCER_MIN_CALC(reducer, v)` sets the current view of the + * reducer to the min of its previous value and a specified new value. + * This is equivalent to + * + * REDUCER_VIEW(reducer) = min(REDUCER_VIEW(reducer), v) + * + * @param reducer The reducer whose contained value is to be updated. + * @param v The value that it is to be minimized with. + */ +#define CILK_C_REDUCER_MIN_CALC(reducer, v) do { \ + _Typeof((reducer).value)* view = &(REDUCER_VIEW(reducer)); \ + _Typeof(v) __value = (v); \ + if (*view > __value) { \ + *view = __value; \ + } } while (0) + +/// @cond internal + +/** Declares the min reducer functions for a numeric type. + * + * This macro expands into external function declarations for functions which implement + * the reducer functionality for the min reducer type for a specified numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer type name, + * function names, etc. + */ +#define CILK_C_REDUCER_MIN_DECLARATION(t,tn,id) \ + typedef CILK_C_DECLARE_REDUCER(t) CILK_C_REDUCER_MIN_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_min,tn,l,r); \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_min,tn); + +/** Defines the min reducer functions for a numeric type. + * + * This macro expands into function definitions for functions which implement the + * reducer functionality for the min reducer type for a specified numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer type name, + * function names, etc. + */ +#define CILK_C_REDUCER_MIN_DEFINITION(t,tn,id) \ + typedef CILK_C_DECLARE_REDUCER(t) CILK_C_REDUCER_MIN_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_min,tn,l,r) \ + { if (*(t*)l > *(t*)r) *(t*)l = *(t*)r; } \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_min,tn) \ + { *(t*)v = id; } + +//@{ +/** @def CILK_C_REDUCER_MIN_INSTANCE + * @brief Declare or define implementation functions for a reducer type. + * + * In the runtime source file c_reducers.c, the macro `CILK_C_DEFINE_REDUCERS` will be defined, and + * this macro will generate reducer implementation functions. Everywhere else, `CILK_C_DEFINE_REDUCERS` + * will be undefined, and this macro will expand into external declarations for the functions. + */ +#ifdef CILK_C_DEFINE_REDUCERS +# define CILK_C_REDUCER_MIN_INSTANCE(t,tn,id) \ + CILK_C_REDUCER_MIN_DEFINITION(t,tn,id) +#else +# define CILK_C_REDUCER_MIN_INSTANCE(t,tn,id) \ + CILK_C_REDUCER_MIN_DECLARATION(t,tn,id) +#endif +//@} + +/* Declares or defines an instance of the reducer type and its functions for each + * numeric type. + */ +__CILKRTS_BEGIN_EXTERN_C +CILK_C_REDUCER_MIN_INSTANCE(char, char, CHAR_MAX) +CILK_C_REDUCER_MIN_INSTANCE(unsigned char, uchar, CHAR_MAX) +CILK_C_REDUCER_MIN_INSTANCE(signed char, schar, SCHAR_MAX) +CILK_C_REDUCER_MIN_INSTANCE(wchar_t, wchar_t, WCHAR_MAX) +CILK_C_REDUCER_MIN_INSTANCE(short, short, SHRT_MAX) +CILK_C_REDUCER_MIN_INSTANCE(unsigned short, ushort, USHRT_MAX) +CILK_C_REDUCER_MIN_INSTANCE(int, int, INT_MAX) +CILK_C_REDUCER_MIN_INSTANCE(unsigned int, uint, UINT_MAX) +CILK_C_REDUCER_MIN_INSTANCE(unsigned int, unsigned, UINT_MAX) // alternate name +CILK_C_REDUCER_MIN_INSTANCE(long, long, LONG_MAX) +CILK_C_REDUCER_MIN_INSTANCE(unsigned long, ulong, ULONG_MAX) +CILK_C_REDUCER_MIN_INSTANCE(long long, longlong, LLONG_MAX) +CILK_C_REDUCER_MIN_INSTANCE(unsigned long long, ulonglong, ULLONG_MAX) +CILK_C_REDUCER_MIN_INSTANCE(float, float, HUGE_VALF) +CILK_C_REDUCER_MIN_INSTANCE(double, double, HUGE_VAL) +CILK_C_REDUCER_MIN_INSTANCE(long double, longdouble, HUGE_VALL) +__CILKRTS_END_EXTERN_C + +/// @endcond + +/** Declares `min_index` reducer type name. + * + * This macro expands into the identifier which is the name of the min_index reducer + * type for a specified numeric type. + * + * @param tn The @ref reducers_c_type_names "numeric type name" specifying the type of the + * reducer. + * + * @see @ref reducers_c_predefined + */ +#define CILK_C_REDUCER_MIN_INDEX_TYPE(tn) \ + __CILKRTS_MKIDENT(cilk_c_reducer_min_index_,tn) + +/** Declares an op_min_index reducer object. + * + * This macro expands into a declaration of a min_index reducer object for a specified + * numeric type. For example: + * + * CILK_C_REDUCER_MIN_INDEX(my_reducer, double, -DBL_MIN_INDEX); + * + * @param obj The variable name to be used for the declared reducer object. + * @param tn The @ref reducers_c_type_names "numeric type name" specifying the type of the + * reducer. + * @param v The initial value for the reducer. (A value which can be assigned to the + * numeric type represented by @a tn.) + * + * @see @ref reducers_c_predefined + */ +#define CILK_C_REDUCER_MIN_INDEX(obj,tn,v) \ + CILK_C_REDUCER_MIN_INDEX_TYPE(tn) obj = \ + CILK_C_INIT_REDUCER(_Typeof(obj.value), \ + __CILKRTS_MKIDENT(cilk_c_reducer_min_index_reduce_,tn), \ + __CILKRTS_MKIDENT(cilk_c_reducer_min_index_identity_,tn), \ + __cilkrts_hyperobject_noop_destroy, {0, v}) + +/** Minimizes with a value. + * + * `CILK_C_REDUCER_MIN_INDEX_CALC(reducer, i, v)` sets the current view of the + * reducer to the min of its previous value and a specified new value. + * This is equivalent to + * + * REDUCER_VIEW(reducer) = min_index(REDUCER_VIEW(reducer), v) + * + * If the value of the reducer is changed to @a v, then the index of the reducer is + * changed to @a i. + * + * @param reducer The reducer whose contained value and index are to be updated. + * @param i The index associated with the new value. + * @param v The value that it is to be minimized with. + */ +#define CILK_C_REDUCER_MIN_INDEX_CALC(reducer, i, v) do { \ + _Typeof((reducer).value)* view = &(REDUCER_VIEW(reducer)); \ + _Typeof(v) __value = (v); \ + if (view->value > __value) { \ + view->index = (i); \ + view->value = __value; \ + } } while (0) + +/// @cond internal + +/** Declares the min_index view type. + * + * The view of a min_index reducer is a structure containing both the + * minimum value for the reducer and the index that was associated with + * that value in the sequence of input values. + */ +#define CILK_C_REDUCER_MIN_INDEX_VIEW(t,tn) \ + typedef struct { \ + __STDNS ptrdiff_t index; \ + t value; \ + } __CILKRTS_MKIDENT(cilk_c_reducer_min_index_view_,tn) + +/** Declares the min_index reducer functions for a numeric type. + * + * This macro expands into external function declarations for functions which implement + * the reducer functionality for the min_index reducer type for a specified numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer type name, + * function names, etc. + */ +#define CILK_C_REDUCER_MIN_INDEX_DECLARATION(t,tn,id) \ + CILK_C_REDUCER_MIN_INDEX_VIEW(t,tn); \ + typedef CILK_C_DECLARE_REDUCER( \ + __CILKRTS_MKIDENT(cilk_c_reducer_min_index_view_,tn)) \ + CILK_C_REDUCER_MIN_INDEX_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_min_index,tn,l,r); \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_min_index,tn); + +/** Defines the min_index reducer functions for a numeric type. + * + * This macro expands into function definitions for functions which implement the + * reducer functionality for the min_index reducer type for a specified numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer type name, + * function names, etc. + */ +#define CILK_C_REDUCER_MIN_INDEX_DEFINITION(t,tn,id) \ + CILK_C_REDUCER_MIN_INDEX_VIEW(t,tn); \ + typedef CILK_C_DECLARE_REDUCER( \ + __CILKRTS_MKIDENT(cilk_c_reducer_min_index_view_,tn)) \ + CILK_C_REDUCER_MIN_INDEX_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_min_index,tn,l,r) \ + { typedef __CILKRTS_MKIDENT(cilk_c_reducer_min_index_view_,tn) view_t; \ + if (((view_t*)l)->value > ((view_t*)r)->value) \ + *(view_t*)l = *(view_t*)r; } \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_min_index,tn) \ + { typedef __CILKRTS_MKIDENT(cilk_c_reducer_min_index_view_,tn) view_t; \ + ((view_t*)v)->index = 0; ((view_t*)v)->value = id; } + +//@{ +/** @def CILK_C_REDUCER_MIN_INDEX_INSTANCE + * @brief Declares or defines implementation functions for a reducer type. + * + * In the runtime source file c_reducers.c, the macro `CILK_C_DEFINE_REDUCERS` will be defined, and + * this macro will generate reducer implementation functions. Everywhere else, `CILK_C_DEFINE_REDUCERS` + * will be undefined, and this macro will expand into external declarations for the functions. + */ +#ifdef CILK_C_DEFINE_REDUCERS +# define CILK_C_REDUCER_MIN_INDEX_INSTANCE(t,tn,id) \ + CILK_C_REDUCER_MIN_INDEX_DEFINITION(t,tn,id) +#else +# define CILK_C_REDUCER_MIN_INDEX_INSTANCE(t,tn,id) \ + CILK_C_REDUCER_MIN_INDEX_DECLARATION(t,tn,id) +#endif +//@} + +/* Declares or defines an instance of the reducer type and its functions for each + * numeric type. + */ +__CILKRTS_BEGIN_EXTERN_C +CILK_C_REDUCER_MIN_INDEX_INSTANCE(char, char, CHAR_MAX) +CILK_C_REDUCER_MIN_INDEX_INSTANCE(unsigned char, uchar, CHAR_MAX) +CILK_C_REDUCER_MIN_INDEX_INSTANCE(signed char, schar, SCHAR_MAX) +CILK_C_REDUCER_MIN_INDEX_INSTANCE(wchar_t, wchar_t, WCHAR_MAX) +CILK_C_REDUCER_MIN_INDEX_INSTANCE(short, short, SHRT_MAX) +CILK_C_REDUCER_MIN_INDEX_INSTANCE(unsigned short, ushort, USHRT_MAX) +CILK_C_REDUCER_MIN_INDEX_INSTANCE(int, int, INT_MAX) +CILK_C_REDUCER_MIN_INDEX_INSTANCE(unsigned int, uint, UINT_MAX) +CILK_C_REDUCER_MIN_INDEX_INSTANCE(unsigned int, unsigned, UINT_MAX) // alternate name +CILK_C_REDUCER_MIN_INDEX_INSTANCE(long, long, LONG_MAX) +CILK_C_REDUCER_MIN_INDEX_INSTANCE(unsigned long, ulong, ULONG_MAX) +CILK_C_REDUCER_MIN_INDEX_INSTANCE(long long, longlong, LLONG_MAX) +CILK_C_REDUCER_MIN_INDEX_INSTANCE(unsigned long long, ulonglong, ULLONG_MAX) +CILK_C_REDUCER_MIN_INDEX_INSTANCE(float, float, HUGE_VALF) +CILK_C_REDUCER_MIN_INDEX_INSTANCE(double, double, HUGE_VAL) +CILK_C_REDUCER_MIN_INDEX_INSTANCE(long double, longdouble, HUGE_VALL) +__CILKRTS_END_EXTERN_C + +/// @endcond + +//@} + +#endif // defined REDUCER_MIN_MAX_H_INCLUDED diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_opadd.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_opadd.h new file mode 100644 index 0000000000..46d4b6e9b5 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_opadd.h @@ -0,0 +1,701 @@ +/* reducer_opadd.h -*- C++ -*- + * + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + */ + +/** @file reducer_opadd.h + * + * @brief Defines classes for doing parallel addition reductions. + * + * @ingroup ReducersAdd + * + * @see ReducersAdd + */ + +#ifndef REDUCER_OPADD_H_INCLUDED +#define REDUCER_OPADD_H_INCLUDED + +#include + +/** @defgroup ReducersAdd Addition Reducers + * + * Addition reducers allow the computation of the sum of a set of values in + * parallel. + * + * @ingroup Reducers + * + * You should be familiar with @ref pagereducers "Intel(R) Cilk(TM) Plus reducers", + * described in file `reducers.md`, and particularly with @ref reducers_using, + * before trying to use the information in this file. + * + * @section redopadd_usage Usage Example + * + * cilk::reducer< cilk::op_add > r; + * cilk_for (int i = 0; i != N; ++i) { + * *r += a[i]; + * } + * return r.get_value(); + * + * @section redopadd_monoid The Monoid + * + * @subsection redopadd_monoid_values Value Set + * + * The value set of an addition reducer is the set of values of `Type`, which + * is expected to be a builtin numeric type (or something like it, such as + * `std::complex`). + * + * @subsection redopadd_monoid_operator Operator + * + * The operator of an addition reducer is the addition operator, defined by + * the "`+`" binary operator on `Type`. + * + * @subsection redopadd_monoid_identity Identity + * + * The identity value of the reducer is the numeric value "`0`". This is + * expected to be the value of the default constructor `Type()`. + * + * @section redopadd_operations Operations + * + * @subsection redopadd_constructors Constructors + * + * reducer() // identity + * reducer(const Type& value) + * reducer(move_in(Type& variable)) + * + * @subsection redopadd_get_set Set and Get + * + * r.set_value(const Type& value) + * const Type& = r.get_value() const + * r.move_in(Type& variable) + * r.move_out(Type& variable) + * + * @subsection redopadd_initial Initial Values + * + * If an addition reducer is constructed without an explicit initial value, + * then its initial value will be its identity value, as long as `Type` + * satisfies the requirements of @ref redopadd_types. + * + * @subsection redopadd_view_ops View Operations + * + * *r += a + * *r -= a + * ++*r + * --*r + * (*r)++ + * (*r)-- + * *r = *r + a + * *r = *r - a + * *r = *r ± a1 ± a2 … ± an + * + * The post-increment and post-decrement operations do not return a value. (If + * they did, they would expose the value contained in the view, which is + * non-deterministic in the middle of a reduction.) + * + * Note that subtraction operations are allowed on an addition reducer because + * subtraction is equivalent to addition with a negated operand. It is true + * that `(x - y) - z` is not equivalent to `x - (y - z)`, but + * `(x + (-y)) + (-z)` _is_ equivalent to `x + ((-y) + (-z))`. + * + * @section redopadd_floating_point Issues with Floating-Point Types + * + * Because of precision and round-off issues, floating-point addition is not + * really associative. For example, `(1e30 + -1e30) + 1 == 1`, but + * `1e30 + (-1e30 + 1) == 0`. + * + * In many cases, this won't matter, but computations which have been + * carefully ordered to control round-off errors may not deal well with + * being reassociated. In general, you should be sure to understand the + * floating-point behavior of your program before doing any transformation + * that will reassociate its computations. + * + * @section redopadd_types Type and Operator Requirements + * + * `Type` must be `Copy Constructible`, `Default Constructible`, and + * `Assignable`. + * + * The operator "`+=`" must be defined on `Type`, with `x += a` having the + * same meaning as `x = x + a`. In addition, if the code uses the "`-=`", + * pre-increment, post-increment, pre-decrement, or post-decrement operators, + * then the corresponding operators must be defined on `Type`. + * + * The expression `Type()` must be a valid expression which yields the + * identity value (the value of `Type` whose numeric value is zero). + * + * @section redopadd_in_c Addition Reducers in C + * + * The @ref CILK_C_REDUCER_OPADD and @ref CILK_C_REDUCER_OPADD_TYPE macros can + * be used to do addition reductions in C. For example: + * + * CILK_C_REDUCER_OPADD(r, double, 0); + * CILK_C_REGISTER_REDUCER(r); + * cilk_for(int i = 0; i != n; ++i) { + * REDUCER_VIEW(r) += a[i]; + * } + * CILK_C_UNREGISTER_REDUCER(r); + * printf("The sum of the elements of a is %f\n", REDUCER_VIEW(r)); + * + * See @ref reducers_c_predefined. + */ + +#ifdef __cplusplus + +namespace cilk { + +/** The addition reducer view class. + * + * This is the view class for reducers created with + * `cilk::reducer< cilk::op_add >`. It holds the accumulator variable + * for the reduction, and allows only addition and subtraction operations to + * be performed on it. + * + * @note The reducer "dereference" operation (`reducer::operator *()`) + * yields a reference to the view. Thus, for example, the view class's + * `+=` operation would be used in an expression like `*r += a`, where + * `r` is an op_add reducer variable. + * + * @tparam Type The type of the contained accumulator variable. This will + * be the value type of a monoid_with_view that is + * instantiated with this view. + * + * @see ReducersAdd + * @see op_add + * + * @ingroup ReducersAdd + */ +template +class op_add_view : public scalar_view +{ + typedef scalar_view base; + +public: + /** Class to represent the right-hand side of + * `*reducer = *reducer ± value`. + * + * The only assignment operator for the op_add_view class takes an + * rhs_proxy as its operand. This results in the syntactic restriction + * that the only expressions that can be assigned to an op_add_view are + * ones which generate an rhs_proxy - that is, expressions of the form + * `op_add_view ± value ... ± value`. + * + * @warning + * The lhs and rhs views in such an assignment must be the same; + * otherwise, the behavior will be undefined. (I.e., `v1 = v1 + x` is + * legal; `v1 = v2 + x` is illegal.) This condition will be checked with a + * runtime assertion when compiled in debug mode. + * + * @see op_add_view + */ + class rhs_proxy { + friend class op_add_view; + + const op_add_view* m_view; + Type m_value; + + // Constructor is invoked only from op_add_view::operator+() and + // op_add_view::operator-(). + // + rhs_proxy(const op_add_view* view, const Type& value) : + m_view(view), m_value(value) {} + + rhs_proxy& operator=(const rhs_proxy&); // Disable assignment operator + rhs_proxy(); // Disable default constructor + + public: + ///@{ + /** Adds or subtracts an additional rhs value. If `v` is an op_add_view + * and `a1` is a value, then the expression `v + a1` invokes the view's + * `operator+()` to create an rhs_proxy for `(v, a1)`; then + * `v + a1 + a2` invokes the rhs_proxy's `operator+()` to create a new + * rhs_proxy for `(v, a1+a2)`. This allows the right-hand side of an + * assignment to be not just `view ± value`, but + * `view ± value ± value ... ± value`. The effect is that + * + * v = v ± a1 ± a2 ... ± an; + * + * is evaluated as + * + * v = v ± (±a1 ± a2 ... ± an); + */ + rhs_proxy& operator+(const Type& x) { m_value += x; return *this; } + rhs_proxy& operator-(const Type& x) { m_value -= x; return *this; } + ///@} + }; + + + /** Default/identity constructor. This constructor initializes the + * contained value to `Type()`, which is expected to be the identity value + * for addition on `Type`. + */ + op_add_view() : base() {} + + /** Construct with a specified initial value. + */ + explicit op_add_view(const Type& v) : base(v) {} + + /** Reduces the views of two strands. + * + * This function is invoked by the @ref op_add monoid to combine the views + * of two strands when the right strand merges with the left one. It adds + * the value contained in the right-strand view to the value contained in + * the left-strand view, and leaves the value in the right-strand view + * undefined. + * + * @param right A pointer to the right-strand view. (`this` points to + * the left-strand view.) + * + * @note Used only by the @ref op_add monoid to implement the monoid + * reduce operation. + */ + void reduce(op_add_view* right) { this->m_value += right->m_value; } + + /** @name Accumulator variable updates. + * + * These functions support the various syntaxes for incrementing or + * decrementing the accumulator variable contained in the view. + */ + ///@{ + + /** Increments the accumulator variable by @a x. + */ + op_add_view& operator+=(const Type& x) { this->m_value += x; return *this; } + + /** Decrements the accumulator variable by @a x. + */ + op_add_view& operator-=(const Type& x) { this->m_value -= x; return *this; } + + /** Pre-increment. + */ + op_add_view& operator++() { ++this->m_value; return *this; } + + /** Post-increments. + * + * @note Conventionally, post-increment operators return the old value + * of the incremented variable. However, reducer views do not + * expose their contained values, so `view++` does not have a + * return value. + */ + void operator++(int) { this->m_value++; } + + /** Pre-decrements. + */ + op_add_view& operator--() { --this->m_value; return *this; } + + /** Post-decrements. + * + * @note Conventionally, post-decrement operators return the old value + * of the decremented variable. However, reducer views do not + * expose their contained values, so `view--` does not have a + * return value. + */ + void operator--(int) { this->m_value--; } + + /** Creates an object representing `*this + x`. + * + * @see rhs_proxy + */ + rhs_proxy operator+(const Type& x) const { return rhs_proxy(this, x); } + + /** Creates an object representing `*this - x`. + * + * @see rhs_proxy + */ + rhs_proxy operator-(const Type& x) const { return rhs_proxy(this, -x); } + + /** Assigns the result of a `view ± value` expression to the view. Note that + * this is the only assignment operator for this class. + * + * @see rhs_proxy + */ + op_add_view& operator=(const rhs_proxy& rhs) { + __CILKRTS_ASSERT(this == rhs.m_view); + this->m_value += rhs.m_value; + return *this; + } + + ///@} +}; + + +/** Monoid class for addition reductions. Instantiate the cilk::reducer + * template class with an op_add monoid to create an addition reducer class. + * For example, to compute + * the sum of a set of `int` values: + * + * cilk::reducer< cilk::op_add > r; + * + * @tparam Type The reducer value type. + * @tparam Align If `false` (the default), reducers instantiated on this + * monoid will be naturally aligned (the Intel Cilk Plus library 1.0 + * behavior). If `true`, reducers instantiated on this monoid + * will be cache-aligned for binary compatibility with + * reducers in Intel Cilk Plus library version 0.9. + * + * @see ReducersAdd + * @see op_add_view + * + * @ingroup ReducersAdd + */ +template +struct op_add : public monoid_with_view, Align> {}; + +/** **Deprecated** addition reducer wrapper class. + * + * reducer_opadd is the same as @ref reducer<@ref op_add>, except that + * reducer_opadd is a proxy for the contained view, so that accumulator + * variable update operations can be applied directly to the reducer. For + * example, a value is added to a `reducer<%op_add>` with `*r += a`, but a + * value can be added to a `%reducer_opadd` with `r += a`. + * + * @deprecated Users are strongly encouraged to use `reducer` + * reducers rather than the old wrappers like reducer_opadd. + * The `reducer` reducers show the reducer/monoid/view + * architecture more clearly, are more consistent in their + * implementation, and present a simpler model for new + * user-implemented reducers. + * + * @note Implicit conversions are provided between `%reducer_opadd` + * and `reducer<%op_add>`. This allows incremental code + * conversion: old code that used `%reducer_opadd` can pass a + * `%reducer_opadd` to a converted function that now expects a + * pointer or reference to a `reducer<%op_add>`, and vice + * versa. + * + * @tparam Type The value type of the reducer. + * + * @see op_add + * @see reducer + * @see ReducersAdd + * + * @ingroup ReducersAdd + */ +template +class reducer_opadd : public reducer< op_add > +{ + typedef reducer< op_add > base; + using base::view; + + public: + /// The view type for the reducer. + typedef typename base::view_type view_type; + + /// The view's rhs proxy type. + typedef typename view_type::rhs_proxy rhs_proxy; + + /// The view type for the reducer. + typedef view_type View; + + /// The monoid type for the reducer. + typedef typename base::monoid_type Monoid; + + /** @name Constructors + */ + ///@{ + + /** Default (identity) constructor. + * + * Constructs the wrapper with the default initial value of `Type()`. + */ + reducer_opadd() {} + + /** Value constructor. + * + * Constructs the wrapper with a specified initial value. + */ + explicit reducer_opadd(const Type& initial_value) : base(initial_value) {} + + ///@} + + /** @name Forwarded functions + * @details Functions that update the contained accumulator variable are + * simply forwarded to the contained @ref op_add_view. */ + ///@{ + + /// @copydoc op_add_view::operator+=(const Type&) + reducer_opadd& operator+=(const Type& x) { view() += x; return *this; } + + /// @copydoc op_add_view::operator-=(const Type&) + reducer_opadd& operator-=(const Type& x) { view() -= x; return *this; } + + /// @copydoc op_add_view::operator++() + reducer_opadd& operator++() { ++view(); return *this; } + + /// @copydoc op_add_view::operator++(int) + void operator++(int) { view()++; } + + /// @copydoc op_add_view::operator-\-() + reducer_opadd& operator--() { --view(); return *this; } + + /// @copydoc op_add_view::operator-\-(int) + void operator--(int) { view()--; } + + // The legacy definitions of reducer_opadd::operator+() and + // reducer_opadd::operator-() have different behavior and a different + // return type than this definition. The legacy version is defined as a + // member function, so this new version is defined as a free function to + // give it a different signature, so that they won't end up sharing a + // single object file entry. + + /// @copydoc op_add_view::operator+(const Type&) const + friend rhs_proxy operator+(const reducer_opadd& r, const Type& x) + { + return r.view() + x; + } + /// @copydoc op_add_view::operator-(const Type&) const + friend rhs_proxy operator-(const reducer_opadd& r, const Type& x) + { + return r.view() - x; + } + /// @copydoc op_add_view::operator=(const rhs_proxy&) + reducer_opadd& operator=(const rhs_proxy& temp) + { + view() = temp; + return *this; + } + ///@} + + /** @name Dereference + * @details Dereferencing a wrapper is a no-op. It simply returns the + * wrapper. Combined with the rule that the wrapper forwards view + * operations to its contained view, this means that view operations can + * be written the same way on reducers and wrappers, which is convenient + * for incrementally converting old code using wrappers to use reducers + * instead. That is: + * + * reducer< op_add > r; + * *r += a; // *r returns the view + * // operator += is a view member function + * + * reducer_opadd w; + * *w += a; // *w returns the wrapper + * // operator += is a wrapper member function that + * // calls the corresponding view function + */ + ///@{ + reducer_opadd& operator*() { return *this; } + reducer_opadd const& operator*() const { return *this; } + + reducer_opadd* operator->() { return this; } + reducer_opadd const* operator->() const { return this; } + ///@} + + /** @name Upcast + * @details In Intel Cilk Plus library 0.9, reducers were always cache-aligned. + * In library 1.0, reducer cache alignment is optional. By default, + * reducers are unaligned (i.e., just naturally aligned), but legacy + * wrappers inherit from cache-aligned reducers for binary compatibility. + * + * This means that a wrapper will automatically be upcast to its aligned + * reducer base class. The following conversion operators provide + * pseudo-upcasts to the corresponding unaligned reducer class. + */ + ///@{ + operator reducer< op_add >& () + { + return *reinterpret_cast< reducer< op_add >* >(this); + } + operator const reducer< op_add >& () const + { + return *reinterpret_cast< const reducer< op_add >* >(this); + } + ///@} +}; + +/// @cond internal +/** Metafunction specialization for reducer conversion. + * + * This specialization of the @ref legacy_reducer_downcast template class + * defined in reducer.h causes the `reducer< op_add >` class to have an + * `operator reducer_opadd& ()` conversion operator that statically + * downcasts the `reducer` to the corresponding `reducer_opadd` type. + * (The reverse conversion, from `reducer_opadd` to `reducer`, is just + * an upcast, which is provided for free by the language.) + * + * @ingroup ReducersAdd + */ +template +struct legacy_reducer_downcast > > +{ + typedef reducer_opadd type; +}; +/// @endcond + +} // namespace cilk + +#endif // __cplusplus + + +/** @ingroup ReducersAdd + */ +///@{ + +/** @name C Language Reducer Macros + * + * These macros are used to declare and work with numeric op_add reducers in + * C code. + * + * @see @ref page_reducers_in_c + */ + ///@{ + +__CILKRTS_BEGIN_EXTERN_C + +/** Declares opadd reducer type name. + * + * This macro expands into the identifier which is the name of the op_add + * reducer type for a specified numeric type. + * + * @param tn The @ref reducers_c_type_names "numeric type name" specifying + * the type of the reducer. + * + * @see @ref reducers_c_predefined + * @see ReducersAdd + */ +#define CILK_C_REDUCER_OPADD_TYPE(tn) \ + __CILKRTS_MKIDENT(cilk_c_reducer_opadd_,tn) + +/** Declares an op_add reducer object. + * + * This macro expands into a declaration of an op_add reducer object for a + * specified numeric type. For example: + * + * CILK_C_REDUCER_OPADD(my_reducer, double, 0.0); + * + * @param obj The variable name to be used for the declared reducer object. + * @param tn The @ref reducers_c_type_names "numeric type name" specifying + * the type of the reducer. + * @param v The initial value for the reducer. (A value which can be + * assigned to the numeric type represented by @a tn.) + * + * @see @ref reducers_c_predefined + * @see ReducersAdd + */ +#define CILK_C_REDUCER_OPADD(obj,tn,v) \ + CILK_C_REDUCER_OPADD_TYPE(tn) obj = \ + CILK_C_INIT_REDUCER(_Typeof(obj.value), \ + __CILKRTS_MKIDENT(cilk_c_reducer_opadd_reduce_,tn), \ + __CILKRTS_MKIDENT(cilk_c_reducer_opadd_identity_,tn), \ + __cilkrts_hyperobject_noop_destroy, v) + +/// @cond internal + +/** Declares the op_add reducer functions for a numeric type. + * + * This macro expands into external function declarations for functions which + * implement the reducer functionality for the op_add reducer type for a + * specified numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer + * type name, function names, etc. + */ +#define CILK_C_REDUCER_OPADD_DECLARATION(t,tn) \ + typedef CILK_C_DECLARE_REDUCER(t) CILK_C_REDUCER_OPADD_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_opadd,tn,l,r); \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_opadd,tn); + +/** Defines the op_add reducer functions for a numeric type. + * + * This macro expands into function definitions for functions which implement + * the reducer functionality for the op_add reducer type for a specified + * numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer + * type name, function names, etc. + */ +#define CILK_C_REDUCER_OPADD_DEFINITION(t,tn) \ + typedef CILK_C_DECLARE_REDUCER(t) CILK_C_REDUCER_OPADD_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_opadd,tn,l,r) \ + { *(t*)l += *(t*)r; } \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_opadd,tn) \ + { *(t*)v = 0; } + +///@{ +/** @def CILK_C_REDUCER_OPADD_INSTANCE + * @brief Declares or defines implementation functions for a reducer type. + * + * In the runtime source file c_reducers.c, the macro `CILK_C_DEFINE_REDUCERS` + * will be defined, and this macro will generate reducer implementation + * functions. Everywhere else, `CILK_C_DEFINE_REDUCERS` will be undefined, + * and this macro will expand into external declarations for the functions. + */ +#ifdef CILK_C_DEFINE_REDUCERS +# define CILK_C_REDUCER_OPADD_INSTANCE(t,tn) \ + CILK_C_REDUCER_OPADD_DEFINITION(t,tn) +#else +# define CILK_C_REDUCER_OPADD_INSTANCE(t,tn) \ + CILK_C_REDUCER_OPADD_DECLARATION(t,tn) +#endif +///@} + +/* Declares or defines an instance of the reducer type and its functions for each + * numeric type. + */ +CILK_C_REDUCER_OPADD_INSTANCE(char, char) +CILK_C_REDUCER_OPADD_INSTANCE(unsigned char, uchar) +CILK_C_REDUCER_OPADD_INSTANCE(signed char, schar) +CILK_C_REDUCER_OPADD_INSTANCE(wchar_t, wchar_t) +CILK_C_REDUCER_OPADD_INSTANCE(short, short) +CILK_C_REDUCER_OPADD_INSTANCE(unsigned short, ushort) +CILK_C_REDUCER_OPADD_INSTANCE(int, int) +CILK_C_REDUCER_OPADD_INSTANCE(unsigned int, uint) +CILK_C_REDUCER_OPADD_INSTANCE(unsigned int, unsigned) /* alternate name */ +CILK_C_REDUCER_OPADD_INSTANCE(long, long) +CILK_C_REDUCER_OPADD_INSTANCE(unsigned long, ulong) +CILK_C_REDUCER_OPADD_INSTANCE(long long, longlong) +CILK_C_REDUCER_OPADD_INSTANCE(unsigned long long, ulonglong) +CILK_C_REDUCER_OPADD_INSTANCE(float, float) +CILK_C_REDUCER_OPADD_INSTANCE(double, double) +CILK_C_REDUCER_OPADD_INSTANCE(long double, longdouble) + +//@endcond + +__CILKRTS_END_EXTERN_C + +///@} + +///@} + +#endif /* REDUCER_OPADD_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_opand.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_opand.h new file mode 100644 index 0000000000..44d537d4f3 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_opand.h @@ -0,0 +1,614 @@ +/* reducer_opand.h -*- C++ -*- + * + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + */ + +/** @file reducer_opand.h + * + * @brief Defines classes for doing parallel bitwise AND reductions. + * + * @ingroup ReducersAnd + * + * @see ReducersAnd + */ + +#ifndef REDUCER_OPAND_H_INCLUDED +#define REDUCER_OPAND_H_INCLUDED + +#include + +/** @defgroup ReducersAnd Bitwise AND Reducers + * + * Bitwise AND reducers allow the computation of the bitwise AND of a set of + * values in parallel. + * + * @ingroup Reducers + * + * You should be familiar with @ref pagereducers "Intel(R) Cilk(TM) Plus reducers", + * described in file `reducers.md`, and particularly with @ref reducers_using, + * before trying to use the information in this file. + * + * @section redopand_usage Usage Example + * + * cilk::reducer< cilk::op_and > r; + * cilk_for (int i = 0; i != N; ++i) { + * *r &= a[i]; + * } + * unsigned result; + * r.move_out(result); + * + * @section redopand_monoid The Monoid + * + * @subsection redopand_monoid_values Value Set + * + * The value set of a bitwise AND reducer is the set of values of `Type`, + * which is expected to be a builtin integer type which has a representation + * as a sequence of bits (or something like it, such as `bool` or + * `std::bitset`). + * + * @subsection redopand_monoid_operator Operator + * + * The bitwise AND operator is defined by the "`&`" binary operator on `Type`. + * + * @subsection redopand_monoid_identity Identity + * + * The identity value of the reducer is the value whose representation + * contains all 1-bits. This is expected to be the value of the expression + * `~Type()` (i.e., the bitwise negation operator applied to the default value + * of the value type). + * + * @section redopand_operations Operations + * + * @subsection redopand_constructors Constructors + * + * reducer() // identity + * reducer(const Type& value) + * reducer(move_in(Type& variable)) + * + * @subsection redopand_get_set Set and Get + * + * r.set_value(const Type& value) + * const Type& = r.get_value() const + * r.move_in(Type& variable) + * r.move_out(Type& variable) + * + * @subsection redopand_initial Initial Values + * + * If a bitwise AND reducer is constructed without an explicit initial value, + * then its initial value will be its identity value, as long as `Type` + * satisfies the requirements of @ref redopand_types. + * + * @subsection redopand_view_ops View Operations + * + * *r &= a + * *r = *r & a + * *r = *r & a1 & a2 … & an + * + * @section redopand_types Type and Operator Requirements + * + * `Type` must be `Copy Constructible`, `Default Constructible`, and + * `Assignable`. + * + * The operator "`&=`" must be defined on `Type`, with `x &= a` having the + * same meaning as `x = x & a`. + * + * The expression `~ Type()` must be a valid expression which yields the + * identity value (the value of `Type` whose representation consists of all + * 1-bits). + * + * @section redopand_in_c Bitwise AND Reducers in C + * + * The @ref CILK_C_REDUCER_OPAND and @ref CILK_C_REDUCER_OPAND_TYPE macros can + * be used to do bitwise AND reductions in C. For example: + * + * CILK_C_REDUCER_OPAND(r, uint, ~0); + * CILK_C_REGISTER_REDUCER(r); + * cilk_for(int i = 0; i != n; ++i) { + * REDUCER_VIEW(r) &= a[i]; + * } + * CILK_C_UNREGISTER_REDUCER(r); + * printf("The bitwise AND of the elements of a is %x\n", REDUCER_VIEW(r)); + * + * See @ref reducers_c_predefined. + */ + +#ifdef __cplusplus + +namespace cilk { + +/** The bitwise AND reducer view class. + * + * This is the view class for reducers created with + * `cilk::reducer< cilk::op_and >`. It holds the accumulator variable + * for the reduction, and allows only AND operations to be performed on it. + * + * @note The reducer "dereference" operation (`reducer::operator *()`) + * yields a reference to the view. Thus, for example, the view class's + * `&=` operation would be used in an expression like `*r &= a`, where + * `r` is an opmod reducer variable. + * + * @tparam Type The type of the contained accumulator variable. This will + * be the value type of a monoid_with_view that is + * instantiated with this view. + * + * @see ReducersAnd + * @see op_and + * + * @ingroup ReducersAnd + */ +template +class op_and_view : public scalar_view +{ + typedef scalar_view base; + +public: + /** Class to represent the right-hand side of `*reducer = *reducer & value`. + * + * The only assignment operator for the op_and_view class takes an + * rhs_proxy as its operand. This results in the syntactic restriction + * that the only expressions that can be assigned to an op_and_view are + * ones which generate an rhs_proxy - that is, expressions of the form + * `op_and_view & value ... & value`. + * + * @warning + * The lhs and rhs views in such an assignment must be the same; + * otherwise, the behavior will be undefined. (I.e., `v1 = v1 & x` is + * legal; `v1 = v2 & x` is illegal.) This condition will be checked with + * a runtime assertion when compiled in debug mode. + * + * @see op_and_view + */ + class rhs_proxy { + private: + friend class op_and_view; + + const op_and_view* m_view; + Type m_value; + + // Constructor is invoked only from op_and_view::operator&(). + // + rhs_proxy(const op_and_view* view, const Type& value) : m_view(view), m_value(value) {} + + rhs_proxy& operator=(const rhs_proxy&); // Disable assignment operator + rhs_proxy(); // Disable default constructor + + public: + /** Bitwise AND with an additional `rhs` value. If `v` is an op_and_view + * and `a1` is a value, then the expression `v & a1` invokes the + * view's `operator&()` to create an rhs_proxy for `(v, a1)`; then + * `v & a1 & a2` invokes the rhs_proxy's `operator&()` to create a new + * rhs_proxy for `(v, a1&a2)`. This allows the right-hand side of an + * assignment to be not just `view & value`, but + * `view & value & value ... & value`. The effect is that + * + * v = v & a1 & a2 ... & an; + * + * is evaluated as + * + * v = v & (a1 & a2 ... & an); + */ + rhs_proxy& operator&(const Type& x) { m_value &= x; return *this; } + }; + + + /** Default/identity constructor. This constructor initializes the + * contained value to `~ Type()`. + */ + op_and_view() : base(~Type()) {} + + /** Construct with a specified initial value. + */ + explicit op_and_view(const Type& v) : base(v) {} + + + /** Reduces the views of two strands. + * + * This function is invoked by the @ref op_and monoid to combine the views + * of two strands when the right strand merges with the left one. It + * "ANDs" the value contained in the left-strand view with the value + * contained in the right-strand view, and leaves the value in the + * right-strand view undefined. + * + * @param right A pointer to the right-strand view. (`this` points to + * the left-strand view.) + * + * @note Used only by the @ref op_and monoid to implement the monoid + * reduce operation. + */ + void reduce(op_and_view* right) { this->m_value &= right->m_value; } + + /** @name Accumulator variable updates. + * + * These functions support the various syntaxes for "ANDing" the + * accumulator variable contained in the view with some value. + */ + ///@{ + + /** Performs AND between the accumulator variable and @a x. + */ + op_and_view& operator&=(const Type& x) { this->m_value &= x; return *this; } + + /** Creates an object representing `*this & x`. + * + * @see rhs_proxy + */ + rhs_proxy operator&(const Type& x) const { return rhs_proxy(this, x); } + + /** Assigns the result of a `view & value` expression to the view. Note that + * this is the only assignment operator for this class. + * + * @see rhs_proxy + */ + op_and_view& operator=(const rhs_proxy& rhs) { + __CILKRTS_ASSERT(this == rhs.m_view); + this->m_value &= rhs.m_value; + return *this; + } + + ///@} +}; + +/** Monoid class for bitwise AND reductions. Instantiate the cilk::reducer + * template class with an op_and monoid to create a bitwise AND reducer + * class. For example, to compute the bitwise AND of a set of `unsigned long` + * values: + * + * cilk::reducer< cilk::op_and > r; + * + * @tparam Type The reducer value type. + * @tparam Align If `false` (the default), reducers instantiated on this + * monoid will be naturally aligned (the Intel Cilk Plus library 1.0 + * behavior). If `true`, reducers instantiated on this monoid + * will be cache-aligned for binary compatibility with + * reducers in Intel Cilk Plus library version 0.9. + * + * @see ReducersAnd + * @see op_and_view + * + * @ingroup ReducersAnd + */ +template +struct op_and : public monoid_with_view, Align> {}; + +/** Deprecated bitwise AND reducer class. + * + * reducer_opand is the same as @ref reducer<@ref op_and>, except that + * reducer_opand is a proxy for the contained view, so that accumulator + * variable update operations can be applied directly to the reducer. For + * example, a value is "ANDed" with a `reducer<%op_and>` with `*r &= a`, but a + * value can be "ANDed" with a `%reducer_opand` with `r &= a`. + * + * @deprecated Users are strongly encouraged to use `reducer` + * reducers rather than the old wrappers like reducer_opand. + * The `reducer` reducers show the reducer/monoid/view + * architecture more clearly, are more consistent in their + * implementation, and present a simpler model for new + * user-implemented reducers. + * + * @note Implicit conversions are provided between `%reducer_opand` + * and `reducer<%op_and>`. This allows incremental code + * conversion: old code that used `%reducer_opand` can pass a + * `%reducer_opand` to a converted function that now expects a + * pointer or reference to a `reducer<%op_and>`, and vice + * versa. + * + * @tparam Type The value type of the reducer. + * + * @see op_and + * @see reducer + * @see ReducersAnd + * + * @ingroup ReducersAnd + */ +template +class reducer_opand : public reducer< op_and > +{ + typedef reducer< op_and > base; + using base::view; + +public: + /// The view type for the reducer. + typedef typename base::view_type view_type; + + /// The view's rhs proxy type. + typedef typename view_type::rhs_proxy rhs_proxy; + + /// The view type for the reducer. + typedef view_type View; + + /// The monoid type for the reducer. + typedef typename base::monoid_type Monoid; + + /** @name Constructors + */ + ///@{ + + /** Default constructor. + * + * Constructs the wrapper with the default initial value of `Type()` + * (not the identity value). + */ + reducer_opand() : base(Type()) {} + + /** Value constructor. + * + * Constructs the wrapper with a specified initial value. + */ + explicit reducer_opand(const Type& initial_value) : base(initial_value) {} + + ///@} + + /** @name Forwarded functions + * @details Functions that update the contained accumulator variable are + * simply forwarded to the contained @ref op_and_view. */ + ///@{ + + /// @copydoc op_and_view::operator&=(const Type&) + reducer_opand& operator&=(const Type& x) + { + view() &= x; + return *this; + } + + // The legacy definition of reducer_opand::operator&() has different + // behavior and a different return type than this definition. The legacy + // version is defined as a member function, so this new version is defined + // as a free function to give it a different signature, so that they won't + // end up sharing a single object file entry. + + /// @copydoc op_and_view::operator&(const Type&) const + friend rhs_proxy operator&(const reducer_opand& r, const Type& x) + { + return r.view() & x; + } + + /// @copydoc op_and_view::operator=(const rhs_proxy&) + reducer_opand& operator=(const rhs_proxy& temp) + { + view() = temp; + return *this; + } + ///@} + + /** @name Dereference + * @details Dereferencing a wrapper is a no-op. It simply returns the + * wrapper. Combined with the rule that the wrapper forwards view + * operations to its contained view, this means that view operations can + * be written the same way on reducers and wrappers, which is convenient + * for incrementally converting old code using wrappers to use reducers + * instead. That is: + * + * reducer< op_and > r; + * *r &= a; // *r returns the view + * // operator &= is a view member function + * + * reducer_opand w; + * *w &= a; // *w returns the wrapper + * // operator &= is a wrapper member function that + * // calls the corresponding view function + */ + ///@{ + reducer_opand& operator*() { return *this; } + reducer_opand const& operator*() const { return *this; } + + reducer_opand* operator->() { return this; } + reducer_opand const* operator->() const { return this; } + ///@} + + /** @name Upcast + * @details In Intel Cilk Plus library 0.9, reducers were always cache-aligned. + * In library 1.0, reducer cache alignment is optional. By default, + * reducers are unaligned (i.e., just naturally aligned), but legacy + * wrappers inherit from cache-aligned reducers for binary compatibility. + * + * This means that a wrapper will automatically be upcast to its aligned + * reducer base class. The following conversion operators provide + * pseudo-upcasts to the corresponding unaligned reducer class. + */ + ///@{ + operator reducer< op_and >& () + { + return *reinterpret_cast< reducer< op_and >* >(this); + } + operator const reducer< op_and >& () const + { + return *reinterpret_cast< const reducer< op_and >* >(this); + } + ///@} +}; + +/// @cond internal +/** Metafunction specialization for reducer conversion. + * + * This specialization of the @ref legacy_reducer_downcast template class + * defined in reducer.h causes the `reducer< op_and >` class to have an + * `operator reducer_opand& ()` conversion operator that statically + * downcasts the `reducer` to the corresponding `reducer_opand` type. + * (The reverse conversion, from `reducer_opand` to `reducer`, is just + * an upcast, which is provided for free by the language.) + * + * @ingroup ReducersAnd + */ +template +struct legacy_reducer_downcast > > +{ + typedef reducer_opand type; +}; +/// @endcond + +} // namespace cilk + +#endif // __cplusplus + + +/** @ingroup ReducersAdd + */ +///@{ + +/** @name C language reducer macros + * + * These macros are used to declare and work with op_and reducers in C code. + * + * @see @ref page_reducers_in_c + */ + ///@{ + +__CILKRTS_BEGIN_EXTERN_C + +/** Declares `opand` reducer type name. + * + * This macro expands into the identifier which is the name of the op_and + * reducer type for a specified numeric type. + * + * @param tn The @ref reducers_c_type_names "numeric type name" specifying + * the type of the reducer. + * + * @see @ref reducers_c_predefined + * @see ReducersAnd + */ +#define CILK_C_REDUCER_OPAND_TYPE(tn) \ + __CILKRTS_MKIDENT(cilk_c_reducer_opand_,tn) + +/** Declares an op_and reducer object. + * + * This macro expands into a declaration of an op_and reducer object for a + * specified numeric type. For example: + * + * CILK_C_REDUCER_OPAND(my_reducer, ulong, ~0UL); + * + * @param obj The variable name to be used for the declared reducer object. + * @param tn The @ref reducers_c_type_names "numeric type name" specifying + * the type of the reducer. + * @param v The initial value for the reducer. (A value which can be + * assigned to the numeric type represented by @a tn.) + * + * @see @ref reducers_c_predefined + * @see ReducersAnd + */ +#define CILK_C_REDUCER_OPAND(obj,tn,v) \ + CILK_C_REDUCER_OPAND_TYPE(tn) obj = \ + CILK_C_INIT_REDUCER(_Typeof(obj.value), \ + __CILKRTS_MKIDENT(cilk_c_reducer_opand_reduce_,tn), \ + __CILKRTS_MKIDENT(cilk_c_reducer_opand_identity_,tn), \ + __cilkrts_hyperobject_noop_destroy, v) + +/// @cond internal + +/** Declares the op_and reducer functions for a numeric type. + * + * This macro expands into external function declarations for functions which + * implement the reducer functionality for the op_and reducer type for a + * specified numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer + * type name, function names, etc. + */ +#define CILK_C_REDUCER_OPAND_DECLARATION(t,tn) \ + typedef CILK_C_DECLARE_REDUCER(t) CILK_C_REDUCER_OPAND_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_opand,tn,l,r); \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_opand,tn); + +/** Defines the op_and reducer functions for a numeric type. + * + * This macro expands into function definitions for functions which implement + * the reducer functionality for the op_and reducer type for a specified + * numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer + * type name, function names, etc. + */ +#define CILK_C_REDUCER_OPAND_DEFINITION(t,tn) \ + typedef CILK_C_DECLARE_REDUCER(t) CILK_C_REDUCER_OPAND_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_opand,tn,l,r) \ + { *(t*)l &= *(t*)r; } \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_opand,tn) \ + { *(t*)v = ~((t)0); } + +///@{ +/** @def CILK_C_REDUCER_OPAND_INSTANCE + * @brief Declares or defines implementation functions for a reducer type. + * + * In the runtime source file c_reducers.c, the macro `CILK_C_DEFINE_REDUCERS` + * will be defined, and this macro will generate reducer implementation + * functions. Everywhere else, `CILK_C_DEFINE_REDUCERS` will be undefined, and + * this macro will expand into external declarations for the functions. + */ +#ifdef CILK_C_DEFINE_REDUCERS +# define CILK_C_REDUCER_OPAND_INSTANCE(t,tn) \ + CILK_C_REDUCER_OPAND_DEFINITION(t,tn) +#else +# define CILK_C_REDUCER_OPAND_INSTANCE(t,tn) \ + CILK_C_REDUCER_OPAND_DECLARATION(t,tn) +#endif +///@} + +/* Declares or defines an instance of the reducer type and its functions for + * each numeric type. + */ +CILK_C_REDUCER_OPAND_INSTANCE(char, char) +CILK_C_REDUCER_OPAND_INSTANCE(unsigned char, uchar) +CILK_C_REDUCER_OPAND_INSTANCE(signed char, schar) +CILK_C_REDUCER_OPAND_INSTANCE(wchar_t, wchar_t) +CILK_C_REDUCER_OPAND_INSTANCE(short, short) +CILK_C_REDUCER_OPAND_INSTANCE(unsigned short, ushort) +CILK_C_REDUCER_OPAND_INSTANCE(int, int) +CILK_C_REDUCER_OPAND_INSTANCE(unsigned int, uint) +CILK_C_REDUCER_OPAND_INSTANCE(unsigned int, unsigned) /* alternate name */ +CILK_C_REDUCER_OPAND_INSTANCE(long, long) +CILK_C_REDUCER_OPAND_INSTANCE(unsigned long, ulong) +CILK_C_REDUCER_OPAND_INSTANCE(long long, longlong) +CILK_C_REDUCER_OPAND_INSTANCE(unsigned long long, ulonglong) + +//@endcond + +__CILKRTS_END_EXTERN_C + +///@} + +///@} + +#endif /* REDUCER_OPAND_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_opmul.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_opmul.h new file mode 100644 index 0000000000..8a3e2d2a2a --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_opmul.h @@ -0,0 +1,453 @@ +/* reducer_opmul.h -*- C++ -*- + * + * Copyright (C) 2012-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + */ + +/** @file reducer_opmul.h + * + * @brief Defines classes for doing parallel multiplication reductions. + * + * @ingroup ReducersMul + * + * @see ReducersMul + */ + +#ifndef REDUCER_OPMUL_H_INCLUDED +#define REDUCER_OPMUL_H_INCLUDED + +#include + +/** @defgroup ReducersMul Multiplication Reducers + * + * Multiplication reducers allow the computation of the product of a set of + * values in parallel. + * + * @ingroup Reducers + * + * You should be familiar with @ref pagereducers "Intel(R) Cilk(TM) Plus reducers", + * described in file `reducers.md`, and particularly with @ref reducers_using, + * before trying to use the information in this file. + * + * @section redopmul_usage Usage Example + * + * cilk::reducer< cilk::op_mul > r; + * cilk_for (int i = 0; i != N; ++i) { + * *r *= a[i]; + * } + * double product; + * r.move_out(product); + * + * @section redopmul_monoid The Monoid + * + * @subsection redopmul_monoid_values Value Set + * + * The value set of a multiplication reducer is the set of values of `Type`, + * which is expected to be a builtin numeric type (or something like it, such + * as `std::complex`). + * + * @subsection redopmul_monoid_operator Operator + * + * The operator of a multiplication reducer is the multiplication operation, + * defined by the "`*`" binary operator on `Type`. + * + * @subsection redopmul_monoid_identity Identity + * + * The identity value of the reducer is the numeric value "`1`". This is + * expected to be the value of the expression `Type(1)`. + * + * @section redopmul_operations Operations + * + * @subsection redopmul_constructors Constructors + * + * reducer() // identity + * reducer(const Type& value) + * reducer(move_in(Type& variable)) + * + * @subsection redopmul_get_set Set and Get + * + * r.set_value(const Type& value) + * const Type& = r.get_value() const + * r.move_in(Type& variable) + * r.move_out(Type& variable) + * + * @subsection redopmul_initial Initial Values + * + * If a multiplication reducer is constructed without an explicit initial + * value, then its initial value will be its identity value, as long as `Type` + * satisfies the requirements of @ref redopmul_types. + * + * @subsection redopmul_view_ops View Operations + * + * *r *= a + * *r = *r * a + * *r = *r * a1 * a2 … * an + * + * @section redopmul_floating_point Issues with Floating-Point Types + * + * Because of overflow and underflow issues, floating-point multiplication is + * not really associative. For example, `(1e200 * 1e-200) * 1e-200 == 1e-200`, + * but `1e200 * (1e-200 * 1e-200 == 0. + * + * In many cases, this won't matter, but computations which have been + * carefully ordered to control overflow and underflow may not deal well with + * being reassociated. In general, you should be sure to understand the + * floating-point behavior of your program before doing any transformation + * that will reassociate its computations. + * + * @section redopmul_types Type and Operator Requirements + * + * `Type` must be `Copy Constructible`, `Default Constructible`, and + * `Assignable`. + * + * The operator "`*=`" must be defined on `Type`, with `x *= a` having the same + * meaning as `x = x * a`. + * + * The expression `Type(1)` must be a valid expression which yields the + * identity value (the value of `Type` whose numeric value is `1`). + * + * @section redopmul_in_c Multiplication Reducers in C + * + * The @ref CILK_C_REDUCER_OPMUL and @ref CILK_C_REDUCER_OPMUL_TYPE macros can + * be used to do multiplication reductions in C. For example: + * + * CILK_C_REDUCER_OPMUL(r, double, 1); + * CILK_C_REGISTER_REDUCER(r); + * cilk_for(int i = 0; i != n; ++i) { + * REDUCER_VIEW(r) *= a[i]; + * } + * CILK_C_UNREGISTER_REDUCER(r); + * printf("The product of the elements of a is %f\n", REDUCER_VIEW(r)); + * + * See @ref reducers_c_predefined. + */ + +#ifdef __cplusplus + +namespace cilk { + +/** The multiplication reducer view class. + * + * This is the view class for reducers created with + * `cilk::reducer< cilk::op_mul >`. It holds the accumulator variable + * for the reduction, and allows only multiplication operations to be + * performed on it. + * + * @note The reducer "dereference" operation (`reducer::operator *()`) + * yields a reference to the view. Thus, for example, the view class's + * `*=` operation would be used in an expression like `*r *= a`, where + * `r` is an op_mul reducer variable. + * + * @tparam Type The type of the contained accumulator variable. This will + * be the value type of a monoid_with_view that is + * instantiated with this view. + * + * @see ReducersMul + * @see op_mul + * + * @ingroup ReducersMul + */ +template +class op_mul_view : public scalar_view +{ + typedef scalar_view base; + +public: + /** Class to represent the right-hand side of `*reducer = *reducer * value`. + * + * The only assignment operator for the op_mul_view class takes an + * rhs_proxy as its operand. This results in the syntactic restriction + * that the only expressions that can be assigned to an op_mul_view are + * ones which generate an rhs_proxy - that is, expressions of the form + * `op_mul_view * value ... * value`. + * + * @warning + * The lhs and rhs views in such an assignment must be the same; + * otherwise, the behavior will be undefined. (I.e., `v1 = v1 * x` is + * legal; `v1 = v2 * x` is illegal.) This condition will be checked with a + * runtime assertion when compiled in debug mode. + * + * @see op_mul_view + */ + class rhs_proxy { + friend class op_mul_view; + + const op_mul_view* m_view; + Type m_value; + + // Constructor is invoked only from op_mul_view::operator*(). + // + rhs_proxy(const op_mul_view* view, const Type& value) : m_view(view), m_value(value) {} + + rhs_proxy& operator=(const rhs_proxy&); // Disable assignment operator + rhs_proxy(); // Disable default constructor + + public: + /** Multiplies by an additional `rhs` value. If `v` is an op_mul_view and + * `a1` is a value, then the expression `v * a1` invokes the view's + * `operator*()` to create an rhs_proxy for `(v, a1)`; then + * `v * a1 * a2` invokes the rhs_proxy's `operator*()` to create a + * new rhs_proxy for `(v, a1*a2)`. This allows the right-hand side of + * an assignment to be not just `view * value`, but + * `view * value * value ... * value`. The effect is that + * + * v = v * a1 * a2 ... * an; + * + * is evaluated as + * + * v = v * (a1 * a2 ... * an); + */ + rhs_proxy& operator*(const Type& x) { m_value *= x; return *this; } + }; + + + /** Default/identity constructor. This constructor initializes the + * contained value to `Type(1)`, which is expected to be the identity + * value for multiplication on `Type`. + */ + op_mul_view() : base(Type(1)) {} + + /** Construct with a specified initial value. + */ + explicit op_mul_view(const Type& v) : base(v) {} + + /** Reduces two strand views. + * + * This function is invoked by the @ref op_mul monoid to combine the views + * of two strands when the right strand merges with the left one. It + * multiplies the value contained in the left-strand view by the value + * contained in the right-strand view, and leaves the value in the + * right-strand view undefined. + * + * @param right A pointer to the right-strand view. (`this` points to + * the left-strand view.) + * + * @note Used only by the @ref op_mul monoid to implement the monoid + * reduce operation. + */ + void reduce(op_mul_view* right) { this->m_value *= right->m_value; } + + /** @name Accumulator variable updates. + * + * These functions support the various syntaxes for multiplying the + * accumulator variable contained in the view by some value. + */ + ///@{ + + /** Multiplies the accumulator variable by @a x. + */ + op_mul_view& operator*=(const Type& x) { this->m_value *= x; return *this; } + + /** Creates an object representing `*this * x`. + * + * @see rhs_proxy + */ + rhs_proxy operator*(const Type& x) const { return rhs_proxy(this, x); } + + /** Assigns the result of a `view * value` expression to the view. Note that + * this is the only assignment operator for this class. + * + * @see rhs_proxy + */ + op_mul_view& operator=(const rhs_proxy& rhs) { + __CILKRTS_ASSERT(this == rhs.m_view); + this->m_value *= rhs.m_value; + return *this; + } + + ///@} +}; + +/** Monoid class for multiplication reductions. Instantiate the cilk::reducer + * template class with an op_mul monoid to create a multiplication reducer + * class. For example, to compute the product of a set of `double` values: + * + * cilk::reducer< cilk::op_mul > r; + * + * @see ReducersMul + * @see op_mul_view + * + * @ingroup ReducersMul + */ +template +struct op_mul : public monoid_with_view< op_mul_view > {}; + +} // namespace cilk + +#endif // __cplusplus + + +/** @ingroup ReducersAdd + */ +///@{ + +/** @name C language reducer macros + * + * These macros are used to declare and work with numeric op_mul reducers in + * C code. + * + * @see @ref page_reducers_in_c + */ + ///@{ + +__CILKRTS_BEGIN_EXTERN_C + +/** Declares `opmul` reducer type name. + * + * This macro expands into the identifier which is the name of the op_mul + * reducer type for a specified numeric type. + * + * @param tn The @ref reducers_c_type_names "numeric type name" specifying + * the type of the reducer. + * + * @see @ref reducers_c_predefined + * @see ReducersMul + */ +#define CILK_C_REDUCER_OPMUL_TYPE(tn) \ + __CILKRTS_MKIDENT(cilk_c_reducer_opmul_,tn) + +/** Declares an op_mul reducer object. + * + * This macro expands into a declaration of an op_mul reducer object for a + * specified numeric type. For example: + * + * CILK_C_REDUCER_OPMUL(my_reducer, double, 1.0); + * + * @param obj The variable name to be used for the declared reducer object. + * @param tn The @ref reducers_c_type_names "numeric type name" specifying + * the type of the reducer. + * @param v The initial value for the reducer. (A value which can be + * assigned to the numeric type represented by @a tn.) + * + * @see @ref reducers_c_predefined + * @see ReducersMul + */ +#define CILK_C_REDUCER_OPMUL(obj,tn,v) \ + CILK_C_REDUCER_OPMUL_TYPE(tn) obj = \ + CILK_C_INIT_REDUCER(_Typeof(obj.value), \ + __CILKRTS_MKIDENT(cilk_c_reducer_opmul_reduce_,tn), \ + __CILKRTS_MKIDENT(cilk_c_reducer_opmul_identity_,tn), \ + __cilkrts_hyperobject_noop_destroy, v) + +/// @cond internal + +/** Declares the op_mul reducer functions for a numeric type. + * + * This macro expands into external function declarations for functions which + * implement the reducer functionality for the op_mul reducer type for a + * specified numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer + * type name, function names, etc. + */ +#define CILK_C_REDUCER_OPMUL_DECLARATION(t,tn) \ + typedef CILK_C_DECLARE_REDUCER(t) CILK_C_REDUCER_OPMUL_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_opmul,tn,l,r); \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_opmul,tn); + +/** Defines the op_mul reducer functions for a numeric type. + * + * This macro expands into function definitions for functions which implement + * the reducer functionality for the op_mul reducer type for a specified + * numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer + * type name, function names, etc. + */ +#define CILK_C_REDUCER_OPMUL_DEFINITION(t,tn) \ + typedef CILK_C_DECLARE_REDUCER(t) CILK_C_REDUCER_OPMUL_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_opmul,tn,l,r) \ + { *(t*)l *= *(t*)r; } \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_opmul,tn) \ + { *(t*)v = 1; } + +///@{ +/** @def CILK_C_REDUCER_OPMUL_INSTANCE + * @brief Declares or defines implementation functions for a reducer type. + * + * In the runtime source file c_reducers.c, the macro `CILK_C_DEFINE_REDUCERS` + * will be defined, and this macro will generate reducer implementation + * functions. Everywhere else, `CILK_C_DEFINE_REDUCERS` will be undefined, and + * this macro will expand into external declarations for the functions. + */ +#ifdef CILK_C_DEFINE_REDUCERS +# define CILK_C_REDUCER_OPMUL_INSTANCE(t,tn) \ + CILK_C_REDUCER_OPMUL_DEFINITION(t,tn) +#else +# define CILK_C_REDUCER_OPMUL_INSTANCE(t,tn) \ + CILK_C_REDUCER_OPMUL_DECLARATION(t,tn) +#endif +///@} + +/* Declares or defines an instance of the reducer type and its functions for each + * numeric type. + */ +CILK_C_REDUCER_OPMUL_INSTANCE(char, char) +CILK_C_REDUCER_OPMUL_INSTANCE(unsigned char, uchar) +CILK_C_REDUCER_OPMUL_INSTANCE(signed char, schar) +CILK_C_REDUCER_OPMUL_INSTANCE(wchar_t, wchar_t) +CILK_C_REDUCER_OPMUL_INSTANCE(short, short) +CILK_C_REDUCER_OPMUL_INSTANCE(unsigned short, ushort) +CILK_C_REDUCER_OPMUL_INSTANCE(int, int) +CILK_C_REDUCER_OPMUL_INSTANCE(unsigned int, uint) +CILK_C_REDUCER_OPMUL_INSTANCE(unsigned int, unsigned) /* alternate name */ +CILK_C_REDUCER_OPMUL_INSTANCE(long, long) +CILK_C_REDUCER_OPMUL_INSTANCE(unsigned long, ulong) +CILK_C_REDUCER_OPMUL_INSTANCE(long long, longlong) +CILK_C_REDUCER_OPMUL_INSTANCE(unsigned long long, ulonglong) +CILK_C_REDUCER_OPMUL_INSTANCE(float, float) +CILK_C_REDUCER_OPMUL_INSTANCE(double, double) +CILK_C_REDUCER_OPMUL_INSTANCE(long double, longdouble) + +//@endcond + +__CILKRTS_END_EXTERN_C + +///@} + +///@} + +#endif /* REDUCER_OPMUL_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_opor.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_opor.h new file mode 100644 index 0000000000..8d6d520248 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_opor.h @@ -0,0 +1,609 @@ +/* reducer_opor.h -*- C++ -*- + * + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + */ + +/** @file reducer_opor.h + * + * @brief Defines classes for doing parallel bitwise OR reductions. + * + * @ingroup ReducersOr + * + * @see ReducersOr + */ + +#ifndef REDUCER_OPOR_H_INCLUDED +#define REDUCER_OPOR_H_INCLUDED + +#include + +/** @defgroup ReducersOr Bitwise `OR` Reducers + * + * Bitwise `OR` reducers allow the computation of the bitwise `OR` of a set of + * values in parallel. + * + * @ingroup Reducers + * + * You should be familiar with @ref pagereducers "Intel(R) Cilk(TM) Plus reducers", + * described in file `reducers.md`, and particularly with @ref reducers_using, + * before trying to use the information in this file. + * + * @section redopor_usage Usage Example + * + * cilk::reducer< cilk::op_or > r; + * cilk_for (int i = 0; i != N; ++i) { + * *r |= a[i]; + * } + * unsigned result; + * r.move_out(result); + * + * @section redopor_monoid The Monoid + * + * @subsection redopor_monoid_values Value Set + * + * The value set of a bitwise `OR` reducer is the set of values of `Type`, which + * is expected to be a builtin integer type which has a representation as a + * sequence of bits (or something like it, such as `bool` or `std::bitset`). + * + * @subsection redopor_monoid_operator Operator + * + * The operator of a bitwise `OR` reducer is the bitwise OR operator, defined by + * the "`|`" binary operator on `Type`. + * + * @subsection redopor_monoid_identity Identity + * + * The identity value of the reducer is the value whose representation + * contains all 0-bits. This is expected to be the value of the default + * constructor `Type()`. + * + * @section redopor_operations Operations + * + * @subsection redopor_constructors Constructors + * + * reducer() // identity + * reducer(const Type& value) + * reducer(move_in(Type& variable)) + * + * @subsection redopor_get_set Set and Get + * + * r.set_value(const Type& value) + * const Type& = r.get_value() const + * r.move_in(Type& variable) + * r.move_out(Type& variable) + * + * @subsection redopor_initial Initial Values + * + * If a bitwise OR reducer is constructed without an explicit initial value, + * then its initial value will be its identity value, as long as `Type` + * satisfies the requirements of @ref redopor_types. + * + * @subsection redopor_view_ops View Operations + * + * *r |= a + * *r = *r | a + * *r = *r | a1 | a2 … | an + * + * @section redopor_types Type and Operator Requirements + * + * `Type` must be `Copy Constructible`, `Default Constructible`, and + * `Assignable`. + * + * The operator "`|=`" must be defined on `Type`, with `x |= a` having the + * same meaning as `x = x | a`. + * + * The expression `Type()` must be a valid expression which yields the + * identity value (the value of `Type` whose representation consists of all + * 0-bits). + * + * @section redopor_in_c Bitwise OR Reducers in C + * + * The @ref CILK_C_REDUCER_OPOR and @ref CILK_C_REDUCER_OPOR_TYPE macros can + * be used to do bitwise OR reductions in C. For example: + * + * CILK_C_REDUCER_OPOR(r, uint, 0); + * CILK_C_REGISTER_REDUCER(r); + * cilk_for(int i = 0; i != n; ++i) { + * REDUCER_VIEW(r) |= a[i]; + * } + * CILK_C_UNREGISTER_REDUCER(r); + * printf("The bitwise OR of the elements of a is %x\n", REDUCER_VIEW(r)); + * + * See @ref reducers_c_predefined. + */ + +#ifdef __cplusplus + +namespace cilk { + +/** The bitwise OR reducer view class. + * + * This is the view class for reducers created with + * `cilk::reducer< cilk::op_or >`. It holds the accumulator variable for + * the reduction, and allows only `or` operations to be performed on it. + * + * @note The reducer "dereference" operation (`reducer::operator *()`) + * yields a reference to the view. Thus, for example, the view class's + * `|=` operation would be used in an expression like `*r |= a`, where + * `r` is an opmod reducer variable. + * + * @tparam Type The type of the contained accumulator variable. This will + * be the value type of a monoid_with_view that is + * instantiated with this view. + * + * @see ReducersOr + * @see op_or + * + * @ingroup ReducersOr + */ +template +class op_or_view : public scalar_view +{ + typedef scalar_view base; + +public: + /** Class to represent the right-hand side of `*reducer = *reducer | value`. + * + * The only assignment operator for the op_or_view class takes an + * rhs_proxy as its operand. This results in the syntactic restriction + * that the only expressions that can be assigned to an op_or_view are + * ones which generate an rhs_proxy - that is, expressions of the form + * `op_or_view | value ... | value`. + * + * @warning + * The lhs and rhs views in such an assignment must be the same; + * otherwise, the behavior will be undefined. (I.e., `v1 = v1 | x` is + * legal; `v1 = v2 | x` is illegal.) This condition will be checked with + * a runtime assertion when compiled in debug mode. + * + * @see op_or_view + */ + class rhs_proxy { + friend class op_or_view; + + const op_or_view* m_view; + Type m_value; + + // Constructor is invoked only from op_or_view::operator|(). + // + rhs_proxy(const op_or_view* view, const Type& value) : m_view(view), m_value(value) {} + + rhs_proxy& operator=(const rhs_proxy&); // Disable assignment operator + rhs_proxy(); // Disable default constructor + + public: + /** bitwise OR with an additional rhs value. If `v` is an op_or_view + * and `a1` is a value, then the expression `v | a1` invokes the + * view's `operator|()` to create an rhs_proxy for `(v, a1)`; then + * `v | a1 | a2` invokes the rhs_proxy's `operator|()` to create a new + * rhs_proxy for `(v, a1|a2)`. This allows the right-hand side of an + * assignment to be not just `view | value`, but + ( `view | value | value ... | value`. The effect is that + * + * v = v | a1 | a2 ... | an; + * + * is evaluated as + * + * v = v | (a1 | a2 ... | an); + */ + rhs_proxy& operator|(const Type& x) { m_value |= x; return *this; } + }; + + + /** Default/identity constructor. This constructor initializes the + * contained value to `Type()`. + */ + op_or_view() : base() {} + + /** Construct with a specified initial value. + */ + explicit op_or_view(const Type& v) : base(v) {} + + /** Reduces the views of two strands. + * + * This function is invoked by the @ref op_or monoid to combine the views + * of two strands when the right strand merges with the left one. It + * "ORs" the value contained in the left-strand view by the value + * contained in the right-strand view, and leaves the value in the + * right-strand view undefined. + * + * @param right A pointer to the right-strand view. (`this` points to + * the left-strand view.) + * + * @note Used only by the @ref op_or monoid to implement the monoid + * reduce operation. + */ + void reduce(op_or_view* right) { this->m_value |= right->m_value; } + + /** @name Accumulator variable updates. + * + * These functions support the various syntaxes for "ORing" the + * accumulator variable contained in the view with some value. + */ + ///@{ + + /** Perfoms an OR operation between the accumulator variable and @a x. + */ + op_or_view& operator|=(const Type& x) { this->m_value |= x; return *this; } + + /** Creates an object representing `*this | x`. + * + * @see rhs_proxy + */ + rhs_proxy operator|(const Type& x) const { return rhs_proxy(this, x); } + + /** Assigns the result of a `view | value` expression to the view. Note that + * this is the only assignment operator for this class. + * + * @see rhs_proxy + */ + op_or_view& operator=(const rhs_proxy& rhs) { + __CILKRTS_ASSERT(this == rhs.m_view); + this->m_value |= rhs.m_value; + return *this; + } + + ///@} +}; + +/** Monoid class for bitwise OR reductions. Instantiate the cilk::reducer + * template class with an op_or monoid to create a bitwise OR reducer + * class. For example, to compute the bitwise OR of a set of `unsigned long` + * values: + * + * cilk::reducer< cilk::op_or > r; + * + * @tparam Type The reducer value type. + * @tparam Align If `false` (the default), reducers instantiated on this + * monoid will be naturally aligned (the Intel Cilk Plus library 1.0 + * behavior). If `true`, reducers instantiated on this monoid + * will be cache-aligned for binary compatibility with + * reducers in Intel Cilk Plus library version 0.9. + * + * @see ReducersOr + * @see op_or_view + * + * @ingroup ReducersOr + */ +template +struct op_or : public monoid_with_view, Align> {}; + +/** Deprecated bitwise OR reducer class. + * + * reducer_opor is the same as @ref reducer<@ref op_or>, except that + * reducer_opor is a proxy for the contained view, so that accumulator + * variable update operations can be applied directly to the reducer. For + * example, a value is "ORed" with a `reducer<%op_or>` with `*r |= a`, but a + * value can be "ORed" with a `%reducer_opor` with `r |= a`. + * + * @deprecated Users are strongly encouraged to use `reducer` + * reducers rather than the old wrappers like reducer_opor. + * The `reducer` reducers show the reducer/monoid/view + * architecture more clearly, are more consistent in their + * implementation, and present a simpler model for new + * user-implemented reducers. + * + * @note Implicit conversions are provided between `%reducer_opor` + * and `reducer<%op_or>`. This allows incremental code + * conversion: old code that used `%reducer_opor` can pass a + * `%reducer_opor` to a converted function that now expects a + * pointer or reference to a `reducer<%op_or>`, and vice + * versa. + * + * @tparam Type The value type of the reducer. + * + * @see op_or + * @see reducer + * @see ReducersOr + * + * @ingroup ReducersOr + */ +template +class reducer_opor : public reducer< op_or > +{ + typedef reducer< op_or > base; + using base::view; + + public: + /// The view type for the reducer. + typedef typename base::view_type view_type; + + /// The view's rhs proxy type. + typedef typename view_type::rhs_proxy rhs_proxy; + + /// The view type for the reducer. + typedef view_type View; + + /// The monoid type for the reducer. + typedef typename base::monoid_type Monoid; + + /** @name Constructors + */ + ///@{ + + /** Default (identity) constructor. + * + * Constructs the wrapper with the default initial value of `Type()`. + */ + reducer_opor() {} + + /** Value constructor. + * + * Constructs the wrapper with a specified initial value. + */ + explicit reducer_opor(const Type& initial_value) : base(initial_value) {} + + ///@} + + /** @name Forwarded functions + * @details Functions that update the contained accumulator variable are + * simply forwarded to the contained @ref op_and_view. */ + ///@{ + + /// @copydoc op_or_view::operator|=(const Type&) + reducer_opor& operator|=(const Type& x) + { + view() |= x; return *this; + } + + // The legacy definition of reducer_opor::operator|() has different + // behavior and a different return type than this definition. The legacy + // version is defined as a member function, so this new version is defined + // as a free function to give it a different signature, so that they won't + // end up sharing a single object file entry. + + /// @copydoc op_or_view::operator|(const Type&) const + friend rhs_proxy operator|(const reducer_opor& r, const Type& x) + { + return r.view() | x; + } + + /// @copydoc op_and_view::operator=(const rhs_proxy&) + reducer_opor& operator=(const rhs_proxy& temp) + { + view() = temp; return *this; + } + ///@} + + /** @name Dereference + * @details Dereferencing a wrapper is a no-op. It simply returns the + * wrapper. Combined with the rule that the wrapper forwards view + * operations to its contained view, this means that view operations can + * be written the same way on reducers and wrappers, which is convenient + * for incrementally converting old code using wrappers to use reducers + * instead. That is: + * + * reducer< op_and > r; + * *r &= a; // *r returns the view + * // operator &= is a view member function + * + * reducer_opand w; + * *w &= a; // *w returns the wrapper + * // operator &= is a wrapper member function that + * // calls the corresponding view function + */ + ///@{ + reducer_opor& operator*() { return *this; } + reducer_opor const& operator*() const { return *this; } + + reducer_opor* operator->() { return this; } + reducer_opor const* operator->() const { return this; } + ///@} + + /** @name Upcast + * @details In Intel Cilk Plus library 0.9, reducers were always cache-aligned. + * In library 1.0, reducer cache alignment is optional. By default, + * reducers are unaligned (i.e., just naturally aligned), but legacy + * wrappers inherit from cache-aligned reducers for binary compatibility. + * + * This means that a wrapper will automatically be upcast to its aligned + * reducer base class. The following conversion operators provide + * pseudo-upcasts to the corresponding unaligned reducer class. + */ + ///@{ + operator reducer< op_or >& () + { + return *reinterpret_cast< reducer< op_or >* >(this); + } + operator const reducer< op_or >& () const + { + return *reinterpret_cast< const reducer< op_or >* >(this); + } + ///@} + +}; + +/// @cond internal +/** Metafunction specialization for reducer conversion. + * + * This specialization of the @ref legacy_reducer_downcast template class + * defined in reducer.h causes the `reducer< op_or >` class to have an + * `operator reducer_opor& ()` conversion operator that statically + * downcasts the `reducer` to the corresponding `reducer_opor` type. + * (The reverse conversion, from `reducer_opor` to `reducer`, is just + * an upcast, which is provided for free by the language.) + * + * @ingroup ReducersOr + */ +template +struct legacy_reducer_downcast > > +{ + typedef reducer_opor type; +}; +/// @endcond + +} // namespace cilk + +#endif /* __cplusplus */ + + +/** @ingroup ReducersOr + */ +///@{ + +/** @name C language reducer macros + * + * These macros are used to declare and work with op_or reducers in C code. + * + * @see @ref page_reducers_in_c + */ + ///@{ + +__CILKRTS_BEGIN_EXTERN_C + +/** Declares OPOR reducer type name. + * + * This macro expands into the identifier which is the name of the op_or + * reducer type for a specified numeric type. + * + * @param tn The @ref reducers_c_type_names "numeric type name" specifying + * the type of the reducer. + * + * @see @ref reducers_c_predefined + * @see ReducersOr + */ +#define CILK_C_REDUCER_OPOR_TYPE(tn) \ + __CILKRTS_MKIDENT(cilk_c_reducer_opor_,tn) + +/** Declares an op_or reducer object. + * + * This macro expands into a declaration of an op_or reducer object for a + * specified numeric type. For example: + * + * CILK_C_REDUCER_OPOR(my_reducer, ulong, 0); + * + * @param obj The variable name to be used for the declared reducer object. + * @param tn The @ref reducers_c_type_names "numeric type name" specifying + * the type of the reducer. + * @param v The initial value for the reducer. (A value which can be + * assigned to the numeric type represented by @a tn.) + * + * @see @ref reducers_c_predefined + * @see ReducersOr + */ +#define CILK_C_REDUCER_OPOR(obj,tn,v) \ + CILK_C_REDUCER_OPOR_TYPE(tn) obj = \ + CILK_C_INIT_REDUCER(_Typeof(obj.value), \ + __CILKRTS_MKIDENT(cilk_c_reducer_opor_reduce_,tn), \ + __CILKRTS_MKIDENT(cilk_c_reducer_opor_identity_,tn), \ + __cilkrts_hyperobject_noop_destroy, v) + +/// @cond internal + +/** Declares the op_or reducer functions for a numeric type. + * + * This macro expands into external function declarations for functions which + * implement the reducer functionality for the op_or reducer type for a + * specified numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer + * type name, function names, etc. + */ +#define CILK_C_REDUCER_OPOR_DECLARATION(t,tn) \ + typedef CILK_C_DECLARE_REDUCER(t) CILK_C_REDUCER_OPOR_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_opor,tn,l,r); \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_opor,tn); + +/** Defines the op_or reducer functions for a numeric type. + * + * This macro expands into function definitions for functions which implement + * the reducer functionality for the op_or reducer type for a specified + * numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer + * type name, function names, etc. + */ +#define CILK_C_REDUCER_OPOR_DEFINITION(t,tn) \ + typedef CILK_C_DECLARE_REDUCER(t) CILK_C_REDUCER_OPOR_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_opor,tn,l,r) \ + { *(t*)l |= *(t*)r; } \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_opor,tn) \ + { *(t*)v = 0; } + +///@{ +/** @def CILK_C_REDUCER_OPOR_INSTANCE + * @brief Declares or defines implementation functions for a reducer type. + * + * In the runtime source file c_reducers.c, the macro `CILK_C_DEFINE_REDUCERS` + * will be defined, and this macro will generate reducer implementation + * functions. Everywhere else, `CILK_C_DEFINE_REDUCERS` will be undefined, and + * this macro will expand into external declarations for the functions. + */ +#ifdef CILK_C_DEFINE_REDUCERS +# define CILK_C_REDUCER_OPOR_INSTANCE(t,tn) \ + CILK_C_REDUCER_OPOR_DEFINITION(t,tn) +#else +# define CILK_C_REDUCER_OPOR_INSTANCE(t,tn) \ + CILK_C_REDUCER_OPOR_DECLARATION(t,tn) +#endif +///@} + +/* Declare or define an instance of the reducer type and its functions for each + * numeric type. + */ +CILK_C_REDUCER_OPOR_INSTANCE(char, char) +CILK_C_REDUCER_OPOR_INSTANCE(unsigned char, uchar) +CILK_C_REDUCER_OPOR_INSTANCE(signed char, schar) +CILK_C_REDUCER_OPOR_INSTANCE(wchar_t, wchar_t) +CILK_C_REDUCER_OPOR_INSTANCE(short, short) +CILK_C_REDUCER_OPOR_INSTANCE(unsigned short, ushort) +CILK_C_REDUCER_OPOR_INSTANCE(int, int) +CILK_C_REDUCER_OPOR_INSTANCE(unsigned int, uint) +CILK_C_REDUCER_OPOR_INSTANCE(unsigned int, unsigned) /* alternate name */ +CILK_C_REDUCER_OPOR_INSTANCE(long, long) +CILK_C_REDUCER_OPOR_INSTANCE(unsigned long, ulong) +CILK_C_REDUCER_OPOR_INSTANCE(long long, longlong) +CILK_C_REDUCER_OPOR_INSTANCE(unsigned long long, ulonglong) + +//@endcond + +__CILKRTS_END_EXTERN_C + +///@} + +///@} + +#endif /* REDUCER_OPOR_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_opxor.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_opxor.h new file mode 100644 index 0000000000..cb6560f9c5 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_opxor.h @@ -0,0 +1,608 @@ +/* reducer_opxor.h -*- C++ -*- + * + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + */ + +/** @file reducer_opxor.h + * + * @brief Defines classes for doing parallel bitwise or reductions. + * + * @ingroup ReducersXor + * + * @see ReducersXor + */ + +#ifndef REDUCER_OPXOR_H_INCLUDED +#define REDUCER_OPXOR_H_INCLUDED + +#include + +/** @defgroup ReducersXor Bitwise XOR Reducers + * + * Bitwise XOR reducers allow the computation of the bitwise XOR of a set of + * values in parallel. + * + * @ingroup Reducers + * + * You should be familiar with @ref pagereducers "Intel(R) Cilk(TM) Plus reducers", + * described in file `reducers.md`, and particularly with @ref reducers_using, + * before trying to use the information in this file. + * + * @section redopxor_usage Usage Example + * + * cilk::reducer< cilk::op_xor > r; + * cilk_for (int i = 0; i != N; ++i) { + * *r ^= a[i]; + * } + * unsigned result; + * r.move_out(result); + * + * @section redopxor_monoid The Monoid + * + * @subsection redopxor_monoid_values Value Set + * + * The value set of a bitwise XOR reducer is the set of values of `Type`, which + * is expected to be a builtin integer type which has a representation as a + * sequence of bits (or something like it, such as `bool` or `std::bitset`). + * + * @subsection redopxor_monoid_operator Operator + * + * The bitwise XOR operator is defined by the "`^`" binary operator on `Type`. + * + * @subsection redopxor_monoid_identity Identity + * + * The identity value of the reducer is the value whose representation + * contains all 0-bits. This is expected to be the value of the default + * constructor `Type()`. + * + * @section redopxor_operations Operations + * + * @subsection redopxor_constructors Constructors + * + * reducer() // identity + * reducer(const Type& value) + * reducer(move_in(Type& variable)) + * + * @subsection redopxor_get_set Set and Get + * + * r.set_value(const Type& value) + * const Type& = r.get_value() const + * r.move_in(Type& variable) + * r.move_out(Type& variable) + * + * @subsection redopxor_initial Initial Values + * + * If a bitwise XOR reducer is constructed without an explicit initial value, + * then its initial value will be its identity value, as long as `Type` + * satisfies the requirements of @ref redopxor_types. + * + * @subsection redopxor_view_ops View Operations + * + * *r ^= a + * *r = *r ^ a + * *r = *r ^ a1 ^ a2 … ^ an + * + * @section redopxor_types Type and Operator Requirements + * + * `Type` must be `Copy Constructible`, `Default Constructible`, and + * `Assignable`. + * + * The operator "`^=`" must be defined on `Type`, with `x ^= a` having the + * same meaning as `x = x ^ a`. + * + * The expression `Type()` must be a valid expression which yields the + * identity value (the value of `Type` whose representation consists of all + * 0-bits). + * + * @section redopxor_in_c Bitwise XOR Reducers in C + * + * The @ref CILK_C_REDUCER_OPXOR and @ref CILK_C_REDUCER_OPXOR_TYPE macros can + * be used to do bitwise XOR reductions in C. For example: + * + * CILK_C_REDUCER_OPXOR(r, uint, 0); + * CILK_C_REGISTER_REDUCER(r); + * cilk_for(int i = 0; i != n; ++i) { + * REDUCER_VIEW(r) ^= a[i]; + * } + * CILK_C_UNREGISTER_REDUCER(r); + * printf("The bitwise XOR of the elements of a is %x\n", REDUCER_VIEW(r)); + * + * See @ref reducers_c_predefined. + */ + +#ifdef __cplusplus + +namespace cilk { + +/** The bitwise XOR reducer view class. + * + * This is the view class for reducers created with + * `cilk::reducer< cilk::op_xor >`. It holds the accumulator variable + * for the reduction, and allows only `xor` operations to be performed on it. + * + * @note The reducer "dereference" operation (`reducer::operator *()`) + * yields a reference to the view. Thus, for example, the view class's + * `^=` operation would be used in an expression like `*r ^= a`, where + * `r` is an opmod reducer variable. + * + * @tparam Type The type of the contained accumulator variable. This will + * be the value type of a monoid_with_view that is + * instantiated with this view. + * + * @see ReducersXor + * @see op_xor + * + * @ingroup ReducersXor + */ +template +class op_xor_view : public scalar_view +{ + typedef scalar_view base; + +public: + /** Class to represent the right-hand side of `*reducer = *reducer ^ value`. + * + * The only assignment operator for the op_xor_view class takes an + * rhs_proxy as its operand. This results in the syntactic restriction + * that the only expressions that can be assigned to an op_xor_view are + * ones which generate an rhs_proxy - that is, expressions of the form + * `op_xor_view ^ value ... ^ value`. + * + * @warning + * The lhs and rhs views in such an assignment must be the same; + * otherwise, the behavior will be undefined. (I.e., `v1 = v1 ^ x` is + * legal; `v1 = v2 ^ x` is illegal.) This condition will be checked with + * a runtime assertion when compiled in debug mode. + * + * @see op_xor_view + */ + class rhs_proxy { + friend class op_xor_view; + + const op_xor_view* m_view; + Type m_value; + + // Constructor is invoked only from op_xor_view::operator^(). + // + rhs_proxy(const op_xor_view* view, const Type& value) : m_view(view), m_value(value) {} + + rhs_proxy& operator=(const rhs_proxy&); // Disable assignment operator + rhs_proxy(); // Disable default constructor + + public: + /** bitwise XOR with an additional rhs value. If `v` is an op_xor_view + * and `a1` is a value, then the expression `v ^ a1` invokes the + * view's `operator^()` to create an rhs_proxy for `(v, a1)`; then + * `v ^ a1 ^ a2` invokes the rhs_proxy's `operator^()` to create a new + * rhs_proxy for `(v, a1^a2)`. This allows the right-hand side of an + * assignment to be not just `view ^ value`, but + ( `view ^ value ^ value ... ^ value`. The effect is that + * + * v = v ^ a1 ^ a2 ... ^ an; + * + * is evaluated as + * + * v = v ^ (a1 ^ a2 ... ^ an); + */ + rhs_proxy& operator^(const Type& x) { m_value ^= x; return *this; } + }; + + + /** Default/identity constructor. This constructor initializes the + * contained value to `Type()`. + */ + op_xor_view() : base() {} + + /** Construct with a specified initial value. + */ + explicit op_xor_view(const Type& v) : base(v) {} + + /** Reduces the views of two strands. + * + * This function is invoked by the @ref op_xor monoid to combine the views + * of two strands when the right strand merges with the left one. It + * "XORs" the value contained in the left-strand view by the value + * contained in the right-strand view, and leaves the value in the + * right-strand view undefined. + * + * @param right A pointer to the right-strand view. (`this` points to + * the left-strand view.) + * + * @note Used only by the @ref op_xor monoid to implement the monoid + * reduce operation. + */ + void reduce(op_xor_view* right) { this->m_value ^= right->m_value; } + + /** @name Accumulator variable updates. + * + * These functions support the various syntaxes for "XORing" the + * accumulator variable contained in the view with some value. + */ + ///@{ + + /** Performs XOR operation between the accumulator variable and @a x. + */ + op_xor_view& operator^=(const Type& x) { this->m_value ^= x; return *this; } + + /** Creates an object representing `*this ^ x`. + * + * @see rhs_proxy + */ + rhs_proxy operator^(const Type& x) const { return rhs_proxy(this, x); } + + /** Assigns the result of a `view ^ value` expression to the view. Note that + * this is the only assignment operator for this class. + * + * @see rhs_proxy + */ + op_xor_view& operator=(const rhs_proxy& rhs) { + __CILKRTS_ASSERT(this == rhs.m_view); + this->m_value ^= rhs.m_value; + return *this; + } + + ///@} +}; + +/** Monoid class for bitwise XOR reductions. Instantiate the cilk::reducer + * template class with an op_xor monoid to create a bitwise XOR reducer + * class. For example, to compute the bitwise XOR of a set of `unsigned long` + * values: + * + * cilk::reducer< cilk::op_xor > r; + * + * @tparam Type The reducer value type. + * @tparam Align If `false` (the default), reducers instantiated on this + * monoid will be naturally aligned (the Intel Cilk Plus library 1.0 + * behavior). If `true`, reducers instantiated on this monoid + * will be cache-aligned for binary compatibility with + * reducers in Intel Cilk Plus library version 0.9. + * + * @see ReducersXor + * @see op_xor_view + * + * @ingroup ReducersXor + */ +template +struct op_xor : public monoid_with_view, Align> {}; + +/** Deprecated bitwise XOR reducer class. + * + * reducer_opxor is the same as @ref reducer<@ref op_xor>, except that + * reducer_opxor is a proxy for the contained view, so that accumulator + * variable update operations can be applied directly to the reducer. For + * example, a value is "XORed" with a `reducer<%op_xor>` with `*r ^= a`, but a + * value can be "XORed" with a `%reducer_opxor` with `r ^= a`. + * + * @deprecated Users are strongly encouraged to use `reducer` + * reducers rather than the old wrappers like reducer_opand. + * The `reducer` reducers show the reducer/monoid/view + * architecture more clearly, are more consistent in their + * implementation, and present a simpler model for new + * user-implemented reducers. + * + * @note Implicit conversions are provided between `%reducer_opxor` + * and `reducer<%op_xor>`. This allows incremental code + * conversion: old code that used `%reducer_opxor` can pass a + * `%reducer_opxor` to a converted function that now expects a + * pointer or reference to a `reducer<%op_xor>`, and vice + * versa. + * + * @tparam Type The value type of the reducer. + * + * @see op_xor + * @see reducer + * @see ReducersXor + * + * @ingroup ReducersXor + */ +template +class reducer_opxor : public reducer< op_xor > +{ + typedef reducer< op_xor > base; + using base::view; + + public: + /// The view type for the reducer. + typedef typename base::view_type view_type; + + /// The view's rhs proxy type. + typedef typename view_type::rhs_proxy rhs_proxy; + + /// The view type for the reducer. + typedef view_type View; + + /// The monoid type for the reducer. + typedef typename base::monoid_type Monoid; + + /** @name Constructors + */ + ///@{ + + /** Default (identity) constructor. + * + * Constructs the wrapper with the default initial value of `Type()`. + */ + reducer_opxor() {} + + /** Value constructor. + * + * Constructs the wrapper with a specified initial value. + */ + explicit reducer_opxor(const Type& initial_value) : base(initial_value) {} + + ///@} + + /** @name Forwarded functions + * @details Functions that update the contained accumulator variable are + * simply forwarded to the contained @ref op_and_view. */ + ///@{ + + /// @copydoc op_xor_view::operator^=(const Type&) + reducer_opxor& operator^=(const Type& x) + { + view() ^= x; return *this; + } + + // The legacy definition of reducer_opxor::operator^() has different + // behavior and a different return type than this definition. The legacy + // version is defined as a member function, so this new version is defined + // as a free function to give it a different signature, so that they won't + // end up sharing a single object file entry. + + /// @copydoc op_xor_view::operator^(const Type&) const + friend rhs_proxy operator^(const reducer_opxor& r, const Type& x) + { + return r.view() ^ x; + } + + /// @copydoc op_and_view::operator=(const rhs_proxy&) + reducer_opxor& operator=(const rhs_proxy& temp) + { + view() = temp; return *this; + } + ///@} + + /** @name Dereference + * @details Dereferencing a wrapper is a no-op. It simply returns the + * wrapper. Combined with the rule that the wrapper forwards view + * operations to its contained view, this means that view operations can + * be written the same way on reducers and wrappers, which is convenient + * for incrementally converting old code using wrappers to use reducers + * instead. That is: + * + * reducer< op_and > r; + * *r &= a; // *r returns the view + * // operator &= is a view member function + * + * reducer_opand w; + * *w &= a; // *w returns the wrapper + * // operator &= is a wrapper member function that + * // calls the corresponding view function + */ + ///@{ + reducer_opxor& operator*() { return *this; } + reducer_opxor const& operator*() const { return *this; } + + reducer_opxor* operator->() { return this; } + reducer_opxor const* operator->() const { return this; } + ///@} + + /** @name Upcast + * @details In Intel Cilk Plus library 0.9, reducers were always cache-aligned. + * In library 1.0, reducer cache alignment is optional. By default, + * reducers are unaligned (i.e., just naturally aligned), but legacy + * wrappers inherit from cache-aligned reducers for binary compatibility. + * + * This means that a wrapper will automatically be upcast to its aligned + * reducer base class. The following conversion operators provide + * pseudo-upcasts to the corresponding unaligned reducer class. + */ + ///@{ + operator reducer< op_xor >& () + { + return *reinterpret_cast< reducer< op_xor >* >(this); + } + operator const reducer< op_xor >& () const + { + return *reinterpret_cast< const reducer< op_xor >* >(this); + } + ///@} + +}; + +/// @cond internal +/** Metafunction specialization for reducer conversion. + * + * This specialization of the @ref legacy_reducer_downcast template class + * defined in reducer.h causes the `reducer< op_xor >` class to have an + * `operator reducer_opxor& ()` conversion operator that statically + * downcasts the `reducer` to the corresponding `reducer_opxor` type. + * (The reverse conversion, from `reducer_opxor` to `reducer`, is just + * an upcast, which is provided for free by the language.) + * + * @ingroup ReducersXor + */ +template +struct legacy_reducer_downcast > > +{ + typedef reducer_opxor type; +}; +/// @endcond + +} // namespace cilk + +#endif /* __cplusplus */ + + +/** @ingroup ReducersXor + */ +///@{ + +/** @name C language reducer macros + * + * These macros are used to declare and work with op_xor reducers in C code. + * + * @see @ref page_reducers_in_c + */ + ///@{ + +__CILKRTS_BEGIN_EXTERN_C + +/** Declares OPXOR reducer type name. + * + * This macro expands into the identifier which is the name of the op_xor + * reducer type for a specified numeric type. + * + * @param tn The @ref reducers_c_type_names "numeric type name" specifying + * the type of the reducer. + * + * @see @ref reducers_c_predefined + * @see ReducersXor + */ +#define CILK_C_REDUCER_OPXOR_TYPE(tn) \ + __CILKRTS_MKIDENT(cilk_c_reducer_opxor_,tn) + +/** Declares an op_xor reducer object. + * + * This macro expands into a declaration of an op_xor reducer object for a + * specified numeric type. For example: + * + * CILK_C_REDUCER_OPXOR(my_reducer, ulong, 0); + * + * @param obj The variable name to be used for the declared reducer object. + * @param tn The @ref reducers_c_type_names "numeric type name" specifying + * the type of the reducer. + * @param v The initial value for the reducer. (A value which can be + * assigned to the numeric type represented by @a tn.) + * + * @see @ref reducers_c_predefined + * @see ReducersXor + */ +#define CILK_C_REDUCER_OPXOR(obj,tn,v) \ + CILK_C_REDUCER_OPXOR_TYPE(tn) obj = \ + CILK_C_INIT_REDUCER(_Typeof(obj.value), \ + __CILKRTS_MKIDENT(cilk_c_reducer_opxor_reduce_,tn), \ + __CILKRTS_MKIDENT(cilk_c_reducer_opxor_identity_,tn), \ + __cilkrts_hyperobject_noop_destroy, v) + +/// @cond internal + +/** Declares the op_xor reducer functions for a numeric type. + * + * This macro expands into external function declarations for functions which + * implement the reducer functionality for the op_xor reducer type for a + * specified numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer + * type name, function names, etc. + */ +#define CILK_C_REDUCER_OPXOR_DECLARATION(t,tn) \ + typedef CILK_C_DECLARE_REDUCER(t) CILK_C_REDUCER_OPXOR_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_opxor,tn,l,r); \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_opxor,tn); + +/** Defines the op_xor reducer functions for a numeric type. + * + * This macro expands into function definitions for functions which implement + * the reducer functionality for the op_xor reducer type for a specified + * numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer + * type name, function names, etc. + */ +#define CILK_C_REDUCER_OPXOR_DEFINITION(t,tn) \ + typedef CILK_C_DECLARE_REDUCER(t) CILK_C_REDUCER_OPXOR_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_opxor,tn,l,r) \ + { *(t*)l ^= *(t*)r; } \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_opxor,tn) \ + { *(t*)v = 0; } + +///@{ +/** @def CILK_C_REDUCER_OPXOR_INSTANCE + * @brief Declares or defines implementation functions for a reducer type. + * + * In the runtime source file c_reducers.c, the macro `CILK_C_DEFINE_REDUCERS` + * will be defined, and this macro will generate reducer implementation + * functions. Everywhere else, `CILK_C_DEFINE_REDUCERS` will be undefined, and + * this macro will expand into external declarations for the functions. + */ +#ifdef CILK_C_DEFINE_REDUCERS +# define CILK_C_REDUCER_OPXOR_INSTANCE(t,tn) \ + CILK_C_REDUCER_OPXOR_DEFINITION(t,tn) +#else +# define CILK_C_REDUCER_OPXOR_INSTANCE(t,tn) \ + CILK_C_REDUCER_OPXOR_DECLARATION(t,tn) +#endif +///@} + +/* Declares or defines an instance of the reducer type and its functions for each + * numeric type. + */ +CILK_C_REDUCER_OPXOR_INSTANCE(char, char) +CILK_C_REDUCER_OPXOR_INSTANCE(unsigned char, uchar) +CILK_C_REDUCER_OPXOR_INSTANCE(signed char, schar) +CILK_C_REDUCER_OPXOR_INSTANCE(wchar_t, wchar_t) +CILK_C_REDUCER_OPXOR_INSTANCE(short, short) +CILK_C_REDUCER_OPXOR_INSTANCE(unsigned short, ushort) +CILK_C_REDUCER_OPXOR_INSTANCE(int, int) +CILK_C_REDUCER_OPXOR_INSTANCE(unsigned int, uint) +CILK_C_REDUCER_OPXOR_INSTANCE(unsigned int, unsigned) /* alternate name */ +CILK_C_REDUCER_OPXOR_INSTANCE(long, long) +CILK_C_REDUCER_OPXOR_INSTANCE(unsigned long, ulong) +CILK_C_REDUCER_OPXOR_INSTANCE(long long, longlong) +CILK_C_REDUCER_OPXOR_INSTANCE(unsigned long long, ulonglong) + +//@endcond + +__CILKRTS_END_EXTERN_C + +///@} + +///@} + +#endif /* REDUCER_OPXOR_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_ostream.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_ostream.h new file mode 100644 index 0000000000..793c3c5020 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_ostream.h @@ -0,0 +1,517 @@ +/* reducer_ostream.h -*- C++ -*- + * + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + */ + +/** @file reducer_ostream.h + * + * @brief Defines a class for writing to an ostream in parallel. + * + * @ingroup ReducersOstream + * + * @see @ref ReducersOstream + */ + +#ifndef REDUCER_OSTREAM_H_INCLUDED +#define REDUCER_OSTREAM_H_INCLUDED + +#include +#include +#include + +/** @defgroup ReducersOstream Ostream Reducers + * + * Ostream reducers allow multiple strands to write to an ostream in parallel. + * + * @ingroup Reducers + * + * You should be familiar with @ref pagereducers "Intel(R) Cilk(TM) Plus reducers", + * described in file reducers.md, and particularly with @ref reducers_using, + * before trying to use the information in this file. + * + * @section redostream_usage Usage Example + * + * One of the most common debugging techniques is adding `print` statements + * to the code being debugged. When the code is parallelized, the results can + * be less than satisfactory, as output from multiple strands is mingled in an + * unpredictable way. Like other reducers, an ostream reducer requires minimal + * recoding to guarantee that the output from parallelized computation will be + * ordered the same as though the computation were executed serially. + * + * cilk::reducer r(std::cerr); + * cilk_for (int i = 0; i != data.size(); ++i) { + * *r << "Iteration " << i << ":\n"; + * ... some computation ... + * *r << " Step 1:" << some information; + * ... some more computation ... + * *r << " Step 2:" << some more information; + * ... still more computation ... + * *r << " Step 3:" << still more information; + * } + * + * Output on standard error: + * + * Iteration 1: + * Step 1: ... + * Step 2: ... + * Step 3: ... + * Iteration 2: + * Step 1: ... + * Step 2: ... + * Step 3: ... + * Iteration 3: + * Step 1: ... + * Step 2: ... + * Step 3: ... + * ... + * + * @section redostream_overview Overview + * + * An "ostream reducer" is not really a reducer. It uses the reducer + * technology to coordinate operations on parallel strands to achieve + * the same behavior in a parallel computation that would be seen in a + * serial computation, but it does not have a monoid. It has a "monoid + * class," because that is part of the implementation framework, but it + * does not represent a mathematical monoid: there is no value type, no + * associative operation, and no identity value. The reducer is used for + * its side effect rather than to construct a value. + * + * You might think of an ostream reducer as a relative of a + * @ref ReducersString "string reducer" which uses stream output + * syntax (`stream << value`) instead of string append syntax + * (`string += value`), and which writes its result string to an + * ostream instead of making it available as the reducer value. + * + * Another difference is that "real" reducers protect their contained + * value quite strongly from improper access by the user. Ostream reducers, + * on the other hand, pretty much have to expose the ostream, since normal + * use of an ostream involves accessing its internal state. Furthermore, + * the ostream reducer just coordinates output to an existing ostream - + * there is nothing to keep the user from writing directly to the attached + * stream, with unpredictable results. + * + * @section redostream_operations Operations + * + * In the operation descriptions below, the type name `Ostream` refers to the + * reducer's ostream type, `std::basic_ostream`. + * + * @subsection redostream_constructors Constructors + * + * The only constructor is + * + * reducer(const Ostream& os) + * + * This creates a reducer that is associated with the existing ostream `os`. + * Anything "written to" the reducer will (eventually) be written to `os`. + * + * @subsection redostream_get_set Set and Get + * + * Just as a stream does not have a "value," neither does an ostream + * reducer. Therefore, none of the usual `set_value`, `get_value`, + * `move_in`, or `move_out` functions are available for ostream reducers. + * + * @subsection redostream_initial Initial Values + * + * Ostream reducers do not have default constructors. + * + * @subsection redostream_view_ops View Operations + * + * An ostream reducer view is actually a kind of `std::ostream`. Therefore, + * any operation that can be used on an ostream can be used on an ostream + * reducer view. For example: + * + * reducer r(cout); + * *r << setw(5) << (x=1) << endl; + * + * + * @section redostream_performance Performance Considerations + * + * Ostream reducers work by creating a string stream for each non-leftmost + * view. When two strands are merged, the contents of the string buffer of the + * right view are written to the left view. Since all non-leftmost strands are + * eventually merged, all output is eventually written to the associated + * ostream. + * + * This implementation has two consequences. + * + * First, all output written to an ostream reducer on a stolen strand is kept + * in memory (in a string buffer) until the strand is merged with the leftmost + * strand. This means that some portion of the output written to an ostream + * reducer during a parallel computation - half of the total output, on + * average - will temporarily be held in memory during the computation. + * Obviously, ostream reducers will work better for small and moderate amounts + * of output. + * + * Second, buffered ostream reducer content must be copied at every merge. + * The total amount of copying is potentially proportional to the total amount + * of output multiplied by the number of strands stolen during the computation. + * + * In short, writing to an ostream in a parallel computation with an ostream + * reducer will always be less efficient than writing the same output directly + * to the ostream in a serial computation. The value of the ostream + * reducer is not in the writing of the ostream itself, but in removing the + * race and serialization obstacles that the ostream output would cause in an + * otherwise parallelizable computation. + * + * + * @section redostream_state Stream State + * + * The reducer implementation can correctly order the output that is written + * to an ostream. However, an ostream has additional state that controls its + * behavior, such as its formatting attributes, error state, extensible arrays, * and registered callbacks. If these are modified during the computation, the * reducer implementation cannot guarantee that they will be the same in a + * parallel computation as in a serial computation. In particular: + * + * - In the serial execution, the ostream state in the continuation of a + * spawn will be the same as the state at the end of the spawned function. + * In the parallel execution, if the continuation is stolen, its view will + * contain a newly created ostream with the default initial state. + * - In the serial execution, the ostream state following a sync is the same + * as the state before the sync. In the parallel execution, if the + * continuation is stolen, then the state following the sync will be the + * same as the state at the end of some spawned function. + * + * In short, you must not make any assumptions about the stream state of an + * ostream reducer: + * + * - Following a `cilk_spawn`. + * - Following a `cilk_sync`. + * - At the start of an iteration of a `cilk_for` loop. + * - Following the completion of a `cilk_for` loop. + * + * @section redostream_types Type and Operator Requirements + * + * `std::basic_ostream` must be a valid type. +*/ + +namespace cilk { + +/** @ingroup ReducersOstream */ +//@{ + +/** The ostream reducer view class. + * + * This is the view class for reducers created with + * `cilk::reducer< cilk::op_basic_ostream >`. It holds the + * actual ostream for a parallel strand, and allows only stream output + * operations to be performed on it. + * + * @note The reducer "dereference" operation (`reducer::operator *()`) + * yields a reference to the view. Thus, for example, the view + * class's `<<` operation would be used in an expression like + * `*r << "x = " << x`, where `r` is an ostream reducer. + * + * @tparam Char The ostream element type (not the ostream type). + * @tparam Traits The character traits type. + * + * @see ReducersOstream + * @see op_basic_ostream + */ +template +class op_basic_ostream_view : public std::basic_ostream +{ + typedef std::basic_ostream base; + typedef std::basic_ostream ostream_type; + + // A non-leftmost view is associated with a private string buffer. (The + // leftmost view is associated with the buffer of the reducer's associated + // ostream, so its private buffer is unused.) + // + std::basic_stringbuf m_buffer; + +public: + + /** Value type. Required by @ref monoid_with_view. + */ + typedef ostream_type value_type; + + /** Reduce operation. Required by @ref monoid_with_view. + */ + void reduce(op_basic_ostream_view* other) + { + // Writing an empty buffer results in failure. Testing `sgetc()` is the + // easiest way of checking for an empty buffer. + if (other->m_buffer.sgetc() != Traits::eof()) { + *this << (&other->m_buffer); + } + } + + /** Non-leftmost (identity) view constructor. The view is associated with + * its internal buffer. Required by @ref monoid_base. + */ + op_basic_ostream_view() : base(&m_buffer) {} + + /** Leftmost view constructor. The view is associated with an existing + * ostream. + */ + op_basic_ostream_view(const ostream_type& os) : base(0) + { + base::rdbuf(os.rdbuf()); // Copy stream buffer + base::flags(os.flags()); // Copy formatting flags + base::setstate(os.rdstate()); // Copy error state + } + + /** Sets/gets. + * + * These are all no-ops. + */ + //@{ + + void view_set_value(const value_type&) + { assert("set_value() is not allowed on ostream reducers" && 0); } + const value_type& view_get_value() const + { assert("get_value() is not allowed on ostream reducers" && 0); + return *this; } + typedef value_type const& return_type_for_get_value; + void view_move_in(const value_type&) + { assert("move_in() is not allowed on ostream reducers" && 0); } + void view_move_out(const value_type&) + { assert("move_out() is not allowed on ostream reducers" && 0); } + + //@} +}; + +/** Ostream monoid class. Instantiate the cilk::reducer template class with an + * op_basic_ostream monoid to create an ostream reducer class: + * + * cilk::reducer< cilk::op_basic_string > r; + * + * @tparam Char The stream element type (not the stream type). + * @tparam Traits The character traits type. + * + * @see ReducersOstream + * @see op_basic_ostream_view + * @see reducer_ostream + * @see op_ostream + * @see op_wostream + */ +template, + bool Align = false> +class op_basic_ostream : + public monoid_with_view< op_basic_ostream_view, Align > +{ + typedef monoid_with_view< op_basic_ostream_view, Align > + base; + typedef std::basic_ostream ostream_type; + typedef provisional_guard view_guard; + +public: + + /** View type of the monoid. + */ + typedef typename base::view_type view_type; + + /** @name Construct function. + * + * The only supported ostream reducer constructor takes a reference to + * an existing ostream. + * + * @param os The ostream destination for receive all data written to the + * reducer. + */ + static void construct( + op_basic_ostream* monoid, + view_type* view, + const ostream_type& os) + { + view_guard vg( new((void*) view) view_type(os) ); + vg.confirm_if( new((void*) monoid) op_basic_ostream ); + } +}; + + +/** + * Convenience typedef for narrow ostreams. + */ +typedef op_basic_ostream op_ostream; + +/** + * Convenience typedef for wide ostreams. + */ +typedef op_basic_ostream op_wostream; + +/// @cond internal + +class reducer_ostream; + +/** Metafunction specialization for reducer conversion. + * + * This specialization of the @ref legacy_reducer_downcast template class + * defined in reducer.h causes the `reducer >` class + * to have an `operator reducer_ostream& ()` conversion operator that + * statically downcasts the `reducer >` to + * `reducer_ostream`. (The reverse conversion, from `reducer_ostream` to + * `reducer >`, is just an upcast, which is provided + * for free by the language.) + */ +template +struct legacy_reducer_downcast< + reducer, Align> > > +{ + typedef reducer_ostream type; +}; + +/// @endcond + +/** Deprecated ostream reducer class. + * + * reducer_ostream is the same as @ref cilk::reducer<@ref op_ostream>, except + * that reducer_ostream is a proxy for the contained view, so that ostream + * operations can be applied directly to the reducer. For example, a number is + * written to a `reducer` with `*r << x`, but a number can be + * written to a `reducer_ostream` with `r << x`. + * + * @deprecated Users are strongly encouraged to use `reducer` + * reducers rather than the old wrappers like reducer_ostream. The + * `reducer` reducers show the reducer/monoid/view + * architecture more clearly, are more consistent in their + * implementation, and present a simpler model for new + * user-implemented reducers. + * + * @note Implicit conversions are provided between `%reducer_ostream` + * and `reducer<%op_ostream>`. This allows incremental code + * conversion: old code that used `%reducer_ostream` can pass a + * `%reducer_ostream` to a converted function that now expects a + * pointer or reference to a `reducer<%op_ostream>`, and vice versa. + * + * @tparam Char The stream element type (not the stream type). + * @tparam Traits The character traits type. + * + * @see op_ostream + * @see reducer + * @see ReducersOstream + */ +class reducer_ostream : + public reducer, true> > +{ + typedef reducer, true> > base; + using base::view; +public: + + /// The view type for the reducer. + typedef base::view_type View; + + /// The monoid type for the reducer. + typedef base::monoid_type Monoid; + + /** Constructs an initial `reducer_ostream` from a `std::ostream`. The + * specified stream is used as the eventual destination for all text + * streamed to this hyperobject. + */ + explicit reducer_ostream(const std::ostream &os) : base(os) {} + + /** Returns a modifiable reference to the underlying 'ostream' object. + */ + std::ostream& get_reference() { return view(); } + + /** Writes to the ostream. + */ + template + std::ostream& operator<< (const T &v) + { + return view() << v; + } + + /** + * Calls a manipulator. + * + * @param _Pfn Pointer to the manipulator function. + */ + reducer_ostream& operator<< (std::ostream &(*_Pfn)(std::ostream &)) + { + (*_Pfn)(view()); + return *this; + } + + /** @name Dereference + * @details Dereferencing a wrapper is a no-op. It simply returns the + * wrapper. Combined with the rule that the wrapper forwards view + * operations to its contained view, this means that view operations can + * be written the same way on reducers and wrappers, which is convenient + * for incrementally converting old code using wrappers to use reducers + * instead. That is: + * + * reducer r; + * *r << "a"; // *r returns the view + * // operator<<() is a view member function + * + * reducer_ostream w; + * *w << "a"; // *w returns the wrapper + * // operator<<() is a wrapper member function + * // that calls the corresponding view function + */ + //@{ + reducer_ostream& operator*() { return *this; } + reducer_ostream const& operator*() const { return *this; } + + reducer_ostream* operator->() { return this; } + reducer_ostream const* operator->() const { return this; } + //@} + + /** @name Upcast + * @details In Intel Cilk Plus library 0.9, reducers were always cache-aligned. + * In library 1.0, reducer cache alignment is optional. By default, + * reducers are unaligned (i.e., just naturally aligned), but legacy + * wrappers inherit from cache-aligned reducers for binary compatibility. + * + * This means that a wrapper will automatically be upcast to its aligned + * reducer base class. The following conversion operators provide + * pseudo-upcasts to the corresponding unaligned reducer class. + */ + //@{ + operator reducer& () + { + return *reinterpret_cast< reducer* >(this); + } + operator const reducer& () const + { + return *reinterpret_cast< const reducer* >(this); + } + //@} +}; + +} // namespace cilk + +#endif // REDUCER_OSTREAM_H_INCLUDED diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_string.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_string.h new file mode 100644 index 0000000000..9af65d5534 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cilk/reducer_string.h @@ -0,0 +1,763 @@ +/* reducer_string.h -*- C++ -*- + * + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + */ + +/** @file reducer_string.h + * + * @brief Defines classes for doing parallel string creation by appending. + * + * @ingroup ReducersString + * + * @see ReducersString + */ + +#ifndef REDUCER_STRING_H_INCLUDED +#define REDUCER_STRING_H_INCLUDED + +#include +#include +#include + +/** @defgroup ReducersString String Reducers + * + * String reducers allow the creation of a string by concatenating a set of + * strings or characters in parallel. + * + * @ingroup Reducers + * + * You should be familiar with @ref pagereducers "Intel(R) Cilk(TM) Plus reducers", + * described in file reducers.md, and particularly with @ref reducers_using, + * before trying to use the information in this file. + * + * @section redstring_usage Usage Example + * + * vector data; + * void expensive_string_computation(const Data& x, string& s); + * cilk::reducer r; + * cilk_for (int i = 0; i != data.size(); ++i) { + * string temp; + * expensive_string_computation(data[i], temp); + * *r += temp; + * } + * string result; + * r.move_out(result); + * + * @section redstring_monoid The Monoid + * + * @subsection redstring_monoid_values Value Set + * + * The value set of a string reducer is the set of values of the class + * `std::basic_string`, which we refer to as "the + * reducer's string type". + * + * @subsection redstring_monoid_operator Operator + * + * The operator of a string reducer is the string concatenation operator, + * defined by the "`+`" binary operator on the reducer's string type. + * + * @subsection redstring_monoid_identity Identity + * + * The identity value of a string reducer is the empty string, which is the + * value of the expression + * `std::basic_string([allocator])`. + * + * @section redstring_operations Operations + * + * In the operation descriptions below, the type name `String` refers to the + * reducer's string type, `std::basic_string`. + * + * @subsection redstring_constructors Constructors + * + * Any argument list which is valid for a `std::basic_string` constructor is + * valid for a string reducer constructor. The usual move-in constructor is + * also provided: + * + * reducer(move_in(String& variable)) + * + * @subsection redstring_get_set Set and Get + * + * r.set_value(const String& value) + * const String& = r.get_value() const + * r.move_in(String& variable) + * r.move_out(String& variable) + * + * @subsection redstring_initial Initial Values + * + * A string reducer with no constructor arguments, or with only an allocator + * argument, will initially contain the identity value, an empty string. + * + * @subsection redstring_view_ops View Operations + * + * *r += a + * r->append(a) + * r->append(a, b) + * r->push_back(a) + * + * These operations on string reducer views are the same as the corresponding + * operations on strings. + * + * @section redstring_performance Performance Considerations + * + * String reducers work by creating a string for each view, collecting those + * strings in a list, and then concatenating them into a single result string + * at the end of the computation. This last step takes place in serial code, + * and necessarily takes time proportional to the length of the result string. + * Thus, a parallel string reducer cannot actually speed up the time spent + * directly creating the string. This trivial example would probably be slower + * (because of reducer overhead) than the corresponding serial code: + * + * vector a; + * reducer r; + * cilk_for (int i = 0; i != a.length(); ++i) { + * *r += a[i]; + * } + * string result; + * r.move_out(result); + * + * What a string reducer _can_ do is to allow the _remainder_ of the + * computation to be done in parallel, without having to worry about managing + * the string computation. + * + * The strings for new views are created (by the view identity constructor) + * using the same allocator as the string that was created when the reducer + * was constructed. Note that this allocator is determined when the reducer is + * constructed. The following two examples may have very different behavior: + * + * string a_string; + * + * reducer< op_string reducer1(move_in(a_string)); + * ... parallel computation ... + * reducer1.move_out(a_string); + * + * reducer< op_string reducer2; + * reducer2.move_in(a_string); + * ... parallel computation ... + * reducer2.move_out(a_string); + * + * * `reducer1` will be constructed with the same allocator as `a_string`, + * because the string was specified in the constructor. The `move_in` + * and `move_out` can therefore be done with a `swap` in constant time. + * * `reducer2` will be constructed with a _default_ allocator of type + * `Allocator`, which may not be the same as the allocator of `a_string`. + * Therefore, the `move_in` and `move_out` may have to be done with a copy + * in _O(N)_ time. + * + * (All instances of an allocator type with no internal state (like + * `std::allocator`) are "the same". You only need to worry about the "same + * allocator" issue when you create string reducers with custom allocator + * types.) + * + * @section redstring_types Type and Operator Requirements + * + * `std::basic_string` must be a valid type. +*/ + +namespace cilk { + +/** @ingroup ReducersString */ +//@{ + +/** The string append reducer view class. + * + * This is the view class for reducers created with + * `cilk::reducer< cilk::op_basic_string >`. It holds + * the accumulator variable for the reduction, and allows only append + * operations to be performed on it. + * + * @note The reducer "dereference" operation (`reducer::operator *()`) + * yields a reference to the view. Thus, for example, the view class's + * `append` operation would be used in an expression like + * `r->append(a)`, where `r` is a string append reducer variable. + * + * @tparam Char The string element type (not the string type). + * @tparam Traits The character traits type. + * @tparam Alloc The string allocator type. + * + * @see ReducersString + * @see op_basic_string + */ +template +class op_basic_string_view +{ + typedef std::basic_string string_type; + typedef std::list list_type; + typedef typename string_type::size_type size_type; + + // The view's value is represented by a list of strings and a single + // string. The value is the concatenation of the strings in the list with + // the single string at the end. All string operations apply to the single + // string; reduce operations cause lists of partial strings from multiple + // strands to be combined. + // + mutable string_type m_string; + mutable list_type m_list; + + // Before returning the value of the reducer, concatenate all the strings + // in the list with the single string. + // + void flatten() const + { + if (m_list.empty()) return; + + typename list_type::iterator i; + + size_type len = m_string.size(); + for (i = m_list.begin(); i != m_list.end(); ++i) + len += i->size(); + + string_type result(get_allocator()); + result.reserve(len); + + for (i = m_list.begin(); i != m_list.end(); ++i) + result += *i; + m_list.clear(); + + result += m_string; + result.swap(m_string); + } + +public: + + /** @name Monoid support. + */ + //@{ + + /// Required by @ref monoid_with_view + typedef string_type value_type; + + /// Required by @ref op_string + Alloc get_allocator() const + { + return m_string.get_allocator(); + } + + /** Reduces the views of two strands. + * + * This function is invoked by the @ref op_basic_string monoid to combine + * the views of two strands when the right strand merges with the left + * one. It appends the value contained in the right-strand view to the + * value contained in the left-strand view, and leaves the value in the + * right-strand view undefined. + * + * @param right A pointer to the right-strand view. (`this` points to + * the left-strand view.) + * + * @note Used only by the @ref op_basic_string monoid to implement the + * monoid reduce operation. + */ + void reduce(op_basic_string_view* right) + { + if (!right->m_string.empty() || !right->m_list.empty()) { + // (list, string) + (right_list, right_string) => + // (list + {string} + right_list, right_string) + if (!m_string.empty()) { + // simulate m_list.push_back(std::move(m_string)) + m_list.push_back(string_type(get_allocator())); + m_list.back().swap(m_string); + } + m_list.splice(m_list.end(), right->m_list); + m_string.swap(right->m_string); + } + } + + //@} + + /** @name Passes constructor arguments to the string constructor. + */ + //@{ + + op_basic_string_view() : m_string() {} + + template + op_basic_string_view(const T1& x1) : m_string(x1) {} + + template + op_basic_string_view(const T1& x1, const T2& x2) : m_string(x1, x2) {} + + template + op_basic_string_view(const T1& x1, const T2& x2, const T3& x3) : m_string(x1, x2, x3) {} + + template + op_basic_string_view(const T1& x1, const T2& x2, const T3& x3, const T4& x4) : + m_string(x1, x2, x3, x4) {} + + //@} + + /** Move-in constructor. + */ + explicit op_basic_string_view(move_in_wrapper w) + : m_string(w.value().get_allocator()) + { + m_string.swap(w.value()); + } + + /** @name @ref reducer support. + */ + //@{ + + void view_move_in(string_type& s) + { + m_list.clear(); + if (m_string.get_allocator() == s.get_allocator()) + // Equal allocators. Do a (fast) swap. + m_string.swap(s); + else + // Unequal allocators. Do a (slow) copy. + m_string = s; + s.clear(); + } + + void view_move_out(string_type& s) + { + flatten(); + if (m_string.get_allocator() == s.get_allocator()) + // Equal allocators. Do a (fast) swap. + m_string.swap(s); + else + // Unequal allocators. Do a (slow) copy. + s = m_string; + m_string.clear(); + } + + void view_set_value(const string_type& s) + { m_list.clear(); m_string = s; } + + string_type const& view_get_value() const + { flatten(); return m_string; } + + typedef string_type const& return_type_for_get_value; + + string_type & view_get_reference() + { flatten(); return m_string; } + + string_type const& view_get_reference() const + { flatten(); return m_string; } + + //@} + + /** @name View modifier operations. + * + * @details These simply wrap the corresponding operations on the underlying string. + */ + //@{ + + template + op_basic_string_view& operator +=(const T& x) + { m_string += x; return *this; } + + template + op_basic_string_view& append(const T1& x1) + { m_string.append(x1); return *this; } + + template + op_basic_string_view& append(const T1& x1, const T2& x2) + { m_string.append(x1, x2); return *this; } + + template + op_basic_string_view& append(const T1& x1, const T2& x2, const T3& x3) + { m_string.append(x1, x2, x3); return *this; } + + void push_back(const Char x) { m_string.push_back(x); } + + //@} +}; + + +/** String append monoid class. Instantiate the cilk::reducer template class + * with an op_basic_string monoid to create a string append reducer class. For + * example, to concatenate a collection of standard strings: + * + * cilk::reducer< cilk::op_basic_string > r; + * + * @tparam Char The string element type (not the string type). + * @tparam Traits The character traits type. + * @tparam Alloc The string allocator type. + * @tparam Align If `false` (the default), reducers instantiated on this + * monoid will be naturally aligned (the Intel Cilk Plus library 1.0 + * behavior). If `true`, reducers instantiated on this monoid + * will be cache-aligned for binary compatibility with + * reducers in Intel Cilk Plus library version 0.9. + * + * @see ReducersString + * @see op_basic_string_view + * @see reducer_basic_string + * @see op_string + * @see op_wstring + */ +template, + typename Alloc = std::allocator, + bool Align = false> +class op_basic_string : + public monoid_with_view< op_basic_string_view, Align > +{ + typedef monoid_with_view< op_basic_string_view, Align > + base; + typedef provisional_guard view_guard; + + Alloc m_allocator; + +public: + + /** View type of the monoid. + */ + typedef typename base::view_type view_type; + + /** Constructor. + * + * There is no default constructor for string monoids, because the + * allocator must always be specified. + * + * @param allocator The list allocator to be used when + * identity-constructing new views. + */ + op_basic_string(const Alloc& allocator = Alloc()) : m_allocator(allocator) + {} + + /** Creates an identity view. + * + * String view identity constructors take the string allocator as an + * argument. + * + * @param v The address of the uninitialized memory in which the view + * will be constructed. + */ + void identity(view_type *v) const + { ::new((void*) v) view_type(m_allocator); } + + /** @name Construct functions + * + * A string append reduction monoid must have a copy of the allocator of + * the leftmost view's string, so that it can use it in the `identity` + * operation. This, in turn, requires that string reduction monoids have a + * specialized `construct()` function. + * + * All string reducer monoid `construct()` functions first construct the + * leftmost view, using the arguments that were passed in from the reducer + * constructor. They then call the view's `get_allocator()` function to + * get the string allocator from the string in the leftmost view, and pass + * that to the monoid constructor. + */ + //@{ + + static void construct(op_basic_string* monoid, view_type* view) + { + view_guard vg( new((void*) view) view_type() ); + vg.confirm_if( + new((void*) monoid) op_basic_string(view->get_allocator()) ); + } + + template + static void construct(op_basic_string* monoid, view_type* view, + const T1& x1) + { + view_guard vg( new((void*) view) view_type(x1) ); + vg.confirm_if( + new((void*) monoid) op_basic_string(view->get_allocator()) ); + } + + template + static void construct(op_basic_string* monoid, view_type* view, + const T1& x1, const T2& x2) + { + view_guard vg( new((void*) view) view_type(x1, x2) ); + vg.confirm_if( + new((void*) monoid) op_basic_string(view->get_allocator()) ); + } + + template + static void construct(op_basic_string* monoid, view_type* view, + const T1& x1, const T2& x2, const T3& x3) + { + view_guard vg( new((void*) view) view_type(x1, x2, x3) ); + vg.confirm_if( + new((void*) monoid) op_basic_string(view->get_allocator()) ); + } + + template + static void construct(op_basic_string* monoid, view_type* view, + const T1& x1, const T2& x2, const T3& x3, + const T4& x4) + { + view_guard vg( new((void*) view) view_type(x1, x2, x3, x4) ); + vg.confirm_if( + new((void*) monoid) op_basic_string(view->get_allocator()) ); + } + + //@} +}; + + +/** Convenience typedef for 8-bit strings + */ +typedef op_basic_string op_string; + +/** Convenience typedef for 16-bit strings + */ +typedef op_basic_string op_wstring; + + +/** Deprecated string append reducer class. + * + * reducer_basic_string is the same as @ref reducer<@ref op_basic_string>, + * except that reducer_basic_string is a proxy for the contained view, so that + * accumulator variable update operations can be applied directly to the + * reducer. For example, a value is appended to a `reducer<%op_basic_string>` + * with `r->push_back(a)`, but a value can be appended to a `%reducer_opand` + * with `r.push_back(a)`. + * + * @deprecated Users are strongly encouraged to use `reducer` + * reducers rather than the old wrappers like reducer_basic_string. + * The `reducer` reducers show the reducer/monoid/view + * architecture more clearly, are more consistent in their + * implementation, and present a simpler model for new + * user-implemented reducers. + * + * @note Implicit conversions are provided between `%reducer_basic_string` + * and `reducer<%op_basic_string>`. This allows incremental code + * conversion: old code that used `%reducer_basic_string` can pass a + * `%reducer_basic_string` to a converted function that now expects a + * pointer or reference to a `reducer<%op_basic_string>`, and vice + * versa. + * + * @tparam Char The string element type (not the string type). + * @tparam Traits The character traits type. + * @tparam Alloc The string allocator type. + * + * @see op_basic_string + * @see reducer + * @see ReducersString + */ +template, + typename Alloc = std::allocator > +class reducer_basic_string : + public reducer< op_basic_string > +{ + typedef reducer< op_basic_string > base; + using base::view; +public: + + /// The reducer's string type. + typedef typename base::value_type string_type; + + /// The reducer's primitive component type. + typedef Char basic_value_type; + + /// The string size type. + typedef typename string_type::size_type size_type; + + /// The view type for the reducer. + typedef typename base::view_type View; + + /// The monoid type for the reducer. + typedef typename base::monoid_type Monoid; + + + /** @name Constructors + */ + //@{ + + /** @name Forward constructor calls to the base class. + * + * All basic_string constructor forms are supported. + */ + //@{ + reducer_basic_string() {} + + template + reducer_basic_string(const T1& x1) : + base(x1) {} + + template + reducer_basic_string(const T1& x1, const T2& x2) : + base(x1, x2) {} + + template + reducer_basic_string(const T1& x1, const T2& x2, const T3& x3) : + base(x1, x2, x3) {} + + template + reducer_basic_string(const T1& x1, const T2& x2, const T3& x3, const T4& x4) : + base(x1, x2, x3, x4) {} + //@} + + /** Allows mutable access to the string within the current view. + * + * @warning If this method is called before the parallel calculation is + * complete, the string returned by this method will be a + * partial result. + * + * @returns A mutable reference to the string within the current view. + */ + string_type &get_reference() + { return view().view_get_reference(); } + + /** Allows read-only access to the string within the current view. + * + * @warning If this method is called before the parallel calculation is + * complete, the string returned by this method will be a + * partial result. + * + * @returns A const reference to the string within the current view. + */ + string_type const &get_reference() const + { return view().view_get_reference(); } + + /** @name Appends to the string. + * + * These operations are simply forwarded to the view. + */ + //@{ + void append(const Char *ptr) + { view().append(ptr); } + void append(const Char *ptr, size_type count) + { view().append(ptr, count); } + void append(const string_type &str, size_type offset, size_type count) + { view().append(str, offset, count); } + void append(const string_type &str) + { view().append(str); } + void append(size_type count, Char ch) + { view().append(count, ch); } + + // Appends to the string + reducer_basic_string &operator+=(Char ch) + { view() += ch; return *this; } + reducer_basic_string &operator+=(const Char *ptr) + { view() += ptr; return *this; } + reducer_basic_string &operator+=(const string_type &right) + { view() += right; return *this; } + //@} + + /** @name Dereference + * @details Dereferencing a wrapper is a no-op. It simply returns the + * wrapper. Combined with the rule that the wrapper forwards view + * operations to its contained view, this means that view operations can + * be written the same way on reducers and wrappers, which is convenient + * for incrementally converting old code using wrappers to use reducers + * instead. That is: + * + * reducer r; + * r->push_back(a); // r-> returns the view + * // push_back() is a view member function + * + * reducer_string w; + * w->push_back(a); // *w returns the wrapper + * // push_back() is a wrapper member function + * // that calls the corresponding view function + */ + //@{ + reducer_basic_string& operator*() { return *this; } + reducer_basic_string const& operator*() const { return *this; } + + reducer_basic_string* operator->() { return this; } + reducer_basic_string const* operator->() const { return this; } + //@} + + /** @name Upcast + * @details In Intel Cilk Plus library 0.9, reducers were always cache-aligned. + * In library 1.0, reducer cache alignment is optional. By default, + * reducers are unaligned (i.e., just naturally aligned), but legacy + * wrappers inherit from cache-aligned reducers for binary compatibility. + * + * This means that a wrapper will automatically be upcast to its aligned + * reducer base class. The following conversion operators provide + * pseudo-upcasts to the corresponding unaligned reducer class. + */ + //@{ + operator reducer< op_basic_string >& () + { + return *reinterpret_cast< reducer< + op_basic_string >* + >(this); + } + operator const reducer< op_basic_string >& () const + { + return *reinterpret_cast< const reducer< + op_basic_string >* + >(this); + } + //@} +}; + + +/** Convenience typedef for 8-bit strings + */ +typedef reducer_basic_string reducer_string; + +/** Convenience typedef for 16-bit strings + */ +typedef reducer_basic_string reducer_wstring; + +/// @cond internal + +/// @cond internal +/** Metafunction specialization for reducer conversion. + * + * This specialization of the @ref legacy_reducer_downcast template class + * defined in reducer.h causes the `reducer< op_basic_string >` class to + * have an `operator reducer_basic_string& ()` conversion operator that + * statically downcasts the `reducer` to the corresponding + * `reducer_basic_string` type. (The reverse conversion, from + * `reducer_basic_string` to `reducer`, is just an upcast, + * which is provided for free by the language.) + * + * @ingroup ReducersString + */ +template +struct legacy_reducer_downcast< + reducer > > +{ + typedef reducer_basic_string type; +}; + +/// @endcond + +//@} + +} // namespace cilk + +#endif // REDUCER_STRING_H_INCLUDED diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/clflushoptintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/clflushoptintrin.h new file mode 100644 index 0000000000..1d56b65e31 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/clflushoptintrin.h @@ -0,0 +1,49 @@ +/* Copyright (C) 2013-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#if !defined _X86INTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _CLFLUSHOPTINTRIN_H_INCLUDED +#define _CLFLUSHOPTINTRIN_H_INCLUDED + +#ifndef __CLFLUSHOPT__ +#pragma GCC push_options +#pragma GCC target("clflushopt") +#define __DISABLE_CLFLUSHOPT__ +#endif /* __CLFLUSHOPT__ */ + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_clflushopt (void *__A) +{ + __builtin_ia32_clflushopt (__A); +} + +#ifdef __DISABLE_CLFLUSHOPT__ +#undef __DISABLE_CLFLUSHOPT__ +#pragma GCC pop_options +#endif /* __DISABLE_CLFLUSHOPT__ */ + +#endif /* _CLFLUSHOPTINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/clwbintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/clwbintrin.h new file mode 100644 index 0000000000..37ba9108eb --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/clwbintrin.h @@ -0,0 +1,49 @@ +/* Copyright (C) 2013-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#if !defined _X86INTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _CLWBINTRIN_H_INCLUDED +#define _CLWBINTRIN_H_INCLUDED + +#ifndef __CLWB__ +#pragma GCC push_options +#pragma GCC target("clwb") +#define __DISABLE_CLWB__ +#endif /* __CLWB__ */ + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_clwb (void *__A) +{ + __builtin_ia32_clwb (__A); +} + +#ifdef __DISABLE_CLWB__ +#undef __DISABLE_CLWB__ +#pragma GCC pop_options +#endif /* __DISABLE_CLWB__ */ + +#endif /* _CLWBINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/clzerointrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/clzerointrin.h new file mode 100644 index 0000000000..dc72b1bec9 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/clzerointrin.h @@ -0,0 +1,44 @@ +/* Copyright (C) 2012-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _CLZEROINTRIN_H_INCLUDED +#define _CLZEROINTRIN_H_INCLUDED + +#ifndef __CLZERO__ +#pragma GCC push_options +#pragma GCC target("clzero") +#define __DISABLE_CLZERO__ +#endif /* __CLZERO__ */ + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_clzero (void * __I) +{ + __builtin_ia32_clzero (__I); +} + +#ifdef __DISABLE_CLZERO__ +#undef __DISABLE_CLZERO__ +#pragma GCC pop_options +#endif /* __DISABLE_CLZERO__ */ + +#endif /* _CLZEROINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cpuid.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cpuid.h new file mode 100644 index 0000000000..b3b0f912c9 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cpuid.h @@ -0,0 +1,273 @@ +/* + * Copyright (C) 2007-2017 Free Software Foundation, Inc. + * + * This file is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 3, or (at your option) any + * later version. + * + * This file is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * Under Section 7 of GPL version 3, you are granted additional + * permissions described in the GCC Runtime Library Exception, version + * 3.1, as published by the Free Software Foundation. + * + * You should have received a copy of the GNU General Public License and + * a copy of the GCC Runtime Library Exception along with this program; + * see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + * . + */ + +/* %ecx */ +#define bit_SSE3 (1 << 0) +#define bit_PCLMUL (1 << 1) +#define bit_LZCNT (1 << 5) +#define bit_SSSE3 (1 << 9) +#define bit_FMA (1 << 12) +#define bit_CMPXCHG16B (1 << 13) +#define bit_SSE4_1 (1 << 19) +#define bit_SSE4_2 (1 << 20) +#define bit_MOVBE (1 << 22) +#define bit_POPCNT (1 << 23) +#define bit_AES (1 << 25) +#define bit_XSAVE (1 << 26) +#define bit_OSXSAVE (1 << 27) +#define bit_AVX (1 << 28) +#define bit_F16C (1 << 29) +#define bit_RDRND (1 << 30) + +/* %edx */ +#define bit_CMPXCHG8B (1 << 8) +#define bit_CMOV (1 << 15) +#define bit_MMX (1 << 23) +#define bit_FXSAVE (1 << 24) +#define bit_SSE (1 << 25) +#define bit_SSE2 (1 << 26) + +/* Extended Features (%eax == 0x80000001) */ +/* %ecx */ +#define bit_LAHF_LM (1 << 0) +#define bit_ABM (1 << 5) +#define bit_SSE4a (1 << 6) +#define bit_PRFCHW (1 << 8) +#define bit_XOP (1 << 11) +#define bit_LWP (1 << 15) +#define bit_FMA4 (1 << 16) +#define bit_TBM (1 << 21) +#define bit_MWAITX (1 << 29) + +/* %edx */ +#define bit_MMXEXT (1 << 22) +#define bit_LM (1 << 29) +#define bit_3DNOWP (1 << 30) +#define bit_3DNOW (1u << 31) + +/* %ebx */ +#define bit_CLZERO (1 << 0) + +/* Extended Features (%eax == 7) */ +/* %ebx */ +#define bit_FSGSBASE (1 << 0) +#define bit_SGX (1 << 2) +#define bit_BMI (1 << 3) +#define bit_HLE (1 << 4) +#define bit_AVX2 (1 << 5) +#define bit_BMI2 (1 << 8) +#define bit_RTM (1 << 11) +#define bit_MPX (1 << 14) +#define bit_AVX512F (1 << 16) +#define bit_AVX512DQ (1 << 17) +#define bit_RDSEED (1 << 18) +#define bit_ADX (1 << 19) +#define bit_AVX512IFMA (1 << 21) +#define bit_CLFLUSHOPT (1 << 23) +#define bit_CLWB (1 << 24) +#define bit_AVX512PF (1 << 26) +#define bit_AVX512ER (1 << 27) +#define bit_AVX512CD (1 << 28) +#define bit_SHA (1 << 29) +#define bit_AVX512BW (1 << 30) +#define bit_AVX512VL (1u << 31) + +/* %ecx */ +#define bit_PREFETCHWT1 (1 << 0) +#define bit_AVX512VBMI (1 << 1) +#define bit_PKU (1 << 3) +#define bit_OSPKE (1 << 4) +#define bit_AVX512VPOPCNTDQ (1 << 14) +#define bit_RDPID (1 << 22) + +/* %edx */ +#define bit_AVX5124VNNIW (1 << 2) +#define bit_AVX5124FMAPS (1 << 3) + +/* XFEATURE_ENABLED_MASK register bits (%eax == 13, %ecx == 0) */ +#define bit_BNDREGS (1 << 3) +#define bit_BNDCSR (1 << 4) + +/* Extended State Enumeration Sub-leaf (%eax == 13, %ecx == 1) */ +#define bit_XSAVEOPT (1 << 0) +#define bit_XSAVEC (1 << 1) +#define bit_XSAVES (1 << 3) + +/* Signatures for different CPU implementations as returned in uses + of cpuid with level 0. */ +#define signature_AMD_ebx 0x68747541 +#define signature_AMD_ecx 0x444d4163 +#define signature_AMD_edx 0x69746e65 + +#define signature_CENTAUR_ebx 0x746e6543 +#define signature_CENTAUR_ecx 0x736c7561 +#define signature_CENTAUR_edx 0x48727561 + +#define signature_CYRIX_ebx 0x69727943 +#define signature_CYRIX_ecx 0x64616574 +#define signature_CYRIX_edx 0x736e4978 + +#define signature_INTEL_ebx 0x756e6547 +#define signature_INTEL_ecx 0x6c65746e +#define signature_INTEL_edx 0x49656e69 + +#define signature_TM1_ebx 0x6e617254 +#define signature_TM1_ecx 0x55504361 +#define signature_TM1_edx 0x74656d73 + +#define signature_TM2_ebx 0x756e6547 +#define signature_TM2_ecx 0x3638784d +#define signature_TM2_edx 0x54656e69 + +#define signature_NSC_ebx 0x646f6547 +#define signature_NSC_ecx 0x43534e20 +#define signature_NSC_edx 0x79622065 + +#define signature_NEXGEN_ebx 0x4778654e +#define signature_NEXGEN_ecx 0x6e657669 +#define signature_NEXGEN_edx 0x72446e65 + +#define signature_RISE_ebx 0x65736952 +#define signature_RISE_ecx 0x65736952 +#define signature_RISE_edx 0x65736952 + +#define signature_SIS_ebx 0x20536953 +#define signature_SIS_ecx 0x20536953 +#define signature_SIS_edx 0x20536953 + +#define signature_UMC_ebx 0x20434d55 +#define signature_UMC_ecx 0x20434d55 +#define signature_UMC_edx 0x20434d55 + +#define signature_VIA_ebx 0x20414956 +#define signature_VIA_ecx 0x20414956 +#define signature_VIA_edx 0x20414956 + +#define signature_VORTEX_ebx 0x74726f56 +#define signature_VORTEX_ecx 0x436f5320 +#define signature_VORTEX_edx 0x36387865 + +#define __cpuid(level, a, b, c, d) \ + __asm__ ("cpuid\n\t" \ + : "=a" (a), "=b" (b), "=c" (c), "=d" (d) \ + : "0" (level)) + +#define __cpuid_count(level, count, a, b, c, d) \ + __asm__ ("cpuid\n\t" \ + : "=a" (a), "=b" (b), "=c" (c), "=d" (d) \ + : "0" (level), "2" (count)) + + +/* Return highest supported input value for cpuid instruction. ext can + be either 0x0 or 0x80000000 to return highest supported value for + basic or extended cpuid information. Function returns 0 if cpuid + is not supported or whatever cpuid returns in eax register. If sig + pointer is non-null, then first four bytes of the signature + (as found in ebx register) are returned in location pointed by sig. */ + +static __inline unsigned int +__get_cpuid_max (unsigned int __ext, unsigned int *__sig) +{ + unsigned int __eax, __ebx, __ecx, __edx; + +#ifndef __x86_64__ + /* See if we can use cpuid. On AMD64 we always can. */ +#if __GNUC__ >= 3 + __asm__ ("pushf{l|d}\n\t" + "pushf{l|d}\n\t" + "pop{l}\t%0\n\t" + "mov{l}\t{%0, %1|%1, %0}\n\t" + "xor{l}\t{%2, %0|%0, %2}\n\t" + "push{l}\t%0\n\t" + "popf{l|d}\n\t" + "pushf{l|d}\n\t" + "pop{l}\t%0\n\t" + "popf{l|d}\n\t" + : "=&r" (__eax), "=&r" (__ebx) + : "i" (0x00200000)); +#else +/* Host GCCs older than 3.0 weren't supporting Intel asm syntax + nor alternatives in i386 code. */ + __asm__ ("pushfl\n\t" + "pushfl\n\t" + "popl\t%0\n\t" + "movl\t%0, %1\n\t" + "xorl\t%2, %0\n\t" + "pushl\t%0\n\t" + "popfl\n\t" + "pushfl\n\t" + "popl\t%0\n\t" + "popfl\n\t" + : "=&r" (__eax), "=&r" (__ebx) + : "i" (0x00200000)); +#endif + + if (!((__eax ^ __ebx) & 0x00200000)) + return 0; +#endif + + /* Host supports cpuid. Return highest supported cpuid input value. */ + __cpuid (__ext, __eax, __ebx, __ecx, __edx); + + if (__sig) + *__sig = __ebx; + + return __eax; +} + +/* Return cpuid data for requested cpuid leaf, as found in returned + eax, ebx, ecx and edx registers. The function checks if cpuid is + supported and returns 1 for valid cpuid information or 0 for + unsupported cpuid leaf. All pointers are required to be non-null. */ + +static __inline int +__get_cpuid (unsigned int __leaf, + unsigned int *__eax, unsigned int *__ebx, + unsigned int *__ecx, unsigned int *__edx) +{ + unsigned int __ext = __leaf & 0x80000000; + unsigned int __maxlevel = __get_cpuid_max (__ext, 0); + + if (__maxlevel == 0 || __maxlevel < __leaf) + return 0; + + __cpuid (__leaf, *__eax, *__ebx, *__ecx, *__edx); + return 1; +} + +/* Same as above, but sub-leaf can be specified. */ + +static __inline int +__get_cpuid_count (unsigned int __leaf, unsigned int __subleaf, + unsigned int *__eax, unsigned int *__ebx, + unsigned int *__ecx, unsigned int *__edx) +{ + unsigned int __ext = __leaf & 0x80000000; + unsigned int __maxlevel = __get_cpuid_max (__ext, 0); + + if (__maxlevel == 0 || __maxlevel < __leaf) + return 0; + + __cpuid_count (__leaf, __subleaf, *__eax, *__ebx, *__ecx, *__edx); + return 1; +} diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cross-stdarg.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cross-stdarg.h new file mode 100644 index 0000000000..67a2150a9d --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/cross-stdarg.h @@ -0,0 +1,72 @@ +/* Copyright (C) 2002-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef __CROSS_STDARG_H_INCLUDED +#define __CROSS_STDARG_H_INCLUDED + +/* Make sure that for non x64 targets cross builtins are defined. */ +#ifndef __x86_64__ +/* Call abi ms_abi. */ +#define __builtin_ms_va_list __builtin_va_list +#define __builtin_ms_va_copy __builtin_va_copy +#define __builtin_ms_va_start __builtin_va_start +#define __builtin_ms_va_end __builtin_va_end + +/* Call abi sysv_abi. */ +#define __builtin_sysv_va_list __builtin_va_list +#define __builtin_sysv_va_copy __builtin_va_copy +#define __builtin_sysv_va_start __builtin_va_start +#define __builtin_sysv_va_end __builtin_va_end +#endif + +#define __ms_va_copy(__d,__s) __builtin_ms_va_copy(__d,__s) +#define __ms_va_start(__v,__l) __builtin_ms_va_start(__v,__l) +#define __ms_va_arg(__v,__l) __builtin_va_arg(__v,__l) +#define __ms_va_end(__v) __builtin_ms_va_end(__v) + +#define __sysv_va_copy(__d,__s) __builtin_sysv_va_copy(__d,__s) +#define __sysv_va_start(__v,__l) __builtin_sysv_va_start(__v,__l) +#define __sysv_va_arg(__v,__l) __builtin_va_arg(__v,__l) +#define __sysv_va_end(__v) __builtin_sysv_va_end(__v) + +#ifndef __GNUC_SYSV_VA_LIST +#define __GNUC_SYSV_VA_LIST + typedef __builtin_sysv_va_list __gnuc_sysv_va_list; +#endif + +#ifndef _SYSV_VA_LIST_DEFINED +#define _SYSV_VA_LIST_DEFINED + typedef __gnuc_sysv_va_list sysv_va_list; +#endif + +#ifndef __GNUC_MS_VA_LIST +#define __GNUC_MS_VA_LIST + typedef __builtin_ms_va_list __gnuc_ms_va_list; +#endif + +#ifndef _MS_VA_LIST_DEFINED +#define _MS_VA_LIST_DEFINED + typedef __gnuc_ms_va_list ms_va_list; +#endif + +#endif /* __CROSS_STDARG_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/emmintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/emmintrin.h new file mode 100644 index 0000000000..828f4a07a9 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/emmintrin.h @@ -0,0 +1,1565 @@ +/* Copyright (C) 2003-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +/* Implemented from the specification included in the Intel C++ Compiler + User Guide and Reference, version 9.0. */ + +#ifndef _EMMINTRIN_H_INCLUDED +#define _EMMINTRIN_H_INCLUDED + +/* We need definitions from the SSE header files*/ +#include + +#ifndef __SSE2__ +#pragma GCC push_options +#pragma GCC target("sse2") +#define __DISABLE_SSE2__ +#endif /* __SSE2__ */ + +/* SSE2 */ +typedef double __v2df __attribute__ ((__vector_size__ (16))); +typedef long long __v2di __attribute__ ((__vector_size__ (16))); +typedef unsigned long long __v2du __attribute__ ((__vector_size__ (16))); +typedef int __v4si __attribute__ ((__vector_size__ (16))); +typedef unsigned int __v4su __attribute__ ((__vector_size__ (16))); +typedef short __v8hi __attribute__ ((__vector_size__ (16))); +typedef unsigned short __v8hu __attribute__ ((__vector_size__ (16))); +typedef char __v16qi __attribute__ ((__vector_size__ (16))); +typedef unsigned char __v16qu __attribute__ ((__vector_size__ (16))); + +/* The Intel API is flexible enough that we must allow aliasing with other + vector types, and their scalar components. */ +typedef long long __m128i __attribute__ ((__vector_size__ (16), __may_alias__)); +typedef double __m128d __attribute__ ((__vector_size__ (16), __may_alias__)); + +/* Unaligned version of the same types. */ +typedef long long __m128i_u __attribute__ ((__vector_size__ (16), __may_alias__, __aligned__ (1))); +typedef double __m128d_u __attribute__ ((__vector_size__ (16), __may_alias__, __aligned__ (1))); + +/* Create a selector for use with the SHUFPD instruction. */ +#define _MM_SHUFFLE2(fp1,fp0) \ + (((fp1) << 1) | (fp0)) + +/* Create a vector with element 0 as F and the rest zero. */ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set_sd (double __F) +{ + return __extension__ (__m128d){ __F, 0.0 }; +} + +/* Create a vector with both elements equal to F. */ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set1_pd (double __F) +{ + return __extension__ (__m128d){ __F, __F }; +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set_pd1 (double __F) +{ + return _mm_set1_pd (__F); +} + +/* Create a vector with the lower value X and upper value W. */ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set_pd (double __W, double __X) +{ + return __extension__ (__m128d){ __X, __W }; +} + +/* Create a vector with the lower value W and upper value X. */ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_setr_pd (double __W, double __X) +{ + return __extension__ (__m128d){ __W, __X }; +} + +/* Create an undefined vector. */ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_undefined_pd (void) +{ + __m128d __Y = __Y; + return __Y; +} + +/* Create a vector of zeros. */ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_setzero_pd (void) +{ + return __extension__ (__m128d){ 0.0, 0.0 }; +} + +/* Sets the low DPFP value of A from the low value of B. */ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_move_sd (__m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_movsd ((__v2df)__A, (__v2df)__B); +} + +/* Load two DPFP values from P. The address must be 16-byte aligned. */ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_load_pd (double const *__P) +{ + return *(__m128d *)__P; +} + +/* Load two DPFP values from P. The address need not be 16-byte aligned. */ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_loadu_pd (double const *__P) +{ + return *(__m128d_u *)__P; +} + +/* Create a vector with all two elements equal to *P. */ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_load1_pd (double const *__P) +{ + return _mm_set1_pd (*__P); +} + +/* Create a vector with element 0 as *P and the rest zero. */ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_load_sd (double const *__P) +{ + return _mm_set_sd (*__P); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_load_pd1 (double const *__P) +{ + return _mm_load1_pd (__P); +} + +/* Load two DPFP values in reverse order. The address must be aligned. */ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_loadr_pd (double const *__P) +{ + __m128d __tmp = _mm_load_pd (__P); + return __builtin_ia32_shufpd (__tmp, __tmp, _MM_SHUFFLE2 (0,1)); +} + +/* Store two DPFP values. The address must be 16-byte aligned. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_store_pd (double *__P, __m128d __A) +{ + *(__m128d *)__P = __A; +} + +/* Store two DPFP values. The address need not be 16-byte aligned. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_storeu_pd (double *__P, __m128d __A) +{ + *(__m128d_u *)__P = __A; +} + +/* Stores the lower DPFP value. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_store_sd (double *__P, __m128d __A) +{ + *__P = ((__v2df)__A)[0]; +} + +extern __inline double __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsd_f64 (__m128d __A) +{ + return ((__v2df)__A)[0]; +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_storel_pd (double *__P, __m128d __A) +{ + _mm_store_sd (__P, __A); +} + +/* Stores the upper DPFP value. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_storeh_pd (double *__P, __m128d __A) +{ + *__P = ((__v2df)__A)[1]; +} + +/* Store the lower DPFP value across two words. + The address must be 16-byte aligned. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_store1_pd (double *__P, __m128d __A) +{ + _mm_store_pd (__P, __builtin_ia32_shufpd (__A, __A, _MM_SHUFFLE2 (0,0))); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_store_pd1 (double *__P, __m128d __A) +{ + _mm_store1_pd (__P, __A); +} + +/* Store two DPFP values in reverse order. The address must be aligned. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_storer_pd (double *__P, __m128d __A) +{ + _mm_store_pd (__P, __builtin_ia32_shufpd (__A, __A, _MM_SHUFFLE2 (0,1))); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi128_si32 (__m128i __A) +{ + return __builtin_ia32_vec_ext_v4si ((__v4si)__A, 0); +} + +#ifdef __x86_64__ +/* Intel intrinsic. */ +extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi128_si64 (__m128i __A) +{ + return ((__v2di)__A)[0]; +} + +/* Microsoft intrinsic. */ +extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi128_si64x (__m128i __A) +{ + return ((__v2di)__A)[0]; +} +#endif + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_add_pd (__m128d __A, __m128d __B) +{ + return (__m128d) ((__v2df)__A + (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_add_sd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_addsd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sub_pd (__m128d __A, __m128d __B) +{ + return (__m128d) ((__v2df)__A - (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sub_sd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_subsd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mul_pd (__m128d __A, __m128d __B) +{ + return (__m128d) ((__v2df)__A * (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mul_sd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_mulsd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_div_pd (__m128d __A, __m128d __B) +{ + return (__m128d) ((__v2df)__A / (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_div_sd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_divsd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sqrt_pd (__m128d __A) +{ + return (__m128d)__builtin_ia32_sqrtpd ((__v2df)__A); +} + +/* Return pair {sqrt (B[0]), A[1]}. */ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sqrt_sd (__m128d __A, __m128d __B) +{ + __v2df __tmp = __builtin_ia32_movsd ((__v2df)__A, (__v2df)__B); + return (__m128d)__builtin_ia32_sqrtsd ((__v2df)__tmp); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_min_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_minpd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_min_sd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_minsd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_max_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_maxpd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_max_sd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_maxsd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_and_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_andpd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_andnot_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_andnpd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_or_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_orpd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_xor_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_xorpd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpeqpd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmplt_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpltpd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmple_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmplepd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpgtpd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpge_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpgepd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpneq_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpneqpd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpnlt_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpnltpd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpnle_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpnlepd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpngt_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpngtpd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpnge_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpngepd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpord_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpordpd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpunord_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpunordpd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_sd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpeqsd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmplt_sd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpltsd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmple_sd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmplesd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_sd (__m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_movsd ((__v2df) __A, + (__v2df) + __builtin_ia32_cmpltsd ((__v2df) __B, + (__v2df) + __A)); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpge_sd (__m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_movsd ((__v2df) __A, + (__v2df) + __builtin_ia32_cmplesd ((__v2df) __B, + (__v2df) + __A)); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpneq_sd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpneqsd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpnlt_sd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpnltsd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpnle_sd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpnlesd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpngt_sd (__m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_movsd ((__v2df) __A, + (__v2df) + __builtin_ia32_cmpnltsd ((__v2df) __B, + (__v2df) + __A)); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpnge_sd (__m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_movsd ((__v2df) __A, + (__v2df) + __builtin_ia32_cmpnlesd ((__v2df) __B, + (__v2df) + __A)); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpord_sd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpordsd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpunord_sd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpunordsd ((__v2df)__A, (__v2df)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comieq_sd (__m128d __A, __m128d __B) +{ + return __builtin_ia32_comisdeq ((__v2df)__A, (__v2df)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comilt_sd (__m128d __A, __m128d __B) +{ + return __builtin_ia32_comisdlt ((__v2df)__A, (__v2df)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comile_sd (__m128d __A, __m128d __B) +{ + return __builtin_ia32_comisdle ((__v2df)__A, (__v2df)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comigt_sd (__m128d __A, __m128d __B) +{ + return __builtin_ia32_comisdgt ((__v2df)__A, (__v2df)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comige_sd (__m128d __A, __m128d __B) +{ + return __builtin_ia32_comisdge ((__v2df)__A, (__v2df)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comineq_sd (__m128d __A, __m128d __B) +{ + return __builtin_ia32_comisdneq ((__v2df)__A, (__v2df)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_ucomieq_sd (__m128d __A, __m128d __B) +{ + return __builtin_ia32_ucomisdeq ((__v2df)__A, (__v2df)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_ucomilt_sd (__m128d __A, __m128d __B) +{ + return __builtin_ia32_ucomisdlt ((__v2df)__A, (__v2df)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_ucomile_sd (__m128d __A, __m128d __B) +{ + return __builtin_ia32_ucomisdle ((__v2df)__A, (__v2df)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_ucomigt_sd (__m128d __A, __m128d __B) +{ + return __builtin_ia32_ucomisdgt ((__v2df)__A, (__v2df)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_ucomige_sd (__m128d __A, __m128d __B) +{ + return __builtin_ia32_ucomisdge ((__v2df)__A, (__v2df)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_ucomineq_sd (__m128d __A, __m128d __B) +{ + return __builtin_ia32_ucomisdneq ((__v2df)__A, (__v2df)__B); +} + +/* Create a vector of Qi, where i is the element number. */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set_epi64x (long long __q1, long long __q0) +{ + return __extension__ (__m128i)(__v2di){ __q0, __q1 }; +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set_epi64 (__m64 __q1, __m64 __q0) +{ + return _mm_set_epi64x ((long long)__q1, (long long)__q0); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set_epi32 (int __q3, int __q2, int __q1, int __q0) +{ + return __extension__ (__m128i)(__v4si){ __q0, __q1, __q2, __q3 }; +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set_epi16 (short __q7, short __q6, short __q5, short __q4, + short __q3, short __q2, short __q1, short __q0) +{ + return __extension__ (__m128i)(__v8hi){ + __q0, __q1, __q2, __q3, __q4, __q5, __q6, __q7 }; +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set_epi8 (char __q15, char __q14, char __q13, char __q12, + char __q11, char __q10, char __q09, char __q08, + char __q07, char __q06, char __q05, char __q04, + char __q03, char __q02, char __q01, char __q00) +{ + return __extension__ (__m128i)(__v16qi){ + __q00, __q01, __q02, __q03, __q04, __q05, __q06, __q07, + __q08, __q09, __q10, __q11, __q12, __q13, __q14, __q15 + }; +} + +/* Set all of the elements of the vector to A. */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set1_epi64x (long long __A) +{ + return _mm_set_epi64x (__A, __A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set1_epi64 (__m64 __A) +{ + return _mm_set_epi64 (__A, __A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set1_epi32 (int __A) +{ + return _mm_set_epi32 (__A, __A, __A, __A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set1_epi16 (short __A) +{ + return _mm_set_epi16 (__A, __A, __A, __A, __A, __A, __A, __A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set1_epi8 (char __A) +{ + return _mm_set_epi8 (__A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A); +} + +/* Create a vector of Qi, where i is the element number. + The parameter order is reversed from the _mm_set_epi* functions. */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_setr_epi64 (__m64 __q0, __m64 __q1) +{ + return _mm_set_epi64 (__q1, __q0); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_setr_epi32 (int __q0, int __q1, int __q2, int __q3) +{ + return _mm_set_epi32 (__q3, __q2, __q1, __q0); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_setr_epi16 (short __q0, short __q1, short __q2, short __q3, + short __q4, short __q5, short __q6, short __q7) +{ + return _mm_set_epi16 (__q7, __q6, __q5, __q4, __q3, __q2, __q1, __q0); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_setr_epi8 (char __q00, char __q01, char __q02, char __q03, + char __q04, char __q05, char __q06, char __q07, + char __q08, char __q09, char __q10, char __q11, + char __q12, char __q13, char __q14, char __q15) +{ + return _mm_set_epi8 (__q15, __q14, __q13, __q12, __q11, __q10, __q09, __q08, + __q07, __q06, __q05, __q04, __q03, __q02, __q01, __q00); +} + +/* Create a vector with element 0 as *P and the rest zero. */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_load_si128 (__m128i const *__P) +{ + return *__P; +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_loadu_si128 (__m128i_u const *__P) +{ + return *__P; +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_loadl_epi64 (__m128i_u const *__P) +{ + return _mm_set_epi64 ((__m64)0LL, *(__m64_u *)__P); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_store_si128 (__m128i *__P, __m128i __B) +{ + *__P = __B; +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_storeu_si128 (__m128i_u *__P, __m128i __B) +{ + *__P = __B; +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_storel_epi64 (__m128i_u *__P, __m128i __B) +{ + *(__m64_u *)__P = (__m64) ((__v2di)__B)[0]; +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movepi64_pi64 (__m128i __B) +{ + return (__m64) ((__v2di)__B)[0]; +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movpi64_epi64 (__m64 __A) +{ + return _mm_set_epi64 ((__m64)0LL, __A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_move_epi64 (__m128i __A) +{ + return (__m128i)__builtin_ia32_movq128 ((__v2di) __A); +} + +/* Create an undefined vector. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_undefined_si128 (void) +{ + __m128i __Y = __Y; + return __Y; +} + +/* Create a vector of zeros. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_setzero_si128 (void) +{ + return __extension__ (__m128i)(__v4si){ 0, 0, 0, 0 }; +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepi32_pd (__m128i __A) +{ + return (__m128d)__builtin_ia32_cvtdq2pd ((__v4si) __A); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepi32_ps (__m128i __A) +{ + return (__m128)__builtin_ia32_cvtdq2ps ((__v4si) __A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtpd_epi32 (__m128d __A) +{ + return (__m128i)__builtin_ia32_cvtpd2dq ((__v2df) __A); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtpd_pi32 (__m128d __A) +{ + return (__m64)__builtin_ia32_cvtpd2pi ((__v2df) __A); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtpd_ps (__m128d __A) +{ + return (__m128)__builtin_ia32_cvtpd2ps ((__v2df) __A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttpd_epi32 (__m128d __A) +{ + return (__m128i)__builtin_ia32_cvttpd2dq ((__v2df) __A); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttpd_pi32 (__m128d __A) +{ + return (__m64)__builtin_ia32_cvttpd2pi ((__v2df) __A); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtpi32_pd (__m64 __A) +{ + return (__m128d)__builtin_ia32_cvtpi2pd ((__v2si) __A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtps_epi32 (__m128 __A) +{ + return (__m128i)__builtin_ia32_cvtps2dq ((__v4sf) __A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttps_epi32 (__m128 __A) +{ + return (__m128i)__builtin_ia32_cvttps2dq ((__v4sf) __A); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtps_pd (__m128 __A) +{ + return (__m128d)__builtin_ia32_cvtps2pd ((__v4sf) __A); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsd_si32 (__m128d __A) +{ + return __builtin_ia32_cvtsd2si ((__v2df) __A); +} + +#ifdef __x86_64__ +/* Intel intrinsic. */ +extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsd_si64 (__m128d __A) +{ + return __builtin_ia32_cvtsd2si64 ((__v2df) __A); +} + +/* Microsoft intrinsic. */ +extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsd_si64x (__m128d __A) +{ + return __builtin_ia32_cvtsd2si64 ((__v2df) __A); +} +#endif + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttsd_si32 (__m128d __A) +{ + return __builtin_ia32_cvttsd2si ((__v2df) __A); +} + +#ifdef __x86_64__ +/* Intel intrinsic. */ +extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttsd_si64 (__m128d __A) +{ + return __builtin_ia32_cvttsd2si64 ((__v2df) __A); +} + +/* Microsoft intrinsic. */ +extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttsd_si64x (__m128d __A) +{ + return __builtin_ia32_cvttsd2si64 ((__v2df) __A); +} +#endif + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsd_ss (__m128 __A, __m128d __B) +{ + return (__m128)__builtin_ia32_cvtsd2ss ((__v4sf) __A, (__v2df) __B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi32_sd (__m128d __A, int __B) +{ + return (__m128d)__builtin_ia32_cvtsi2sd ((__v2df) __A, __B); +} + +#ifdef __x86_64__ +/* Intel intrinsic. */ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi64_sd (__m128d __A, long long __B) +{ + return (__m128d)__builtin_ia32_cvtsi642sd ((__v2df) __A, __B); +} + +/* Microsoft intrinsic. */ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi64x_sd (__m128d __A, long long __B) +{ + return (__m128d)__builtin_ia32_cvtsi642sd ((__v2df) __A, __B); +} +#endif + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtss_sd (__m128d __A, __m128 __B) +{ + return (__m128d)__builtin_ia32_cvtss2sd ((__v2df) __A, (__v4sf)__B); +} + +#ifdef __OPTIMIZE__ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_shuffle_pd(__m128d __A, __m128d __B, const int __mask) +{ + return (__m128d)__builtin_ia32_shufpd ((__v2df)__A, (__v2df)__B, __mask); +} +#else +#define _mm_shuffle_pd(A, B, N) \ + ((__m128d)__builtin_ia32_shufpd ((__v2df)(__m128d)(A), \ + (__v2df)(__m128d)(B), (int)(N))) +#endif + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpackhi_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_unpckhpd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpacklo_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_unpcklpd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_loadh_pd (__m128d __A, double const *__B) +{ + return (__m128d)__builtin_ia32_loadhpd ((__v2df)__A, __B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_loadl_pd (__m128d __A, double const *__B) +{ + return (__m128d)__builtin_ia32_loadlpd ((__v2df)__A, __B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movemask_pd (__m128d __A) +{ + return __builtin_ia32_movmskpd ((__v2df)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_packs_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_packsswb128 ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_packs_epi32 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_packssdw128 ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_packus_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_packuswb128 ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpackhi_epi8 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_punpckhbw128 ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpackhi_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_punpckhwd128 ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpackhi_epi32 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_punpckhdq128 ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpackhi_epi64 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_punpckhqdq128 ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpacklo_epi8 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_punpcklbw128 ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpacklo_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_punpcklwd128 ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpacklo_epi32 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_punpckldq128 ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpacklo_epi64 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_punpcklqdq128 ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_add_epi8 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v16qu)__A + (__v16qu)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_add_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v8hu)__A + (__v8hu)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_add_epi32 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v4su)__A + (__v4su)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_add_epi64 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v2du)__A + (__v2du)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_adds_epi8 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_paddsb128 ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_adds_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_paddsw128 ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_adds_epu8 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_paddusb128 ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_adds_epu16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_paddusw128 ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sub_epi8 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v16qu)__A - (__v16qu)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sub_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v8hu)__A - (__v8hu)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sub_epi32 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v4su)__A - (__v4su)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sub_epi64 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v2du)__A - (__v2du)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_subs_epi8 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_psubsb128 ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_subs_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_psubsw128 ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_subs_epu8 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_psubusb128 ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_subs_epu16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_psubusw128 ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_madd_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_pmaddwd128 ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mulhi_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_pmulhw128 ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mullo_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v8hu)__A * (__v8hu)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mul_su32 (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pmuludq ((__v2si)__A, (__v2si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mul_epu32 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_pmuludq128 ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_slli_epi16 (__m128i __A, int __B) +{ + return (__m128i)__builtin_ia32_psllwi128 ((__v8hi)__A, __B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_slli_epi32 (__m128i __A, int __B) +{ + return (__m128i)__builtin_ia32_pslldi128 ((__v4si)__A, __B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_slli_epi64 (__m128i __A, int __B) +{ + return (__m128i)__builtin_ia32_psllqi128 ((__v2di)__A, __B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srai_epi16 (__m128i __A, int __B) +{ + return (__m128i)__builtin_ia32_psrawi128 ((__v8hi)__A, __B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srai_epi32 (__m128i __A, int __B) +{ + return (__m128i)__builtin_ia32_psradi128 ((__v4si)__A, __B); +} + +#ifdef __OPTIMIZE__ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_bsrli_si128 (__m128i __A, const int __N) +{ + return (__m128i)__builtin_ia32_psrldqi128 (__A, __N * 8); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_bslli_si128 (__m128i __A, const int __N) +{ + return (__m128i)__builtin_ia32_pslldqi128 (__A, __N * 8); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srli_si128 (__m128i __A, const int __N) +{ + return (__m128i)__builtin_ia32_psrldqi128 (__A, __N * 8); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_slli_si128 (__m128i __A, const int __N) +{ + return (__m128i)__builtin_ia32_pslldqi128 (__A, __N * 8); +} +#else +#define _mm_bsrli_si128(A, N) \ + ((__m128i)__builtin_ia32_psrldqi128 ((__m128i)(A), (int)(N) * 8)) +#define _mm_bslli_si128(A, N) \ + ((__m128i)__builtin_ia32_pslldqi128 ((__m128i)(A), (int)(N) * 8)) +#define _mm_srli_si128(A, N) \ + ((__m128i)__builtin_ia32_psrldqi128 ((__m128i)(A), (int)(N) * 8)) +#define _mm_slli_si128(A, N) \ + ((__m128i)__builtin_ia32_pslldqi128 ((__m128i)(A), (int)(N) * 8)) +#endif + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srli_epi16 (__m128i __A, int __B) +{ + return (__m128i)__builtin_ia32_psrlwi128 ((__v8hi)__A, __B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srli_epi32 (__m128i __A, int __B) +{ + return (__m128i)__builtin_ia32_psrldi128 ((__v4si)__A, __B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srli_epi64 (__m128i __A, int __B) +{ + return (__m128i)__builtin_ia32_psrlqi128 ((__v2di)__A, __B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sll_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_psllw128((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sll_epi32 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_pslld128((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sll_epi64 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_psllq128((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sra_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_psraw128 ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sra_epi32 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_psrad128 ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srl_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_psrlw128 ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srl_epi32 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_psrld128 ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srl_epi64 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_psrlq128 ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_and_si128 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v2du)__A & (__v2du)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_andnot_si128 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_pandn128 ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_or_si128 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v2du)__A | (__v2du)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_xor_si128 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v2du)__A ^ (__v2du)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_epi8 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v16qi)__A == (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v8hi)__A == (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_epi32 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v4si)__A == (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmplt_epi8 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v16qi)__A < (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmplt_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v8hi)__A < (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmplt_epi32 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v4si)__A < (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_epi8 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v16qi)__A > (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v8hi)__A > (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_epi32 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v4si)__A > (__v4si)__B); +} + +#ifdef __OPTIMIZE__ +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_extract_epi16 (__m128i const __A, int const __N) +{ + return (unsigned short) __builtin_ia32_vec_ext_v8hi ((__v8hi)__A, __N); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_insert_epi16 (__m128i const __A, int const __D, int const __N) +{ + return (__m128i) __builtin_ia32_vec_set_v8hi ((__v8hi)__A, __D, __N); +} +#else +#define _mm_extract_epi16(A, N) \ + ((int) (unsigned short) __builtin_ia32_vec_ext_v8hi ((__v8hi)(__m128i)(A), (int)(N))) +#define _mm_insert_epi16(A, D, N) \ + ((__m128i) __builtin_ia32_vec_set_v8hi ((__v8hi)(__m128i)(A), \ + (int)(D), (int)(N))) +#endif + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_max_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_pmaxsw128 ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_max_epu8 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_pmaxub128 ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_min_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_pminsw128 ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_min_epu8 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_pminub128 ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movemask_epi8 (__m128i __A) +{ + return __builtin_ia32_pmovmskb128 ((__v16qi)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mulhi_epu16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_pmulhuw128 ((__v8hi)__A, (__v8hi)__B); +} + +#ifdef __OPTIMIZE__ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_shufflehi_epi16 (__m128i __A, const int __mask) +{ + return (__m128i)__builtin_ia32_pshufhw ((__v8hi)__A, __mask); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_shufflelo_epi16 (__m128i __A, const int __mask) +{ + return (__m128i)__builtin_ia32_pshuflw ((__v8hi)__A, __mask); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_shuffle_epi32 (__m128i __A, const int __mask) +{ + return (__m128i)__builtin_ia32_pshufd ((__v4si)__A, __mask); +} +#else +#define _mm_shufflehi_epi16(A, N) \ + ((__m128i)__builtin_ia32_pshufhw ((__v8hi)(__m128i)(A), (int)(N))) +#define _mm_shufflelo_epi16(A, N) \ + ((__m128i)__builtin_ia32_pshuflw ((__v8hi)(__m128i)(A), (int)(N))) +#define _mm_shuffle_epi32(A, N) \ + ((__m128i)__builtin_ia32_pshufd ((__v4si)(__m128i)(A), (int)(N))) +#endif + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskmoveu_si128 (__m128i __A, __m128i __B, char *__C) +{ + __builtin_ia32_maskmovdqu ((__v16qi)__A, (__v16qi)__B, __C); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_avg_epu8 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_pavgb128 ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_avg_epu16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_pavgw128 ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sad_epu8 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_psadbw128 ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_stream_si32 (int *__A, int __B) +{ + __builtin_ia32_movnti (__A, __B); +} + +#ifdef __x86_64__ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_stream_si64 (long long int *__A, long long int __B) +{ + __builtin_ia32_movnti64 (__A, __B); +} +#endif + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_stream_si128 (__m128i *__A, __m128i __B) +{ + __builtin_ia32_movntdq ((__v2di *)__A, (__v2di)__B); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_stream_pd (double *__A, __m128d __B) +{ + __builtin_ia32_movntpd (__A, (__v2df)__B); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_clflush (void const *__A) +{ + __builtin_ia32_clflush (__A); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_lfence (void) +{ + __builtin_ia32_lfence (); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mfence (void) +{ + __builtin_ia32_mfence (); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi32_si128 (int __A) +{ + return _mm_set_epi32 (0, 0, 0, __A); +} + +#ifdef __x86_64__ +/* Intel intrinsic. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi64_si128 (long long __A) +{ + return _mm_set_epi64x (0, __A); +} + +/* Microsoft intrinsic. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi64x_si128 (long long __A) +{ + return _mm_set_epi64x (0, __A); +} +#endif + +/* Casts between various SP, DP, INT vector types. Note that these do no + conversion of values, they just change the type. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_castpd_ps(__m128d __A) +{ + return (__m128) __A; +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_castpd_si128(__m128d __A) +{ + return (__m128i) __A; +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_castps_pd(__m128 __A) +{ + return (__m128d) __A; +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_castps_si128(__m128 __A) +{ + return (__m128i) __A; +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_castsi128_ps(__m128i __A) +{ + return (__m128) __A; +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_castsi128_pd(__m128i __A) +{ + return (__m128d) __A; +} + +#ifdef __DISABLE_SSE2__ +#undef __DISABLE_SSE2__ +#pragma GCC pop_options +#endif /* __DISABLE_SSE2__ */ + +#endif /* _EMMINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/f16cintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/f16cintrin.h new file mode 100644 index 0000000000..488e97d674 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/f16cintrin.h @@ -0,0 +1,98 @@ +/* Copyright (C) 2011-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#if !defined _X86INTRIN_H_INCLUDED && !defined _IMMINTRIN_H_INCLUDED +# error "Never use directly; include or instead." +#endif + +#ifndef _F16CINTRIN_H_INCLUDED +#define _F16CINTRIN_H_INCLUDED + +#ifndef __F16C__ +#pragma GCC push_options +#pragma GCC target("f16c") +#define __DISABLE_F16C__ +#endif /* __F16C__ */ + +extern __inline float __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_cvtsh_ss (unsigned short __S) +{ + __v8hi __H = __extension__ (__v8hi){ (short) __S, 0, 0, 0, 0, 0, 0, 0 }; + __v4sf __A = __builtin_ia32_vcvtph2ps (__H); + return __builtin_ia32_vec_ext_v4sf (__A, 0); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtph_ps (__m128i __A) +{ + return (__m128) __builtin_ia32_vcvtph2ps ((__v8hi) __A); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtph_ps (__m128i __A) +{ + return (__m256) __builtin_ia32_vcvtph2ps256 ((__v8hi) __A); +} + +#ifdef __OPTIMIZE__ +extern __inline unsigned short __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_cvtss_sh (float __F, const int __I) +{ + __v4sf __A = __extension__ (__v4sf){ __F, 0, 0, 0 }; + __v8hi __H = __builtin_ia32_vcvtps2ph (__A, __I); + return (unsigned short) __builtin_ia32_vec_ext_v8hi (__H, 0); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtps_ph (__m128 __A, const int __I) +{ + return (__m128i) __builtin_ia32_vcvtps2ph ((__v4sf) __A, __I); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtps_ph (__m256 __A, const int __I) +{ + return (__m128i) __builtin_ia32_vcvtps2ph256 ((__v8sf) __A, __I); +} +#else +#define _cvtss_sh(__F, __I) \ + (__extension__ \ + ({ \ + __v4sf __A = __extension__ (__v4sf){ __F, 0, 0, 0 }; \ + __v8hi __H = __builtin_ia32_vcvtps2ph (__A, __I); \ + (unsigned short) __builtin_ia32_vec_ext_v8hi (__H, 0); \ + })) + +#define _mm_cvtps_ph(A, I) \ + ((__m128i) __builtin_ia32_vcvtps2ph ((__v4sf)(__m128) A, (int) (I))) + +#define _mm256_cvtps_ph(A, I) \ + ((__m128i) __builtin_ia32_vcvtps2ph256 ((__v8sf)(__m256) A, (int) (I))) +#endif /* __OPTIMIZE */ + +#ifdef __DISABLE_F16C__ +#undef __DISABLE_F16C__ +#pragma GCC pop_options +#endif /* __DISABLE_F16C__ */ + +#endif /* _F16CINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/float.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/float.h new file mode 100644 index 0000000000..658017fd63 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/float.h @@ -0,0 +1,506 @@ +/* Copyright (C) 2002-2017 Free Software Foundation, Inc. + +This file is part of GCC. + +GCC is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3, or (at your option) +any later version. + +GCC is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +Under Section 7 of GPL version 3, you are granted additional +permissions described in the GCC Runtime Library Exception, version +3.1, as published by the Free Software Foundation. + +You should have received a copy of the GNU General Public License and +a copy of the GCC Runtime Library Exception along with this program; +see the files COPYING3 and COPYING.RUNTIME respectively. If not, see +. */ + +/* + * ISO C Standard: 5.2.4.2.2 Characteristics of floating types + */ + +#ifndef _FLOAT_H___ +#define _FLOAT_H___ + +/* Radix of exponent representation, b. */ +#undef FLT_RADIX +#define FLT_RADIX __FLT_RADIX__ + +/* Number of base-FLT_RADIX digits in the significand, p. */ +#undef FLT_MANT_DIG +#undef DBL_MANT_DIG +#undef LDBL_MANT_DIG +#define FLT_MANT_DIG __FLT_MANT_DIG__ +#define DBL_MANT_DIG __DBL_MANT_DIG__ +#define LDBL_MANT_DIG __LDBL_MANT_DIG__ + +/* Number of decimal digits, q, such that any floating-point number with q + decimal digits can be rounded into a floating-point number with p radix b + digits and back again without change to the q decimal digits, + + p * log10(b) if b is a power of 10 + floor((p - 1) * log10(b)) otherwise +*/ +#undef FLT_DIG +#undef DBL_DIG +#undef LDBL_DIG +#define FLT_DIG __FLT_DIG__ +#define DBL_DIG __DBL_DIG__ +#define LDBL_DIG __LDBL_DIG__ + +/* Minimum int x such that FLT_RADIX**(x-1) is a normalized float, emin */ +#undef FLT_MIN_EXP +#undef DBL_MIN_EXP +#undef LDBL_MIN_EXP +#define FLT_MIN_EXP __FLT_MIN_EXP__ +#define DBL_MIN_EXP __DBL_MIN_EXP__ +#define LDBL_MIN_EXP __LDBL_MIN_EXP__ + +/* Minimum negative integer such that 10 raised to that power is in the + range of normalized floating-point numbers, + + ceil(log10(b) * (emin - 1)) +*/ +#undef FLT_MIN_10_EXP +#undef DBL_MIN_10_EXP +#undef LDBL_MIN_10_EXP +#define FLT_MIN_10_EXP __FLT_MIN_10_EXP__ +#define DBL_MIN_10_EXP __DBL_MIN_10_EXP__ +#define LDBL_MIN_10_EXP __LDBL_MIN_10_EXP__ + +/* Maximum int x such that FLT_RADIX**(x-1) is a representable float, emax. */ +#undef FLT_MAX_EXP +#undef DBL_MAX_EXP +#undef LDBL_MAX_EXP +#define FLT_MAX_EXP __FLT_MAX_EXP__ +#define DBL_MAX_EXP __DBL_MAX_EXP__ +#define LDBL_MAX_EXP __LDBL_MAX_EXP__ + +/* Maximum integer such that 10 raised to that power is in the range of + representable finite floating-point numbers, + + floor(log10((1 - b**-p) * b**emax)) +*/ +#undef FLT_MAX_10_EXP +#undef DBL_MAX_10_EXP +#undef LDBL_MAX_10_EXP +#define FLT_MAX_10_EXP __FLT_MAX_10_EXP__ +#define DBL_MAX_10_EXP __DBL_MAX_10_EXP__ +#define LDBL_MAX_10_EXP __LDBL_MAX_10_EXP__ + +/* Maximum representable finite floating-point number, + + (1 - b**-p) * b**emax +*/ +#undef FLT_MAX +#undef DBL_MAX +#undef LDBL_MAX +#define FLT_MAX __FLT_MAX__ +#define DBL_MAX __DBL_MAX__ +#define LDBL_MAX __LDBL_MAX__ + +/* The difference between 1 and the least value greater than 1 that is + representable in the given floating point type, b**1-p. */ +#undef FLT_EPSILON +#undef DBL_EPSILON +#undef LDBL_EPSILON +#define FLT_EPSILON __FLT_EPSILON__ +#define DBL_EPSILON __DBL_EPSILON__ +#define LDBL_EPSILON __LDBL_EPSILON__ + +/* Minimum normalized positive floating-point number, b**(emin - 1). */ +#undef FLT_MIN +#undef DBL_MIN +#undef LDBL_MIN +#define FLT_MIN __FLT_MIN__ +#define DBL_MIN __DBL_MIN__ +#define LDBL_MIN __LDBL_MIN__ + +/* Addition rounds to 0: zero, 1: nearest, 2: +inf, 3: -inf, -1: unknown. */ +/* ??? This is supposed to change with calls to fesetround in . */ +#undef FLT_ROUNDS +#define FLT_ROUNDS 1 + +#if (defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) \ + || (defined (__cplusplus) && __cplusplus >= 201103L) +/* The floating-point expression evaluation method. The precise + definitions of these values are generalised to include support for + the interchange and extended types defined in ISO/IEC TS 18661-3. + Prior to this (for C99/C11) the definitions were: + + -1 indeterminate + 0 evaluate all operations and constants just to the range and + precision of the type + 1 evaluate operations and constants of type float and double + to the range and precision of the double type, evaluate + long double operations and constants to the range and + precision of the long double type + 2 evaluate all operations and constants to the range and + precision of the long double type + + The TS 18661-3 definitions are: + + -1 indeterminate + 0 evaluate all operations and constants, whose semantic type has + at most the range and precision of float, to the range and + precision of float; evaluate all other operations and constants + to the range and precision of the semantic type. + 1 evaluate all operations and constants, whose semantic type has + at most the range and precision of double, to the range and + precision of double; evaluate all other operations and constants + to the range and precision of the semantic type. + 2 evaluate all operations and constants, whose semantic type has + at most the range and precision of long double, to the range and + precision of long double; evaluate all other operations and + constants to the range and precision of the semantic type. + N where _FloatN is a supported interchange floating type + evaluate all operations and constants, whose semantic type has + at most the range and precision of the _FloatN type, to the + range and precision of the _FloatN type; evaluate all other + operations and constants to the range and precision of the + semantic type. + N + 1, where _FloatNx is a supported extended floating type + evaluate operations and constants, whose semantic type has at + most the range and precision of the _FloatNx type, to the range + and precision of the _FloatNx type; evaluate all other + operations and constants to the range and precision of the + semantic type. + + The compiler predefines two macros: + + __FLT_EVAL_METHOD__ + Which, depending on the value given for + -fpermitted-flt-eval-methods, may be limited to only those values + for FLT_EVAL_METHOD defined in C99/C11. + + __FLT_EVAL_METHOD_TS_18661_3__ + Which always permits the values for FLT_EVAL_METHOD defined in + ISO/IEC TS 18661-3. + + Here we want to use __FLT_EVAL_METHOD__, unless + __STDC_WANT_IEC_60559_TYPES_EXT__ is defined, in which case the user + is specifically asking for the ISO/IEC TS 18661-3 types, so we use + __FLT_EVAL_METHOD_TS_18661_3__. + + ??? This ought to change with the setting of the fp control word; + the value provided by the compiler assumes the widest setting. */ +#undef FLT_EVAL_METHOD +#ifdef __STDC_WANT_IEC_60559_TYPES_EXT__ +#define FLT_EVAL_METHOD __FLT_EVAL_METHOD_TS_18661_3__ +#else +#define FLT_EVAL_METHOD __FLT_EVAL_METHOD__ +#endif + +/* Number of decimal digits, n, such that any floating-point number in the + widest supported floating type with pmax radix b digits can be rounded + to a floating-point number with n decimal digits and back again without + change to the value, + + pmax * log10(b) if b is a power of 10 + ceil(1 + pmax * log10(b)) otherwise +*/ +#undef DECIMAL_DIG +#define DECIMAL_DIG __DECIMAL_DIG__ + +#endif /* C99 */ + +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +/* Versions of DECIMAL_DIG for each floating-point type. */ +#undef FLT_DECIMAL_DIG +#undef DBL_DECIMAL_DIG +#undef LDBL_DECIMAL_DIG +#define FLT_DECIMAL_DIG __FLT_DECIMAL_DIG__ +#define DBL_DECIMAL_DIG __DBL_DECIMAL_DIG__ +#define LDBL_DECIMAL_DIG __LDBL_DECIMAL_DIG__ + +/* Whether types support subnormal numbers. */ +#undef FLT_HAS_SUBNORM +#undef DBL_HAS_SUBNORM +#undef LDBL_HAS_SUBNORM +#define FLT_HAS_SUBNORM __FLT_HAS_DENORM__ +#define DBL_HAS_SUBNORM __DBL_HAS_DENORM__ +#define LDBL_HAS_SUBNORM __LDBL_HAS_DENORM__ + +/* Minimum positive values, including subnormals. */ +#undef FLT_TRUE_MIN +#undef DBL_TRUE_MIN +#undef LDBL_TRUE_MIN +#define FLT_TRUE_MIN __FLT_DENORM_MIN__ +#define DBL_TRUE_MIN __DBL_DENORM_MIN__ +#define LDBL_TRUE_MIN __LDBL_DENORM_MIN__ + +#endif /* C11 */ + +#ifdef __STDC_WANT_IEC_60559_BFP_EXT__ +/* Number of decimal digits for which conversions between decimal + character strings and binary formats, in both directions, are + correctly rounded. */ +#define CR_DECIMAL_DIG __UINTMAX_MAX__ +#endif + +#ifdef __STDC_WANT_IEC_60559_TYPES_EXT__ +/* Constants for _FloatN and _FloatNx types from TS 18661-3. See + comments above for their semantics. */ + +#ifdef __FLT16_MANT_DIG__ +#undef FLT16_MANT_DIG +#define FLT16_MANT_DIG __FLT16_MANT_DIG__ +#undef FLT16_DIG +#define FLT16_DIG __FLT16_DIG__ +#undef FLT16_MIN_EXP +#define FLT16_MIN_EXP __FLT16_MIN_EXP__ +#undef FLT16_MIN_10_EXP +#define FLT16_MIN_10_EXP __FLT16_MIN_10_EXP__ +#undef FLT16_MAX_EXP +#define FLT16_MAX_EXP __FLT16_MAX_EXP__ +#undef FLT16_MAX_10_EXP +#define FLT16_MAX_10_EXP __FLT16_MAX_10_EXP__ +#undef FLT16_MAX +#define FLT16_MAX __FLT16_MAX__ +#undef FLT16_EPSILON +#define FLT16_EPSILON __FLT16_EPSILON__ +#undef FLT16_MIN +#define FLT16_MIN __FLT16_MIN__ +#undef FLT16_DECIMAL_DIG +#define FLT16_DECIMAL_DIG __FLT16_DECIMAL_DIG__ +#undef FLT16_TRUE_MIN +#define FLT16_TRUE_MIN __FLT16_DENORM_MIN__ +#endif /* __FLT16_MANT_DIG__. */ + +#ifdef __FLT32_MANT_DIG__ +#undef FLT32_MANT_DIG +#define FLT32_MANT_DIG __FLT32_MANT_DIG__ +#undef FLT32_DIG +#define FLT32_DIG __FLT32_DIG__ +#undef FLT32_MIN_EXP +#define FLT32_MIN_EXP __FLT32_MIN_EXP__ +#undef FLT32_MIN_10_EXP +#define FLT32_MIN_10_EXP __FLT32_MIN_10_EXP__ +#undef FLT32_MAX_EXP +#define FLT32_MAX_EXP __FLT32_MAX_EXP__ +#undef FLT32_MAX_10_EXP +#define FLT32_MAX_10_EXP __FLT32_MAX_10_EXP__ +#undef FLT32_MAX +#define FLT32_MAX __FLT32_MAX__ +#undef FLT32_EPSILON +#define FLT32_EPSILON __FLT32_EPSILON__ +#undef FLT32_MIN +#define FLT32_MIN __FLT32_MIN__ +#undef FLT32_DECIMAL_DIG +#define FLT32_DECIMAL_DIG __FLT32_DECIMAL_DIG__ +#undef FLT32_TRUE_MIN +#define FLT32_TRUE_MIN __FLT32_DENORM_MIN__ +#endif /* __FLT32_MANT_DIG__. */ + +#ifdef __FLT64_MANT_DIG__ +#undef FLT64_MANT_DIG +#define FLT64_MANT_DIG __FLT64_MANT_DIG__ +#undef FLT64_DIG +#define FLT64_DIG __FLT64_DIG__ +#undef FLT64_MIN_EXP +#define FLT64_MIN_EXP __FLT64_MIN_EXP__ +#undef FLT64_MIN_10_EXP +#define FLT64_MIN_10_EXP __FLT64_MIN_10_EXP__ +#undef FLT64_MAX_EXP +#define FLT64_MAX_EXP __FLT64_MAX_EXP__ +#undef FLT64_MAX_10_EXP +#define FLT64_MAX_10_EXP __FLT64_MAX_10_EXP__ +#undef FLT64_MAX +#define FLT64_MAX __FLT64_MAX__ +#undef FLT64_EPSILON +#define FLT64_EPSILON __FLT64_EPSILON__ +#undef FLT64_MIN +#define FLT64_MIN __FLT64_MIN__ +#undef FLT64_DECIMAL_DIG +#define FLT64_DECIMAL_DIG __FLT64_DECIMAL_DIG__ +#undef FLT64_TRUE_MIN +#define FLT64_TRUE_MIN __FLT64_DENORM_MIN__ +#endif /* __FLT64_MANT_DIG__. */ + +#ifdef __FLT128_MANT_DIG__ +#undef FLT128_MANT_DIG +#define FLT128_MANT_DIG __FLT128_MANT_DIG__ +#undef FLT128_DIG +#define FLT128_DIG __FLT128_DIG__ +#undef FLT128_MIN_EXP +#define FLT128_MIN_EXP __FLT128_MIN_EXP__ +#undef FLT128_MIN_10_EXP +#define FLT128_MIN_10_EXP __FLT128_MIN_10_EXP__ +#undef FLT128_MAX_EXP +#define FLT128_MAX_EXP __FLT128_MAX_EXP__ +#undef FLT128_MAX_10_EXP +#define FLT128_MAX_10_EXP __FLT128_MAX_10_EXP__ +#undef FLT128_MAX +#define FLT128_MAX __FLT128_MAX__ +#undef FLT128_EPSILON +#define FLT128_EPSILON __FLT128_EPSILON__ +#undef FLT128_MIN +#define FLT128_MIN __FLT128_MIN__ +#undef FLT128_DECIMAL_DIG +#define FLT128_DECIMAL_DIG __FLT128_DECIMAL_DIG__ +#undef FLT128_TRUE_MIN +#define FLT128_TRUE_MIN __FLT128_DENORM_MIN__ +#endif /* __FLT128_MANT_DIG__. */ + +#ifdef __FLT32X_MANT_DIG__ +#undef FLT32X_MANT_DIG +#define FLT32X_MANT_DIG __FLT32X_MANT_DIG__ +#undef FLT32X_DIG +#define FLT32X_DIG __FLT32X_DIG__ +#undef FLT32X_MIN_EXP +#define FLT32X_MIN_EXP __FLT32X_MIN_EXP__ +#undef FLT32X_MIN_10_EXP +#define FLT32X_MIN_10_EXP __FLT32X_MIN_10_EXP__ +#undef FLT32X_MAX_EXP +#define FLT32X_MAX_EXP __FLT32X_MAX_EXP__ +#undef FLT32X_MAX_10_EXP +#define FLT32X_MAX_10_EXP __FLT32X_MAX_10_EXP__ +#undef FLT32X_MAX +#define FLT32X_MAX __FLT32X_MAX__ +#undef FLT32X_EPSILON +#define FLT32X_EPSILON __FLT32X_EPSILON__ +#undef FLT32X_MIN +#define FLT32X_MIN __FLT32X_MIN__ +#undef FLT32X_DECIMAL_DIG +#define FLT32X_DECIMAL_DIG __FLT32X_DECIMAL_DIG__ +#undef FLT32X_TRUE_MIN +#define FLT32X_TRUE_MIN __FLT32X_DENORM_MIN__ +#endif /* __FLT32X_MANT_DIG__. */ + +#ifdef __FLT64X_MANT_DIG__ +#undef FLT64X_MANT_DIG +#define FLT64X_MANT_DIG __FLT64X_MANT_DIG__ +#undef FLT64X_DIG +#define FLT64X_DIG __FLT64X_DIG__ +#undef FLT64X_MIN_EXP +#define FLT64X_MIN_EXP __FLT64X_MIN_EXP__ +#undef FLT64X_MIN_10_EXP +#define FLT64X_MIN_10_EXP __FLT64X_MIN_10_EXP__ +#undef FLT64X_MAX_EXP +#define FLT64X_MAX_EXP __FLT64X_MAX_EXP__ +#undef FLT64X_MAX_10_EXP +#define FLT64X_MAX_10_EXP __FLT64X_MAX_10_EXP__ +#undef FLT64X_MAX +#define FLT64X_MAX __FLT64X_MAX__ +#undef FLT64X_EPSILON +#define FLT64X_EPSILON __FLT64X_EPSILON__ +#undef FLT64X_MIN +#define FLT64X_MIN __FLT64X_MIN__ +#undef FLT64X_DECIMAL_DIG +#define FLT64X_DECIMAL_DIG __FLT64X_DECIMAL_DIG__ +#undef FLT64X_TRUE_MIN +#define FLT64X_TRUE_MIN __FLT64X_DENORM_MIN__ +#endif /* __FLT64X_MANT_DIG__. */ + +#ifdef __FLT128X_MANT_DIG__ +#undef FLT128X_MANT_DIG +#define FLT128X_MANT_DIG __FLT128X_MANT_DIG__ +#undef FLT128X_DIG +#define FLT128X_DIG __FLT128X_DIG__ +#undef FLT128X_MIN_EXP +#define FLT128X_MIN_EXP __FLT128X_MIN_EXP__ +#undef FLT128X_MIN_10_EXP +#define FLT128X_MIN_10_EXP __FLT128X_MIN_10_EXP__ +#undef FLT128X_MAX_EXP +#define FLT128X_MAX_EXP __FLT128X_MAX_EXP__ +#undef FLT128X_MAX_10_EXP +#define FLT128X_MAX_10_EXP __FLT128X_MAX_10_EXP__ +#undef FLT128X_MAX +#define FLT128X_MAX __FLT128X_MAX__ +#undef FLT128X_EPSILON +#define FLT128X_EPSILON __FLT128X_EPSILON__ +#undef FLT128X_MIN +#define FLT128X_MIN __FLT128X_MIN__ +#undef FLT128X_DECIMAL_DIG +#define FLT128X_DECIMAL_DIG __FLT128X_DECIMAL_DIG__ +#undef FLT128X_TRUE_MIN +#define FLT128X_TRUE_MIN __FLT128X_DENORM_MIN__ +#endif /* __FLT128X_MANT_DIG__. */ + +#endif /* __STDC_WANT_IEC_60559_TYPES_EXT__. */ + +#ifdef __STDC_WANT_DEC_FP__ +/* Draft Technical Report 24732, extension for decimal floating-point + arithmetic: Characteristic of decimal floating types . */ + +/* Number of base-FLT_RADIX digits in the significand, p. */ +#undef DEC32_MANT_DIG +#undef DEC64_MANT_DIG +#undef DEC128_MANT_DIG +#define DEC32_MANT_DIG __DEC32_MANT_DIG__ +#define DEC64_MANT_DIG __DEC64_MANT_DIG__ +#define DEC128_MANT_DIG __DEC128_MANT_DIG__ + +/* Minimum exponent. */ +#undef DEC32_MIN_EXP +#undef DEC64_MIN_EXP +#undef DEC128_MIN_EXP +#define DEC32_MIN_EXP __DEC32_MIN_EXP__ +#define DEC64_MIN_EXP __DEC64_MIN_EXP__ +#define DEC128_MIN_EXP __DEC128_MIN_EXP__ + +/* Maximum exponent. */ +#undef DEC32_MAX_EXP +#undef DEC64_MAX_EXP +#undef DEC128_MAX_EXP +#define DEC32_MAX_EXP __DEC32_MAX_EXP__ +#define DEC64_MAX_EXP __DEC64_MAX_EXP__ +#define DEC128_MAX_EXP __DEC128_MAX_EXP__ + +/* Maximum representable finite decimal floating-point number + (there are 6, 15, and 33 9s after the decimal points respectively). */ +#undef DEC32_MAX +#undef DEC64_MAX +#undef DEC128_MAX +#define DEC32_MAX __DEC32_MAX__ +#define DEC64_MAX __DEC64_MAX__ +#define DEC128_MAX __DEC128_MAX__ + +/* The difference between 1 and the least value greater than 1 that is + representable in the given floating point type. */ +#undef DEC32_EPSILON +#undef DEC64_EPSILON +#undef DEC128_EPSILON +#define DEC32_EPSILON __DEC32_EPSILON__ +#define DEC64_EPSILON __DEC64_EPSILON__ +#define DEC128_EPSILON __DEC128_EPSILON__ + +/* Minimum normalized positive floating-point number. */ +#undef DEC32_MIN +#undef DEC64_MIN +#undef DEC128_MIN +#define DEC32_MIN __DEC32_MIN__ +#define DEC64_MIN __DEC64_MIN__ +#define DEC128_MIN __DEC128_MIN__ + +/* Minimum subnormal positive floating-point number. */ +#undef DEC32_SUBNORMAL_MIN +#undef DEC64_SUBNORMAL_MIN +#undef DEC128_SUBNORMAL_MIN +#define DEC32_SUBNORMAL_MIN __DEC32_SUBNORMAL_MIN__ +#define DEC64_SUBNORMAL_MIN __DEC64_SUBNORMAL_MIN__ +#define DEC128_SUBNORMAL_MIN __DEC128_SUBNORMAL_MIN__ + +/* The floating-point expression evaluation method. + -1 indeterminate + 0 evaluate all operations and constants just to the range and + precision of the type + 1 evaluate operations and constants of type _Decimal32 + and _Decimal64 to the range and precision of the _Decimal64 + type, evaluate _Decimal128 operations and constants to the + range and precision of the _Decimal128 type; + 2 evaluate all operations and constants to the range and + precision of the _Decimal128 type. */ + +#undef DEC_EVAL_METHOD +#define DEC_EVAL_METHOD __DEC_EVAL_METHOD__ + +#endif /* __STDC_WANT_DEC_FP__ */ + +#endif /* _FLOAT_H___ */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/fma4intrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/fma4intrin.h new file mode 100644 index 0000000000..7230c9c9ae --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/fma4intrin.h @@ -0,0 +1,241 @@ +/* Copyright (C) 2007-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _X86INTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _FMA4INTRIN_H_INCLUDED +#define _FMA4INTRIN_H_INCLUDED + +/* We need definitions from the SSE4A, SSE3, SSE2 and SSE header files. */ +#include + +#ifndef __FMA4__ +#pragma GCC push_options +#pragma GCC target("fma4") +#define __DISABLE_FMA4__ +#endif /* __FMA4__ */ + +/* 128b Floating point multiply/add type instructions. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_macc_ps (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddps ((__v4sf)__A, (__v4sf)__B, (__v4sf)__C); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_macc_pd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddpd ((__v2df)__A, (__v2df)__B, (__v2df)__C); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_macc_ss (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddss ((__v4sf)__A, (__v4sf)__B, (__v4sf)__C); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_macc_sd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddsd ((__v2df)__A, (__v2df)__B, (__v2df)__C); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_msub_ps (__m128 __A, __m128 __B, __m128 __C) + +{ + return (__m128) __builtin_ia32_vfmaddps ((__v4sf)__A, (__v4sf)__B, -(__v4sf)__C); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_msub_pd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddpd ((__v2df)__A, (__v2df)__B, -(__v2df)__C); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_msub_ss (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddss ((__v4sf)__A, (__v4sf)__B, -(__v4sf)__C); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_msub_sd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddsd ((__v2df)__A, (__v2df)__B, -(__v2df)__C); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_nmacc_ps (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddps (-(__v4sf)__A, (__v4sf)__B, (__v4sf)__C); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_nmacc_pd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddpd (-(__v2df)__A, (__v2df)__B, (__v2df)__C); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_nmacc_ss (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddss (-(__v4sf)__A, (__v4sf)__B, (__v4sf)__C); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_nmacc_sd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddsd (-(__v2df)__A, (__v2df)__B, (__v2df)__C); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_nmsub_ps (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddps (-(__v4sf)__A, (__v4sf)__B, -(__v4sf)__C); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_nmsub_pd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddpd (-(__v2df)__A, (__v2df)__B, -(__v2df)__C); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_nmsub_ss (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddss (-(__v4sf)__A, (__v4sf)__B, -(__v4sf)__C); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_nmsub_sd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddsd (-(__v2df)__A, (__v2df)__B, -(__v2df)__C); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maddsub_ps (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddsubps ((__v4sf)__A, (__v4sf)__B, (__v4sf)__C); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maddsub_pd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddsubpd ((__v2df)__A, (__v2df)__B, (__v2df)__C); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_msubadd_ps (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddsubps ((__v4sf)__A, (__v4sf)__B, -(__v4sf)__C); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_msubadd_pd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddsubpd ((__v2df)__A, (__v2df)__B, -(__v2df)__C); +} + +/* 256b Floating point multiply/add type instructions. */ +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_macc_ps (__m256 __A, __m256 __B, __m256 __C) +{ + return (__m256) __builtin_ia32_vfmaddps256 ((__v8sf)__A, (__v8sf)__B, (__v8sf)__C); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_macc_pd (__m256d __A, __m256d __B, __m256d __C) +{ + return (__m256d) __builtin_ia32_vfmaddpd256 ((__v4df)__A, (__v4df)__B, (__v4df)__C); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_msub_ps (__m256 __A, __m256 __B, __m256 __C) + +{ + return (__m256) __builtin_ia32_vfmaddps256 ((__v8sf)__A, (__v8sf)__B, -(__v8sf)__C); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_msub_pd (__m256d __A, __m256d __B, __m256d __C) +{ + return (__m256d) __builtin_ia32_vfmaddpd256 ((__v4df)__A, (__v4df)__B, -(__v4df)__C); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_nmacc_ps (__m256 __A, __m256 __B, __m256 __C) +{ + return (__m256) __builtin_ia32_vfmaddps256 (-(__v8sf)__A, (__v8sf)__B, (__v8sf)__C); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_nmacc_pd (__m256d __A, __m256d __B, __m256d __C) +{ + return (__m256d) __builtin_ia32_vfmaddpd256 (-(__v4df)__A, (__v4df)__B, (__v4df)__C); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_nmsub_ps (__m256 __A, __m256 __B, __m256 __C) +{ + return (__m256) __builtin_ia32_vfmaddps256 (-(__v8sf)__A, (__v8sf)__B, -(__v8sf)__C); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_nmsub_pd (__m256d __A, __m256d __B, __m256d __C) +{ + return (__m256d) __builtin_ia32_vfmaddpd256 (-(__v4df)__A, (__v4df)__B, -(__v4df)__C); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maddsub_ps (__m256 __A, __m256 __B, __m256 __C) +{ + return (__m256) __builtin_ia32_vfmaddsubps256 ((__v8sf)__A, (__v8sf)__B, (__v8sf)__C); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maddsub_pd (__m256d __A, __m256d __B, __m256d __C) +{ + return (__m256d) __builtin_ia32_vfmaddsubpd256 ((__v4df)__A, (__v4df)__B, (__v4df)__C); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_msubadd_ps (__m256 __A, __m256 __B, __m256 __C) +{ + return (__m256) __builtin_ia32_vfmaddsubps256 ((__v8sf)__A, (__v8sf)__B, -(__v8sf)__C); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_msubadd_pd (__m256d __A, __m256d __B, __m256d __C) +{ + return (__m256d) __builtin_ia32_vfmaddsubpd256 ((__v4df)__A, (__v4df)__B, -(__v4df)__C); +} + +#ifdef __DISABLE_FMA4__ +#undef __DISABLE_FMA4__ +#pragma GCC pop_options +#endif /* __DISABLE_FMA4__ */ + +#endif diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/fmaintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/fmaintrin.h new file mode 100644 index 0000000000..2b1531e433 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/fmaintrin.h @@ -0,0 +1,302 @@ +/* Copyright (C) 2011-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _FMAINTRIN_H_INCLUDED +#define _FMAINTRIN_H_INCLUDED + +#ifndef __FMA__ +#pragma GCC push_options +#pragma GCC target("fma") +#define __DISABLE_FMA__ +#endif /* __FMA__ */ + +extern __inline __m128d +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fmadd_pd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d)__builtin_ia32_vfmaddpd ((__v2df)__A, (__v2df)__B, + (__v2df)__C); +} + +extern __inline __m256d +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_fmadd_pd (__m256d __A, __m256d __B, __m256d __C) +{ + return (__m256d)__builtin_ia32_vfmaddpd256 ((__v4df)__A, (__v4df)__B, + (__v4df)__C); +} + +extern __inline __m128 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fmadd_ps (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128)__builtin_ia32_vfmaddps ((__v4sf)__A, (__v4sf)__B, + (__v4sf)__C); +} + +extern __inline __m256 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_fmadd_ps (__m256 __A, __m256 __B, __m256 __C) +{ + return (__m256)__builtin_ia32_vfmaddps256 ((__v8sf)__A, (__v8sf)__B, + (__v8sf)__C); +} + +extern __inline __m128d +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fmadd_sd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddsd3 ((__v2df)__A, (__v2df)__B, + (__v2df)__C); +} + +extern __inline __m128 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fmadd_ss (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddss3 ((__v4sf)__A, (__v4sf)__B, + (__v4sf)__C); +} + +extern __inline __m128d +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fmsub_pd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d)__builtin_ia32_vfmaddpd ((__v2df)__A, (__v2df)__B, + -(__v2df)__C); +} + +extern __inline __m256d +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_fmsub_pd (__m256d __A, __m256d __B, __m256d __C) +{ + return (__m256d)__builtin_ia32_vfmaddpd256 ((__v4df)__A, (__v4df)__B, + -(__v4df)__C); +} + +extern __inline __m128 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fmsub_ps (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128)__builtin_ia32_vfmaddps ((__v4sf)__A, (__v4sf)__B, + -(__v4sf)__C); +} + +extern __inline __m256 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_fmsub_ps (__m256 __A, __m256 __B, __m256 __C) +{ + return (__m256)__builtin_ia32_vfmaddps256 ((__v8sf)__A, (__v8sf)__B, + -(__v8sf)__C); +} + +extern __inline __m128d +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fmsub_sd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d)__builtin_ia32_vfmaddsd3 ((__v2df)__A, (__v2df)__B, + -(__v2df)__C); +} + +extern __inline __m128 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fmsub_ss (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128)__builtin_ia32_vfmaddss3 ((__v4sf)__A, (__v4sf)__B, + -(__v4sf)__C); +} + +extern __inline __m128d +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fnmadd_pd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d)__builtin_ia32_vfmaddpd (-(__v2df)__A, (__v2df)__B, + (__v2df)__C); +} + +extern __inline __m256d +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_fnmadd_pd (__m256d __A, __m256d __B, __m256d __C) +{ + return (__m256d)__builtin_ia32_vfmaddpd256 (-(__v4df)__A, (__v4df)__B, + (__v4df)__C); +} + +extern __inline __m128 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fnmadd_ps (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128)__builtin_ia32_vfmaddps (-(__v4sf)__A, (__v4sf)__B, + (__v4sf)__C); +} + +extern __inline __m256 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_fnmadd_ps (__m256 __A, __m256 __B, __m256 __C) +{ + return (__m256)__builtin_ia32_vfmaddps256 (-(__v8sf)__A, (__v8sf)__B, + (__v8sf)__C); +} + +extern __inline __m128d +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fnmadd_sd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d)__builtin_ia32_vfmaddsd3 ((__v2df)__A, -(__v2df)__B, + (__v2df)__C); +} + +extern __inline __m128 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fnmadd_ss (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128)__builtin_ia32_vfmaddss3 ((__v4sf)__A, -(__v4sf)__B, + (__v4sf)__C); +} + +extern __inline __m128d +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fnmsub_pd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d)__builtin_ia32_vfmaddpd (-(__v2df)__A, (__v2df)__B, + -(__v2df)__C); +} + +extern __inline __m256d +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_fnmsub_pd (__m256d __A, __m256d __B, __m256d __C) +{ + return (__m256d)__builtin_ia32_vfmaddpd256 (-(__v4df)__A, (__v4df)__B, + -(__v4df)__C); +} + +extern __inline __m128 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fnmsub_ps (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128)__builtin_ia32_vfmaddps (-(__v4sf)__A, (__v4sf)__B, + -(__v4sf)__C); +} + +extern __inline __m256 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_fnmsub_ps (__m256 __A, __m256 __B, __m256 __C) +{ + return (__m256)__builtin_ia32_vfmaddps256 (-(__v8sf)__A, (__v8sf)__B, + -(__v8sf)__C); +} + +extern __inline __m128d +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fnmsub_sd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d)__builtin_ia32_vfmaddsd3 ((__v2df)__A, -(__v2df)__B, + -(__v2df)__C); +} + +extern __inline __m128 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fnmsub_ss (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128)__builtin_ia32_vfmaddss3 ((__v4sf)__A, -(__v4sf)__B, + -(__v4sf)__C); +} + +extern __inline __m128d +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fmaddsub_pd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d)__builtin_ia32_vfmaddsubpd ((__v2df)__A, (__v2df)__B, + (__v2df)__C); +} + +extern __inline __m256d +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_fmaddsub_pd (__m256d __A, __m256d __B, __m256d __C) +{ + return (__m256d)__builtin_ia32_vfmaddsubpd256 ((__v4df)__A, + (__v4df)__B, + (__v4df)__C); +} + +extern __inline __m128 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fmaddsub_ps (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128)__builtin_ia32_vfmaddsubps ((__v4sf)__A, (__v4sf)__B, + (__v4sf)__C); +} + +extern __inline __m256 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_fmaddsub_ps (__m256 __A, __m256 __B, __m256 __C) +{ + return (__m256)__builtin_ia32_vfmaddsubps256 ((__v8sf)__A, + (__v8sf)__B, + (__v8sf)__C); +} + +extern __inline __m128d +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fmsubadd_pd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d)__builtin_ia32_vfmaddsubpd ((__v2df)__A, (__v2df)__B, + -(__v2df)__C); +} + +extern __inline __m256d +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_fmsubadd_pd (__m256d __A, __m256d __B, __m256d __C) +{ + return (__m256d)__builtin_ia32_vfmaddsubpd256 ((__v4df)__A, + (__v4df)__B, + -(__v4df)__C); +} + +extern __inline __m128 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fmsubadd_ps (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128)__builtin_ia32_vfmaddsubps ((__v4sf)__A, (__v4sf)__B, + -(__v4sf)__C); +} + +extern __inline __m256 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_fmsubadd_ps (__m256 __A, __m256 __B, __m256 __C) +{ + return (__m256)__builtin_ia32_vfmaddsubps256 ((__v8sf)__A, + (__v8sf)__B, + -(__v8sf)__C); +} + +#ifdef __DISABLE_FMA__ +#undef __DISABLE_FMA__ +#pragma GCC pop_options +#endif /* __DISABLE_FMA__ */ + +#endif diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/fxsrintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/fxsrintrin.h new file mode 100644 index 0000000000..d04cf9c12f --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/fxsrintrin.h @@ -0,0 +1,73 @@ +/* Copyright (C) 2012-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +/* #if !defined _X86INTRIN_H_INCLUDED && !defined _IMMINTRIN_H_INCLUDED */ +/* # error "Never use directly; include instead." */ +/* #endif */ + +#ifndef _FXSRINTRIN_H_INCLUDED +#define _FXSRINTRIN_H_INCLUDED + +#ifndef __FXSR__ +#pragma GCC push_options +#pragma GCC target("fxsr") +#define __DISABLE_FXSR__ +#endif /* __FXSR__ */ + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_fxsave (void *__P) +{ + __builtin_ia32_fxsave (__P); +} + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_fxrstor (void *__P) +{ + __builtin_ia32_fxrstor (__P); +} + +#ifdef __x86_64__ +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_fxsave64 (void *__P) +{ + __builtin_ia32_fxsave64 (__P); +} + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_fxrstor64 (void *__P) +{ + __builtin_ia32_fxrstor64 (__P); +} +#endif + +#ifdef __DISABLE_FXSR__ +#undef __DISABLE_FXSR__ +#pragma GCC pop_options +#endif /* __DISABLE_FXSR__ */ + + +#endif /* _FXSRINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/ia32intrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/ia32intrin.h new file mode 100644 index 0000000000..5f954fce85 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/ia32intrin.h @@ -0,0 +1,313 @@ +/* Copyright (C) 2009-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _X86INTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +/* 32bit bsf */ +extern __inline int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__bsfd (int __X) +{ + return __builtin_ctz (__X); +} + +/* 32bit bsr */ +extern __inline int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__bsrd (int __X) +{ + return __builtin_ia32_bsrsi (__X); +} + +/* 32bit bswap */ +extern __inline int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__bswapd (int __X) +{ + return __builtin_bswap32 (__X); +} + +#ifndef __iamcu__ + +#ifndef __SSE4_2__ +#pragma GCC push_options +#pragma GCC target("sse4.2") +#define __DISABLE_SSE4_2__ +#endif /* __SSE4_2__ */ + +/* 32bit accumulate CRC32 (polynomial 0x11EDC6F41) value. */ +extern __inline unsigned int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__crc32b (unsigned int __C, unsigned char __V) +{ + return __builtin_ia32_crc32qi (__C, __V); +} + +extern __inline unsigned int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__crc32w (unsigned int __C, unsigned short __V) +{ + return __builtin_ia32_crc32hi (__C, __V); +} + +extern __inline unsigned int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__crc32d (unsigned int __C, unsigned int __V) +{ + return __builtin_ia32_crc32si (__C, __V); +} + +#ifdef __DISABLE_SSE4_2__ +#undef __DISABLE_SSE4_2__ +#pragma GCC pop_options +#endif /* __DISABLE_SSE4_2__ */ + +#endif /* __iamcu__ */ + +/* 32bit popcnt */ +extern __inline int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__popcntd (unsigned int __X) +{ + return __builtin_popcount (__X); +} + +#ifndef __iamcu__ + +/* rdpmc */ +extern __inline unsigned long long +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__rdpmc (int __S) +{ + return __builtin_ia32_rdpmc (__S); +} + +#endif /* __iamcu__ */ + +/* rdtsc */ +extern __inline unsigned long long +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__rdtsc (void) +{ + return __builtin_ia32_rdtsc (); +} + +#ifndef __iamcu__ + +/* rdtscp */ +extern __inline unsigned long long +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__rdtscp (unsigned int *__A) +{ + return __builtin_ia32_rdtscp (__A); +} + +#endif /* __iamcu__ */ + +/* 8bit rol */ +extern __inline unsigned char +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__rolb (unsigned char __X, int __C) +{ + return __builtin_ia32_rolqi (__X, __C); +} + +/* 16bit rol */ +extern __inline unsigned short +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__rolw (unsigned short __X, int __C) +{ + return __builtin_ia32_rolhi (__X, __C); +} + +/* 32bit rol */ +extern __inline unsigned int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__rold (unsigned int __X, int __C) +{ + return (__X << __C) | (__X >> (32 - __C)); +} + +/* 8bit ror */ +extern __inline unsigned char +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__rorb (unsigned char __X, int __C) +{ + return __builtin_ia32_rorqi (__X, __C); +} + +/* 16bit ror */ +extern __inline unsigned short +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__rorw (unsigned short __X, int __C) +{ + return __builtin_ia32_rorhi (__X, __C); +} + +/* 32bit ror */ +extern __inline unsigned int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__rord (unsigned int __X, int __C) +{ + return (__X >> __C) | (__X << (32 - __C)); +} + +/* Pause */ +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__pause (void) +{ + __builtin_ia32_pause (); +} + +#ifdef __x86_64__ +/* 64bit bsf */ +extern __inline int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__bsfq (long long __X) +{ + return __builtin_ctzll (__X); +} + +/* 64bit bsr */ +extern __inline int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__bsrq (long long __X) +{ + return __builtin_ia32_bsrdi (__X); +} + +/* 64bit bswap */ +extern __inline long long +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__bswapq (long long __X) +{ + return __builtin_bswap64 (__X); +} + +#ifndef __SSE4_2__ +#pragma GCC push_options +#pragma GCC target("sse4.2") +#define __DISABLE_SSE4_2__ +#endif /* __SSE4_2__ */ + +/* 64bit accumulate CRC32 (polynomial 0x11EDC6F41) value. */ +extern __inline unsigned long long +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__crc32q (unsigned long long __C, unsigned long long __V) +{ + return __builtin_ia32_crc32di (__C, __V); +} + +#ifdef __DISABLE_SSE4_2__ +#undef __DISABLE_SSE4_2__ +#pragma GCC pop_options +#endif /* __DISABLE_SSE4_2__ */ + +/* 64bit popcnt */ +extern __inline long long +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__popcntq (unsigned long long __X) +{ + return __builtin_popcountll (__X); +} + +/* 64bit rol */ +extern __inline unsigned long long +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__rolq (unsigned long long __X, int __C) +{ + return (__X << __C) | (__X >> (64 - __C)); +} + +/* 64bit ror */ +extern __inline unsigned long long +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__rorq (unsigned long long __X, int __C) +{ + return (__X >> __C) | (__X << (64 - __C)); +} + +/* Read flags register */ +extern __inline unsigned long long +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__readeflags (void) +{ + return __builtin_ia32_readeflags_u64 (); +} + +/* Write flags register */ +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__writeeflags (unsigned long long __X) +{ + __builtin_ia32_writeeflags_u64 (__X); +} + +#define _bswap64(a) __bswapq(a) +#define _popcnt64(a) __popcntq(a) +#else + +/* Read flags register */ +extern __inline unsigned int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__readeflags (void) +{ + return __builtin_ia32_readeflags_u32 (); +} + +/* Write flags register */ +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__writeeflags (unsigned int __X) +{ + __builtin_ia32_writeeflags_u32 (__X); +} + +#endif + +/* On LP64 systems, longs are 64-bit. Use the appropriate rotate + * function. */ +#ifdef __LP64__ +#define _lrotl(a,b) __rolq((a), (b)) +#define _lrotr(a,b) __rorq((a), (b)) +#else +#define _lrotl(a,b) __rold((a), (b)) +#define _lrotr(a,b) __rord((a), (b)) +#endif + +#define _bit_scan_forward(a) __bsfd(a) +#define _bit_scan_reverse(a) __bsrd(a) +#define _bswap(a) __bswapd(a) +#define _popcnt32(a) __popcntd(a) +#ifndef __iamcu__ +#define _rdpmc(a) __rdpmc(a) +#define _rdtscp(a) __rdtscp(a) +#endif /* __iamcu__ */ +#define _rdtsc() __rdtsc() +#define _rotwl(a,b) __rolw((a), (b)) +#define _rotwr(a,b) __rorw((a), (b)) +#define _rotl(a,b) __rold((a), (b)) +#define _rotr(a,b) __rord((a), (b)) diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/immintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/immintrin.h new file mode 100644 index 0000000000..b52f58efa4 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/immintrin.h @@ -0,0 +1,217 @@ +/* Copyright (C) 2008-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +#define _IMMINTRIN_H_INCLUDED + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#ifndef __RDRND__ +#pragma GCC push_options +#pragma GCC target("rdrnd") +#define __DISABLE_RDRND__ +#endif /* __RDRND__ */ +extern __inline int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_rdrand16_step (unsigned short *__P) +{ + return __builtin_ia32_rdrand16_step (__P); +} + +extern __inline int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_rdrand32_step (unsigned int *__P) +{ + return __builtin_ia32_rdrand32_step (__P); +} +#ifdef __DISABLE_RDRND__ +#undef __DISABLE_RDRND__ +#pragma GCC pop_options +#endif /* __DISABLE_RDRND__ */ + +#ifndef __RDPID__ +#pragma GCC push_options +#pragma GCC target("rdpid") +#define __DISABLE_RDPID__ +#endif /* __RDPID__ */ +extern __inline unsigned int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_rdpid_u32 (void) +{ + return __builtin_ia32_rdpid (); +} +#ifdef __DISABLE_RDPID__ +#undef __DISABLE_RDPID__ +#pragma GCC pop_options +#endif /* __DISABLE_RDPID__ */ + +#ifdef __x86_64__ + +#ifndef __FSGSBASE__ +#pragma GCC push_options +#pragma GCC target("fsgsbase") +#define __DISABLE_FSGSBASE__ +#endif /* __FSGSBASE__ */ +extern __inline unsigned int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_readfsbase_u32 (void) +{ + return __builtin_ia32_rdfsbase32 (); +} + +extern __inline unsigned long long +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_readfsbase_u64 (void) +{ + return __builtin_ia32_rdfsbase64 (); +} + +extern __inline unsigned int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_readgsbase_u32 (void) +{ + return __builtin_ia32_rdgsbase32 (); +} + +extern __inline unsigned long long +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_readgsbase_u64 (void) +{ + return __builtin_ia32_rdgsbase64 (); +} + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_writefsbase_u32 (unsigned int __B) +{ + __builtin_ia32_wrfsbase32 (__B); +} + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_writefsbase_u64 (unsigned long long __B) +{ + __builtin_ia32_wrfsbase64 (__B); +} + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_writegsbase_u32 (unsigned int __B) +{ + __builtin_ia32_wrgsbase32 (__B); +} + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_writegsbase_u64 (unsigned long long __B) +{ + __builtin_ia32_wrgsbase64 (__B); +} +#ifdef __DISABLE_FSGSBASE__ +#undef __DISABLE_FSGSBASE__ +#pragma GCC pop_options +#endif /* __DISABLE_FSGSBASE__ */ + +#ifndef __RDRND__ +#pragma GCC push_options +#pragma GCC target("rdrnd") +#define __DISABLE_RDRND__ +#endif /* __RDRND__ */ +extern __inline int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_rdrand64_step (unsigned long long *__P) +{ + return __builtin_ia32_rdrand64_step (__P); +} +#ifdef __DISABLE_RDRND__ +#undef __DISABLE_RDRND__ +#pragma GCC pop_options +#endif /* __DISABLE_RDRND__ */ + +#endif /* __x86_64__ */ + +#endif /* _IMMINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/iso646.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/iso646.h new file mode 100644 index 0000000000..5a6b50335e --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/iso646.h @@ -0,0 +1,45 @@ +/* Copyright (C) 1997-2017 Free Software Foundation, Inc. + +This file is part of GCC. + +GCC is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3, or (at your option) +any later version. + +GCC is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +Under Section 7 of GPL version 3, you are granted additional +permissions described in the GCC Runtime Library Exception, version +3.1, as published by the Free Software Foundation. + +You should have received a copy of the GNU General Public License and +a copy of the GCC Runtime Library Exception along with this program; +see the files COPYING3 and COPYING.RUNTIME respectively. If not, see +. */ + +/* + * ISO C Standard: 7.9 Alternative spellings + */ + +#ifndef _ISO646_H +#define _ISO646_H + +#ifndef __cplusplus +#define and && +#define and_eq &= +#define bitand & +#define bitor | +#define compl ~ +#define not ! +#define not_eq != +#define or || +#define or_eq |= +#define xor ^ +#define xor_eq ^= +#endif + +#endif diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/limits.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/limits.h new file mode 100644 index 0000000000..743481d784 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/limits.h @@ -0,0 +1,197 @@ +/* Copyright (C) 1992-2017 Free Software Foundation, Inc. + +This file is part of GCC. + +GCC is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 3, or (at your option) any later +version. + +GCC is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +Under Section 7 of GPL version 3, you are granted additional +permissions described in the GCC Runtime Library Exception, version +3.1, as published by the Free Software Foundation. + +You should have received a copy of the GNU General Public License and +a copy of the GCC Runtime Library Exception along with this program; +see the files COPYING3 and COPYING.RUNTIME respectively. If not, see +. */ + +/* This administrivia gets added to the beginning of limits.h + if the system has its own version of limits.h. */ + +/* We use _GCC_LIMITS_H_ because we want this not to match + any macros that the system's limits.h uses for its own purposes. */ +#ifndef _GCC_LIMITS_H_ /* Terminated in limity.h. */ +#define _GCC_LIMITS_H_ + +#ifndef _LIBC_LIMITS_H_ +/* Use "..." so that we find syslimits.h only in this same directory. */ +#include "syslimits.h" +#endif +/* Copyright (C) 1991-2017 Free Software Foundation, Inc. + +This file is part of GCC. + +GCC is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 3, or (at your option) any later +version. + +GCC is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +Under Section 7 of GPL version 3, you are granted additional +permissions described in the GCC Runtime Library Exception, version +3.1, as published by the Free Software Foundation. + +You should have received a copy of the GNU General Public License and +a copy of the GCC Runtime Library Exception along with this program; +see the files COPYING3 and COPYING.RUNTIME respectively. If not, see +. */ + +#ifndef _LIMITS_H___ +#define _LIMITS_H___ + +/* Number of bits in a `char'. */ +#undef CHAR_BIT +#define CHAR_BIT __CHAR_BIT__ + +/* Maximum length of a multibyte character. */ +#ifndef MB_LEN_MAX +#define MB_LEN_MAX 1 +#endif + +/* Minimum and maximum values a `signed char' can hold. */ +#undef SCHAR_MIN +#define SCHAR_MIN (-SCHAR_MAX - 1) +#undef SCHAR_MAX +#define SCHAR_MAX __SCHAR_MAX__ + +/* Maximum value an `unsigned char' can hold. (Minimum is 0). */ +#undef UCHAR_MAX +#if __SCHAR_MAX__ == __INT_MAX__ +# define UCHAR_MAX (SCHAR_MAX * 2U + 1U) +#else +# define UCHAR_MAX (SCHAR_MAX * 2 + 1) +#endif + +/* Minimum and maximum values a `char' can hold. */ +#ifdef __CHAR_UNSIGNED__ +# undef CHAR_MIN +# if __SCHAR_MAX__ == __INT_MAX__ +# define CHAR_MIN 0U +# else +# define CHAR_MIN 0 +# endif +# undef CHAR_MAX +# define CHAR_MAX UCHAR_MAX +#else +# undef CHAR_MIN +# define CHAR_MIN SCHAR_MIN +# undef CHAR_MAX +# define CHAR_MAX SCHAR_MAX +#endif + +/* Minimum and maximum values a `signed short int' can hold. */ +#undef SHRT_MIN +#define SHRT_MIN (-SHRT_MAX - 1) +#undef SHRT_MAX +#define SHRT_MAX __SHRT_MAX__ + +/* Maximum value an `unsigned short int' can hold. (Minimum is 0). */ +#undef USHRT_MAX +#if __SHRT_MAX__ == __INT_MAX__ +# define USHRT_MAX (SHRT_MAX * 2U + 1U) +#else +# define USHRT_MAX (SHRT_MAX * 2 + 1) +#endif + +/* Minimum and maximum values a `signed int' can hold. */ +#undef INT_MIN +#define INT_MIN (-INT_MAX - 1) +#undef INT_MAX +#define INT_MAX __INT_MAX__ + +/* Maximum value an `unsigned int' can hold. (Minimum is 0). */ +#undef UINT_MAX +#define UINT_MAX (INT_MAX * 2U + 1U) + +/* Minimum and maximum values a `signed long int' can hold. + (Same as `int'). */ +#undef LONG_MIN +#define LONG_MIN (-LONG_MAX - 1L) +#undef LONG_MAX +#define LONG_MAX __LONG_MAX__ + +/* Maximum value an `unsigned long int' can hold. (Minimum is 0). */ +#undef ULONG_MAX +#define ULONG_MAX (LONG_MAX * 2UL + 1UL) + +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L +/* Minimum and maximum values a `signed long long int' can hold. */ +# undef LLONG_MIN +# define LLONG_MIN (-LLONG_MAX - 1LL) +# undef LLONG_MAX +# define LLONG_MAX __LONG_LONG_MAX__ + +/* Maximum value an `unsigned long long int' can hold. (Minimum is 0). */ +# undef ULLONG_MAX +# define ULLONG_MAX (LLONG_MAX * 2ULL + 1ULL) +#endif + +#if defined (__GNU_LIBRARY__) ? defined (__USE_GNU) : !defined (__STRICT_ANSI__) +/* Minimum and maximum values a `signed long long int' can hold. */ +# undef LONG_LONG_MIN +# define LONG_LONG_MIN (-LONG_LONG_MAX - 1LL) +# undef LONG_LONG_MAX +# define LONG_LONG_MAX __LONG_LONG_MAX__ + +/* Maximum value an `unsigned long long int' can hold. (Minimum is 0). */ +# undef ULONG_LONG_MAX +# define ULONG_LONG_MAX (LONG_LONG_MAX * 2ULL + 1ULL) +#endif + +#ifdef __STDC_WANT_IEC_60559_BFP_EXT__ +/* TS 18661-1 widths of integer types. */ +# undef CHAR_WIDTH +# define CHAR_WIDTH __SCHAR_WIDTH__ +# undef SCHAR_WIDTH +# define SCHAR_WIDTH __SCHAR_WIDTH__ +# undef UCHAR_WIDTH +# define UCHAR_WIDTH __SCHAR_WIDTH__ +# undef SHRT_WIDTH +# define SHRT_WIDTH __SHRT_WIDTH__ +# undef USHRT_WIDTH +# define USHRT_WIDTH __SHRT_WIDTH__ +# undef INT_WIDTH +# define INT_WIDTH __INT_WIDTH__ +# undef UINT_WIDTH +# define UINT_WIDTH __INT_WIDTH__ +# undef LONG_WIDTH +# define LONG_WIDTH __LONG_WIDTH__ +# undef ULONG_WIDTH +# define ULONG_WIDTH __LONG_WIDTH__ +# undef LLONG_WIDTH +# define LLONG_WIDTH __LONG_LONG_WIDTH__ +# undef ULLONG_WIDTH +# define ULLONG_WIDTH __LONG_LONG_WIDTH__ +#endif + +#endif /* _LIMITS_H___ */ +/* This administrivia gets added to the end of limits.h + if the system has its own version of limits.h. */ + +#else /* not _GCC_LIMITS_H_ */ + +#ifdef _GCC_NEXT_LIMITS_H +#include_next /* recurse down to the real one */ +#endif + +#endif /* not _GCC_LIMITS_H_ */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/lwpintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/lwpintrin.h new file mode 100644 index 0000000000..5c095aba18 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/lwpintrin.h @@ -0,0 +1,107 @@ +/* Copyright (C) 2007-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _X86INTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _LWPINTRIN_H_INCLUDED +#define _LWPINTRIN_H_INCLUDED + +#ifndef __LWP__ +#pragma GCC push_options +#pragma GCC target("lwp") +#define __DISABLE_LWP__ +#endif /* __LWP__ */ + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__llwpcb (void *__pcbAddress) +{ + __builtin_ia32_llwpcb (__pcbAddress); +} + +extern __inline void * __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__slwpcb (void) +{ + return __builtin_ia32_slwpcb (); +} + +#ifdef __OPTIMIZE__ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__lwpval32 (unsigned int __data2, unsigned int __data1, unsigned int __flags) +{ + __builtin_ia32_lwpval32 (__data2, __data1, __flags); +} + +#ifdef __x86_64__ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__lwpval64 (unsigned long long __data2, unsigned int __data1, + unsigned int __flags) +{ + __builtin_ia32_lwpval64 (__data2, __data1, __flags); +} +#endif +#else +#define __lwpval32(D2, D1, F) \ + (__builtin_ia32_lwpval32 ((unsigned int) (D2), (unsigned int) (D1), \ + (unsigned int) (F))) +#ifdef __x86_64__ +#define __lwpval64(D2, D1, F) \ + (__builtin_ia32_lwpval64 ((unsigned long long) (D2), (unsigned int) (D1), \ + (unsigned int) (F))) +#endif +#endif + + +#ifdef __OPTIMIZE__ +extern __inline unsigned char __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__lwpins32 (unsigned int __data2, unsigned int __data1, unsigned int __flags) +{ + return __builtin_ia32_lwpins32 (__data2, __data1, __flags); +} + +#ifdef __x86_64__ +extern __inline unsigned char __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__lwpins64 (unsigned long long __data2, unsigned int __data1, + unsigned int __flags) +{ + return __builtin_ia32_lwpins64 (__data2, __data1, __flags); +} +#endif +#else +#define __lwpins32(D2, D1, F) \ + (__builtin_ia32_lwpins32 ((unsigned int) (D2), (unsigned int) (D1), \ + (unsigned int) (F))) +#ifdef __x86_64__ +#define __lwpins64(D2, D1, F) \ + (__builtin_ia32_lwpins64 ((unsigned long long) (D2), (unsigned int) (D1), \ + (unsigned int) (F))) +#endif +#endif + +#ifdef __DISABLE_LWP__ +#undef __DISABLE_LWP__ +#pragma GCC pop_options +#endif /* __DISABLE_LWP__ */ + +#endif /* _LWPINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/lzcntintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/lzcntintrin.h new file mode 100644 index 0000000000..401656cdab --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/lzcntintrin.h @@ -0,0 +1,75 @@ +/* Copyright (C) 2009-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#if !defined _X86INTRIN_H_INCLUDED && !defined _IMMINTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + + +#ifndef _LZCNTINTRIN_H_INCLUDED +#define _LZCNTINTRIN_H_INCLUDED + +#ifndef __LZCNT__ +#pragma GCC push_options +#pragma GCC target("lzcnt") +#define __DISABLE_LZCNT__ +#endif /* __LZCNT__ */ + +extern __inline unsigned short __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__lzcnt16 (unsigned short __X) +{ + return __builtin_ia32_lzcnt_u16 (__X); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__lzcnt32 (unsigned int __X) +{ + return __builtin_ia32_lzcnt_u32 (__X); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_lzcnt_u32 (unsigned int __X) +{ + return __builtin_ia32_lzcnt_u32 (__X); +} + +#ifdef __x86_64__ +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__lzcnt64 (unsigned long long __X) +{ + return __builtin_ia32_lzcnt_u64 (__X); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_lzcnt_u64 (unsigned long long __X) +{ + return __builtin_ia32_lzcnt_u64 (__X); +} +#endif + +#ifdef __DISABLE_LZCNT__ +#undef __DISABLE_LZCNT__ +#pragma GCC pop_options +#endif /* __DISABLE_LZCNT__ */ + +#endif /* _LZCNTINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/mm3dnow.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/mm3dnow.h new file mode 100644 index 0000000000..2d5c538493 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/mm3dnow.h @@ -0,0 +1,233 @@ +/* Copyright (C) 2004-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +/* Implemented from the mm3dnow.h (of supposedly AMD origin) included with + MSVC 7.1. */ + +#ifndef _MM3DNOW_H_INCLUDED +#define _MM3DNOW_H_INCLUDED + +#include +#include + +#if defined __x86_64__ && !defined __SSE__ || !defined __3dNOW__ +#pragma GCC push_options +#ifdef __x86_64__ +#pragma GCC target("sse,3dnow") +#else +#pragma GCC target("3dnow") +#endif +#define __DISABLE_3dNOW__ +#endif /* __3dNOW__ */ + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_femms (void) +{ + __builtin_ia32_femms(); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pavgusb (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pavgusb ((__v8qi)__A, (__v8qi)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pf2id (__m64 __A) +{ + return (__m64)__builtin_ia32_pf2id ((__v2sf)__A); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfacc (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pfacc ((__v2sf)__A, (__v2sf)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfadd (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pfadd ((__v2sf)__A, (__v2sf)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfcmpeq (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pfcmpeq ((__v2sf)__A, (__v2sf)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfcmpge (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pfcmpge ((__v2sf)__A, (__v2sf)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfcmpgt (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pfcmpgt ((__v2sf)__A, (__v2sf)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfmax (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pfmax ((__v2sf)__A, (__v2sf)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfmin (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pfmin ((__v2sf)__A, (__v2sf)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfmul (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pfmul ((__v2sf)__A, (__v2sf)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfrcp (__m64 __A) +{ + return (__m64)__builtin_ia32_pfrcp ((__v2sf)__A); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfrcpit1 (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pfrcpit1 ((__v2sf)__A, (__v2sf)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfrcpit2 (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pfrcpit2 ((__v2sf)__A, (__v2sf)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfrsqrt (__m64 __A) +{ + return (__m64)__builtin_ia32_pfrsqrt ((__v2sf)__A); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfrsqit1 (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pfrsqit1 ((__v2sf)__A, (__v2sf)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfsub (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pfsub ((__v2sf)__A, (__v2sf)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfsubr (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pfsubr ((__v2sf)__A, (__v2sf)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pi2fd (__m64 __A) +{ + return (__m64)__builtin_ia32_pi2fd ((__v2si)__A); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pmulhrw (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pmulhrw ((__v4hi)__A, (__v4hi)__B); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_prefetch (void *__P) +{ + __builtin_prefetch (__P, 0, 3 /* _MM_HINT_T0 */); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_from_float (float __A) +{ + return __extension__ (__m64)(__v2sf){ __A, 0.0f }; +} + +extern __inline float __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_to_float (__m64 __A) +{ + union { __v2sf v; float a[2]; } __tmp; + __tmp.v = (__v2sf)__A; + return __tmp.a[0]; +} + +#ifdef __DISABLE_3dNOW__ +#undef __DISABLE_3dNOW__ +#pragma GCC pop_options +#endif /* __DISABLE_3dNOW__ */ + +#if defined __x86_64__ && !defined __SSE__ || !defined __3dNOW_A__ +#pragma GCC push_options +#ifdef __x86_64__ +#pragma GCC target("sse,3dnowa") +#else +#pragma GCC target("3dnowa") +#endif +#define __DISABLE_3dNOW_A__ +#endif /* __3dNOW_A__ */ + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pf2iw (__m64 __A) +{ + return (__m64)__builtin_ia32_pf2iw ((__v2sf)__A); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfnacc (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pfnacc ((__v2sf)__A, (__v2sf)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfpnacc (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pfpnacc ((__v2sf)__A, (__v2sf)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pi2fw (__m64 __A) +{ + return (__m64)__builtin_ia32_pi2fw ((__v2si)__A); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pswapd (__m64 __A) +{ + return (__m64)__builtin_ia32_pswapdsf ((__v2sf)__A); +} + +#ifdef __DISABLE_3dNOW_A__ +#undef __DISABLE_3dNOW_A__ +#pragma GCC pop_options +#endif /* __DISABLE_3dNOW_A__ */ + +#endif /* _MM3DNOW_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/mm_malloc.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/mm_malloc.h new file mode 100644 index 0000000000..73d9e887b4 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/mm_malloc.h @@ -0,0 +1,57 @@ +/* Copyright (C) 2004-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _MM_MALLOC_H_INCLUDED +#define _MM_MALLOC_H_INCLUDED + +#include + +/* We can't depend on since the prototype of posix_memalign + may not be visible. */ +#ifndef __cplusplus +extern int posix_memalign (void **, size_t, size_t); +#else +extern "C" int posix_memalign (void **, size_t, size_t) throw (); +#endif + +static __inline void * +_mm_malloc (size_t __size, size_t __alignment) +{ + void *__ptr; + if (__alignment == 1) + return malloc (__size); + if (__alignment == 2 || (sizeof (void *) == 8 && __alignment == 4)) + __alignment = sizeof (void *); + if (posix_memalign (&__ptr, __alignment, __size) == 0) + return __ptr; + else + return NULL; +} + +static __inline void +_mm_free (void *__ptr) +{ + free (__ptr); +} + +#endif /* _MM_MALLOC_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/mmintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/mmintrin.h new file mode 100644 index 0000000000..2cb73e34e9 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/mmintrin.h @@ -0,0 +1,949 @@ +/* Copyright (C) 2002-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +/* Implemented from the specification included in the Intel C++ Compiler + User Guide and Reference, version 9.0. */ + +#ifndef _MMINTRIN_H_INCLUDED +#define _MMINTRIN_H_INCLUDED + +#if defined __x86_64__ && !defined __SSE__ || !defined __MMX__ +#pragma GCC push_options +#ifdef __x86_64__ +#pragma GCC target("sse,mmx") +#else +#pragma GCC target("mmx") +#endif +#define __DISABLE_MMX__ +#endif /* __MMX__ */ + +/* The Intel API is flexible enough that we must allow aliasing with other + vector types, and their scalar components. */ +typedef int __m64 __attribute__ ((__vector_size__ (8), __may_alias__)); + +/* Unaligned version of the same type */ +typedef int __m64_u __attribute__ ((__vector_size__ (8), __may_alias__, __aligned__ (1))); + +/* Internal data types for implementing the intrinsics. */ +typedef int __v2si __attribute__ ((__vector_size__ (8))); +typedef short __v4hi __attribute__ ((__vector_size__ (8))); +typedef char __v8qi __attribute__ ((__vector_size__ (8))); +typedef long long __v1di __attribute__ ((__vector_size__ (8))); +typedef float __v2sf __attribute__ ((__vector_size__ (8))); + +/* Empty the multimedia state. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_empty (void) +{ + __builtin_ia32_emms (); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_empty (void) +{ + _mm_empty (); +} + +/* Convert I to a __m64 object. The integer is zero-extended to 64-bits. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi32_si64 (int __i) +{ + return (__m64) __builtin_ia32_vec_init_v2si (__i, 0); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_from_int (int __i) +{ + return _mm_cvtsi32_si64 (__i); +} + +#ifdef __x86_64__ +/* Convert I to a __m64 object. */ + +/* Intel intrinsic. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_from_int64 (long long __i) +{ + return (__m64) __i; +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi64_m64 (long long __i) +{ + return (__m64) __i; +} + +/* Microsoft intrinsic. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi64x_si64 (long long __i) +{ + return (__m64) __i; +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set_pi64x (long long __i) +{ + return (__m64) __i; +} +#endif + +/* Convert the lower 32 bits of the __m64 object into an integer. */ +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi64_si32 (__m64 __i) +{ + return __builtin_ia32_vec_ext_v2si ((__v2si)__i, 0); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_to_int (__m64 __i) +{ + return _mm_cvtsi64_si32 (__i); +} + +#ifdef __x86_64__ +/* Convert the __m64 object to a 64bit integer. */ + +/* Intel intrinsic. */ +extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_to_int64 (__m64 __i) +{ + return (long long)__i; +} + +extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtm64_si64 (__m64 __i) +{ + return (long long)__i; +} + +/* Microsoft intrinsic. */ +extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi64_si64x (__m64 __i) +{ + return (long long)__i; +} +#endif + +/* Pack the four 16-bit values from M1 into the lower four 8-bit values of + the result, and the four 16-bit values from M2 into the upper four 8-bit + values of the result, all with signed saturation. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_packs_pi16 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_packsswb ((__v4hi)__m1, (__v4hi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_packsswb (__m64 __m1, __m64 __m2) +{ + return _mm_packs_pi16 (__m1, __m2); +} + +/* Pack the two 32-bit values from M1 in to the lower two 16-bit values of + the result, and the two 32-bit values from M2 into the upper two 16-bit + values of the result, all with signed saturation. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_packs_pi32 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_packssdw ((__v2si)__m1, (__v2si)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_packssdw (__m64 __m1, __m64 __m2) +{ + return _mm_packs_pi32 (__m1, __m2); +} + +/* Pack the four 16-bit values from M1 into the lower four 8-bit values of + the result, and the four 16-bit values from M2 into the upper four 8-bit + values of the result, all with unsigned saturation. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_packs_pu16 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_packuswb ((__v4hi)__m1, (__v4hi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_packuswb (__m64 __m1, __m64 __m2) +{ + return _mm_packs_pu16 (__m1, __m2); +} + +/* Interleave the four 8-bit values from the high half of M1 with the four + 8-bit values from the high half of M2. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpackhi_pi8 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_punpckhbw ((__v8qi)__m1, (__v8qi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_punpckhbw (__m64 __m1, __m64 __m2) +{ + return _mm_unpackhi_pi8 (__m1, __m2); +} + +/* Interleave the two 16-bit values from the high half of M1 with the two + 16-bit values from the high half of M2. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpackhi_pi16 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_punpckhwd ((__v4hi)__m1, (__v4hi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_punpckhwd (__m64 __m1, __m64 __m2) +{ + return _mm_unpackhi_pi16 (__m1, __m2); +} + +/* Interleave the 32-bit value from the high half of M1 with the 32-bit + value from the high half of M2. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpackhi_pi32 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_punpckhdq ((__v2si)__m1, (__v2si)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_punpckhdq (__m64 __m1, __m64 __m2) +{ + return _mm_unpackhi_pi32 (__m1, __m2); +} + +/* Interleave the four 8-bit values from the low half of M1 with the four + 8-bit values from the low half of M2. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpacklo_pi8 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_punpcklbw ((__v8qi)__m1, (__v8qi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_punpcklbw (__m64 __m1, __m64 __m2) +{ + return _mm_unpacklo_pi8 (__m1, __m2); +} + +/* Interleave the two 16-bit values from the low half of M1 with the two + 16-bit values from the low half of M2. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpacklo_pi16 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_punpcklwd ((__v4hi)__m1, (__v4hi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_punpcklwd (__m64 __m1, __m64 __m2) +{ + return _mm_unpacklo_pi16 (__m1, __m2); +} + +/* Interleave the 32-bit value from the low half of M1 with the 32-bit + value from the low half of M2. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpacklo_pi32 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_punpckldq ((__v2si)__m1, (__v2si)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_punpckldq (__m64 __m1, __m64 __m2) +{ + return _mm_unpacklo_pi32 (__m1, __m2); +} + +/* Add the 8-bit values in M1 to the 8-bit values in M2. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_add_pi8 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_paddb ((__v8qi)__m1, (__v8qi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_paddb (__m64 __m1, __m64 __m2) +{ + return _mm_add_pi8 (__m1, __m2); +} + +/* Add the 16-bit values in M1 to the 16-bit values in M2. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_add_pi16 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_paddw ((__v4hi)__m1, (__v4hi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_paddw (__m64 __m1, __m64 __m2) +{ + return _mm_add_pi16 (__m1, __m2); +} + +/* Add the 32-bit values in M1 to the 32-bit values in M2. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_add_pi32 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_paddd ((__v2si)__m1, (__v2si)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_paddd (__m64 __m1, __m64 __m2) +{ + return _mm_add_pi32 (__m1, __m2); +} + +/* Add the 64-bit values in M1 to the 64-bit values in M2. */ +#ifndef __SSE2__ +#pragma GCC push_options +#pragma GCC target("sse2,mmx") +#define __DISABLE_SSE2__ +#endif /* __SSE2__ */ + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_add_si64 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_paddq ((__v1di)__m1, (__v1di)__m2); +} +#ifdef __DISABLE_SSE2__ +#undef __DISABLE_SSE2__ +#pragma GCC pop_options +#endif /* __DISABLE_SSE2__ */ + +/* Add the 8-bit values in M1 to the 8-bit values in M2 using signed + saturated arithmetic. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_adds_pi8 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_paddsb ((__v8qi)__m1, (__v8qi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_paddsb (__m64 __m1, __m64 __m2) +{ + return _mm_adds_pi8 (__m1, __m2); +} + +/* Add the 16-bit values in M1 to the 16-bit values in M2 using signed + saturated arithmetic. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_adds_pi16 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_paddsw ((__v4hi)__m1, (__v4hi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_paddsw (__m64 __m1, __m64 __m2) +{ + return _mm_adds_pi16 (__m1, __m2); +} + +/* Add the 8-bit values in M1 to the 8-bit values in M2 using unsigned + saturated arithmetic. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_adds_pu8 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_paddusb ((__v8qi)__m1, (__v8qi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_paddusb (__m64 __m1, __m64 __m2) +{ + return _mm_adds_pu8 (__m1, __m2); +} + +/* Add the 16-bit values in M1 to the 16-bit values in M2 using unsigned + saturated arithmetic. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_adds_pu16 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_paddusw ((__v4hi)__m1, (__v4hi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_paddusw (__m64 __m1, __m64 __m2) +{ + return _mm_adds_pu16 (__m1, __m2); +} + +/* Subtract the 8-bit values in M2 from the 8-bit values in M1. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sub_pi8 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_psubb ((__v8qi)__m1, (__v8qi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psubb (__m64 __m1, __m64 __m2) +{ + return _mm_sub_pi8 (__m1, __m2); +} + +/* Subtract the 16-bit values in M2 from the 16-bit values in M1. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sub_pi16 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_psubw ((__v4hi)__m1, (__v4hi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psubw (__m64 __m1, __m64 __m2) +{ + return _mm_sub_pi16 (__m1, __m2); +} + +/* Subtract the 32-bit values in M2 from the 32-bit values in M1. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sub_pi32 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_psubd ((__v2si)__m1, (__v2si)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psubd (__m64 __m1, __m64 __m2) +{ + return _mm_sub_pi32 (__m1, __m2); +} + +/* Add the 64-bit values in M1 to the 64-bit values in M2. */ +#ifndef __SSE2__ +#pragma GCC push_options +#pragma GCC target("sse2,mmx") +#define __DISABLE_SSE2__ +#endif /* __SSE2__ */ + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sub_si64 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_psubq ((__v1di)__m1, (__v1di)__m2); +} +#ifdef __DISABLE_SSE2__ +#undef __DISABLE_SSE2__ +#pragma GCC pop_options +#endif /* __DISABLE_SSE2__ */ + +/* Subtract the 8-bit values in M2 from the 8-bit values in M1 using signed + saturating arithmetic. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_subs_pi8 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_psubsb ((__v8qi)__m1, (__v8qi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psubsb (__m64 __m1, __m64 __m2) +{ + return _mm_subs_pi8 (__m1, __m2); +} + +/* Subtract the 16-bit values in M2 from the 16-bit values in M1 using + signed saturating arithmetic. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_subs_pi16 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_psubsw ((__v4hi)__m1, (__v4hi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psubsw (__m64 __m1, __m64 __m2) +{ + return _mm_subs_pi16 (__m1, __m2); +} + +/* Subtract the 8-bit values in M2 from the 8-bit values in M1 using + unsigned saturating arithmetic. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_subs_pu8 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_psubusb ((__v8qi)__m1, (__v8qi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psubusb (__m64 __m1, __m64 __m2) +{ + return _mm_subs_pu8 (__m1, __m2); +} + +/* Subtract the 16-bit values in M2 from the 16-bit values in M1 using + unsigned saturating arithmetic. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_subs_pu16 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_psubusw ((__v4hi)__m1, (__v4hi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psubusw (__m64 __m1, __m64 __m2) +{ + return _mm_subs_pu16 (__m1, __m2); +} + +/* Multiply four 16-bit values in M1 by four 16-bit values in M2 producing + four 32-bit intermediate results, which are then summed by pairs to + produce two 32-bit results. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_madd_pi16 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_pmaddwd ((__v4hi)__m1, (__v4hi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pmaddwd (__m64 __m1, __m64 __m2) +{ + return _mm_madd_pi16 (__m1, __m2); +} + +/* Multiply four signed 16-bit values in M1 by four signed 16-bit values in + M2 and produce the high 16 bits of the 32-bit results. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mulhi_pi16 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_pmulhw ((__v4hi)__m1, (__v4hi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pmulhw (__m64 __m1, __m64 __m2) +{ + return _mm_mulhi_pi16 (__m1, __m2); +} + +/* Multiply four 16-bit values in M1 by four 16-bit values in M2 and produce + the low 16 bits of the results. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mullo_pi16 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_pmullw ((__v4hi)__m1, (__v4hi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pmullw (__m64 __m1, __m64 __m2) +{ + return _mm_mullo_pi16 (__m1, __m2); +} + +/* Shift four 16-bit values in M left by COUNT. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sll_pi16 (__m64 __m, __m64 __count) +{ + return (__m64) __builtin_ia32_psllw ((__v4hi)__m, (__v4hi)__count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psllw (__m64 __m, __m64 __count) +{ + return _mm_sll_pi16 (__m, __count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_slli_pi16 (__m64 __m, int __count) +{ + return (__m64) __builtin_ia32_psllwi ((__v4hi)__m, __count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psllwi (__m64 __m, int __count) +{ + return _mm_slli_pi16 (__m, __count); +} + +/* Shift two 32-bit values in M left by COUNT. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sll_pi32 (__m64 __m, __m64 __count) +{ + return (__m64) __builtin_ia32_pslld ((__v2si)__m, (__v2si)__count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pslld (__m64 __m, __m64 __count) +{ + return _mm_sll_pi32 (__m, __count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_slli_pi32 (__m64 __m, int __count) +{ + return (__m64) __builtin_ia32_pslldi ((__v2si)__m, __count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pslldi (__m64 __m, int __count) +{ + return _mm_slli_pi32 (__m, __count); +} + +/* Shift the 64-bit value in M left by COUNT. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sll_si64 (__m64 __m, __m64 __count) +{ + return (__m64) __builtin_ia32_psllq ((__v1di)__m, (__v1di)__count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psllq (__m64 __m, __m64 __count) +{ + return _mm_sll_si64 (__m, __count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_slli_si64 (__m64 __m, int __count) +{ + return (__m64) __builtin_ia32_psllqi ((__v1di)__m, __count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psllqi (__m64 __m, int __count) +{ + return _mm_slli_si64 (__m, __count); +} + +/* Shift four 16-bit values in M right by COUNT; shift in the sign bit. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sra_pi16 (__m64 __m, __m64 __count) +{ + return (__m64) __builtin_ia32_psraw ((__v4hi)__m, (__v4hi)__count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psraw (__m64 __m, __m64 __count) +{ + return _mm_sra_pi16 (__m, __count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srai_pi16 (__m64 __m, int __count) +{ + return (__m64) __builtin_ia32_psrawi ((__v4hi)__m, __count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psrawi (__m64 __m, int __count) +{ + return _mm_srai_pi16 (__m, __count); +} + +/* Shift two 32-bit values in M right by COUNT; shift in the sign bit. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sra_pi32 (__m64 __m, __m64 __count) +{ + return (__m64) __builtin_ia32_psrad ((__v2si)__m, (__v2si)__count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psrad (__m64 __m, __m64 __count) +{ + return _mm_sra_pi32 (__m, __count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srai_pi32 (__m64 __m, int __count) +{ + return (__m64) __builtin_ia32_psradi ((__v2si)__m, __count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psradi (__m64 __m, int __count) +{ + return _mm_srai_pi32 (__m, __count); +} + +/* Shift four 16-bit values in M right by COUNT; shift in zeros. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srl_pi16 (__m64 __m, __m64 __count) +{ + return (__m64) __builtin_ia32_psrlw ((__v4hi)__m, (__v4hi)__count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psrlw (__m64 __m, __m64 __count) +{ + return _mm_srl_pi16 (__m, __count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srli_pi16 (__m64 __m, int __count) +{ + return (__m64) __builtin_ia32_psrlwi ((__v4hi)__m, __count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psrlwi (__m64 __m, int __count) +{ + return _mm_srli_pi16 (__m, __count); +} + +/* Shift two 32-bit values in M right by COUNT; shift in zeros. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srl_pi32 (__m64 __m, __m64 __count) +{ + return (__m64) __builtin_ia32_psrld ((__v2si)__m, (__v2si)__count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psrld (__m64 __m, __m64 __count) +{ + return _mm_srl_pi32 (__m, __count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srli_pi32 (__m64 __m, int __count) +{ + return (__m64) __builtin_ia32_psrldi ((__v2si)__m, __count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psrldi (__m64 __m, int __count) +{ + return _mm_srli_pi32 (__m, __count); +} + +/* Shift the 64-bit value in M left by COUNT; shift in zeros. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srl_si64 (__m64 __m, __m64 __count) +{ + return (__m64) __builtin_ia32_psrlq ((__v1di)__m, (__v1di)__count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psrlq (__m64 __m, __m64 __count) +{ + return _mm_srl_si64 (__m, __count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srli_si64 (__m64 __m, int __count) +{ + return (__m64) __builtin_ia32_psrlqi ((__v1di)__m, __count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psrlqi (__m64 __m, int __count) +{ + return _mm_srli_si64 (__m, __count); +} + +/* Bit-wise AND the 64-bit values in M1 and M2. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_and_si64 (__m64 __m1, __m64 __m2) +{ + return __builtin_ia32_pand (__m1, __m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pand (__m64 __m1, __m64 __m2) +{ + return _mm_and_si64 (__m1, __m2); +} + +/* Bit-wise complement the 64-bit value in M1 and bit-wise AND it with the + 64-bit value in M2. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_andnot_si64 (__m64 __m1, __m64 __m2) +{ + return __builtin_ia32_pandn (__m1, __m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pandn (__m64 __m1, __m64 __m2) +{ + return _mm_andnot_si64 (__m1, __m2); +} + +/* Bit-wise inclusive OR the 64-bit values in M1 and M2. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_or_si64 (__m64 __m1, __m64 __m2) +{ + return __builtin_ia32_por (__m1, __m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_por (__m64 __m1, __m64 __m2) +{ + return _mm_or_si64 (__m1, __m2); +} + +/* Bit-wise exclusive OR the 64-bit values in M1 and M2. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_xor_si64 (__m64 __m1, __m64 __m2) +{ + return __builtin_ia32_pxor (__m1, __m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pxor (__m64 __m1, __m64 __m2) +{ + return _mm_xor_si64 (__m1, __m2); +} + +/* Compare eight 8-bit values. The result of the comparison is 0xFF if the + test is true and zero if false. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_pi8 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_pcmpeqb ((__v8qi)__m1, (__v8qi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pcmpeqb (__m64 __m1, __m64 __m2) +{ + return _mm_cmpeq_pi8 (__m1, __m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_pi8 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_pcmpgtb ((__v8qi)__m1, (__v8qi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pcmpgtb (__m64 __m1, __m64 __m2) +{ + return _mm_cmpgt_pi8 (__m1, __m2); +} + +/* Compare four 16-bit values. The result of the comparison is 0xFFFF if + the test is true and zero if false. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_pi16 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_pcmpeqw ((__v4hi)__m1, (__v4hi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pcmpeqw (__m64 __m1, __m64 __m2) +{ + return _mm_cmpeq_pi16 (__m1, __m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_pi16 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_pcmpgtw ((__v4hi)__m1, (__v4hi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pcmpgtw (__m64 __m1, __m64 __m2) +{ + return _mm_cmpgt_pi16 (__m1, __m2); +} + +/* Compare two 32-bit values. The result of the comparison is 0xFFFFFFFF if + the test is true and zero if false. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_pi32 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_pcmpeqd ((__v2si)__m1, (__v2si)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pcmpeqd (__m64 __m1, __m64 __m2) +{ + return _mm_cmpeq_pi32 (__m1, __m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_pi32 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_pcmpgtd ((__v2si)__m1, (__v2si)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pcmpgtd (__m64 __m1, __m64 __m2) +{ + return _mm_cmpgt_pi32 (__m1, __m2); +} + +/* Creates a 64-bit zero. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_setzero_si64 (void) +{ + return (__m64)0LL; +} + +/* Creates a vector of two 32-bit values; I0 is least significant. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set_pi32 (int __i1, int __i0) +{ + return (__m64) __builtin_ia32_vec_init_v2si (__i0, __i1); +} + +/* Creates a vector of four 16-bit values; W0 is least significant. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set_pi16 (short __w3, short __w2, short __w1, short __w0) +{ + return (__m64) __builtin_ia32_vec_init_v4hi (__w0, __w1, __w2, __w3); +} + +/* Creates a vector of eight 8-bit values; B0 is least significant. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set_pi8 (char __b7, char __b6, char __b5, char __b4, + char __b3, char __b2, char __b1, char __b0) +{ + return (__m64) __builtin_ia32_vec_init_v8qi (__b0, __b1, __b2, __b3, + __b4, __b5, __b6, __b7); +} + +/* Similar, but with the arguments in reverse order. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_setr_pi32 (int __i0, int __i1) +{ + return _mm_set_pi32 (__i1, __i0); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_setr_pi16 (short __w0, short __w1, short __w2, short __w3) +{ + return _mm_set_pi16 (__w3, __w2, __w1, __w0); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_setr_pi8 (char __b0, char __b1, char __b2, char __b3, + char __b4, char __b5, char __b6, char __b7) +{ + return _mm_set_pi8 (__b7, __b6, __b5, __b4, __b3, __b2, __b1, __b0); +} + +/* Creates a vector of two 32-bit values, both elements containing I. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set1_pi32 (int __i) +{ + return _mm_set_pi32 (__i, __i); +} + +/* Creates a vector of four 16-bit values, all elements containing W. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set1_pi16 (short __w) +{ + return _mm_set_pi16 (__w, __w, __w, __w); +} + +/* Creates a vector of eight 8-bit values, all elements containing B. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set1_pi8 (char __b) +{ + return _mm_set_pi8 (__b, __b, __b, __b, __b, __b, __b, __b); +} +#ifdef __DISABLE_MMX__ +#undef __DISABLE_MMX__ +#pragma GCC pop_options +#endif /* __DISABLE_MMX__ */ + +#endif /* _MMINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/mwaitxintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/mwaitxintrin.h new file mode 100644 index 0000000000..aaa4c29e6f --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/mwaitxintrin.h @@ -0,0 +1,50 @@ +/* Copyright (C) 2012-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _MWAITXINTRIN_H_INCLUDED +#define _MWAITXINTRIN_H_INCLUDED + +#ifndef __MWAITX__ +#pragma GCC push_options +#pragma GCC target("mwaitx") +#define __DISABLE_MWAITX__ +#endif /* __MWAITX__ */ + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_monitorx (void const * __P, unsigned int __E, unsigned int __H) +{ + __builtin_ia32_monitorx (__P, __E, __H); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mwaitx (unsigned int __E, unsigned int __H, unsigned int __C) +{ + __builtin_ia32_mwaitx (__E, __H, __C); +} + +#ifdef __DISABLE_MWAITX__ +#undef __DISABLE_MWAITX__ +#pragma GCC pop_options +#endif /* __DISABLE_MWAITX__ */ + +#endif /* _MWAITXINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/nmmintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/nmmintrin.h new file mode 100644 index 0000000000..6ea57d326d --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/nmmintrin.h @@ -0,0 +1,33 @@ +/* Copyright (C) 2007-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +/* Implemented from the specification included in the Intel C++ Compiler + User Guide and Reference, version 10.0. */ + +#ifndef _NMMINTRIN_H_INCLUDED +#define _NMMINTRIN_H_INCLUDED + +/* We just include SSE4.1 header file. */ +#include + +#endif /* _NMMINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/omp.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/omp.h new file mode 100644 index 0000000000..04f170b690 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/omp.h @@ -0,0 +1,165 @@ +/* Copyright (C) 2005-2017 Free Software Foundation, Inc. + Contributed by Richard Henderson . + + This file is part of the GNU Offloading and Multi Processing Library + (libgomp). + + Libgomp is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _OMP_H +#define _OMP_H 1 + +#ifndef _LIBGOMP_OMP_LOCK_DEFINED +#define _LIBGOMP_OMP_LOCK_DEFINED 1 +/* These two structures get edited by the libgomp build process to + reflect the shape of the two types. Their internals are private + to the library. */ + +typedef struct +{ + unsigned char _x[4] + __attribute__((__aligned__(4))); +} omp_lock_t; + +typedef struct +{ + unsigned char _x[8 + sizeof (void *)] + __attribute__((__aligned__(sizeof (void *)))); +} omp_nest_lock_t; +#endif + +typedef enum omp_sched_t +{ + omp_sched_static = 1, + omp_sched_dynamic = 2, + omp_sched_guided = 3, + omp_sched_auto = 4 +} omp_sched_t; + +typedef enum omp_proc_bind_t +{ + omp_proc_bind_false = 0, + omp_proc_bind_true = 1, + omp_proc_bind_master = 2, + omp_proc_bind_close = 3, + omp_proc_bind_spread = 4 +} omp_proc_bind_t; + +typedef enum omp_lock_hint_t +{ + omp_lock_hint_none = 0, + omp_lock_hint_uncontended = 1, + omp_lock_hint_contended = 2, + omp_lock_hint_nonspeculative = 4, + omp_lock_hint_speculative = 8, +} omp_lock_hint_t; + +#ifdef __cplusplus +extern "C" { +# define __GOMP_NOTHROW throw () +#else +# define __GOMP_NOTHROW __attribute__((__nothrow__)) +#endif + +extern void omp_set_num_threads (int) __GOMP_NOTHROW; +extern int omp_get_num_threads (void) __GOMP_NOTHROW; +extern int omp_get_max_threads (void) __GOMP_NOTHROW; +extern int omp_get_thread_num (void) __GOMP_NOTHROW; +extern int omp_get_num_procs (void) __GOMP_NOTHROW; + +extern int omp_in_parallel (void) __GOMP_NOTHROW; + +extern void omp_set_dynamic (int) __GOMP_NOTHROW; +extern int omp_get_dynamic (void) __GOMP_NOTHROW; + +extern void omp_set_nested (int) __GOMP_NOTHROW; +extern int omp_get_nested (void) __GOMP_NOTHROW; + +extern void omp_init_lock (omp_lock_t *) __GOMP_NOTHROW; +extern void omp_init_lock_with_hint (omp_lock_t *, omp_lock_hint_t) + __GOMP_NOTHROW; +extern void omp_destroy_lock (omp_lock_t *) __GOMP_NOTHROW; +extern void omp_set_lock (omp_lock_t *) __GOMP_NOTHROW; +extern void omp_unset_lock (omp_lock_t *) __GOMP_NOTHROW; +extern int omp_test_lock (omp_lock_t *) __GOMP_NOTHROW; + +extern void omp_init_nest_lock (omp_nest_lock_t *) __GOMP_NOTHROW; +extern void omp_init_nest_lock_with_hint (omp_nest_lock_t *, omp_lock_hint_t) + __GOMP_NOTHROW; +extern void omp_destroy_nest_lock (omp_nest_lock_t *) __GOMP_NOTHROW; +extern void omp_set_nest_lock (omp_nest_lock_t *) __GOMP_NOTHROW; +extern void omp_unset_nest_lock (omp_nest_lock_t *) __GOMP_NOTHROW; +extern int omp_test_nest_lock (omp_nest_lock_t *) __GOMP_NOTHROW; + +extern double omp_get_wtime (void) __GOMP_NOTHROW; +extern double omp_get_wtick (void) __GOMP_NOTHROW; + +extern void omp_set_schedule (omp_sched_t, int) __GOMP_NOTHROW; +extern void omp_get_schedule (omp_sched_t *, int *) __GOMP_NOTHROW; +extern int omp_get_thread_limit (void) __GOMP_NOTHROW; +extern void omp_set_max_active_levels (int) __GOMP_NOTHROW; +extern int omp_get_max_active_levels (void) __GOMP_NOTHROW; +extern int omp_get_level (void) __GOMP_NOTHROW; +extern int omp_get_ancestor_thread_num (int) __GOMP_NOTHROW; +extern int omp_get_team_size (int) __GOMP_NOTHROW; +extern int omp_get_active_level (void) __GOMP_NOTHROW; + +extern int omp_in_final (void) __GOMP_NOTHROW; + +extern int omp_get_cancellation (void) __GOMP_NOTHROW; +extern omp_proc_bind_t omp_get_proc_bind (void) __GOMP_NOTHROW; +extern int omp_get_num_places (void) __GOMP_NOTHROW; +extern int omp_get_place_num_procs (int) __GOMP_NOTHROW; +extern void omp_get_place_proc_ids (int, int *) __GOMP_NOTHROW; +extern int omp_get_place_num (void) __GOMP_NOTHROW; +extern int omp_get_partition_num_places (void) __GOMP_NOTHROW; +extern void omp_get_partition_place_nums (int *) __GOMP_NOTHROW; + +extern void omp_set_default_device (int) __GOMP_NOTHROW; +extern int omp_get_default_device (void) __GOMP_NOTHROW; +extern int omp_get_num_devices (void) __GOMP_NOTHROW; +extern int omp_get_num_teams (void) __GOMP_NOTHROW; +extern int omp_get_team_num (void) __GOMP_NOTHROW; + +extern int omp_is_initial_device (void) __GOMP_NOTHROW; +extern int omp_get_initial_device (void) __GOMP_NOTHROW; +extern int omp_get_max_task_priority (void) __GOMP_NOTHROW; + +extern void *omp_target_alloc (__SIZE_TYPE__, int) __GOMP_NOTHROW; +extern void omp_target_free (void *, int) __GOMP_NOTHROW; +extern int omp_target_is_present (void *, int) __GOMP_NOTHROW; +extern int omp_target_memcpy (void *, void *, __SIZE_TYPE__, __SIZE_TYPE__, + __SIZE_TYPE__, int, int) __GOMP_NOTHROW; +extern int omp_target_memcpy_rect (void *, void *, __SIZE_TYPE__, int, + const __SIZE_TYPE__ *, + const __SIZE_TYPE__ *, + const __SIZE_TYPE__ *, + const __SIZE_TYPE__ *, + const __SIZE_TYPE__ *, int, int) + __GOMP_NOTHROW; +extern int omp_target_associate_ptr (void *, void *, __SIZE_TYPE__, + __SIZE_TYPE__, int) __GOMP_NOTHROW; +extern int omp_target_disassociate_ptr (void *, int) __GOMP_NOTHROW; + +#ifdef __cplusplus +} +#endif + +#endif /* _OMP_H */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/openacc.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/openacc.h new file mode 100644 index 0000000000..53d0c3955c --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/openacc.h @@ -0,0 +1,131 @@ +/* OpenACC Runtime Library User-facing Declarations + + Copyright (C) 2013-2017 Free Software Foundation, Inc. + + Contributed by Mentor Embedded. + + This file is part of the GNU Offloading and Multi Processing Library + (libgomp). + + Libgomp is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _OPENACC_H +#define _OPENACC_H 1 + +/* The OpenACC standard is silent on whether or not including + might or must not include other header files. We chose to include + some. */ +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#if __cplusplus >= 201103 +# define __GOACC_NOTHROW noexcept +#elif __cplusplus +# define __GOACC_NOTHROW throw () +#else /* Not C++ */ +# define __GOACC_NOTHROW __attribute__ ((__nothrow__)) +#endif + +/* Types */ +typedef enum acc_device_t { + /* Keep in sync with include/gomp-constants.h. */ + acc_device_none = 0, + acc_device_default = 1, + acc_device_host = 2, + /* acc_device_host_nonshm = 3 removed. */ + acc_device_not_host = 4, + acc_device_nvidia = 5, + _ACC_device_hwm, + /* Ensure enumeration is layout compatible with int. */ + _ACC_highest = __INT_MAX__, + _ACC_neg = -1 +} acc_device_t; + +typedef enum acc_async_t { + /* Keep in sync with include/gomp-constants.h. */ + acc_async_noval = -1, + acc_async_sync = -2 +} acc_async_t; + +int acc_get_num_devices (acc_device_t) __GOACC_NOTHROW; +void acc_set_device_type (acc_device_t) __GOACC_NOTHROW; +acc_device_t acc_get_device_type (void) __GOACC_NOTHROW; +void acc_set_device_num (int, acc_device_t) __GOACC_NOTHROW; +int acc_get_device_num (acc_device_t) __GOACC_NOTHROW; +int acc_async_test (int) __GOACC_NOTHROW; +int acc_async_test_all (void) __GOACC_NOTHROW; +void acc_wait (int) __GOACC_NOTHROW; +void acc_wait_async (int, int) __GOACC_NOTHROW; +void acc_wait_all (void) __GOACC_NOTHROW; +void acc_wait_all_async (int) __GOACC_NOTHROW; +void acc_init (acc_device_t) __GOACC_NOTHROW; +void acc_shutdown (acc_device_t) __GOACC_NOTHROW; +#ifdef __cplusplus +int acc_on_device (int __arg) __GOACC_NOTHROW; +#else +int acc_on_device (acc_device_t __arg) __GOACC_NOTHROW; +#endif +void *acc_malloc (size_t) __GOACC_NOTHROW; +void acc_free (void *) __GOACC_NOTHROW; +/* Some of these would be more correct with const qualifiers, but + the standard specifies otherwise. */ +void *acc_copyin (void *, size_t) __GOACC_NOTHROW; +void *acc_present_or_copyin (void *, size_t) __GOACC_NOTHROW; +void *acc_create (void *, size_t) __GOACC_NOTHROW; +void *acc_present_or_create (void *, size_t) __GOACC_NOTHROW; +void acc_copyout (void *, size_t) __GOACC_NOTHROW; +void acc_delete (void *, size_t) __GOACC_NOTHROW; +void acc_update_device (void *, size_t) __GOACC_NOTHROW; +void acc_update_self (void *, size_t) __GOACC_NOTHROW; +void acc_map_data (void *, void *, size_t) __GOACC_NOTHROW; +void acc_unmap_data (void *) __GOACC_NOTHROW; +void *acc_deviceptr (void *) __GOACC_NOTHROW; +void *acc_hostptr (void *) __GOACC_NOTHROW; +int acc_is_present (void *, size_t) __GOACC_NOTHROW; +void acc_memcpy_to_device (void *, void *, size_t) __GOACC_NOTHROW; +void acc_memcpy_from_device (void *, void *, size_t) __GOACC_NOTHROW; + +/* Old names. OpenACC does not specify whether these can or must + not be macros, inlines or aliases for the new names. */ +#define acc_pcreate acc_present_or_create +#define acc_pcopyin acc_present_or_copyin + +/* CUDA-specific routines. */ +void *acc_get_current_cuda_device (void) __GOACC_NOTHROW; +void *acc_get_current_cuda_context (void) __GOACC_NOTHROW; +void *acc_get_cuda_stream (int) __GOACC_NOTHROW; +int acc_set_cuda_stream (int, void *) __GOACC_NOTHROW; + +#ifdef __cplusplus +} + +/* Forwarding function with correctly typed arg. */ + +#pragma acc routine seq +inline int acc_on_device (acc_device_t __arg) __GOACC_NOTHROW +{ + return acc_on_device ((int) __arg); +} +#endif + +#endif /* _OPENACC_H */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/pkuintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/pkuintrin.h new file mode 100644 index 0000000000..0f4886ca82 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/pkuintrin.h @@ -0,0 +1,56 @@ +/* Copyright (C) 2015-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#if !defined _X86INTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _PKUINTRIN_H_INCLUDED +#define _PKUINTRIN_H_INCLUDED + +#ifndef __PKU__ +#pragma GCC push_options +#pragma GCC target("pku") +#define __DISABLE_PKU__ +#endif /* __PKU__ */ + +extern __inline unsigned int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_rdpkru_u32 (void) +{ + return __builtin_ia32_rdpkru (); +} + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_wrpkru (unsigned int __key) +{ + __builtin_ia32_wrpkru (__key); +} + +#ifdef __DISABLE_PKU__ +#undef __DISABLE_PKU__ +#pragma GCC pop_options +#endif /* __DISABLE_PKU__ */ + +#endif /* _PKUINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/pmmintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/pmmintrin.h new file mode 100644 index 0000000000..39cc1c6761 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/pmmintrin.h @@ -0,0 +1,132 @@ +/* Copyright (C) 2003-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +/* Implemented from the specification included in the Intel C++ Compiler + User Guide and Reference, version 9.0. */ + +#ifndef _PMMINTRIN_H_INCLUDED +#define _PMMINTRIN_H_INCLUDED + +/* We need definitions from the SSE2 and SSE header files*/ +#include + +#ifndef __SSE3__ +#pragma GCC push_options +#pragma GCC target("sse3") +#define __DISABLE_SSE3__ +#endif /* __SSE3__ */ + +/* Additional bits in the MXCSR. */ +#define _MM_DENORMALS_ZERO_MASK 0x0040 +#define _MM_DENORMALS_ZERO_ON 0x0040 +#define _MM_DENORMALS_ZERO_OFF 0x0000 + +#define _MM_SET_DENORMALS_ZERO_MODE(mode) \ + _mm_setcsr ((_mm_getcsr () & ~_MM_DENORMALS_ZERO_MASK) | (mode)) +#define _MM_GET_DENORMALS_ZERO_MODE() \ + (_mm_getcsr() & _MM_DENORMALS_ZERO_MASK) + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_addsub_ps (__m128 __X, __m128 __Y) +{ + return (__m128) __builtin_ia32_addsubps ((__v4sf)__X, (__v4sf)__Y); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hadd_ps (__m128 __X, __m128 __Y) +{ + return (__m128) __builtin_ia32_haddps ((__v4sf)__X, (__v4sf)__Y); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hsub_ps (__m128 __X, __m128 __Y) +{ + return (__m128) __builtin_ia32_hsubps ((__v4sf)__X, (__v4sf)__Y); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movehdup_ps (__m128 __X) +{ + return (__m128) __builtin_ia32_movshdup ((__v4sf)__X); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_moveldup_ps (__m128 __X) +{ + return (__m128) __builtin_ia32_movsldup ((__v4sf)__X); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_addsub_pd (__m128d __X, __m128d __Y) +{ + return (__m128d) __builtin_ia32_addsubpd ((__v2df)__X, (__v2df)__Y); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hadd_pd (__m128d __X, __m128d __Y) +{ + return (__m128d) __builtin_ia32_haddpd ((__v2df)__X, (__v2df)__Y); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hsub_pd (__m128d __X, __m128d __Y) +{ + return (__m128d) __builtin_ia32_hsubpd ((__v2df)__X, (__v2df)__Y); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_loaddup_pd (double const *__P) +{ + return _mm_load1_pd (__P); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movedup_pd (__m128d __X) +{ + return _mm_shuffle_pd (__X, __X, _MM_SHUFFLE2 (0,0)); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_lddqu_si128 (__m128i const *__P) +{ + return (__m128i) __builtin_ia32_lddqu ((char const *)__P); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_monitor (void const * __P, unsigned int __E, unsigned int __H) +{ + __builtin_ia32_monitor (__P, __E, __H); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mwait (unsigned int __E, unsigned int __H) +{ + __builtin_ia32_mwait (__E, __H); +} + +#ifdef __DISABLE_SSE3__ +#undef __DISABLE_SSE3__ +#pragma GCC pop_options +#endif /* __DISABLE_SSE3__ */ + +#endif /* _PMMINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/popcntintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/popcntintrin.h new file mode 100644 index 0000000000..733fd75553 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/popcntintrin.h @@ -0,0 +1,53 @@ +/* Copyright (C) 2009-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _POPCNTINTRIN_H_INCLUDED +#define _POPCNTINTRIN_H_INCLUDED + +#ifndef __POPCNT__ +#pragma GCC push_options +#pragma GCC target("popcnt") +#define __DISABLE_POPCNT__ +#endif /* __POPCNT__ */ + +/* Calculate a number of bits set to 1. */ +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_popcnt_u32 (unsigned int __X) +{ + return __builtin_popcount (__X); +} + +#ifdef __x86_64__ +extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_popcnt_u64 (unsigned long long __X) +{ + return __builtin_popcountll (__X); +} +#endif + +#ifdef __DISABLE_POPCNT__ +#undef __DISABLE_POPCNT__ +#pragma GCC pop_options +#endif /* __DISABLE_POPCNT__ */ + +#endif /* _POPCNTINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/prfchwintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/prfchwintrin.h new file mode 100644 index 0000000000..11a06c8951 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/prfchwintrin.h @@ -0,0 +1,37 @@ +/* Copyright (C) 2012-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#if !defined _X86INTRIN_H_INCLUDED && !defined _MM3DNOW_H_INCLUDED +# error "Never use directly; include or instead." +#endif + +#ifndef _PRFCHWINTRIN_H_INCLUDED +#define _PRFCHWINTRIN_H_INCLUDED + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_prefetchw (void *__P) +{ + __builtin_prefetch (__P, 1, 3 /* _MM_HINT_T0 */); +} + +#endif /* _PRFCHWINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/rdseedintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/rdseedintrin.h new file mode 100644 index 0000000000..2b731df7d8 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/rdseedintrin.h @@ -0,0 +1,66 @@ +/* Copyright (C) 2012-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#if !defined _X86INTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _RDSEEDINTRIN_H_INCLUDED +#define _RDSEEDINTRIN_H_INCLUDED + +#ifndef __RDSEED__ +#pragma GCC push_options +#pragma GCC target("rdseed") +#define __DISABLE_RDSEED__ +#endif /* __RDSEED__ */ + + +extern __inline int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_rdseed16_step (unsigned short *__p) +{ + return __builtin_ia32_rdseed_hi_step (__p); +} + +extern __inline int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_rdseed32_step (unsigned int *__p) +{ + return __builtin_ia32_rdseed_si_step (__p); +} + +#ifdef __x86_64__ +extern __inline int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_rdseed64_step (unsigned long long *__p) +{ + return __builtin_ia32_rdseed_di_step (__p); +} +#endif + +#ifdef __DISABLE_RDSEED__ +#undef __DISABLE_RDSEED__ +#pragma GCC pop_options +#endif /* __DISABLE_RDSEED__ */ + +#endif /* _RDSEEDINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/rtmintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/rtmintrin.h new file mode 100644 index 0000000000..ef2601eed9 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/rtmintrin.h @@ -0,0 +1,84 @@ +/* Copyright (C) 2012-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _RTMINTRIN_H_INCLUDED +#define _RTMINTRIN_H_INCLUDED + +#ifndef __RTM__ +#pragma GCC push_options +#pragma GCC target("rtm") +#define __DISABLE_RTM__ +#endif /* __RTM__ */ + +#define _XBEGIN_STARTED (~0u) +#define _XABORT_EXPLICIT (1 << 0) +#define _XABORT_RETRY (1 << 1) +#define _XABORT_CONFLICT (1 << 2) +#define _XABORT_CAPACITY (1 << 3) +#define _XABORT_DEBUG (1 << 4) +#define _XABORT_NESTED (1 << 5) +#define _XABORT_CODE(x) (((x) >> 24) & 0xFF) + +/* Start an RTM code region. Return _XBEGIN_STARTED on success and the + abort condition otherwise. */ +extern __inline unsigned int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_xbegin (void) +{ + return __builtin_ia32_xbegin (); +} + +/* Specify the end of an RTM code region. If it corresponds to the + outermost transaction, then attempts the transaction commit. If the + commit fails, then control is transferred to the outermost transaction + fallback handler. */ +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_xend (void) +{ + __builtin_ia32_xend (); +} + +/* Force an RTM abort condition. The control is transferred to the + outermost transaction fallback handler with the abort condition IMM. */ +#ifdef __OPTIMIZE__ +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_xabort (const unsigned int __imm) +{ + __builtin_ia32_xabort (__imm); +} +#else +#define _xabort(N) __builtin_ia32_xabort (N) +#endif /* __OPTIMIZE__ */ + +#ifdef __DISABLE_RTM__ +#undef __DISABLE_RTM__ +#pragma GCC pop_options +#endif /* __DISABLE_RTM__ */ + +#endif /* _RTMINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/sanitizer/asan_interface.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/sanitizer/asan_interface.h new file mode 100644 index 0000000000..448a0bcba0 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/sanitizer/asan_interface.h @@ -0,0 +1,149 @@ +//===-- sanitizer/asan_interface.h ------------------------------*- C++ -*-===// +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of AddressSanitizer. +// +// Public interface header. +//===----------------------------------------------------------------------===// +#ifndef SANITIZER_ASAN_INTERFACE_H +#define SANITIZER_ASAN_INTERFACE_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + // Marks memory region [addr, addr+size) as unaddressable. + // This memory must be previously allocated by the user program. Accessing + // addresses in this region from instrumented code is forbidden until + // this region is unpoisoned. This function is not guaranteed to poison + // the whole region - it may poison only subregion of [addr, addr+size) due + // to ASan alignment restrictions. + // Method is NOT thread-safe in the sense that no two threads can + // (un)poison memory in the same memory region simultaneously. + void __asan_poison_memory_region(void const volatile *addr, size_t size); + // Marks memory region [addr, addr+size) as addressable. + // This memory must be previously allocated by the user program. Accessing + // addresses in this region is allowed until this region is poisoned again. + // This function may unpoison a superregion of [addr, addr+size) due to + // ASan alignment restrictions. + // Method is NOT thread-safe in the sense that no two threads can + // (un)poison memory in the same memory region simultaneously. + void __asan_unpoison_memory_region(void const volatile *addr, size_t size); + +// User code should use macros instead of functions. +#if __has_feature(address_sanitizer) || defined(__SANITIZE_ADDRESS__) +#define ASAN_POISON_MEMORY_REGION(addr, size) \ + __asan_poison_memory_region((addr), (size)) +#define ASAN_UNPOISON_MEMORY_REGION(addr, size) \ + __asan_unpoison_memory_region((addr), (size)) +#else +#define ASAN_POISON_MEMORY_REGION(addr, size) \ + ((void)(addr), (void)(size)) +#define ASAN_UNPOISON_MEMORY_REGION(addr, size) \ + ((void)(addr), (void)(size)) +#endif + + // Returns 1 if addr is poisoned (i.e. 1-byte read/write access to this + // address will result in error report from AddressSanitizer). + // Otherwise returns 0. + int __asan_address_is_poisoned(void const volatile *addr); + + // If at least one byte in [beg, beg+size) is poisoned, return the address + // of the first such byte. Otherwise return 0. + void *__asan_region_is_poisoned(void *beg, size_t size); + + // Print the description of addr (useful when debugging in gdb). + void __asan_describe_address(void *addr); + + // Useful for calling from a debugger to get information about an ASan error. + // Returns 1 if an error has been (or is being) reported, otherwise returns 0. + int __asan_report_present(); + + // Useful for calling from a debugger to get information about an ASan error. + // If an error has been (or is being) reported, the following functions return + // the pc, bp, sp, address, access type (0 = read, 1 = write), access size and + // bug description (e.g. "heap-use-after-free"). Otherwise they return 0. + void *__asan_get_report_pc(); + void *__asan_get_report_bp(); + void *__asan_get_report_sp(); + void *__asan_get_report_address(); + int __asan_get_report_access_type(); + size_t __asan_get_report_access_size(); + const char *__asan_get_report_description(); + + // Useful for calling from the debugger to get information about a pointer. + // Returns the category of the given pointer as a constant string. + // Possible return values are "global", "stack", "stack-fake", "heap", + // "heap-invalid", "shadow-low", "shadow-gap", "shadow-high", "unknown". + // If global or stack, tries to also return the variable name, address and + // size. If heap, tries to return the chunk address and size. 'name' should + // point to an allocated buffer of size 'name_size'. + const char *__asan_locate_address(void *addr, char *name, size_t name_size, + void **region_address, size_t *region_size); + + // Useful for calling from the debugger to get the allocation stack trace + // and thread ID for a heap address. Stores up to 'size' frames into 'trace', + // returns the number of stored frames or 0 on error. + size_t __asan_get_alloc_stack(void *addr, void **trace, size_t size, + int *thread_id); + + // Useful for calling from the debugger to get the free stack trace + // and thread ID for a heap address. Stores up to 'size' frames into 'trace', + // returns the number of stored frames or 0 on error. + size_t __asan_get_free_stack(void *addr, void **trace, size_t size, + int *thread_id); + + // Useful for calling from the debugger to get the current shadow memory + // mapping. + void __asan_get_shadow_mapping(size_t *shadow_scale, size_t *shadow_offset); + + // This is an internal function that is called to report an error. + // However it is still a part of the interface because users may want to + // set a breakpoint on this function in a debugger. + void __asan_report_error(void *pc, void *bp, void *sp, + void *addr, int is_write, size_t access_size); + + // Deprecated. Call __sanitizer_set_death_callback instead. + void __asan_set_death_callback(void (*callback)(void)); + + void __asan_set_error_report_callback(void (*callback)(const char*)); + + // User may provide function that would be called right when ASan detects + // an error. This can be used to notice cases when ASan detects an error, but + // the program crashes before ASan report is printed. + void __asan_on_error(); + + // Prints accumulated stats to stderr. Used for debugging. + void __asan_print_accumulated_stats(); + + // This function may be optionally provided by user and should return + // a string containing ASan runtime options. See asan_flags.h for details. + const char* __asan_default_options(); + + // The following 2 functions facilitate garbage collection in presence of + // asan's fake stack. + + // Returns an opaque handler to be used later in __asan_addr_is_in_fake_stack. + // Returns NULL if the current thread does not have a fake stack. + void *__asan_get_current_fake_stack(); + + // If fake_stack is non-NULL and addr belongs to a fake frame in + // fake_stack, returns the address on real stack that corresponds to + // the fake frame and sets beg/end to the boundaries of this fake frame. + // Otherwise returns NULL and does not touch beg/end. + // If beg/end are NULL, they are not touched. + // This function may be called from a thread other than the owner of + // fake_stack, but the owner thread need to be alive. + void *__asan_addr_is_in_fake_stack(void *fake_stack, void *addr, void **beg, + void **end); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // SANITIZER_ASAN_INTERFACE_H diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/sanitizer/common_interface_defs.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/sanitizer/common_interface_defs.h new file mode 100644 index 0000000000..fd38c55a23 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/sanitizer/common_interface_defs.h @@ -0,0 +1,187 @@ +//===-- sanitizer/common_interface_defs.h -----------------------*- C++ -*-===// +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Common part of the public sanitizer interface. +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_COMMON_INTERFACE_DEFS_H +#define SANITIZER_COMMON_INTERFACE_DEFS_H + +#include +#include + +// GCC does not understand __has_feature. +#if !defined(__has_feature) +# define __has_feature(x) 0 +#endif + +#ifdef __cplusplus +extern "C" { +#endif + // Arguments for __sanitizer_sandbox_on_notify() below. + typedef struct { + // Enable sandbox support in sanitizer coverage. + int coverage_sandboxed; + // File descriptor to write coverage data to. If -1 is passed, a file will + // be pre-opened by __sanitizer_sandobx_on_notify(). This field has no + // effect if coverage_sandboxed == 0. + intptr_t coverage_fd; + // If non-zero, split the coverage data into well-formed blocks. This is + // useful when coverage_fd is a socket descriptor. Each block will contain + // a header, allowing data from multiple processes to be sent over the same + // socket. + unsigned int coverage_max_block_size; + } __sanitizer_sandbox_arguments; + + // Tell the tools to write their reports to "path." instead of stderr. + void __sanitizer_set_report_path(const char *path); + // Tell the tools to write their reports to the provided file descriptor + // (casted to void *). + void __sanitizer_set_report_fd(void *fd); + + // Notify the tools that the sandbox is going to be turned on. The reserved + // parameter will be used in the future to hold a structure with functions + // that the tools may call to bypass the sandbox. + void __sanitizer_sandbox_on_notify(__sanitizer_sandbox_arguments *args); + + // This function is called by the tool when it has just finished reporting + // an error. 'error_summary' is a one-line string that summarizes + // the error message. This function can be overridden by the client. + void __sanitizer_report_error_summary(const char *error_summary); + + // Some of the sanitizers (e.g. asan/tsan) may miss bugs that happen + // in unaligned loads/stores. In order to find such bugs reliably one needs + // to replace plain unaligned loads/stores with these calls. + uint16_t __sanitizer_unaligned_load16(const void *p); + uint32_t __sanitizer_unaligned_load32(const void *p); + uint64_t __sanitizer_unaligned_load64(const void *p); + void __sanitizer_unaligned_store16(void *p, uint16_t x); + void __sanitizer_unaligned_store32(void *p, uint32_t x); + void __sanitizer_unaligned_store64(void *p, uint64_t x); + + // Annotate the current state of a contiguous container, such as + // std::vector, std::string or similar. + // A contiguous container is a container that keeps all of its elements + // in a contiguous region of memory. The container owns the region of memory + // [beg, end); the memory [beg, mid) is used to store the current elements + // and the memory [mid, end) is reserved for future elements; + // beg <= mid <= end. For example, in "std::vector<> v" + // beg = &v[0]; + // end = beg + v.capacity() * sizeof(v[0]); + // mid = beg + v.size() * sizeof(v[0]); + // + // This annotation tells the Sanitizer tool about the current state of the + // container so that the tool can report errors when memory from [mid, end) + // is accessed. Insert this annotation into methods like push_back/pop_back. + // Supply the old and the new values of mid (old_mid/new_mid). + // In the initial state mid == end and so should be the final + // state when the container is destroyed or when it reallocates the storage. + // + // Use with caution and don't use for anything other than vector-like classes. + // + // For AddressSanitizer, 'beg' should be 8-aligned and 'end' should + // be either 8-aligned or it should point to the end of a separate heap-, + // stack-, or global- allocated buffer. I.e. the following will not work: + // int64_t x[2]; // 16 bytes, 8-aligned. + // char *beg = (char *)&x[0]; + // char *end = beg + 12; // Not 8 aligned, not the end of the buffer. + // This however will work fine: + // int32_t x[3]; // 12 bytes, but 8-aligned under AddressSanitizer. + // char *beg = (char*)&x[0]; + // char *end = beg + 12; // Not 8-aligned, but is the end of the buffer. + void __sanitizer_annotate_contiguous_container(const void *beg, + const void *end, + const void *old_mid, + const void *new_mid); + // Returns true if the contiguous container [beg, end) is properly poisoned + // (e.g. with __sanitizer_annotate_contiguous_container), i.e. if + // - [beg, mid) is addressable, + // - [mid, end) is unaddressable. + // Full verification requires O(end-beg) time; this function tries to avoid + // such complexity by touching only parts of the container around beg/mid/end. + int __sanitizer_verify_contiguous_container(const void *beg, const void *mid, + const void *end); + + // Similar to __sanitizer_verify_contiguous_container but returns the address + // of the first improperly poisoned byte otherwise. Returns null if the area + // is poisoned properly. + const void *__sanitizer_contiguous_container_find_bad_address( + const void *beg, const void *mid, const void *end); + + // Print the stack trace leading to this call. Useful for debugging user code. + void __sanitizer_print_stack_trace(); + + // Symbolizes the supplied 'pc' using the format string 'fmt'. + // Outputs at most 'out_buf_size' bytes into 'out_buf'. + // The format syntax is described in + // lib/sanitizer_common/sanitizer_stacktrace_printer.h. + void __sanitizer_symbolize_pc(void *pc, const char *fmt, char *out_buf, + size_t out_buf_size); + // Same as __sanitizer_symbolize_pc, but for data section (i.e. globals). + void __sanitizer_symbolize_global(void *data_ptr, const char *fmt, + char *out_buf, size_t out_buf_size); + + // Sets the callback to be called right before death on error. + // Passing 0 will unset the callback. + void __sanitizer_set_death_callback(void (*callback)(void)); + + // Interceptor hooks. + // Whenever a libc function interceptor is called it checks if the + // corresponding weak hook is defined, and it so -- calls it. + // The primary use case is data-flow-guided fuzzing, where the fuzzer needs + // to know what is being passed to libc functions, e.g. memcmp. + // FIXME: implement more hooks. + void __sanitizer_weak_hook_memcmp(void *called_pc, const void *s1, + const void *s2, size_t n, int result); + void __sanitizer_weak_hook_strncmp(void *called_pc, const char *s1, + const char *s2, size_t n, int result); + void __sanitizer_weak_hook_strncasecmp(void *called_pc, const char *s1, + const char *s2, size_t n, int result); + void __sanitizer_weak_hook_strcmp(void *called_pc, const char *s1, + const char *s2, int result); + void __sanitizer_weak_hook_strcasecmp(void *called_pc, const char *s1, + const char *s2, int result); + void __sanitizer_weak_hook_strstr(void *called_pc, const char *s1, + const char *s2, char *result); + void __sanitizer_weak_hook_strcasestr(void *called_pc, const char *s1, + const char *s2, char *result); + void __sanitizer_weak_hook_memmem(void *called_pc, + const void *s1, size_t len1, + const void *s2, size_t len2, void *result); + + // Prints stack traces for all live heap allocations ordered by total + // allocation size until `top_percent` of total live heap is shown. + // `top_percent` should be between 1 and 100. + // Experimental feature currently available only with asan on Linux/x86_64. + void __sanitizer_print_memory_profile(size_t top_percent); + + // Fiber annotation interface. + // Before switching to a different stack, one must call + // __sanitizer_start_switch_fiber with a pointer to the bottom of the + // destination stack and its size. When code starts running on the new stack, + // it must call __sanitizer_finish_switch_fiber to finalize the switch. + // The start_switch function takes a void** to store the current fake stack if + // there is one (it is needed when detect_stack_use_after_return is enabled). + // When restoring a stack, this pointer must be given to the finish_switch + // function. In most cases, this void* can be stored on the stack just before + // switching. When leaving a fiber definitely, null must be passed as first + // argument to the start_switch function so that the fake stack is destroyed. + // If you do not want support for stack use-after-return detection, you can + // always pass null to these two functions. + // Note that the fake stack mechanism is disabled during fiber switch, so if a + // signal callback runs during the switch, it will not benefit from the stack + // use-after-return detection. + void __sanitizer_start_switch_fiber(void **fake_stack_save, + const void *bottom, size_t size); + void __sanitizer_finish_switch_fiber(void *fake_stack_save, + const void **bottom_old, + size_t *size_old); +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // SANITIZER_COMMON_INTERFACE_DEFS_H diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/sanitizer/lsan_interface.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/sanitizer/lsan_interface.h new file mode 100644 index 0000000000..bdbe39084f --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/sanitizer/lsan_interface.h @@ -0,0 +1,82 @@ +//===-- sanitizer/lsan_interface.h ------------------------------*- C++ -*-===// +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of LeakSanitizer. +// +// Public interface header. +//===----------------------------------------------------------------------===// +#ifndef SANITIZER_LSAN_INTERFACE_H +#define SANITIZER_LSAN_INTERFACE_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + // Allocations made between calls to __lsan_disable() and __lsan_enable() will + // be treated as non-leaks. Disable/enable pairs may be nested. + void __lsan_disable(); + void __lsan_enable(); + + // The heap object into which p points will be treated as a non-leak. + void __lsan_ignore_object(const void *p); + + // Memory regions registered through this interface will be treated as sources + // of live pointers during leak checking. Useful if you store pointers in + // mapped memory. + // Points of note: + // - __lsan_unregister_root_region() must be called with the same pointer and + // size that have earlier been passed to __lsan_register_root_region() + // - LSan will skip any inaccessible memory when scanning a root region. E.g., + // if you map memory within a larger region that you have mprotect'ed, you can + // register the entire large region. + // - the implementation is not optimized for performance. This interface is + // intended to be used for a small number of relatively static regions. + void __lsan_register_root_region(const void *p, size_t size); + void __lsan_unregister_root_region(const void *p, size_t size); + + // Check for leaks now. This function behaves identically to the default + // end-of-process leak check. In particular, it will terminate the process if + // leaks are found and the exitcode runtime flag is non-zero. + // Subsequent calls to this function will have no effect and end-of-process + // leak check will not run. Effectively, end-of-process leak check is moved to + // the time of first invocation of this function. + // By calling this function early during process shutdown, you can instruct + // LSan to ignore shutdown-only leaks which happen later on. + void __lsan_do_leak_check(); + + // Check for leaks now. Returns zero if no leaks have been found or if leak + // detection is disabled, non-zero otherwise. + // This function may be called repeatedly, e.g. to periodically check a + // long-running process. It prints a leak report if appropriate, but does not + // terminate the process. It does not affect the behavior of + // __lsan_do_leak_check() or the end-of-process leak check, and is not + // affected by them. + int __lsan_do_recoverable_leak_check(); + + // The user may optionally provide this function to disallow leak checking + // for the program it is linked into (if the return value is non-zero). This + // function must be defined as returning a constant value; any behavior beyond + // that is unsupported. + int __lsan_is_turned_off(); + + // This function may be optionally provided by the user and should return + // a string containing LSan suppressions. + const char *__lsan_default_suppressions(); +#ifdef __cplusplus +} // extern "C" + +namespace __lsan { +class ScopedDisabler { + public: + ScopedDisabler() { __lsan_disable(); } + ~ScopedDisabler() { __lsan_enable(); } +}; +} // namespace __lsan +#endif + +#endif // SANITIZER_LSAN_INTERFACE_H diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/sgxintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/sgxintrin.h new file mode 100644 index 0000000000..4248076971 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/sgxintrin.h @@ -0,0 +1,177 @@ +#ifndef _SGXINTRIN_H_INCLUDED +#define _SGXINTRIN_H_INCLUDED + +#ifndef __SGX__ +#pragma GCC push_options +#pragma GCC target("sgx") +#define __DISABLE_SGX__ +#endif /* __SGX__ */ + +#define __encls_bc(leaf, b, c, retval) \ + __asm__ __volatile__ ("encls\n\t" \ + : "=a" (retval) \ + : "a" (leaf), "b" (b), "c" (c) \ + : "cc") + +#define __encls_bcd(leaf, b, c, d, retval) \ + __asm__ __volatile__("encls\n\t" \ + : "=a" (retval) \ + : "a" (leaf), "b" (b), "c" (c), "d" (d) \ + : "cc") + +#define __encls_c(leaf, c, retval) \ + __asm__ __volatile__("encls\n\t" \ + : "=a" (retval) \ + : "a" (leaf), "c" (c) \ + : "cc") + +#define __encls_edbgrd(leaf, b, c, retval) \ + __asm__ __volatile__("encls\n\t" \ + : "=a" (retval), "=b" (b) \ + : "a" (leaf), "c" (c)) + +#define __encls_generic(leaf, b, c, d, retval) \ + __asm__ __volatile__("encls\n\t" \ + : "=a" (retval), "=b" (b), "=c" (c), "=d" (d)\ + : "a" (leaf), "b" (b), "c" (c), "d" (d) \ + : "cc") + +#define __enclu_bc(leaf, b, c, retval) \ + __asm__ __volatile__("enclu\n\t" \ + : "=a" (retval) \ + : "a" (leaf), "b" (b), "c" (c) \ + : "cc") + +#define __enclu_bcd(leaf, b, c, d, retval) \ + __asm__ __volatile__("enclu\n\t" \ + : "=a" (retval) \ + : "a" (leaf), "b" (b), "c" (c), "d" (d) \ + : "cc") + +#define __enclu_eenter(leaf, b, c, retval) \ + __asm__ __volatile__("enclu\n\t" \ + : "=a" (retval), "=c" (c) \ + : "a" (leaf), "b" (b), "c" (c) \ + : "cc") + +#define __enclu_eexit(leaf, b, c, retval) \ + __asm__ __volatile__("enclu\n\t" \ + : "=a" (retval), "=c" (c) \ + : "a" (leaf), "b" (b) \ + : "cc") + +#define __enclu_generic(leaf, b, c, d, retval) \ + __asm__ __volatile__("enclu\n\t" \ + : "=a" (retval), "=b" (b), "=c" (c), "=d" (d)\ + : "a" (leaf), "b" (b), "c" (c), "d" (d) \ + : "cc") + + +extern __inline int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_encls_u32 (const int __L, size_t __D[]) +{ + enum __encls_type + { + __SGX_ECREATE = 0x00, + __SGX_EADD = 0x01, + __SGX_EINIT = 0x02, + __SGX_EREMOVE = 0x03, + __SGX_EDBGRD = 0x04, + __SGX_EDBGWR = 0x05, + __SGX_EEXTEND = 0x06, + __SGX_ELDB = 0x07, + __SGX_ELDU = 0x08, + __SGX_EBLOCK = 0x09, + __SGX_EPA = 0x0A, + __SGX_EWB = 0x0B, + __SGX_ETRACK = 0x0C, + __SGX_EAUG = 0x0D, + __SGX_EMODPR = 0x0E, + __SGX_EMODT = 0x0F + }; + enum __encls_type __T = (enum __encls_type)__L; + int __R = 0; + if (!__builtin_constant_p (__T)) + __encls_generic (__L, __D[0], __D[1], __D[2], __R); + else switch (__T) + { + case __SGX_ECREATE: + case __SGX_EADD: + case __SGX_EDBGWR: + case __SGX_EEXTEND: + case __SGX_EPA: + case __SGX_EMODPR: + case __SGX_EMODT: + case __SGX_EAUG: + __encls_bc (__L, __D[0], __D[1], __R); + break; + case __SGX_EINIT: + case __SGX_ELDB: + case __SGX_ELDU: + case __SGX_EWB: + __encls_bcd (__L, __D[0], __D[1], __D[2], __R); + break; + case __SGX_EREMOVE: + case __SGX_EBLOCK: + case __SGX_ETRACK: + __encls_c (__L, __D[1], __R); + break; + case __SGX_EDBGRD: + __encls_edbgrd (__L, __D[0], __D[1], __R); + break; + default: + return -1; + } + return __R; +} + +extern __inline int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_enclu_u32 (const int __L, size_t __D[]) +{ + enum __enclu_type + { + __SGX_EREPORT = 0x00, + __SGX_EGETKEY = 0x01, + __SGX_EENTER = 0x02, + __SGX_ERESUME = 0x03, + __SGX_EEXIT = 0x04, + __SGX_EACCEPT = 0x05, + __SGX_EMODPE = 0x06, + __SGX_EACCEPTCOPY = 0x07 + }; + enum __enclu_type __T = (enum __enclu_type) __L; + int __R = 0; + if (!__builtin_constant_p (__T)) + __enclu_generic (__L, __D[0], __D[1], __D[2], __R); + else switch (__T) + { + case __SGX_EREPORT: + case __SGX_EACCEPTCOPY: + __enclu_bcd (__L, __D[0], __D[1], __D[2], __R); + break; + case __SGX_EGETKEY: + case __SGX_ERESUME: + case __SGX_EACCEPT: + case __SGX_EMODPE: + __enclu_bc (__L, __D[0], __D[1], __R); + break; + case __SGX_EENTER: + __enclu_eenter (__L, __D[0], __D[1], __R); + break; + case __SGX_EEXIT: + __enclu_eexit (__L, __D[0], __D[1], __R); + break; + default: + return -1; + } + return __R; +} + +#ifdef __DISABLE_SGX__ +#undef __DISABLE_SGX__ +#pragma GCC pop_options +#endif /* __DISABLE_SGX__ */ + +#endif /* _SGXINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/shaintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/shaintrin.h new file mode 100644 index 0000000000..dfcf7be321 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/shaintrin.h @@ -0,0 +1,98 @@ +/* Copyright (C) 2013-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +#error "Never use directly; include instead." +#endif + +#ifndef _SHAINTRIN_H_INCLUDED +#define _SHAINTRIN_H_INCLUDED + +#ifndef __SHA__ +#pragma GCC push_options +#pragma GCC target("sha") +#define __DISABLE_SHA__ +#endif /* __SHA__ */ + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sha1msg1_epu32 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_sha1msg1 ((__v4si) __A, (__v4si) __B); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sha1msg2_epu32 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_sha1msg2 ((__v4si) __A, (__v4si) __B); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sha1nexte_epu32 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_sha1nexte ((__v4si) __A, (__v4si) __B); +} + +#ifdef __OPTIMIZE__ +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sha1rnds4_epu32 (__m128i __A, __m128i __B, const int __I) +{ + return (__m128i) __builtin_ia32_sha1rnds4 ((__v4si) __A, (__v4si) __B, __I); +} +#else +#define _mm_sha1rnds4_epu32(A, B, I) \ + ((__m128i) __builtin_ia32_sha1rnds4 ((__v4si)(__m128i)A, \ + (__v4si)(__m128i)B, (int)I)) +#endif + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sha256msg1_epu32 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_sha256msg1 ((__v4si) __A, (__v4si) __B); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sha256msg2_epu32 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_sha256msg2 ((__v4si) __A, (__v4si) __B); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sha256rnds2_epu32 (__m128i __A, __m128i __B, __m128i __C) +{ + return (__m128i) __builtin_ia32_sha256rnds2 ((__v4si) __A, (__v4si) __B, + (__v4si) __C); +} + +#ifdef __DISABLE_SHA__ +#undef __DISABLE_SHA__ +#pragma GCC pop_options +#endif /* __DISABLE_SHA__ */ + +#endif /* _SHAINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/smmintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/smmintrin.h new file mode 100644 index 0000000000..f4a273f8cb --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/smmintrin.h @@ -0,0 +1,862 @@ +/* Copyright (C) 2007-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +/* Implemented from the specification included in the Intel C++ Compiler + User Guide and Reference, version 10.0. */ + +#ifndef _SMMINTRIN_H_INCLUDED +#define _SMMINTRIN_H_INCLUDED + +/* We need definitions from the SSSE3, SSE3, SSE2 and SSE header + files. */ +#include + +#ifndef __SSE4_1__ +#pragma GCC push_options +#pragma GCC target("sse4.1") +#define __DISABLE_SSE4_1__ +#endif /* __SSE4_1__ */ + +/* Rounding mode macros. */ +#define _MM_FROUND_TO_NEAREST_INT 0x00 +#define _MM_FROUND_TO_NEG_INF 0x01 +#define _MM_FROUND_TO_POS_INF 0x02 +#define _MM_FROUND_TO_ZERO 0x03 +#define _MM_FROUND_CUR_DIRECTION 0x04 + +#define _MM_FROUND_RAISE_EXC 0x00 +#define _MM_FROUND_NO_EXC 0x08 + +#define _MM_FROUND_NINT \ + (_MM_FROUND_TO_NEAREST_INT | _MM_FROUND_RAISE_EXC) +#define _MM_FROUND_FLOOR \ + (_MM_FROUND_TO_NEG_INF | _MM_FROUND_RAISE_EXC) +#define _MM_FROUND_CEIL \ + (_MM_FROUND_TO_POS_INF | _MM_FROUND_RAISE_EXC) +#define _MM_FROUND_TRUNC \ + (_MM_FROUND_TO_ZERO | _MM_FROUND_RAISE_EXC) +#define _MM_FROUND_RINT \ + (_MM_FROUND_CUR_DIRECTION | _MM_FROUND_RAISE_EXC) +#define _MM_FROUND_NEARBYINT \ + (_MM_FROUND_CUR_DIRECTION | _MM_FROUND_NO_EXC) + +/* Test Instruction */ +/* Packed integer 128-bit bitwise comparison. Return 1 if + (__V & __M) == 0. */ +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_testz_si128 (__m128i __M, __m128i __V) +{ + return __builtin_ia32_ptestz128 ((__v2di)__M, (__v2di)__V); +} + +/* Packed integer 128-bit bitwise comparison. Return 1 if + (__V & ~__M) == 0. */ +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_testc_si128 (__m128i __M, __m128i __V) +{ + return __builtin_ia32_ptestc128 ((__v2di)__M, (__v2di)__V); +} + +/* Packed integer 128-bit bitwise comparison. Return 1 if + (__V & __M) != 0 && (__V & ~__M) != 0. */ +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_testnzc_si128 (__m128i __M, __m128i __V) +{ + return __builtin_ia32_ptestnzc128 ((__v2di)__M, (__v2di)__V); +} + +/* Macros for packed integer 128-bit comparison intrinsics. */ +#define _mm_test_all_zeros(M, V) _mm_testz_si128 ((M), (V)) + +#define _mm_test_all_ones(V) \ + _mm_testc_si128 ((V), _mm_cmpeq_epi32 ((V), (V))) + +#define _mm_test_mix_ones_zeros(M, V) _mm_testnzc_si128 ((M), (V)) + +/* Packed/scalar double precision floating point rounding. */ + +#ifdef __OPTIMIZE__ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_round_pd (__m128d __V, const int __M) +{ + return (__m128d) __builtin_ia32_roundpd ((__v2df)__V, __M); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_round_sd(__m128d __D, __m128d __V, const int __M) +{ + return (__m128d) __builtin_ia32_roundsd ((__v2df)__D, + (__v2df)__V, + __M); +} +#else +#define _mm_round_pd(V, M) \ + ((__m128d) __builtin_ia32_roundpd ((__v2df)(__m128d)(V), (int)(M))) + +#define _mm_round_sd(D, V, M) \ + ((__m128d) __builtin_ia32_roundsd ((__v2df)(__m128d)(D), \ + (__v2df)(__m128d)(V), (int)(M))) +#endif + +/* Packed/scalar single precision floating point rounding. */ + +#ifdef __OPTIMIZE__ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_round_ps (__m128 __V, const int __M) +{ + return (__m128) __builtin_ia32_roundps ((__v4sf)__V, __M); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_round_ss (__m128 __D, __m128 __V, const int __M) +{ + return (__m128) __builtin_ia32_roundss ((__v4sf)__D, + (__v4sf)__V, + __M); +} +#else +#define _mm_round_ps(V, M) \ + ((__m128) __builtin_ia32_roundps ((__v4sf)(__m128)(V), (int)(M))) + +#define _mm_round_ss(D, V, M) \ + ((__m128) __builtin_ia32_roundss ((__v4sf)(__m128)(D), \ + (__v4sf)(__m128)(V), (int)(M))) +#endif + +/* Macros for ceil/floor intrinsics. */ +#define _mm_ceil_pd(V) _mm_round_pd ((V), _MM_FROUND_CEIL) +#define _mm_ceil_sd(D, V) _mm_round_sd ((D), (V), _MM_FROUND_CEIL) + +#define _mm_floor_pd(V) _mm_round_pd((V), _MM_FROUND_FLOOR) +#define _mm_floor_sd(D, V) _mm_round_sd ((D), (V), _MM_FROUND_FLOOR) + +#define _mm_ceil_ps(V) _mm_round_ps ((V), _MM_FROUND_CEIL) +#define _mm_ceil_ss(D, V) _mm_round_ss ((D), (V), _MM_FROUND_CEIL) + +#define _mm_floor_ps(V) _mm_round_ps ((V), _MM_FROUND_FLOOR) +#define _mm_floor_ss(D, V) _mm_round_ss ((D), (V), _MM_FROUND_FLOOR) + +/* SSE4.1 */ + +/* Integer blend instructions - select data from 2 sources using + constant/variable mask. */ + +#ifdef __OPTIMIZE__ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_blend_epi16 (__m128i __X, __m128i __Y, const int __M) +{ + return (__m128i) __builtin_ia32_pblendw128 ((__v8hi)__X, + (__v8hi)__Y, + __M); +} +#else +#define _mm_blend_epi16(X, Y, M) \ + ((__m128i) __builtin_ia32_pblendw128 ((__v8hi)(__m128i)(X), \ + (__v8hi)(__m128i)(Y), (int)(M))) +#endif + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_blendv_epi8 (__m128i __X, __m128i __Y, __m128i __M) +{ + return (__m128i) __builtin_ia32_pblendvb128 ((__v16qi)__X, + (__v16qi)__Y, + (__v16qi)__M); +} + +/* Single precision floating point blend instructions - select data + from 2 sources using constant/variable mask. */ + +#ifdef __OPTIMIZE__ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_blend_ps (__m128 __X, __m128 __Y, const int __M) +{ + return (__m128) __builtin_ia32_blendps ((__v4sf)__X, + (__v4sf)__Y, + __M); +} +#else +#define _mm_blend_ps(X, Y, M) \ + ((__m128) __builtin_ia32_blendps ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (int)(M))) +#endif + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_blendv_ps (__m128 __X, __m128 __Y, __m128 __M) +{ + return (__m128) __builtin_ia32_blendvps ((__v4sf)__X, + (__v4sf)__Y, + (__v4sf)__M); +} + +/* Double precision floating point blend instructions - select data + from 2 sources using constant/variable mask. */ + +#ifdef __OPTIMIZE__ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_blend_pd (__m128d __X, __m128d __Y, const int __M) +{ + return (__m128d) __builtin_ia32_blendpd ((__v2df)__X, + (__v2df)__Y, + __M); +} +#else +#define _mm_blend_pd(X, Y, M) \ + ((__m128d) __builtin_ia32_blendpd ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (int)(M))) +#endif + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_blendv_pd (__m128d __X, __m128d __Y, __m128d __M) +{ + return (__m128d) __builtin_ia32_blendvpd ((__v2df)__X, + (__v2df)__Y, + (__v2df)__M); +} + +/* Dot product instructions with mask-defined summing and zeroing parts + of result. */ + +#ifdef __OPTIMIZE__ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_dp_ps (__m128 __X, __m128 __Y, const int __M) +{ + return (__m128) __builtin_ia32_dpps ((__v4sf)__X, + (__v4sf)__Y, + __M); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_dp_pd (__m128d __X, __m128d __Y, const int __M) +{ + return (__m128d) __builtin_ia32_dppd ((__v2df)__X, + (__v2df)__Y, + __M); +} +#else +#define _mm_dp_ps(X, Y, M) \ + ((__m128) __builtin_ia32_dpps ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (int)(M))) + +#define _mm_dp_pd(X, Y, M) \ + ((__m128d) __builtin_ia32_dppd ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (int)(M))) +#endif + +/* Packed integer 64-bit comparison, zeroing or filling with ones + corresponding parts of result. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_epi64 (__m128i __X, __m128i __Y) +{ + return (__m128i) ((__v2di)__X == (__v2di)__Y); +} + +/* Min/max packed integer instructions. */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_min_epi8 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_pminsb128 ((__v16qi)__X, (__v16qi)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_max_epi8 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_pmaxsb128 ((__v16qi)__X, (__v16qi)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_min_epu16 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_pminuw128 ((__v8hi)__X, (__v8hi)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_max_epu16 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_pmaxuw128 ((__v8hi)__X, (__v8hi)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_min_epi32 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_pminsd128 ((__v4si)__X, (__v4si)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_max_epi32 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_pmaxsd128 ((__v4si)__X, (__v4si)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_min_epu32 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_pminud128 ((__v4si)__X, (__v4si)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_max_epu32 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_pmaxud128 ((__v4si)__X, (__v4si)__Y); +} + +/* Packed integer 32-bit multiplication with truncation of upper + halves of results. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mullo_epi32 (__m128i __X, __m128i __Y) +{ + return (__m128i) ((__v4su)__X * (__v4su)__Y); +} + +/* Packed integer 32-bit multiplication of 2 pairs of operands + with two 64-bit results. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mul_epi32 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_pmuldq128 ((__v4si)__X, (__v4si)__Y); +} + +/* Insert single precision float into packed single precision array + element selected by index N. The bits [7-6] of N define S + index, the bits [5-4] define D index, and bits [3-0] define + zeroing mask for D. */ + +#ifdef __OPTIMIZE__ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_insert_ps (__m128 __D, __m128 __S, const int __N) +{ + return (__m128) __builtin_ia32_insertps128 ((__v4sf)__D, + (__v4sf)__S, + __N); +} +#else +#define _mm_insert_ps(D, S, N) \ + ((__m128) __builtin_ia32_insertps128 ((__v4sf)(__m128)(D), \ + (__v4sf)(__m128)(S), (int)(N))) +#endif + +/* Helper macro to create the N value for _mm_insert_ps. */ +#define _MM_MK_INSERTPS_NDX(S, D, M) (((S) << 6) | ((D) << 4) | (M)) + +/* Extract binary representation of single precision float from packed + single precision array element of X selected by index N. */ + +#ifdef __OPTIMIZE__ +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_extract_ps (__m128 __X, const int __N) +{ + union { int i; float f; } __tmp; + __tmp.f = __builtin_ia32_vec_ext_v4sf ((__v4sf)__X, __N); + return __tmp.i; +} +#else +#define _mm_extract_ps(X, N) \ + (__extension__ \ + ({ \ + union { int i; float f; } __tmp; \ + __tmp.f = __builtin_ia32_vec_ext_v4sf ((__v4sf)(__m128)(X), (int)(N)); \ + __tmp.i; \ + })) +#endif + +/* Extract binary representation of single precision float into + D from packed single precision array element of S selected + by index N. */ +#define _MM_EXTRACT_FLOAT(D, S, N) \ + { (D) = __builtin_ia32_vec_ext_v4sf ((__v4sf)(S), (N)); } + +/* Extract specified single precision float element into the lower + part of __m128. */ +#define _MM_PICK_OUT_PS(X, N) \ + _mm_insert_ps (_mm_setzero_ps (), (X), \ + _MM_MK_INSERTPS_NDX ((N), 0, 0x0e)) + +/* Insert integer, S, into packed integer array element of D + selected by index N. */ + +#ifdef __OPTIMIZE__ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_insert_epi8 (__m128i __D, int __S, const int __N) +{ + return (__m128i) __builtin_ia32_vec_set_v16qi ((__v16qi)__D, + __S, __N); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_insert_epi32 (__m128i __D, int __S, const int __N) +{ + return (__m128i) __builtin_ia32_vec_set_v4si ((__v4si)__D, + __S, __N); +} + +#ifdef __x86_64__ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_insert_epi64 (__m128i __D, long long __S, const int __N) +{ + return (__m128i) __builtin_ia32_vec_set_v2di ((__v2di)__D, + __S, __N); +} +#endif +#else +#define _mm_insert_epi8(D, S, N) \ + ((__m128i) __builtin_ia32_vec_set_v16qi ((__v16qi)(__m128i)(D), \ + (int)(S), (int)(N))) + +#define _mm_insert_epi32(D, S, N) \ + ((__m128i) __builtin_ia32_vec_set_v4si ((__v4si)(__m128i)(D), \ + (int)(S), (int)(N))) + +#ifdef __x86_64__ +#define _mm_insert_epi64(D, S, N) \ + ((__m128i) __builtin_ia32_vec_set_v2di ((__v2di)(__m128i)(D), \ + (long long)(S), (int)(N))) +#endif +#endif + +/* Extract integer from packed integer array element of X selected by + index N. */ + +#ifdef __OPTIMIZE__ +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_extract_epi8 (__m128i __X, const int __N) +{ + return (unsigned char) __builtin_ia32_vec_ext_v16qi ((__v16qi)__X, __N); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_extract_epi32 (__m128i __X, const int __N) +{ + return __builtin_ia32_vec_ext_v4si ((__v4si)__X, __N); +} + +#ifdef __x86_64__ +extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_extract_epi64 (__m128i __X, const int __N) +{ + return __builtin_ia32_vec_ext_v2di ((__v2di)__X, __N); +} +#endif +#else +#define _mm_extract_epi8(X, N) \ + ((int) (unsigned char) __builtin_ia32_vec_ext_v16qi ((__v16qi)(__m128i)(X), (int)(N))) +#define _mm_extract_epi32(X, N) \ + ((int) __builtin_ia32_vec_ext_v4si ((__v4si)(__m128i)(X), (int)(N))) + +#ifdef __x86_64__ +#define _mm_extract_epi64(X, N) \ + ((long long) __builtin_ia32_vec_ext_v2di ((__v2di)(__m128i)(X), (int)(N))) +#endif +#endif + +/* Return horizontal packed word minimum and its index in bits [15:0] + and bits [18:16] respectively. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_minpos_epu16 (__m128i __X) +{ + return (__m128i) __builtin_ia32_phminposuw128 ((__v8hi)__X); +} + +/* Packed integer sign-extension. */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepi8_epi32 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pmovsxbd128 ((__v16qi)__X); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepi16_epi32 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pmovsxwd128 ((__v8hi)__X); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepi8_epi64 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pmovsxbq128 ((__v16qi)__X); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepi32_epi64 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pmovsxdq128 ((__v4si)__X); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepi16_epi64 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pmovsxwq128 ((__v8hi)__X); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepi8_epi16 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pmovsxbw128 ((__v16qi)__X); +} + +/* Packed integer zero-extension. */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepu8_epi32 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pmovzxbd128 ((__v16qi)__X); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepu16_epi32 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pmovzxwd128 ((__v8hi)__X); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepu8_epi64 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pmovzxbq128 ((__v16qi)__X); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepu32_epi64 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pmovzxdq128 ((__v4si)__X); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepu16_epi64 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pmovzxwq128 ((__v8hi)__X); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepu8_epi16 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pmovzxbw128 ((__v16qi)__X); +} + +/* Pack 8 double words from 2 operands into 8 words of result with + unsigned saturation. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_packus_epi32 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_packusdw128 ((__v4si)__X, (__v4si)__Y); +} + +/* Sum absolute 8-bit integer difference of adjacent groups of 4 + byte integers in the first 2 operands. Starting offsets within + operands are determined by the 3rd mask operand. */ + +#ifdef __OPTIMIZE__ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mpsadbw_epu8 (__m128i __X, __m128i __Y, const int __M) +{ + return (__m128i) __builtin_ia32_mpsadbw128 ((__v16qi)__X, + (__v16qi)__Y, __M); +} +#else +#define _mm_mpsadbw_epu8(X, Y, M) \ + ((__m128i) __builtin_ia32_mpsadbw128 ((__v16qi)(__m128i)(X), \ + (__v16qi)(__m128i)(Y), (int)(M))) +#endif + +/* Load double quadword using non-temporal aligned hint. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_stream_load_si128 (__m128i *__X) +{ + return (__m128i) __builtin_ia32_movntdqa ((__v2di *) __X); +} + +#ifndef __SSE4_2__ +#pragma GCC push_options +#pragma GCC target("sse4.2") +#define __DISABLE_SSE4_2__ +#endif /* __SSE4_2__ */ + +/* These macros specify the source data format. */ +#define _SIDD_UBYTE_OPS 0x00 +#define _SIDD_UWORD_OPS 0x01 +#define _SIDD_SBYTE_OPS 0x02 +#define _SIDD_SWORD_OPS 0x03 + +/* These macros specify the comparison operation. */ +#define _SIDD_CMP_EQUAL_ANY 0x00 +#define _SIDD_CMP_RANGES 0x04 +#define _SIDD_CMP_EQUAL_EACH 0x08 +#define _SIDD_CMP_EQUAL_ORDERED 0x0c + +/* These macros specify the polarity. */ +#define _SIDD_POSITIVE_POLARITY 0x00 +#define _SIDD_NEGATIVE_POLARITY 0x10 +#define _SIDD_MASKED_POSITIVE_POLARITY 0x20 +#define _SIDD_MASKED_NEGATIVE_POLARITY 0x30 + +/* These macros specify the output selection in _mm_cmpXstri (). */ +#define _SIDD_LEAST_SIGNIFICANT 0x00 +#define _SIDD_MOST_SIGNIFICANT 0x40 + +/* These macros specify the output selection in _mm_cmpXstrm (). */ +#define _SIDD_BIT_MASK 0x00 +#define _SIDD_UNIT_MASK 0x40 + +/* Intrinsics for text/string processing. */ + +#ifdef __OPTIMIZE__ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpistrm (__m128i __X, __m128i __Y, const int __M) +{ + return (__m128i) __builtin_ia32_pcmpistrm128 ((__v16qi)__X, + (__v16qi)__Y, + __M); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpistri (__m128i __X, __m128i __Y, const int __M) +{ + return __builtin_ia32_pcmpistri128 ((__v16qi)__X, + (__v16qi)__Y, + __M); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpestrm (__m128i __X, int __LX, __m128i __Y, int __LY, const int __M) +{ + return (__m128i) __builtin_ia32_pcmpestrm128 ((__v16qi)__X, __LX, + (__v16qi)__Y, __LY, + __M); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpestri (__m128i __X, int __LX, __m128i __Y, int __LY, const int __M) +{ + return __builtin_ia32_pcmpestri128 ((__v16qi)__X, __LX, + (__v16qi)__Y, __LY, + __M); +} +#else +#define _mm_cmpistrm(X, Y, M) \ + ((__m128i) __builtin_ia32_pcmpistrm128 ((__v16qi)(__m128i)(X), \ + (__v16qi)(__m128i)(Y), (int)(M))) +#define _mm_cmpistri(X, Y, M) \ + ((int) __builtin_ia32_pcmpistri128 ((__v16qi)(__m128i)(X), \ + (__v16qi)(__m128i)(Y), (int)(M))) + +#define _mm_cmpestrm(X, LX, Y, LY, M) \ + ((__m128i) __builtin_ia32_pcmpestrm128 ((__v16qi)(__m128i)(X), \ + (int)(LX), (__v16qi)(__m128i)(Y), \ + (int)(LY), (int)(M))) +#define _mm_cmpestri(X, LX, Y, LY, M) \ + ((int) __builtin_ia32_pcmpestri128 ((__v16qi)(__m128i)(X), (int)(LX), \ + (__v16qi)(__m128i)(Y), (int)(LY), \ + (int)(M))) +#endif + +/* Intrinsics for text/string processing and reading values of + EFlags. */ + +#ifdef __OPTIMIZE__ +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpistra (__m128i __X, __m128i __Y, const int __M) +{ + return __builtin_ia32_pcmpistria128 ((__v16qi)__X, + (__v16qi)__Y, + __M); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpistrc (__m128i __X, __m128i __Y, const int __M) +{ + return __builtin_ia32_pcmpistric128 ((__v16qi)__X, + (__v16qi)__Y, + __M); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpistro (__m128i __X, __m128i __Y, const int __M) +{ + return __builtin_ia32_pcmpistrio128 ((__v16qi)__X, + (__v16qi)__Y, + __M); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpistrs (__m128i __X, __m128i __Y, const int __M) +{ + return __builtin_ia32_pcmpistris128 ((__v16qi)__X, + (__v16qi)__Y, + __M); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpistrz (__m128i __X, __m128i __Y, const int __M) +{ + return __builtin_ia32_pcmpistriz128 ((__v16qi)__X, + (__v16qi)__Y, + __M); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpestra (__m128i __X, int __LX, __m128i __Y, int __LY, const int __M) +{ + return __builtin_ia32_pcmpestria128 ((__v16qi)__X, __LX, + (__v16qi)__Y, __LY, + __M); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpestrc (__m128i __X, int __LX, __m128i __Y, int __LY, const int __M) +{ + return __builtin_ia32_pcmpestric128 ((__v16qi)__X, __LX, + (__v16qi)__Y, __LY, + __M); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpestro (__m128i __X, int __LX, __m128i __Y, int __LY, const int __M) +{ + return __builtin_ia32_pcmpestrio128 ((__v16qi)__X, __LX, + (__v16qi)__Y, __LY, + __M); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpestrs (__m128i __X, int __LX, __m128i __Y, int __LY, const int __M) +{ + return __builtin_ia32_pcmpestris128 ((__v16qi)__X, __LX, + (__v16qi)__Y, __LY, + __M); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpestrz (__m128i __X, int __LX, __m128i __Y, int __LY, const int __M) +{ + return __builtin_ia32_pcmpestriz128 ((__v16qi)__X, __LX, + (__v16qi)__Y, __LY, + __M); +} +#else +#define _mm_cmpistra(X, Y, M) \ + ((int) __builtin_ia32_pcmpistria128 ((__v16qi)(__m128i)(X), \ + (__v16qi)(__m128i)(Y), (int)(M))) +#define _mm_cmpistrc(X, Y, M) \ + ((int) __builtin_ia32_pcmpistric128 ((__v16qi)(__m128i)(X), \ + (__v16qi)(__m128i)(Y), (int)(M))) +#define _mm_cmpistro(X, Y, M) \ + ((int) __builtin_ia32_pcmpistrio128 ((__v16qi)(__m128i)(X), \ + (__v16qi)(__m128i)(Y), (int)(M))) +#define _mm_cmpistrs(X, Y, M) \ + ((int) __builtin_ia32_pcmpistris128 ((__v16qi)(__m128i)(X), \ + (__v16qi)(__m128i)(Y), (int)(M))) +#define _mm_cmpistrz(X, Y, M) \ + ((int) __builtin_ia32_pcmpistriz128 ((__v16qi)(__m128i)(X), \ + (__v16qi)(__m128i)(Y), (int)(M))) + +#define _mm_cmpestra(X, LX, Y, LY, M) \ + ((int) __builtin_ia32_pcmpestria128 ((__v16qi)(__m128i)(X), (int)(LX), \ + (__v16qi)(__m128i)(Y), (int)(LY), \ + (int)(M))) +#define _mm_cmpestrc(X, LX, Y, LY, M) \ + ((int) __builtin_ia32_pcmpestric128 ((__v16qi)(__m128i)(X), (int)(LX), \ + (__v16qi)(__m128i)(Y), (int)(LY), \ + (int)(M))) +#define _mm_cmpestro(X, LX, Y, LY, M) \ + ((int) __builtin_ia32_pcmpestrio128 ((__v16qi)(__m128i)(X), (int)(LX), \ + (__v16qi)(__m128i)(Y), (int)(LY), \ + (int)(M))) +#define _mm_cmpestrs(X, LX, Y, LY, M) \ + ((int) __builtin_ia32_pcmpestris128 ((__v16qi)(__m128i)(X), (int)(LX), \ + (__v16qi)(__m128i)(Y), (int)(LY), \ + (int)(M))) +#define _mm_cmpestrz(X, LX, Y, LY, M) \ + ((int) __builtin_ia32_pcmpestriz128 ((__v16qi)(__m128i)(X), (int)(LX), \ + (__v16qi)(__m128i)(Y), (int)(LY), \ + (int)(M))) +#endif + +/* Packed integer 64-bit comparison, zeroing or filling with ones + corresponding parts of result. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_epi64 (__m128i __X, __m128i __Y) +{ + return (__m128i) ((__v2di)__X > (__v2di)__Y); +} + +#ifdef __DISABLE_SSE4_2__ +#undef __DISABLE_SSE4_2__ +#pragma GCC pop_options +#endif /* __DISABLE_SSE4_2__ */ + +#ifdef __DISABLE_SSE4_1__ +#undef __DISABLE_SSE4_1__ +#pragma GCC pop_options +#endif /* __DISABLE_SSE4_1__ */ + +#include + +#ifndef __SSE4_1__ +#pragma GCC push_options +#pragma GCC target("sse4.1") +#define __DISABLE_SSE4_1__ +#endif /* __SSE4_1__ */ + +#ifndef __SSE4_2__ +#pragma GCC push_options +#pragma GCC target("sse4.2") +#define __DISABLE_SSE4_2__ +#endif /* __SSE4_1__ */ + +/* Accumulate CRC32 (polynomial 0x11EDC6F41) value. */ +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_crc32_u8 (unsigned int __C, unsigned char __V) +{ + return __builtin_ia32_crc32qi (__C, __V); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_crc32_u16 (unsigned int __C, unsigned short __V) +{ + return __builtin_ia32_crc32hi (__C, __V); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_crc32_u32 (unsigned int __C, unsigned int __V) +{ + return __builtin_ia32_crc32si (__C, __V); +} + +#ifdef __x86_64__ +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_crc32_u64 (unsigned long long __C, unsigned long long __V) +{ + return __builtin_ia32_crc32di (__C, __V); +} +#endif + +#ifdef __DISABLE_SSE4_2__ +#undef __DISABLE_SSE4_2__ +#pragma GCC pop_options +#endif /* __DISABLE_SSE4_2__ */ + +#ifdef __DISABLE_SSE4_1__ +#undef __DISABLE_SSE4_1__ +#pragma GCC pop_options +#endif /* __DISABLE_SSE4_1__ */ + +#endif /* _SMMINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/stdalign.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/stdalign.h new file mode 100644 index 0000000000..243d40e54e --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/stdalign.h @@ -0,0 +1,39 @@ +/* Copyright (C) 2011-2017 Free Software Foundation, Inc. + +This file is part of GCC. + +GCC is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3, or (at your option) +any later version. + +GCC is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +Under Section 7 of GPL version 3, you are granted additional +permissions described in the GCC Runtime Library Exception, version +3.1, as published by the Free Software Foundation. + +You should have received a copy of the GNU General Public License and +a copy of the GCC Runtime Library Exception along with this program; +see the files COPYING3 and COPYING.RUNTIME respectively. If not, see +. */ + +/* ISO C1X: 7.15 Alignment . */ + +#ifndef _STDALIGN_H +#define _STDALIGN_H + +#ifndef __cplusplus + +#define alignas _Alignas +#define alignof _Alignof + +#define __alignas_is_defined 1 +#define __alignof_is_defined 1 + +#endif + +#endif /* stdalign.h */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/stdarg.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/stdarg.h new file mode 100644 index 0000000000..aa248be5da --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/stdarg.h @@ -0,0 +1,127 @@ +/* Copyright (C) 1989-2017 Free Software Foundation, Inc. + +This file is part of GCC. + +GCC is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3, or (at your option) +any later version. + +GCC is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +Under Section 7 of GPL version 3, you are granted additional +permissions described in the GCC Runtime Library Exception, version +3.1, as published by the Free Software Foundation. + +You should have received a copy of the GNU General Public License and +a copy of the GCC Runtime Library Exception along with this program; +see the files COPYING3 and COPYING.RUNTIME respectively. If not, see +. */ + +/* + * ISO C Standard: 7.15 Variable arguments + */ + +#ifndef _STDARG_H +#ifndef _ANSI_STDARG_H_ +#ifndef __need___va_list +#define _STDARG_H +#define _ANSI_STDARG_H_ +#endif /* not __need___va_list */ +#undef __need___va_list + +/* Define __gnuc_va_list. */ + +#ifndef __GNUC_VA_LIST +#define __GNUC_VA_LIST +typedef __builtin_va_list __gnuc_va_list; +#endif + +/* Define the standard macros for the user, + if this invocation was from the user program. */ +#ifdef _STDARG_H + +#define va_start(v,l) __builtin_va_start(v,l) +#define va_end(v) __builtin_va_end(v) +#define va_arg(v,l) __builtin_va_arg(v,l) +#if !defined(__STRICT_ANSI__) || __STDC_VERSION__ + 0 >= 199900L \ + || __cplusplus + 0 >= 201103L +#define va_copy(d,s) __builtin_va_copy(d,s) +#endif +#define __va_copy(d,s) __builtin_va_copy(d,s) + +/* Define va_list, if desired, from __gnuc_va_list. */ +/* We deliberately do not define va_list when called from + stdio.h, because ANSI C says that stdio.h is not supposed to define + va_list. stdio.h needs to have access to that data type, + but must not use that name. It should use the name __gnuc_va_list, + which is safe because it is reserved for the implementation. */ + +#ifdef _BSD_VA_LIST +#undef _BSD_VA_LIST +#endif + +#if defined(__svr4__) || (defined(_SCO_DS) && !defined(__VA_LIST)) +/* SVR4.2 uses _VA_LIST for an internal alias for va_list, + so we must avoid testing it and setting it here. + SVR4 uses _VA_LIST as a flag in stdarg.h, but we should + have no conflict with that. */ +#ifndef _VA_LIST_ +#define _VA_LIST_ +#ifdef __i860__ +#ifndef _VA_LIST +#define _VA_LIST va_list +#endif +#endif /* __i860__ */ +typedef __gnuc_va_list va_list; +#ifdef _SCO_DS +#define __VA_LIST +#endif +#endif /* _VA_LIST_ */ +#else /* not __svr4__ || _SCO_DS */ + +/* The macro _VA_LIST_ is the same thing used by this file in Ultrix. + But on BSD NET2 we must not test or define or undef it. + (Note that the comments in NET 2's ansi.h + are incorrect for _VA_LIST_--see stdio.h!) */ +#if !defined (_VA_LIST_) || defined (__BSD_NET2__) || defined (____386BSD____) || defined (__bsdi__) || defined (__sequent__) || defined (__FreeBSD__) || defined(WINNT) +/* The macro _VA_LIST_DEFINED is used in Windows NT 3.5 */ +#ifndef _VA_LIST_DEFINED +/* The macro _VA_LIST is used in SCO Unix 3.2. */ +#ifndef _VA_LIST +/* The macro _VA_LIST_T_H is used in the Bull dpx2 */ +#ifndef _VA_LIST_T_H +/* The macro __va_list__ is used by BeOS. */ +#ifndef __va_list__ +typedef __gnuc_va_list va_list; +#endif /* not __va_list__ */ +#endif /* not _VA_LIST_T_H */ +#endif /* not _VA_LIST */ +#endif /* not _VA_LIST_DEFINED */ +#if !(defined (__BSD_NET2__) || defined (____386BSD____) || defined (__bsdi__) || defined (__sequent__) || defined (__FreeBSD__)) +#define _VA_LIST_ +#endif +#ifndef _VA_LIST +#define _VA_LIST +#endif +#ifndef _VA_LIST_DEFINED +#define _VA_LIST_DEFINED +#endif +#ifndef _VA_LIST_T_H +#define _VA_LIST_T_H +#endif +#ifndef __va_list__ +#define __va_list__ +#endif + +#endif /* not _VA_LIST_, except on certain systems */ + +#endif /* not __svr4__ */ + +#endif /* _STDARG_H */ + +#endif /* not _ANSI_STDARG_H_ */ +#endif /* not _STDARG_H */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/stdatomic.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/stdatomic.h new file mode 100644 index 0000000000..fb29a142fc --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/stdatomic.h @@ -0,0 +1,243 @@ +/* Copyright (C) 2013-2017 Free Software Foundation, Inc. + +This file is part of GCC. + +GCC is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3, or (at your option) +any later version. + +GCC is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +Under Section 7 of GPL version 3, you are granted additional +permissions described in the GCC Runtime Library Exception, version +3.1, as published by the Free Software Foundation. + +You should have received a copy of the GNU General Public License and +a copy of the GCC Runtime Library Exception along with this program; +see the files COPYING3 and COPYING.RUNTIME respectively. If not, see +. */ + +/* ISO C11 Standard: 7.17 Atomics . */ + +#ifndef _STDATOMIC_H +#define _STDATOMIC_H + +typedef enum + { + memory_order_relaxed = __ATOMIC_RELAXED, + memory_order_consume = __ATOMIC_CONSUME, + memory_order_acquire = __ATOMIC_ACQUIRE, + memory_order_release = __ATOMIC_RELEASE, + memory_order_acq_rel = __ATOMIC_ACQ_REL, + memory_order_seq_cst = __ATOMIC_SEQ_CST + } memory_order; + + +typedef _Atomic _Bool atomic_bool; +typedef _Atomic char atomic_char; +typedef _Atomic signed char atomic_schar; +typedef _Atomic unsigned char atomic_uchar; +typedef _Atomic short atomic_short; +typedef _Atomic unsigned short atomic_ushort; +typedef _Atomic int atomic_int; +typedef _Atomic unsigned int atomic_uint; +typedef _Atomic long atomic_long; +typedef _Atomic unsigned long atomic_ulong; +typedef _Atomic long long atomic_llong; +typedef _Atomic unsigned long long atomic_ullong; +typedef _Atomic __CHAR16_TYPE__ atomic_char16_t; +typedef _Atomic __CHAR32_TYPE__ atomic_char32_t; +typedef _Atomic __WCHAR_TYPE__ atomic_wchar_t; +typedef _Atomic __INT_LEAST8_TYPE__ atomic_int_least8_t; +typedef _Atomic __UINT_LEAST8_TYPE__ atomic_uint_least8_t; +typedef _Atomic __INT_LEAST16_TYPE__ atomic_int_least16_t; +typedef _Atomic __UINT_LEAST16_TYPE__ atomic_uint_least16_t; +typedef _Atomic __INT_LEAST32_TYPE__ atomic_int_least32_t; +typedef _Atomic __UINT_LEAST32_TYPE__ atomic_uint_least32_t; +typedef _Atomic __INT_LEAST64_TYPE__ atomic_int_least64_t; +typedef _Atomic __UINT_LEAST64_TYPE__ atomic_uint_least64_t; +typedef _Atomic __INT_FAST8_TYPE__ atomic_int_fast8_t; +typedef _Atomic __UINT_FAST8_TYPE__ atomic_uint_fast8_t; +typedef _Atomic __INT_FAST16_TYPE__ atomic_int_fast16_t; +typedef _Atomic __UINT_FAST16_TYPE__ atomic_uint_fast16_t; +typedef _Atomic __INT_FAST32_TYPE__ atomic_int_fast32_t; +typedef _Atomic __UINT_FAST32_TYPE__ atomic_uint_fast32_t; +typedef _Atomic __INT_FAST64_TYPE__ atomic_int_fast64_t; +typedef _Atomic __UINT_FAST64_TYPE__ atomic_uint_fast64_t; +typedef _Atomic __INTPTR_TYPE__ atomic_intptr_t; +typedef _Atomic __UINTPTR_TYPE__ atomic_uintptr_t; +typedef _Atomic __SIZE_TYPE__ atomic_size_t; +typedef _Atomic __PTRDIFF_TYPE__ atomic_ptrdiff_t; +typedef _Atomic __INTMAX_TYPE__ atomic_intmax_t; +typedef _Atomic __UINTMAX_TYPE__ atomic_uintmax_t; + + +#define ATOMIC_VAR_INIT(VALUE) (VALUE) + +/* Initialize an atomic object pointed to by PTR with VAL. */ +#define atomic_init(PTR, VAL) \ + atomic_store_explicit (PTR, VAL, __ATOMIC_RELAXED) + +#define kill_dependency(Y) \ + __extension__ \ + ({ \ + __auto_type __kill_dependency_tmp = (Y); \ + __kill_dependency_tmp; \ + }) + +extern void atomic_thread_fence (memory_order); +#define atomic_thread_fence(MO) __atomic_thread_fence (MO) +extern void atomic_signal_fence (memory_order); +#define atomic_signal_fence(MO) __atomic_signal_fence (MO) +#define atomic_is_lock_free(OBJ) __atomic_is_lock_free (sizeof (*(OBJ)), (OBJ)) + +#define ATOMIC_BOOL_LOCK_FREE __GCC_ATOMIC_BOOL_LOCK_FREE +#define ATOMIC_CHAR_LOCK_FREE __GCC_ATOMIC_CHAR_LOCK_FREE +#define ATOMIC_CHAR16_T_LOCK_FREE __GCC_ATOMIC_CHAR16_T_LOCK_FREE +#define ATOMIC_CHAR32_T_LOCK_FREE __GCC_ATOMIC_CHAR32_T_LOCK_FREE +#define ATOMIC_WCHAR_T_LOCK_FREE __GCC_ATOMIC_WCHAR_T_LOCK_FREE +#define ATOMIC_SHORT_LOCK_FREE __GCC_ATOMIC_SHORT_LOCK_FREE +#define ATOMIC_INT_LOCK_FREE __GCC_ATOMIC_INT_LOCK_FREE +#define ATOMIC_LONG_LOCK_FREE __GCC_ATOMIC_LONG_LOCK_FREE +#define ATOMIC_LLONG_LOCK_FREE __GCC_ATOMIC_LLONG_LOCK_FREE +#define ATOMIC_POINTER_LOCK_FREE __GCC_ATOMIC_POINTER_LOCK_FREE + + +/* Note that these macros require __typeof__ and __auto_type to remove + _Atomic qualifiers (and const qualifiers, if those are valid on + macro operands). + + Also note that the header file uses the generic form of __atomic + builtins, which requires the address to be taken of the value + parameter, and then we pass that value on. This allows the macros + to work for any type, and the compiler is smart enough to convert + these to lock-free _N variants if possible, and throw away the + temps. */ + +#define atomic_store_explicit(PTR, VAL, MO) \ + __extension__ \ + ({ \ + __auto_type __atomic_store_ptr = (PTR); \ + __typeof__ (*__atomic_store_ptr) __atomic_store_tmp = (VAL); \ + __atomic_store (__atomic_store_ptr, &__atomic_store_tmp, (MO)); \ + }) + +#define atomic_store(PTR, VAL) \ + atomic_store_explicit (PTR, VAL, __ATOMIC_SEQ_CST) + + +#define atomic_load_explicit(PTR, MO) \ + __extension__ \ + ({ \ + __auto_type __atomic_load_ptr = (PTR); \ + __typeof__ (*__atomic_load_ptr) __atomic_load_tmp; \ + __atomic_load (__atomic_load_ptr, &__atomic_load_tmp, (MO)); \ + __atomic_load_tmp; \ + }) + +#define atomic_load(PTR) atomic_load_explicit (PTR, __ATOMIC_SEQ_CST) + + +#define atomic_exchange_explicit(PTR, VAL, MO) \ + __extension__ \ + ({ \ + __auto_type __atomic_exchange_ptr = (PTR); \ + __typeof__ (*__atomic_exchange_ptr) __atomic_exchange_val = (VAL); \ + __typeof__ (*__atomic_exchange_ptr) __atomic_exchange_tmp; \ + __atomic_exchange (__atomic_exchange_ptr, &__atomic_exchange_val, \ + &__atomic_exchange_tmp, (MO)); \ + __atomic_exchange_tmp; \ + }) + +#define atomic_exchange(PTR, VAL) \ + atomic_exchange_explicit (PTR, VAL, __ATOMIC_SEQ_CST) + + +#define atomic_compare_exchange_strong_explicit(PTR, VAL, DES, SUC, FAIL) \ + __extension__ \ + ({ \ + __auto_type __atomic_compare_exchange_ptr = (PTR); \ + __typeof__ (*__atomic_compare_exchange_ptr) __atomic_compare_exchange_tmp \ + = (DES); \ + __atomic_compare_exchange (__atomic_compare_exchange_ptr, (VAL), \ + &__atomic_compare_exchange_tmp, 0, \ + (SUC), (FAIL)); \ + }) + +#define atomic_compare_exchange_strong(PTR, VAL, DES) \ + atomic_compare_exchange_strong_explicit (PTR, VAL, DES, __ATOMIC_SEQ_CST, \ + __ATOMIC_SEQ_CST) + +#define atomic_compare_exchange_weak_explicit(PTR, VAL, DES, SUC, FAIL) \ + __extension__ \ + ({ \ + __auto_type __atomic_compare_exchange_ptr = (PTR); \ + __typeof__ (*__atomic_compare_exchange_ptr) __atomic_compare_exchange_tmp \ + = (DES); \ + __atomic_compare_exchange (__atomic_compare_exchange_ptr, (VAL), \ + &__atomic_compare_exchange_tmp, 1, \ + (SUC), (FAIL)); \ + }) + +#define atomic_compare_exchange_weak(PTR, VAL, DES) \ + atomic_compare_exchange_weak_explicit (PTR, VAL, DES, __ATOMIC_SEQ_CST, \ + __ATOMIC_SEQ_CST) + + + +#define atomic_fetch_add(PTR, VAL) __atomic_fetch_add ((PTR), (VAL), \ + __ATOMIC_SEQ_CST) +#define atomic_fetch_add_explicit(PTR, VAL, MO) \ + __atomic_fetch_add ((PTR), (VAL), (MO)) + +#define atomic_fetch_sub(PTR, VAL) __atomic_fetch_sub ((PTR), (VAL), \ + __ATOMIC_SEQ_CST) +#define atomic_fetch_sub_explicit(PTR, VAL, MO) \ + __atomic_fetch_sub ((PTR), (VAL), (MO)) + +#define atomic_fetch_or(PTR, VAL) __atomic_fetch_or ((PTR), (VAL), \ + __ATOMIC_SEQ_CST) +#define atomic_fetch_or_explicit(PTR, VAL, MO) \ + __atomic_fetch_or ((PTR), (VAL), (MO)) + +#define atomic_fetch_xor(PTR, VAL) __atomic_fetch_xor ((PTR), (VAL), \ + __ATOMIC_SEQ_CST) +#define atomic_fetch_xor_explicit(PTR, VAL, MO) \ + __atomic_fetch_xor ((PTR), (VAL), (MO)) + +#define atomic_fetch_and(PTR, VAL) __atomic_fetch_and ((PTR), (VAL), \ + __ATOMIC_SEQ_CST) +#define atomic_fetch_and_explicit(PTR, VAL, MO) \ + __atomic_fetch_and ((PTR), (VAL), (MO)) + + +typedef _Atomic struct +{ +#if __GCC_ATOMIC_TEST_AND_SET_TRUEVAL == 1 + _Bool __val; +#else + unsigned char __val; +#endif +} atomic_flag; + +#define ATOMIC_FLAG_INIT { 0 } + + +extern _Bool atomic_flag_test_and_set (volatile atomic_flag *); +#define atomic_flag_test_and_set(PTR) \ + __atomic_test_and_set ((PTR), __ATOMIC_SEQ_CST) +extern _Bool atomic_flag_test_and_set_explicit (volatile atomic_flag *, + memory_order); +#define atomic_flag_test_and_set_explicit(PTR, MO) \ + __atomic_test_and_set ((PTR), (MO)) + +extern void atomic_flag_clear (volatile atomic_flag *); +#define atomic_flag_clear(PTR) __atomic_clear ((PTR), __ATOMIC_SEQ_CST) +extern void atomic_flag_clear_explicit (volatile atomic_flag *, memory_order); +#define atomic_flag_clear_explicit(PTR, MO) __atomic_clear ((PTR), (MO)) + +#endif /* _STDATOMIC_H */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/stdbool.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/stdbool.h new file mode 100644 index 0000000000..a69fc3a444 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/stdbool.h @@ -0,0 +1,54 @@ +/* Copyright (C) 1998-2017 Free Software Foundation, Inc. + +This file is part of GCC. + +GCC is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3, or (at your option) +any later version. + +GCC is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +Under Section 7 of GPL version 3, you are granted additional +permissions described in the GCC Runtime Library Exception, version +3.1, as published by the Free Software Foundation. + +You should have received a copy of the GNU General Public License and +a copy of the GCC Runtime Library Exception along with this program; +see the files COPYING3 and COPYING.RUNTIME respectively. If not, see +. */ + +/* + * ISO C Standard: 7.16 Boolean type and values + */ + +#ifndef _STDBOOL_H +#define _STDBOOL_H + +#ifndef __cplusplus + +#define bool _Bool +#define true 1 +#define false 0 + +#else /* __cplusplus */ + +/* Supporting _Bool in C++ is a GCC extension. */ +#define _Bool bool + +#if __cplusplus < 201103L +/* Defining these macros in C++98 is a GCC extension. */ +#define bool bool +#define false false +#define true true +#endif + +#endif /* __cplusplus */ + +/* Signal that all the definitions are present. */ +#define __bool_true_false_are_defined 1 + +#endif /* stdbool.h */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/stddef.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/stddef.h new file mode 100644 index 0000000000..872f451cac --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/stddef.h @@ -0,0 +1,451 @@ +/* Copyright (C) 1989-2017 Free Software Foundation, Inc. + +This file is part of GCC. + +GCC is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3, or (at your option) +any later version. + +GCC is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +Under Section 7 of GPL version 3, you are granted additional +permissions described in the GCC Runtime Library Exception, version +3.1, as published by the Free Software Foundation. + +You should have received a copy of the GNU General Public License and +a copy of the GCC Runtime Library Exception along with this program; +see the files COPYING3 and COPYING.RUNTIME respectively. If not, see +. */ + +/* + * ISO C Standard: 7.17 Common definitions + */ +#if (!defined(_STDDEF_H) && !defined(_STDDEF_H_) && !defined(_ANSI_STDDEF_H) \ + && !defined(__STDDEF_H__)) \ + || defined(__need_wchar_t) || defined(__need_size_t) \ + || defined(__need_ptrdiff_t) || defined(__need_NULL) \ + || defined(__need_wint_t) + +/* Any one of these symbols __need_* means that GNU libc + wants us just to define one data type. So don't define + the symbols that indicate this file's entire job has been done. */ +#if (!defined(__need_wchar_t) && !defined(__need_size_t) \ + && !defined(__need_ptrdiff_t) && !defined(__need_NULL) \ + && !defined(__need_wint_t)) +#define _STDDEF_H +#define _STDDEF_H_ +/* snaroff@next.com says the NeXT needs this. */ +#define _ANSI_STDDEF_H +#endif + +#ifndef __sys_stdtypes_h +/* This avoids lossage on SunOS but only if stdtypes.h comes first. + There's no way to win with the other order! Sun lossage. */ + +/* On 4.3bsd-net2, make sure ansi.h is included, so we have + one less case to deal with in the following. */ +#if defined (__BSD_NET2__) || defined (____386BSD____) || (defined (__FreeBSD__) && (__FreeBSD__ < 5)) || defined(__NetBSD__) +#include +#endif +/* On FreeBSD 5, machine/ansi.h does not exist anymore... */ +#if defined (__FreeBSD__) && (__FreeBSD__ >= 5) +#include +#endif + +/* In 4.3bsd-net2, machine/ansi.h defines these symbols, which are + defined if the corresponding type is *not* defined. + FreeBSD-2.1 defines _MACHINE_ANSI_H_ instead of _ANSI_H_. + NetBSD defines _I386_ANSI_H_ and _X86_64_ANSI_H_ instead of _ANSI_H_ */ +#if defined(_ANSI_H_) || defined(_MACHINE_ANSI_H_) || defined(_X86_64_ANSI_H_) || defined(_I386_ANSI_H_) +#if !defined(_SIZE_T_) && !defined(_BSD_SIZE_T_) +#define _SIZE_T +#endif +#if !defined(_PTRDIFF_T_) && !defined(_BSD_PTRDIFF_T_) +#define _PTRDIFF_T +#endif +/* On BSD/386 1.1, at least, machine/ansi.h defines _BSD_WCHAR_T_ + instead of _WCHAR_T_. */ +#if !defined(_WCHAR_T_) && !defined(_BSD_WCHAR_T_) +#ifndef _BSD_WCHAR_T_ +#define _WCHAR_T +#endif +#endif +/* Undef _FOO_T_ if we are supposed to define foo_t. */ +#if defined (__need_ptrdiff_t) || defined (_STDDEF_H_) +#undef _PTRDIFF_T_ +#undef _BSD_PTRDIFF_T_ +#endif +#if defined (__need_size_t) || defined (_STDDEF_H_) +#undef _SIZE_T_ +#undef _BSD_SIZE_T_ +#endif +#if defined (__need_wchar_t) || defined (_STDDEF_H_) +#undef _WCHAR_T_ +#undef _BSD_WCHAR_T_ +#endif +#endif /* defined(_ANSI_H_) || defined(_MACHINE_ANSI_H_) || defined(_X86_64_ANSI_H_) || defined(_I386_ANSI_H_) */ + +/* Sequent's header files use _PTRDIFF_T_ in some conflicting way. + Just ignore it. */ +#if defined (__sequent__) && defined (_PTRDIFF_T_) +#undef _PTRDIFF_T_ +#endif + +/* On VxWorks, may have defined macros like + _TYPE_size_t which will typedef size_t. fixincludes patched the + vxTypesBase.h so that this macro is only defined if _GCC_SIZE_T is + not defined, and so that defining this macro defines _GCC_SIZE_T. + If we find that the macros are still defined at this point, we must + invoke them so that the type is defined as expected. */ +#if defined (_TYPE_ptrdiff_t) && (defined (__need_ptrdiff_t) || defined (_STDDEF_H_)) +_TYPE_ptrdiff_t; +#undef _TYPE_ptrdiff_t +#endif +#if defined (_TYPE_size_t) && (defined (__need_size_t) || defined (_STDDEF_H_)) +_TYPE_size_t; +#undef _TYPE_size_t +#endif +#if defined (_TYPE_wchar_t) && (defined (__need_wchar_t) || defined (_STDDEF_H_)) +_TYPE_wchar_t; +#undef _TYPE_wchar_t +#endif + +/* In case nobody has defined these types, but we aren't running under + GCC 2.00, make sure that __PTRDIFF_TYPE__, __SIZE_TYPE__, and + __WCHAR_TYPE__ have reasonable values. This can happen if the + parts of GCC is compiled by an older compiler, that actually + include gstddef.h, such as collect2. */ + +/* Signed type of difference of two pointers. */ + +/* Define this type if we are doing the whole job, + or if we want this type in particular. */ +#if defined (_STDDEF_H) || defined (__need_ptrdiff_t) +#ifndef _PTRDIFF_T /* in case has defined it. */ +#ifndef _T_PTRDIFF_ +#ifndef _T_PTRDIFF +#ifndef __PTRDIFF_T +#ifndef _PTRDIFF_T_ +#ifndef _BSD_PTRDIFF_T_ +#ifndef ___int_ptrdiff_t_h +#ifndef _GCC_PTRDIFF_T +#ifndef _PTRDIFF_T_DECLARED /* DragonFly */ +#define _PTRDIFF_T +#define _T_PTRDIFF_ +#define _T_PTRDIFF +#define __PTRDIFF_T +#define _PTRDIFF_T_ +#define _BSD_PTRDIFF_T_ +#define ___int_ptrdiff_t_h +#define _GCC_PTRDIFF_T +#define _PTRDIFF_T_DECLARED +#ifndef __PTRDIFF_TYPE__ +#define __PTRDIFF_TYPE__ long int +#endif +typedef __PTRDIFF_TYPE__ ptrdiff_t; +#endif /* _PTRDIFF_T_DECLARED */ +#endif /* _GCC_PTRDIFF_T */ +#endif /* ___int_ptrdiff_t_h */ +#endif /* _BSD_PTRDIFF_T_ */ +#endif /* _PTRDIFF_T_ */ +#endif /* __PTRDIFF_T */ +#endif /* _T_PTRDIFF */ +#endif /* _T_PTRDIFF_ */ +#endif /* _PTRDIFF_T */ + +/* If this symbol has done its job, get rid of it. */ +#undef __need_ptrdiff_t + +#endif /* _STDDEF_H or __need_ptrdiff_t. */ + +/* Unsigned type of `sizeof' something. */ + +/* Define this type if we are doing the whole job, + or if we want this type in particular. */ +#if defined (_STDDEF_H) || defined (__need_size_t) +#ifndef __size_t__ /* BeOS */ +#ifndef __SIZE_T__ /* Cray Unicos/Mk */ +#ifndef _SIZE_T /* in case has defined it. */ +#ifndef _SYS_SIZE_T_H +#ifndef _T_SIZE_ +#ifndef _T_SIZE +#ifndef __SIZE_T +#ifndef _SIZE_T_ +#ifndef _BSD_SIZE_T_ +#ifndef _SIZE_T_DEFINED_ +#ifndef _SIZE_T_DEFINED +#ifndef _BSD_SIZE_T_DEFINED_ /* Darwin */ +#ifndef _SIZE_T_DECLARED /* FreeBSD 5 */ +#ifndef ___int_size_t_h +#ifndef _GCC_SIZE_T +#ifndef _SIZET_ +#ifndef __size_t +#define __size_t__ /* BeOS */ +#define __SIZE_T__ /* Cray Unicos/Mk */ +#define _SIZE_T +#define _SYS_SIZE_T_H +#define _T_SIZE_ +#define _T_SIZE +#define __SIZE_T +#define _SIZE_T_ +#define _BSD_SIZE_T_ +#define _SIZE_T_DEFINED_ +#define _SIZE_T_DEFINED +#define _BSD_SIZE_T_DEFINED_ /* Darwin */ +#define _SIZE_T_DECLARED /* FreeBSD 5 */ +#define ___int_size_t_h +#define _GCC_SIZE_T +#define _SIZET_ +#if (defined (__FreeBSD__) && (__FreeBSD__ >= 5)) \ + || defined(__DragonFly__) \ + || defined(__FreeBSD_kernel__) +/* __size_t is a typedef on FreeBSD 5, must not trash it. */ +#elif defined (__VMS__) +/* __size_t is also a typedef on VMS. */ +#else +#define __size_t +#endif +#ifndef __SIZE_TYPE__ +#define __SIZE_TYPE__ long unsigned int +#endif +#if !(defined (__GNUG__) && defined (size_t)) +typedef __SIZE_TYPE__ size_t; +#ifdef __BEOS__ +typedef long ssize_t; +#endif /* __BEOS__ */ +#endif /* !(defined (__GNUG__) && defined (size_t)) */ +#endif /* __size_t */ +#endif /* _SIZET_ */ +#endif /* _GCC_SIZE_T */ +#endif /* ___int_size_t_h */ +#endif /* _SIZE_T_DECLARED */ +#endif /* _BSD_SIZE_T_DEFINED_ */ +#endif /* _SIZE_T_DEFINED */ +#endif /* _SIZE_T_DEFINED_ */ +#endif /* _BSD_SIZE_T_ */ +#endif /* _SIZE_T_ */ +#endif /* __SIZE_T */ +#endif /* _T_SIZE */ +#endif /* _T_SIZE_ */ +#endif /* _SYS_SIZE_T_H */ +#endif /* _SIZE_T */ +#endif /* __SIZE_T__ */ +#endif /* __size_t__ */ +#undef __need_size_t +#endif /* _STDDEF_H or __need_size_t. */ + + +/* Wide character type. + Locale-writers should change this as necessary to + be big enough to hold unique values not between 0 and 127, + and not (wchar_t) -1, for each defined multibyte character. */ + +/* Define this type if we are doing the whole job, + or if we want this type in particular. */ +#if defined (_STDDEF_H) || defined (__need_wchar_t) +#ifndef __wchar_t__ /* BeOS */ +#ifndef __WCHAR_T__ /* Cray Unicos/Mk */ +#ifndef _WCHAR_T +#ifndef _T_WCHAR_ +#ifndef _T_WCHAR +#ifndef __WCHAR_T +#ifndef _WCHAR_T_ +#ifndef _BSD_WCHAR_T_ +#ifndef _BSD_WCHAR_T_DEFINED_ /* Darwin */ +#ifndef _BSD_RUNE_T_DEFINED_ /* Darwin */ +#ifndef _WCHAR_T_DECLARED /* FreeBSD 5 */ +#ifndef _WCHAR_T_DEFINED_ +#ifndef _WCHAR_T_DEFINED +#ifndef _WCHAR_T_H +#ifndef ___int_wchar_t_h +#ifndef __INT_WCHAR_T_H +#ifndef _GCC_WCHAR_T +#define __wchar_t__ /* BeOS */ +#define __WCHAR_T__ /* Cray Unicos/Mk */ +#define _WCHAR_T +#define _T_WCHAR_ +#define _T_WCHAR +#define __WCHAR_T +#define _WCHAR_T_ +#define _BSD_WCHAR_T_ +#define _WCHAR_T_DEFINED_ +#define _WCHAR_T_DEFINED +#define _WCHAR_T_H +#define ___int_wchar_t_h +#define __INT_WCHAR_T_H +#define _GCC_WCHAR_T +#define _WCHAR_T_DECLARED + +/* On BSD/386 1.1, at least, machine/ansi.h defines _BSD_WCHAR_T_ + instead of _WCHAR_T_, and _BSD_RUNE_T_ (which, unlike the other + symbols in the _FOO_T_ family, stays defined even after its + corresponding type is defined). If we define wchar_t, then we + must undef _WCHAR_T_; for BSD/386 1.1 (and perhaps others), if + we undef _WCHAR_T_, then we must also define rune_t, since + headers like runetype.h assume that if machine/ansi.h is included, + and _BSD_WCHAR_T_ is not defined, then rune_t is available. + machine/ansi.h says, "Note that _WCHAR_T_ and _RUNE_T_ must be of + the same type." */ +#ifdef _BSD_WCHAR_T_ +#undef _BSD_WCHAR_T_ +#ifdef _BSD_RUNE_T_ +#if !defined (_ANSI_SOURCE) && !defined (_POSIX_SOURCE) +typedef _BSD_RUNE_T_ rune_t; +#define _BSD_WCHAR_T_DEFINED_ +#define _BSD_RUNE_T_DEFINED_ /* Darwin */ +#if defined (__FreeBSD__) && (__FreeBSD__ < 5) +/* Why is this file so hard to maintain properly? In contrast to + the comment above regarding BSD/386 1.1, on FreeBSD for as long + as the symbol has existed, _BSD_RUNE_T_ must not stay defined or + redundant typedefs will occur when stdlib.h is included after this file. */ +#undef _BSD_RUNE_T_ +#endif +#endif +#endif +#endif +/* FreeBSD 5 can't be handled well using "traditional" logic above + since it no longer defines _BSD_RUNE_T_ yet still desires to export + rune_t in some cases... */ +#if defined (__FreeBSD__) && (__FreeBSD__ >= 5) +#if !defined (_ANSI_SOURCE) && !defined (_POSIX_SOURCE) +#if __BSD_VISIBLE +#ifndef _RUNE_T_DECLARED +typedef __rune_t rune_t; +#define _RUNE_T_DECLARED +#endif +#endif +#endif +#endif + +#ifndef __WCHAR_TYPE__ +#define __WCHAR_TYPE__ int +#endif +#ifndef __cplusplus +typedef __WCHAR_TYPE__ wchar_t; +#endif +#endif +#endif +#endif +#endif +#endif +#endif +#endif /* _WCHAR_T_DECLARED */ +#endif /* _BSD_RUNE_T_DEFINED_ */ +#endif +#endif +#endif +#endif +#endif +#endif +#endif +#endif /* __WCHAR_T__ */ +#endif /* __wchar_t__ */ +#undef __need_wchar_t +#endif /* _STDDEF_H or __need_wchar_t. */ + +#if defined (__need_wint_t) +#ifndef _WINT_T +#define _WINT_T + +#ifndef __WINT_TYPE__ +#define __WINT_TYPE__ unsigned int +#endif +typedef __WINT_TYPE__ wint_t; +#endif +#undef __need_wint_t +#endif + +/* In 4.3bsd-net2, leave these undefined to indicate that size_t, etc. + are already defined. */ +/* BSD/OS 3.1 and FreeBSD [23].x require the MACHINE_ANSI_H check here. */ +/* NetBSD 5 requires the I386_ANSI_H and X86_64_ANSI_H checks here. */ +#if defined(_ANSI_H_) || defined(_MACHINE_ANSI_H_) || defined(_X86_64_ANSI_H_) || defined(_I386_ANSI_H_) +/* The references to _GCC_PTRDIFF_T_, _GCC_SIZE_T_, and _GCC_WCHAR_T_ + are probably typos and should be removed before 2.8 is released. */ +#ifdef _GCC_PTRDIFF_T_ +#undef _PTRDIFF_T_ +#undef _BSD_PTRDIFF_T_ +#endif +#ifdef _GCC_SIZE_T_ +#undef _SIZE_T_ +#undef _BSD_SIZE_T_ +#endif +#ifdef _GCC_WCHAR_T_ +#undef _WCHAR_T_ +#undef _BSD_WCHAR_T_ +#endif +/* The following ones are the real ones. */ +#ifdef _GCC_PTRDIFF_T +#undef _PTRDIFF_T_ +#undef _BSD_PTRDIFF_T_ +#endif +#ifdef _GCC_SIZE_T +#undef _SIZE_T_ +#undef _BSD_SIZE_T_ +#endif +#ifdef _GCC_WCHAR_T +#undef _WCHAR_T_ +#undef _BSD_WCHAR_T_ +#endif +#endif /* _ANSI_H_ || _MACHINE_ANSI_H_ || _X86_64_ANSI_H_ || _I386_ANSI_H_ */ + +#endif /* __sys_stdtypes_h */ + +/* A null pointer constant. */ + +#if defined (_STDDEF_H) || defined (__need_NULL) +#undef NULL /* in case has defined it. */ +#ifdef __GNUG__ +#define NULL __null +#else /* G++ */ +#ifndef __cplusplus +#define NULL ((void *)0) +#else /* C++ */ +#define NULL 0 +#endif /* C++ */ +#endif /* G++ */ +#endif /* NULL not defined and or need NULL. */ +#undef __need_NULL + +#ifdef _STDDEF_H + +/* Offset of member MEMBER in a struct of type TYPE. */ +#define offsetof(TYPE, MEMBER) __builtin_offsetof (TYPE, MEMBER) + +#if (defined (__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \ + || (defined(__cplusplus) && __cplusplus >= 201103L) +#ifndef _GCC_MAX_ALIGN_T +#define _GCC_MAX_ALIGN_T +/* Type whose alignment is supported in every context and is at least + as great as that of any standard type not using alignment + specifiers. */ +typedef struct { + long long __max_align_ll __attribute__((__aligned__(__alignof__(long long)))); + long double __max_align_ld __attribute__((__aligned__(__alignof__(long double)))); + /* _Float128 is defined as a basic type, so max_align_t must be + sufficiently aligned for it. This code must work in C++, so we + use __float128 here; that is only available on some + architectures, but only on i386 is extra alignment needed for + __float128. */ +#ifdef __i386__ + __float128 __max_align_f128 __attribute__((__aligned__(__alignof(__float128)))); +#endif +} max_align_t; +#endif +#endif /* C11 or C++11. */ + +#if defined(__cplusplus) && __cplusplus >= 201103L +#ifndef _GXX_NULLPTR_T +#define _GXX_NULLPTR_T + typedef decltype(nullptr) nullptr_t; +#endif +#endif /* C++11. */ + +#endif /* _STDDEF_H was defined this time */ + +#endif /* !_STDDEF_H && !_STDDEF_H_ && !_ANSI_STDDEF_H && !__STDDEF_H__ + || __need_XXX was not defined before */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/stdfix.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/stdfix.h new file mode 100644 index 0000000000..9de1b72dca --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/stdfix.h @@ -0,0 +1,204 @@ +/* Copyright (C) 2007-2017 Free Software Foundation, Inc. + +This file is part of GCC. + +GCC is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3, or (at your option) +any later version. + +GCC is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +Under Section 7 of GPL version 3, you are granted additional +permissions described in the GCC Runtime Library Exception, version +3.1, as published by the Free Software Foundation. + +You should have received a copy of the GNU General Public License and +a copy of the GCC Runtime Library Exception along with this program; +see the files COPYING3 and COPYING.RUNTIME respectively. If not, see +. */ + +/* ISO/IEC JTC1 SC22 WG14 N1169 + * Date: 2006-04-04 + * ISO/IEC TR 18037 + * Programming languages - C - Extensions to support embedded processors + */ + +#ifndef _STDFIX_H +#define _STDFIX_H + +/* 7.18a.1 Introduction. */ + +#undef fract +#undef accum +#undef sat +#define fract _Fract +#define accum _Accum +#define sat _Sat + +/* 7.18a.3 Precision macros. */ + +#undef SFRACT_FBIT +#undef SFRACT_MIN +#undef SFRACT_MAX +#undef SFRACT_EPSILON +#define SFRACT_FBIT __SFRACT_FBIT__ +#define SFRACT_MIN __SFRACT_MIN__ +#define SFRACT_MAX __SFRACT_MAX__ +#define SFRACT_EPSILON __SFRACT_EPSILON__ + +#undef USFRACT_FBIT +#undef USFRACT_MIN +#undef USFRACT_MAX +#undef USFRACT_EPSILON +#define USFRACT_FBIT __USFRACT_FBIT__ +#define USFRACT_MIN __USFRACT_MIN__ /* GCC extension. */ +#define USFRACT_MAX __USFRACT_MAX__ +#define USFRACT_EPSILON __USFRACT_EPSILON__ + +#undef FRACT_FBIT +#undef FRACT_MIN +#undef FRACT_MAX +#undef FRACT_EPSILON +#define FRACT_FBIT __FRACT_FBIT__ +#define FRACT_MIN __FRACT_MIN__ +#define FRACT_MAX __FRACT_MAX__ +#define FRACT_EPSILON __FRACT_EPSILON__ + +#undef UFRACT_FBIT +#undef UFRACT_MIN +#undef UFRACT_MAX +#undef UFRACT_EPSILON +#define UFRACT_FBIT __UFRACT_FBIT__ +#define UFRACT_MIN __UFRACT_MIN__ /* GCC extension. */ +#define UFRACT_MAX __UFRACT_MAX__ +#define UFRACT_EPSILON __UFRACT_EPSILON__ + +#undef LFRACT_FBIT +#undef LFRACT_MIN +#undef LFRACT_MAX +#undef LFRACT_EPSILON +#define LFRACT_FBIT __LFRACT_FBIT__ +#define LFRACT_MIN __LFRACT_MIN__ +#define LFRACT_MAX __LFRACT_MAX__ +#define LFRACT_EPSILON __LFRACT_EPSILON__ + +#undef ULFRACT_FBIT +#undef ULFRACT_MIN +#undef ULFRACT_MAX +#undef ULFRACT_EPSILON +#define ULFRACT_FBIT __ULFRACT_FBIT__ +#define ULFRACT_MIN __ULFRACT_MIN__ /* GCC extension. */ +#define ULFRACT_MAX __ULFRACT_MAX__ +#define ULFRACT_EPSILON __ULFRACT_EPSILON__ + +#undef LLFRACT_FBIT +#undef LLFRACT_MIN +#undef LLFRACT_MAX +#undef LLFRACT_EPSILON +#define LLFRACT_FBIT __LLFRACT_FBIT__ /* GCC extension. */ +#define LLFRACT_MIN __LLFRACT_MIN__ /* GCC extension. */ +#define LLFRACT_MAX __LLFRACT_MAX__ /* GCC extension. */ +#define LLFRACT_EPSILON __LLFRACT_EPSILON__ /* GCC extension. */ + +#undef ULLFRACT_FBIT +#undef ULLFRACT_MIN +#undef ULLFRACT_MAX +#undef ULLFRACT_EPSILON +#define ULLFRACT_FBIT __ULLFRACT_FBIT__ /* GCC extension. */ +#define ULLFRACT_MIN __ULLFRACT_MIN__ /* GCC extension. */ +#define ULLFRACT_MAX __ULLFRACT_MAX__ /* GCC extension. */ +#define ULLFRACT_EPSILON __ULLFRACT_EPSILON__ /* GCC extension. */ + +#undef SACCUM_FBIT +#undef SACCUM_IBIT +#undef SACCUM_MIN +#undef SACCUM_MAX +#undef SACCUM_EPSILON +#define SACCUM_FBIT __SACCUM_FBIT__ +#define SACCUM_IBIT __SACCUM_IBIT__ +#define SACCUM_MIN __SACCUM_MIN__ +#define SACCUM_MAX __SACCUM_MAX__ +#define SACCUM_EPSILON __SACCUM_EPSILON__ + +#undef USACCUM_FBIT +#undef USACCUM_IBIT +#undef USACCUM_MIN +#undef USACCUM_MAX +#undef USACCUM_EPSILON +#define USACCUM_FBIT __USACCUM_FBIT__ +#define USACCUM_IBIT __USACCUM_IBIT__ +#define USACCUM_MIN __USACCUM_MIN__ /* GCC extension. */ +#define USACCUM_MAX __USACCUM_MAX__ +#define USACCUM_EPSILON __USACCUM_EPSILON__ + +#undef ACCUM_FBIT +#undef ACCUM_IBIT +#undef ACCUM_MIN +#undef ACCUM_MAX +#undef ACCUM_EPSILON +#define ACCUM_FBIT __ACCUM_FBIT__ +#define ACCUM_IBIT __ACCUM_IBIT__ +#define ACCUM_MIN __ACCUM_MIN__ +#define ACCUM_MAX __ACCUM_MAX__ +#define ACCUM_EPSILON __ACCUM_EPSILON__ + +#undef UACCUM_FBIT +#undef UACCUM_IBIT +#undef UACCUM_MIN +#undef UACCUM_MAX +#undef UACCUM_EPSILON +#define UACCUM_FBIT __UACCUM_FBIT__ +#define UACCUM_IBIT __UACCUM_IBIT__ +#define UACCUM_MIN __UACCUM_MIN__ /* GCC extension. */ +#define UACCUM_MAX __UACCUM_MAX__ +#define UACCUM_EPSILON __UACCUM_EPSILON__ + +#undef LACCUM_FBIT +#undef LACCUM_IBIT +#undef LACCUM_MIN +#undef LACCUM_MAX +#undef LACCUM_EPSILON +#define LACCUM_FBIT __LACCUM_FBIT__ +#define LACCUM_IBIT __LACCUM_IBIT__ +#define LACCUM_MIN __LACCUM_MIN__ +#define LACCUM_MAX __LACCUM_MAX__ +#define LACCUM_EPSILON __LACCUM_EPSILON__ + +#undef ULACCUM_FBIT +#undef ULACCUM_IBIT +#undef ULACCUM_MIN +#undef ULACCUM_MAX +#undef ULACCUM_EPSILON +#define ULACCUM_FBIT __ULACCUM_FBIT__ +#define ULACCUM_IBIT __ULACCUM_IBIT__ +#define ULACCUM_MIN __ULACCUM_MIN__ /* GCC extension. */ +#define ULACCUM_MAX __ULACCUM_MAX__ +#define ULACCUM_EPSILON __ULACCUM_EPSILON__ + +#undef LLACCUM_FBIT +#undef LLACCUM_IBIT +#undef LLACCUM_MIN +#undef LLACCUM_MAX +#undef LLACCUM_EPSILON +#define LLACCUM_FBIT __LLACCUM_FBIT__ /* GCC extension. */ +#define LLACCUM_IBIT __LLACCUM_IBIT__ /* GCC extension. */ +#define LLACCUM_MIN __LLACCUM_MIN__ /* GCC extension. */ +#define LLACCUM_MAX __LLACCUM_MAX__ /* GCC extension. */ +#define LLACCUM_EPSILON __LLACCUM_EPSILON__ /* GCC extension. */ + +#undef ULLACCUM_FBIT +#undef ULLACCUM_IBIT +#undef ULLACCUM_MIN +#undef ULLACCUM_MAX +#undef ULLACCUM_EPSILON +#define ULLACCUM_FBIT __ULLACCUM_FBIT__ /* GCC extension. */ +#define ULLACCUM_IBIT __ULLACCUM_IBIT__ /* GCC extension. */ +#define ULLACCUM_MIN __ULLACCUM_MIN__ /* GCC extension. */ +#define ULLACCUM_MAX __ULLACCUM_MAX__ /* GCC extension. */ +#define ULLACCUM_EPSILON __ULLACCUM_EPSILON__ /* GCC extension. */ + +#endif /* _STDFIX_H */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/stdint-gcc.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/stdint-gcc.h new file mode 100644 index 0000000000..0ee7e35234 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/stdint-gcc.h @@ -0,0 +1,364 @@ +/* Copyright (C) 2008-2017 Free Software Foundation, Inc. + +This file is part of GCC. + +GCC is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3, or (at your option) +any later version. + +GCC is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +Under Section 7 of GPL version 3, you are granted additional +permissions described in the GCC Runtime Library Exception, version +3.1, as published by the Free Software Foundation. + +You should have received a copy of the GNU General Public License and +a copy of the GCC Runtime Library Exception along with this program; +see the files COPYING3 and COPYING.RUNTIME respectively. If not, see +. */ + +/* + * ISO C Standard: 7.18 Integer types + */ + +#ifndef _GCC_STDINT_H +#define _GCC_STDINT_H + +/* 7.8.1.1 Exact-width integer types */ + +#ifdef __INT8_TYPE__ +typedef __INT8_TYPE__ int8_t; +#endif +#ifdef __INT16_TYPE__ +typedef __INT16_TYPE__ int16_t; +#endif +#ifdef __INT32_TYPE__ +typedef __INT32_TYPE__ int32_t; +#endif +#ifdef __INT64_TYPE__ +typedef __INT64_TYPE__ int64_t; +#endif +#ifdef __UINT8_TYPE__ +typedef __UINT8_TYPE__ uint8_t; +#endif +#ifdef __UINT16_TYPE__ +typedef __UINT16_TYPE__ uint16_t; +#endif +#ifdef __UINT32_TYPE__ +typedef __UINT32_TYPE__ uint32_t; +#endif +#ifdef __UINT64_TYPE__ +typedef __UINT64_TYPE__ uint64_t; +#endif + +/* 7.8.1.2 Minimum-width integer types */ + +typedef __INT_LEAST8_TYPE__ int_least8_t; +typedef __INT_LEAST16_TYPE__ int_least16_t; +typedef __INT_LEAST32_TYPE__ int_least32_t; +typedef __INT_LEAST64_TYPE__ int_least64_t; +typedef __UINT_LEAST8_TYPE__ uint_least8_t; +typedef __UINT_LEAST16_TYPE__ uint_least16_t; +typedef __UINT_LEAST32_TYPE__ uint_least32_t; +typedef __UINT_LEAST64_TYPE__ uint_least64_t; + +/* 7.8.1.3 Fastest minimum-width integer types */ + +typedef __INT_FAST8_TYPE__ int_fast8_t; +typedef __INT_FAST16_TYPE__ int_fast16_t; +typedef __INT_FAST32_TYPE__ int_fast32_t; +typedef __INT_FAST64_TYPE__ int_fast64_t; +typedef __UINT_FAST8_TYPE__ uint_fast8_t; +typedef __UINT_FAST16_TYPE__ uint_fast16_t; +typedef __UINT_FAST32_TYPE__ uint_fast32_t; +typedef __UINT_FAST64_TYPE__ uint_fast64_t; + +/* 7.8.1.4 Integer types capable of holding object pointers */ + +#ifdef __INTPTR_TYPE__ +typedef __INTPTR_TYPE__ intptr_t; +#endif +#ifdef __UINTPTR_TYPE__ +typedef __UINTPTR_TYPE__ uintptr_t; +#endif + +/* 7.8.1.5 Greatest-width integer types */ + +typedef __INTMAX_TYPE__ intmax_t; +typedef __UINTMAX_TYPE__ uintmax_t; + +#if (!defined __cplusplus || __cplusplus >= 201103L \ + || defined __STDC_LIMIT_MACROS) + +/* 7.18.2 Limits of specified-width integer types */ + +#ifdef __INT8_MAX__ +# undef INT8_MAX +# define INT8_MAX __INT8_MAX__ +# undef INT8_MIN +# define INT8_MIN (-INT8_MAX - 1) +#endif +#ifdef __UINT8_MAX__ +# undef UINT8_MAX +# define UINT8_MAX __UINT8_MAX__ +#endif +#ifdef __INT16_MAX__ +# undef INT16_MAX +# define INT16_MAX __INT16_MAX__ +# undef INT16_MIN +# define INT16_MIN (-INT16_MAX - 1) +#endif +#ifdef __UINT16_MAX__ +# undef UINT16_MAX +# define UINT16_MAX __UINT16_MAX__ +#endif +#ifdef __INT32_MAX__ +# undef INT32_MAX +# define INT32_MAX __INT32_MAX__ +# undef INT32_MIN +# define INT32_MIN (-INT32_MAX - 1) +#endif +#ifdef __UINT32_MAX__ +# undef UINT32_MAX +# define UINT32_MAX __UINT32_MAX__ +#endif +#ifdef __INT64_MAX__ +# undef INT64_MAX +# define INT64_MAX __INT64_MAX__ +# undef INT64_MIN +# define INT64_MIN (-INT64_MAX - 1) +#endif +#ifdef __UINT64_MAX__ +# undef UINT64_MAX +# define UINT64_MAX __UINT64_MAX__ +#endif + +#undef INT_LEAST8_MAX +#define INT_LEAST8_MAX __INT_LEAST8_MAX__ +#undef INT_LEAST8_MIN +#define INT_LEAST8_MIN (-INT_LEAST8_MAX - 1) +#undef UINT_LEAST8_MAX +#define UINT_LEAST8_MAX __UINT_LEAST8_MAX__ +#undef INT_LEAST16_MAX +#define INT_LEAST16_MAX __INT_LEAST16_MAX__ +#undef INT_LEAST16_MIN +#define INT_LEAST16_MIN (-INT_LEAST16_MAX - 1) +#undef UINT_LEAST16_MAX +#define UINT_LEAST16_MAX __UINT_LEAST16_MAX__ +#undef INT_LEAST32_MAX +#define INT_LEAST32_MAX __INT_LEAST32_MAX__ +#undef INT_LEAST32_MIN +#define INT_LEAST32_MIN (-INT_LEAST32_MAX - 1) +#undef UINT_LEAST32_MAX +#define UINT_LEAST32_MAX __UINT_LEAST32_MAX__ +#undef INT_LEAST64_MAX +#define INT_LEAST64_MAX __INT_LEAST64_MAX__ +#undef INT_LEAST64_MIN +#define INT_LEAST64_MIN (-INT_LEAST64_MAX - 1) +#undef UINT_LEAST64_MAX +#define UINT_LEAST64_MAX __UINT_LEAST64_MAX__ + +#undef INT_FAST8_MAX +#define INT_FAST8_MAX __INT_FAST8_MAX__ +#undef INT_FAST8_MIN +#define INT_FAST8_MIN (-INT_FAST8_MAX - 1) +#undef UINT_FAST8_MAX +#define UINT_FAST8_MAX __UINT_FAST8_MAX__ +#undef INT_FAST16_MAX +#define INT_FAST16_MAX __INT_FAST16_MAX__ +#undef INT_FAST16_MIN +#define INT_FAST16_MIN (-INT_FAST16_MAX - 1) +#undef UINT_FAST16_MAX +#define UINT_FAST16_MAX __UINT_FAST16_MAX__ +#undef INT_FAST32_MAX +#define INT_FAST32_MAX __INT_FAST32_MAX__ +#undef INT_FAST32_MIN +#define INT_FAST32_MIN (-INT_FAST32_MAX - 1) +#undef UINT_FAST32_MAX +#define UINT_FAST32_MAX __UINT_FAST32_MAX__ +#undef INT_FAST64_MAX +#define INT_FAST64_MAX __INT_FAST64_MAX__ +#undef INT_FAST64_MIN +#define INT_FAST64_MIN (-INT_FAST64_MAX - 1) +#undef UINT_FAST64_MAX +#define UINT_FAST64_MAX __UINT_FAST64_MAX__ + +#ifdef __INTPTR_MAX__ +# undef INTPTR_MAX +# define INTPTR_MAX __INTPTR_MAX__ +# undef INTPTR_MIN +# define INTPTR_MIN (-INTPTR_MAX - 1) +#endif +#ifdef __UINTPTR_MAX__ +# undef UINTPTR_MAX +# define UINTPTR_MAX __UINTPTR_MAX__ +#endif + +#undef INTMAX_MAX +#define INTMAX_MAX __INTMAX_MAX__ +#undef INTMAX_MIN +#define INTMAX_MIN (-INTMAX_MAX - 1) +#undef UINTMAX_MAX +#define UINTMAX_MAX __UINTMAX_MAX__ + +/* 7.18.3 Limits of other integer types */ + +#undef PTRDIFF_MAX +#define PTRDIFF_MAX __PTRDIFF_MAX__ +#undef PTRDIFF_MIN +#define PTRDIFF_MIN (-PTRDIFF_MAX - 1) + +#undef SIG_ATOMIC_MAX +#define SIG_ATOMIC_MAX __SIG_ATOMIC_MAX__ +#undef SIG_ATOMIC_MIN +#define SIG_ATOMIC_MIN __SIG_ATOMIC_MIN__ + +#undef SIZE_MAX +#define SIZE_MAX __SIZE_MAX__ + +#undef WCHAR_MAX +#define WCHAR_MAX __WCHAR_MAX__ +#undef WCHAR_MIN +#define WCHAR_MIN __WCHAR_MIN__ + +#undef WINT_MAX +#define WINT_MAX __WINT_MAX__ +#undef WINT_MIN +#define WINT_MIN __WINT_MIN__ + +#endif /* (!defined __cplusplus || __cplusplus >= 201103L + || defined __STDC_LIMIT_MACROS) */ + +#if (!defined __cplusplus || __cplusplus >= 201103L \ + || defined __STDC_CONSTANT_MACROS) + +#undef INT8_C +#define INT8_C(c) __INT8_C(c) +#undef INT16_C +#define INT16_C(c) __INT16_C(c) +#undef INT32_C +#define INT32_C(c) __INT32_C(c) +#undef INT64_C +#define INT64_C(c) __INT64_C(c) +#undef UINT8_C +#define UINT8_C(c) __UINT8_C(c) +#undef UINT16_C +#define UINT16_C(c) __UINT16_C(c) +#undef UINT32_C +#define UINT32_C(c) __UINT32_C(c) +#undef UINT64_C +#define UINT64_C(c) __UINT64_C(c) +#undef INTMAX_C +#define INTMAX_C(c) __INTMAX_C(c) +#undef UINTMAX_C +#define UINTMAX_C(c) __UINTMAX_C(c) + +#endif /* (!defined __cplusplus || __cplusplus >= 201103L + || defined __STDC_CONSTANT_MACROS) */ + +#ifdef __STDC_WANT_IEC_60559_BFP_EXT__ +/* TS 18661-1 widths of integer types. */ + +#ifdef __INT8_TYPE__ +# undef INT8_WIDTH +# define INT8_WIDTH 8 +#endif +#ifdef __UINT8_TYPE__ +# undef UINT8_WIDTH +# define UINT8_WIDTH 8 +#endif +#ifdef __INT16_TYPE__ +# undef INT16_WIDTH +# define INT16_WIDTH 16 +#endif +#ifdef __UINT16_TYPE__ +# undef UINT16_WIDTH +# define UINT16_WIDTH 16 +#endif +#ifdef __INT32_TYPE__ +# undef INT32_WIDTH +# define INT32_WIDTH 32 +#endif +#ifdef __UINT32_TYPE__ +# undef UINT32_WIDTH +# define UINT32_WIDTH 32 +#endif +#ifdef __INT64_TYPE__ +# undef INT64_WIDTH +# define INT64_WIDTH 64 +#endif +#ifdef __UINT64_TYPE__ +# undef UINT64_WIDTH +# define UINT64_WIDTH 64 +#endif + +#undef INT_LEAST8_WIDTH +#define INT_LEAST8_WIDTH __INT_LEAST8_WIDTH__ +#undef UINT_LEAST8_WIDTH +#define UINT_LEAST8_WIDTH __INT_LEAST8_WIDTH__ +#undef INT_LEAST16_WIDTH +#define INT_LEAST16_WIDTH __INT_LEAST16_WIDTH__ +#undef UINT_LEAST16_WIDTH +#define UINT_LEAST16_WIDTH __INT_LEAST16_WIDTH__ +#undef INT_LEAST32_WIDTH +#define INT_LEAST32_WIDTH __INT_LEAST32_WIDTH__ +#undef UINT_LEAST32_WIDTH +#define UINT_LEAST32_WIDTH __INT_LEAST32_WIDTH__ +#undef INT_LEAST64_WIDTH +#define INT_LEAST64_WIDTH __INT_LEAST64_WIDTH__ +#undef UINT_LEAST64_WIDTH +#define UINT_LEAST64_WIDTH __INT_LEAST64_WIDTH__ + +#undef INT_FAST8_WIDTH +#define INT_FAST8_WIDTH __INT_FAST8_WIDTH__ +#undef UINT_FAST8_WIDTH +#define UINT_FAST8_WIDTH __INT_FAST8_WIDTH__ +#undef INT_FAST16_WIDTH +#define INT_FAST16_WIDTH __INT_FAST16_WIDTH__ +#undef UINT_FAST16_WIDTH +#define UINT_FAST16_WIDTH __INT_FAST16_WIDTH__ +#undef INT_FAST32_WIDTH +#define INT_FAST32_WIDTH __INT_FAST32_WIDTH__ +#undef UINT_FAST32_WIDTH +#define UINT_FAST32_WIDTH __INT_FAST32_WIDTH__ +#undef INT_FAST64_WIDTH +#define INT_FAST64_WIDTH __INT_FAST64_WIDTH__ +#undef UINT_FAST64_WIDTH +#define UINT_FAST64_WIDTH __INT_FAST64_WIDTH__ + +#ifdef __INTPTR_TYPE__ +# undef INTPTR_WIDTH +# define INTPTR_WIDTH __INTPTR_WIDTH__ +#endif +#ifdef __UINTPTR_TYPE__ +# undef UINTPTR_WIDTH +# define UINTPTR_WIDTH __INTPTR_WIDTH__ +#endif + +#undef INTMAX_WIDTH +#define INTMAX_WIDTH __INTMAX_WIDTH__ +#undef UINTMAX_WIDTH +#define UINTMAX_WIDTH __INTMAX_WIDTH__ + +#undef PTRDIFF_WIDTH +#define PTRDIFF_WIDTH __PTRDIFF_WIDTH__ + +#undef SIG_ATOMIC_WIDTH +#define SIG_ATOMIC_WIDTH __SIG_ATOMIC_WIDTH__ + +#undef SIZE_WIDTH +#define SIZE_WIDTH __SIZE_WIDTH__ + +#undef WCHAR_WIDTH +#define WCHAR_WIDTH __WCHAR_WIDTH__ + +#undef WINT_WIDTH +#define WINT_WIDTH __WINT_WIDTH__ + +#endif + +#endif /* _GCC_STDINT_H */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/stdint.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/stdint.h new file mode 100644 index 0000000000..83b6f70c83 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/stdint.h @@ -0,0 +1,14 @@ +#ifndef _GCC_WRAP_STDINT_H +#if __STDC_HOSTED__ +# if defined __cplusplus && __cplusplus >= 201103L +# undef __STDC_LIMIT_MACROS +# define __STDC_LIMIT_MACROS +# undef __STDC_CONSTANT_MACROS +# define __STDC_CONSTANT_MACROS +# endif +# include_next +#else +# include "stdint-gcc.h" +#endif +#define _GCC_WRAP_STDINT_H +#endif diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/stdnoreturn.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/stdnoreturn.h new file mode 100644 index 0000000000..739c2e31fc --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/stdnoreturn.h @@ -0,0 +1,35 @@ +/* Copyright (C) 2011-2017 Free Software Foundation, Inc. + +This file is part of GCC. + +GCC is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3, or (at your option) +any later version. + +GCC is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +Under Section 7 of GPL version 3, you are granted additional +permissions described in the GCC Runtime Library Exception, version +3.1, as published by the Free Software Foundation. + +You should have received a copy of the GNU General Public License and +a copy of the GCC Runtime Library Exception along with this program; +see the files COPYING3 and COPYING.RUNTIME respectively. If not, see +. */ + +/* ISO C1X: 7.23 _Noreturn . */ + +#ifndef _STDNORETURN_H +#define _STDNORETURN_H + +#ifndef __cplusplus + +#define noreturn _Noreturn + +#endif + +#endif /* stdnoreturn.h */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/syslimits.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/syslimits.h new file mode 100644 index 0000000000..a3628025e8 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/syslimits.h @@ -0,0 +1,8 @@ +/* syslimits.h stands for the system's own limits.h file. + If we can use it ok unmodified, then we install this text. + If fixincludes fixes it, then the fixed version is installed + instead of this text. */ + +#define _GCC_NEXT_LIMITS_H /* tell gcc's limits.h to recurse */ +#include_next +#undef _GCC_NEXT_LIMITS_H diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/tbmintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/tbmintrin.h new file mode 100644 index 0000000000..dfdf163f9c --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/tbmintrin.h @@ -0,0 +1,180 @@ +/* Copyright (C) 2010-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _X86INTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _TBMINTRIN_H_INCLUDED +#define _TBMINTRIN_H_INCLUDED + +#ifndef __TBM__ +#pragma GCC push_options +#pragma GCC target("tbm") +#define __DISABLE_TBM__ +#endif /* __TBM__ */ + +#ifdef __OPTIMIZE__ +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__bextri_u32 (unsigned int __X, const unsigned int __I) +{ + return __builtin_ia32_bextri_u32 (__X, __I); +} +#else +#define __bextri_u32(X, I) \ + ((unsigned int)__builtin_ia32_bextri_u32 ((unsigned int)(X), \ + (unsigned int)(I))) +#endif /*__OPTIMIZE__ */ + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blcfill_u32 (unsigned int __X) +{ + return __X & (__X + 1); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blci_u32 (unsigned int __X) +{ + return __X | ~(__X + 1); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blcic_u32 (unsigned int __X) +{ + return ~__X & (__X + 1); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blcmsk_u32 (unsigned int __X) +{ + return __X ^ (__X + 1); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blcs_u32 (unsigned int __X) +{ + return __X | (__X + 1); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blsfill_u32 (unsigned int __X) +{ + return __X | (__X - 1); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blsic_u32 (unsigned int __X) +{ + return ~__X | (__X - 1); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__t1mskc_u32 (unsigned int __X) +{ + return ~__X | (__X + 1); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__tzmsk_u32 (unsigned int __X) +{ + return ~__X & (__X - 1); +} + + + +#ifdef __x86_64__ +#ifdef __OPTIMIZE__ +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__bextri_u64 (unsigned long long __X, const unsigned int __I) +{ + return __builtin_ia32_bextri_u64 (__X, __I); +} +#else +#define __bextri_u64(X, I) \ + ((unsigned long long)__builtin_ia32_bextri_u64 ((unsigned long long)(X), \ + (unsigned long long)(I))) +#endif /*__OPTIMIZE__ */ + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blcfill_u64 (unsigned long long __X) +{ + return __X & (__X + 1); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blci_u64 (unsigned long long __X) +{ + return __X | ~(__X + 1); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blcic_u64 (unsigned long long __X) +{ + return ~__X & (__X + 1); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blcmsk_u64 (unsigned long long __X) +{ + return __X ^ (__X + 1); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blcs_u64 (unsigned long long __X) +{ + return __X | (__X + 1); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blsfill_u64 (unsigned long long __X) +{ + return __X | (__X - 1); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blsic_u64 (unsigned long long __X) +{ + return ~__X | (__X - 1); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__t1mskc_u64 (unsigned long long __X) +{ + return ~__X | (__X + 1); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__tzmsk_u64 (unsigned long long __X) +{ + return ~__X & (__X - 1); +} + + +#endif /* __x86_64__ */ + +#ifdef __DISABLE_TBM__ +#undef __DISABLE_TBM__ +#pragma GCC pop_options +#endif /* __DISABLE_TBM__ */ + +#endif /* _TBMINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/tmmintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/tmmintrin.h new file mode 100644 index 0000000000..93b016d291 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/tmmintrin.h @@ -0,0 +1,249 @@ +/* Copyright (C) 2006-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +/* Implemented from the specification included in the Intel C++ Compiler + User Guide and Reference, version 9.1. */ + +#ifndef _TMMINTRIN_H_INCLUDED +#define _TMMINTRIN_H_INCLUDED + +/* We need definitions from the SSE3, SSE2 and SSE header files*/ +#include + +#ifndef __SSSE3__ +#pragma GCC push_options +#pragma GCC target("ssse3") +#define __DISABLE_SSSE3__ +#endif /* __SSSE3__ */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hadd_epi16 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_phaddw128 ((__v8hi)__X, (__v8hi)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hadd_epi32 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_phaddd128 ((__v4si)__X, (__v4si)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hadds_epi16 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_phaddsw128 ((__v8hi)__X, (__v8hi)__Y); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hadd_pi16 (__m64 __X, __m64 __Y) +{ + return (__m64) __builtin_ia32_phaddw ((__v4hi)__X, (__v4hi)__Y); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hadd_pi32 (__m64 __X, __m64 __Y) +{ + return (__m64) __builtin_ia32_phaddd ((__v2si)__X, (__v2si)__Y); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hadds_pi16 (__m64 __X, __m64 __Y) +{ + return (__m64) __builtin_ia32_phaddsw ((__v4hi)__X, (__v4hi)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hsub_epi16 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_phsubw128 ((__v8hi)__X, (__v8hi)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hsub_epi32 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_phsubd128 ((__v4si)__X, (__v4si)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hsubs_epi16 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_phsubsw128 ((__v8hi)__X, (__v8hi)__Y); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hsub_pi16 (__m64 __X, __m64 __Y) +{ + return (__m64) __builtin_ia32_phsubw ((__v4hi)__X, (__v4hi)__Y); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hsub_pi32 (__m64 __X, __m64 __Y) +{ + return (__m64) __builtin_ia32_phsubd ((__v2si)__X, (__v2si)__Y); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hsubs_pi16 (__m64 __X, __m64 __Y) +{ + return (__m64) __builtin_ia32_phsubsw ((__v4hi)__X, (__v4hi)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maddubs_epi16 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_pmaddubsw128 ((__v16qi)__X, (__v16qi)__Y); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maddubs_pi16 (__m64 __X, __m64 __Y) +{ + return (__m64) __builtin_ia32_pmaddubsw ((__v8qi)__X, (__v8qi)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mulhrs_epi16 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_pmulhrsw128 ((__v8hi)__X, (__v8hi)__Y); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mulhrs_pi16 (__m64 __X, __m64 __Y) +{ + return (__m64) __builtin_ia32_pmulhrsw ((__v4hi)__X, (__v4hi)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_shuffle_epi8 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_pshufb128 ((__v16qi)__X, (__v16qi)__Y); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_shuffle_pi8 (__m64 __X, __m64 __Y) +{ + return (__m64) __builtin_ia32_pshufb ((__v8qi)__X, (__v8qi)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sign_epi8 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_psignb128 ((__v16qi)__X, (__v16qi)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sign_epi16 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_psignw128 ((__v8hi)__X, (__v8hi)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sign_epi32 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_psignd128 ((__v4si)__X, (__v4si)__Y); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sign_pi8 (__m64 __X, __m64 __Y) +{ + return (__m64) __builtin_ia32_psignb ((__v8qi)__X, (__v8qi)__Y); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sign_pi16 (__m64 __X, __m64 __Y) +{ + return (__m64) __builtin_ia32_psignw ((__v4hi)__X, (__v4hi)__Y); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sign_pi32 (__m64 __X, __m64 __Y) +{ + return (__m64) __builtin_ia32_psignd ((__v2si)__X, (__v2si)__Y); +} + +#ifdef __OPTIMIZE__ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_alignr_epi8(__m128i __X, __m128i __Y, const int __N) +{ + return (__m128i) __builtin_ia32_palignr128 ((__v2di)__X, + (__v2di)__Y, __N * 8); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_alignr_pi8(__m64 __X, __m64 __Y, const int __N) +{ + return (__m64) __builtin_ia32_palignr ((__v1di)__X, + (__v1di)__Y, __N * 8); +} +#else +#define _mm_alignr_epi8(X, Y, N) \ + ((__m128i) __builtin_ia32_palignr128 ((__v2di)(__m128i)(X), \ + (__v2di)(__m128i)(Y), \ + (int)(N) * 8)) +#define _mm_alignr_pi8(X, Y, N) \ + ((__m64) __builtin_ia32_palignr ((__v1di)(__m64)(X), \ + (__v1di)(__m64)(Y), \ + (int)(N) * 8)) +#endif + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_abs_epi8 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pabsb128 ((__v16qi)__X); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_abs_epi16 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pabsw128 ((__v8hi)__X); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_abs_epi32 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pabsd128 ((__v4si)__X); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_abs_pi8 (__m64 __X) +{ + return (__m64) __builtin_ia32_pabsb ((__v8qi)__X); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_abs_pi16 (__m64 __X) +{ + return (__m64) __builtin_ia32_pabsw ((__v4hi)__X); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_abs_pi32 (__m64 __X) +{ + return (__m64) __builtin_ia32_pabsd ((__v2si)__X); +} + +#ifdef __DISABLE_SSSE3__ +#undef __DISABLE_SSSE3__ +#pragma GCC pop_options +#endif /* __DISABLE_SSSE3__ */ + +#endif /* _TMMINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/unwind.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/unwind.h new file mode 100644 index 0000000000..77dd5a9020 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/unwind.h @@ -0,0 +1,291 @@ +/* Exception handling and frame unwind runtime interface routines. + Copyright (C) 2001-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public + License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +/* This is derived from the C++ ABI for IA-64. Where we diverge + for cross-architecture compatibility are noted with "@@@". */ + +#ifndef _UNWIND_H +#define _UNWIND_H + +#if defined (__SEH__) && !defined (__USING_SJLJ_EXCEPTIONS__) +/* Only for _GCC_specific_handler. */ +#include +#endif + +#ifndef HIDE_EXPORTS +#pragma GCC visibility push(default) +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* Level 1: Base ABI */ + +/* @@@ The IA-64 ABI uses uint64 throughout. Most places this is + inefficient for 32-bit and smaller machines. */ +typedef unsigned _Unwind_Word __attribute__((__mode__(__unwind_word__))); +typedef signed _Unwind_Sword __attribute__((__mode__(__unwind_word__))); +#if defined(__ia64__) && defined(__hpux__) +typedef unsigned _Unwind_Ptr __attribute__((__mode__(__word__))); +#else +typedef unsigned _Unwind_Ptr __attribute__((__mode__(__pointer__))); +#endif +typedef unsigned _Unwind_Internal_Ptr __attribute__((__mode__(__pointer__))); + +/* @@@ The IA-64 ABI uses a 64-bit word to identify the producer and + consumer of an exception. We'll go along with this for now even on + 32-bit machines. We'll need to provide some other option for + 16-bit machines and for machines with > 8 bits per byte. */ +typedef unsigned _Unwind_Exception_Class __attribute__((__mode__(__DI__))); + +/* The unwind interface uses reason codes in several contexts to + identify the reasons for failures or other actions. */ +typedef enum +{ + _URC_NO_REASON = 0, + _URC_FOREIGN_EXCEPTION_CAUGHT = 1, + _URC_FATAL_PHASE2_ERROR = 2, + _URC_FATAL_PHASE1_ERROR = 3, + _URC_NORMAL_STOP = 4, + _URC_END_OF_STACK = 5, + _URC_HANDLER_FOUND = 6, + _URC_INSTALL_CONTEXT = 7, + _URC_CONTINUE_UNWIND = 8 +} _Unwind_Reason_Code; + + +/* The unwind interface uses a pointer to an exception header object + as its representation of an exception being thrown. In general, the + full representation of an exception object is language- and + implementation-specific, but it will be prefixed by a header + understood by the unwind interface. */ + +struct _Unwind_Exception; + +typedef void (*_Unwind_Exception_Cleanup_Fn) (_Unwind_Reason_Code, + struct _Unwind_Exception *); + +struct _Unwind_Exception +{ + _Unwind_Exception_Class exception_class; + _Unwind_Exception_Cleanup_Fn exception_cleanup; + +#if !defined (__USING_SJLJ_EXCEPTIONS__) && defined (__SEH__) + _Unwind_Word private_[6]; +#else + _Unwind_Word private_1; + _Unwind_Word private_2; +#endif + + /* @@@ The IA-64 ABI says that this structure must be double-word aligned. + Taking that literally does not make much sense generically. Instead we + provide the maximum alignment required by any type for the machine. */ +} __attribute__((__aligned__)); + + +/* The ACTIONS argument to the personality routine is a bitwise OR of one + or more of the following constants. */ +typedef int _Unwind_Action; + +#define _UA_SEARCH_PHASE 1 +#define _UA_CLEANUP_PHASE 2 +#define _UA_HANDLER_FRAME 4 +#define _UA_FORCE_UNWIND 8 +#define _UA_END_OF_STACK 16 + +/* The target can override this macro to define any back-end-specific + attributes required for the lowest-level stack frame. */ +#ifndef LIBGCC2_UNWIND_ATTRIBUTE +#define LIBGCC2_UNWIND_ATTRIBUTE +#endif + +/* This is an opaque type used to refer to a system-specific data + structure used by the system unwinder. This context is created and + destroyed by the system, and passed to the personality routine + during unwinding. */ +struct _Unwind_Context; + +/* Raise an exception, passing along the given exception object. */ +extern _Unwind_Reason_Code LIBGCC2_UNWIND_ATTRIBUTE +_Unwind_RaiseException (struct _Unwind_Exception *); + +/* Raise an exception for forced unwinding. */ + +typedef _Unwind_Reason_Code (*_Unwind_Stop_Fn) + (int, _Unwind_Action, _Unwind_Exception_Class, + struct _Unwind_Exception *, struct _Unwind_Context *, void *); + +extern _Unwind_Reason_Code LIBGCC2_UNWIND_ATTRIBUTE +_Unwind_ForcedUnwind (struct _Unwind_Exception *, _Unwind_Stop_Fn, void *); + +/* Helper to invoke the exception_cleanup routine. */ +extern void _Unwind_DeleteException (struct _Unwind_Exception *); + +/* Resume propagation of an existing exception. This is used after + e.g. executing cleanup code, and not to implement rethrowing. */ +extern void LIBGCC2_UNWIND_ATTRIBUTE +_Unwind_Resume (struct _Unwind_Exception *); + +/* @@@ Resume propagation of a FORCE_UNWIND exception, or to rethrow + a normal exception that was handled. */ +extern _Unwind_Reason_Code LIBGCC2_UNWIND_ATTRIBUTE +_Unwind_Resume_or_Rethrow (struct _Unwind_Exception *); + +/* @@@ Use unwind data to perform a stack backtrace. The trace callback + is called for every stack frame in the call chain, but no cleanup + actions are performed. */ +typedef _Unwind_Reason_Code (*_Unwind_Trace_Fn) + (struct _Unwind_Context *, void *); + +extern _Unwind_Reason_Code LIBGCC2_UNWIND_ATTRIBUTE +_Unwind_Backtrace (_Unwind_Trace_Fn, void *); + +/* These functions are used for communicating information about the unwind + context (i.e. the unwind descriptors and the user register state) between + the unwind library and the personality routine and landing pad. Only + selected registers may be manipulated. */ + +extern _Unwind_Word _Unwind_GetGR (struct _Unwind_Context *, int); +extern void _Unwind_SetGR (struct _Unwind_Context *, int, _Unwind_Word); + +extern _Unwind_Ptr _Unwind_GetIP (struct _Unwind_Context *); +extern _Unwind_Ptr _Unwind_GetIPInfo (struct _Unwind_Context *, int *); +extern void _Unwind_SetIP (struct _Unwind_Context *, _Unwind_Ptr); + +/* @@@ Retrieve the CFA of the given context. */ +extern _Unwind_Word _Unwind_GetCFA (struct _Unwind_Context *); + +extern void *_Unwind_GetLanguageSpecificData (struct _Unwind_Context *); + +extern _Unwind_Ptr _Unwind_GetRegionStart (struct _Unwind_Context *); + + +/* The personality routine is the function in the C++ (or other language) + runtime library which serves as an interface between the system unwind + library and language-specific exception handling semantics. It is + specific to the code fragment described by an unwind info block, and + it is always referenced via the pointer in the unwind info block, and + hence it has no ABI-specified name. + + Note that this implies that two different C++ implementations can + use different names, and have different contents in the language + specific data area. Moreover, that the language specific data + area contains no version info because name of the function invoked + provides more effective versioning by detecting at link time the + lack of code to handle the different data format. */ + +typedef _Unwind_Reason_Code (*_Unwind_Personality_Fn) + (int, _Unwind_Action, _Unwind_Exception_Class, + struct _Unwind_Exception *, struct _Unwind_Context *); + +/* @@@ The following alternate entry points are for setjmp/longjmp + based unwinding. */ + +struct SjLj_Function_Context; +extern void _Unwind_SjLj_Register (struct SjLj_Function_Context *); +extern void _Unwind_SjLj_Unregister (struct SjLj_Function_Context *); + +extern _Unwind_Reason_Code LIBGCC2_UNWIND_ATTRIBUTE +_Unwind_SjLj_RaiseException (struct _Unwind_Exception *); +extern _Unwind_Reason_Code LIBGCC2_UNWIND_ATTRIBUTE +_Unwind_SjLj_ForcedUnwind (struct _Unwind_Exception *, _Unwind_Stop_Fn, void *); +extern void LIBGCC2_UNWIND_ATTRIBUTE +_Unwind_SjLj_Resume (struct _Unwind_Exception *); +extern _Unwind_Reason_Code LIBGCC2_UNWIND_ATTRIBUTE +_Unwind_SjLj_Resume_or_Rethrow (struct _Unwind_Exception *); + +/* @@@ The following provide access to the base addresses for text + and data-relative addressing in the LDSA. In order to stay link + compatible with the standard ABI for IA-64, we inline these. */ + +#ifdef __ia64__ +static inline _Unwind_Ptr +_Unwind_GetDataRelBase (struct _Unwind_Context *_C) +{ + /* The GP is stored in R1. */ + return _Unwind_GetGR (_C, 1); +} + +static inline _Unwind_Ptr +_Unwind_GetTextRelBase (struct _Unwind_Context *_C __attribute__ ((__unused__))) +{ + __builtin_abort (); + return 0; +} + +/* @@@ Retrieve the Backing Store Pointer of the given context. */ +extern _Unwind_Word _Unwind_GetBSP (struct _Unwind_Context *); +#else +extern _Unwind_Ptr _Unwind_GetDataRelBase (struct _Unwind_Context *); +extern _Unwind_Ptr _Unwind_GetTextRelBase (struct _Unwind_Context *); +#endif + +/* @@@ Given an address, return the entry point of the function that + contains it. */ +extern void * _Unwind_FindEnclosingFunction (void *pc); + +#ifndef __SIZEOF_LONG__ + #error "__SIZEOF_LONG__ macro not defined" +#endif + +#ifndef __SIZEOF_POINTER__ + #error "__SIZEOF_POINTER__ macro not defined" +#endif + + +/* leb128 type numbers have a potentially unlimited size. + The target of the following definitions of _sleb128_t and _uleb128_t + is to have efficient data types large enough to hold the leb128 type + numbers used in the unwind code. + Mostly these types will simply be defined to long and unsigned long + except when a unsigned long data type on the target machine is not + capable of storing a pointer. */ + +#if __SIZEOF_LONG__ >= __SIZEOF_POINTER__ + typedef long _sleb128_t; + typedef unsigned long _uleb128_t; +#elif __SIZEOF_LONG_LONG__ >= __SIZEOF_POINTER__ + typedef long long _sleb128_t; + typedef unsigned long long _uleb128_t; +#else +# error "What type shall we use for _sleb128_t?" +#endif + +#if defined (__SEH__) && !defined (__USING_SJLJ_EXCEPTIONS__) +/* Handles the mapping from SEH to GCC interfaces. */ +EXCEPTION_DISPOSITION _GCC_specific_handler (PEXCEPTION_RECORD, void *, + PCONTEXT, PDISPATCHER_CONTEXT, + _Unwind_Personality_Fn); +#endif + +#ifdef __cplusplus +} +#endif + +#ifndef HIDE_EXPORTS +#pragma GCC visibility pop +#endif + +#endif /* unwind.h */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/varargs.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/varargs.h new file mode 100644 index 0000000000..4b9803e71a --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/varargs.h @@ -0,0 +1,7 @@ +#ifndef _VARARGS_H +#define _VARARGS_H + +#error "GCC no longer implements ." +#error "Revise your code to use ." + +#endif diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/wmmintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/wmmintrin.h new file mode 100644 index 0000000000..53de853a56 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/wmmintrin.h @@ -0,0 +1,132 @@ +/* Copyright (C) 2008-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +/* Implemented from the specification included in the Intel C++ Compiler + User Guide and Reference, version 10.1. */ + +#ifndef _WMMINTRIN_H_INCLUDED +#define _WMMINTRIN_H_INCLUDED + +/* We need definitions from the SSE2 header file. */ +#include + +/* AES */ + +#ifndef __AES__ +#pragma GCC push_options +#pragma GCC target("aes") +#define __DISABLE_AES__ +#endif /* __AES__ */ + +/* Performs 1 round of AES decryption of the first m128i using + the second m128i as a round key. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_aesdec_si128 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_aesdec128 ((__v2di)__X, (__v2di)__Y); +} + +/* Performs the last round of AES decryption of the first m128i + using the second m128i as a round key. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_aesdeclast_si128 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_aesdeclast128 ((__v2di)__X, + (__v2di)__Y); +} + +/* Performs 1 round of AES encryption of the first m128i using + the second m128i as a round key. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_aesenc_si128 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_aesenc128 ((__v2di)__X, (__v2di)__Y); +} + +/* Performs the last round of AES encryption of the first m128i + using the second m128i as a round key. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_aesenclast_si128 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_aesenclast128 ((__v2di)__X, (__v2di)__Y); +} + +/* Performs the InverseMixColumn operation on the source m128i + and stores the result into m128i destination. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_aesimc_si128 (__m128i __X) +{ + return (__m128i) __builtin_ia32_aesimc128 ((__v2di)__X); +} + +/* Generates a m128i round key for the input m128i AES cipher key and + byte round constant. The second parameter must be a compile time + constant. */ +#ifdef __OPTIMIZE__ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_aeskeygenassist_si128 (__m128i __X, const int __C) +{ + return (__m128i) __builtin_ia32_aeskeygenassist128 ((__v2di)__X, __C); +} +#else +#define _mm_aeskeygenassist_si128(X, C) \ + ((__m128i) __builtin_ia32_aeskeygenassist128 ((__v2di)(__m128i)(X), \ + (int)(C))) +#endif + +#ifdef __DISABLE_AES__ +#undef __DISABLE_AES__ +#pragma GCC pop_options +#endif /* __DISABLE_AES__ */ + +/* PCLMUL */ + +#ifndef __PCLMUL__ +#pragma GCC push_options +#pragma GCC target("pclmul") +#define __DISABLE_PCLMUL__ +#endif /* __PCLMUL__ */ + +/* Performs carry-less integer multiplication of 64-bit halves of + 128-bit input operands. The third parameter inducates which 64-bit + haves of the input parameters v1 and v2 should be used. It must be + a compile time constant. */ +#ifdef __OPTIMIZE__ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_clmulepi64_si128 (__m128i __X, __m128i __Y, const int __I) +{ + return (__m128i) __builtin_ia32_pclmulqdq128 ((__v2di)__X, + (__v2di)__Y, __I); +} +#else +#define _mm_clmulepi64_si128(X, Y, I) \ + ((__m128i) __builtin_ia32_pclmulqdq128 ((__v2di)(__m128i)(X), \ + (__v2di)(__m128i)(Y), (int)(I))) +#endif + +#ifdef __DISABLE_PCLMUL__ +#undef __DISABLE_PCLMUL__ +#pragma GCC pop_options +#endif /* __DISABLE_PCLMUL__ */ + +#endif /* _WMMINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/x86intrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/x86intrin.h new file mode 100644 index 0000000000..780edbbe54 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/x86intrin.h @@ -0,0 +1,102 @@ +/* Copyright (C) 2008-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _X86INTRIN_H_INCLUDED +#define _X86INTRIN_H_INCLUDED + +#include + +#ifndef __iamcu__ + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +/* For including AVX instructions */ +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#endif /* __iamcu__ */ + +#include + +#ifndef __iamcu__ + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#endif /* __iamcu__ */ + +#endif /* _X86INTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/xmmintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/xmmintrin.h new file mode 100644 index 0000000000..5f58d24430 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/xmmintrin.h @@ -0,0 +1,1270 @@ +/* Copyright (C) 2002-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +/* Implemented from the specification included in the Intel C++ Compiler + User Guide and Reference, version 9.0. */ + +#ifndef _XMMINTRIN_H_INCLUDED +#define _XMMINTRIN_H_INCLUDED + +/* We need type definitions from the MMX header file. */ +#include + +/* Get _mm_malloc () and _mm_free (). */ +#include + +/* Constants for use with _mm_prefetch. */ +enum _mm_hint +{ + /* _MM_HINT_ET is _MM_HINT_T with set 3rd bit. */ + _MM_HINT_ET0 = 7, + _MM_HINT_ET1 = 6, + _MM_HINT_T0 = 3, + _MM_HINT_T1 = 2, + _MM_HINT_T2 = 1, + _MM_HINT_NTA = 0 +}; + +/* Loads one cache line from address P to a location "closer" to the + processor. The selector I specifies the type of prefetch operation. */ +#ifdef __OPTIMIZE__ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_prefetch (const void *__P, enum _mm_hint __I) +{ + __builtin_prefetch (__P, (__I & 0x4) >> 2, __I & 0x3); +} +#else +#define _mm_prefetch(P, I) \ + __builtin_prefetch ((P), ((I & 0x4) >> 2), (I & 0x3)) +#endif + +#ifndef __SSE__ +#pragma GCC push_options +#pragma GCC target("sse") +#define __DISABLE_SSE__ +#endif /* __SSE__ */ + +/* The Intel API is flexible enough that we must allow aliasing with other + vector types, and their scalar components. */ +typedef float __m128 __attribute__ ((__vector_size__ (16), __may_alias__)); + +/* Unaligned version of the same type. */ +typedef float __m128_u __attribute__ ((__vector_size__ (16), __may_alias__, __aligned__ (1))); + +/* Internal data types for implementing the intrinsics. */ +typedef float __v4sf __attribute__ ((__vector_size__ (16))); + +/* Create a selector for use with the SHUFPS instruction. */ +#define _MM_SHUFFLE(fp3,fp2,fp1,fp0) \ + (((fp3) << 6) | ((fp2) << 4) | ((fp1) << 2) | (fp0)) + +/* Bits in the MXCSR. */ +#define _MM_EXCEPT_MASK 0x003f +#define _MM_EXCEPT_INVALID 0x0001 +#define _MM_EXCEPT_DENORM 0x0002 +#define _MM_EXCEPT_DIV_ZERO 0x0004 +#define _MM_EXCEPT_OVERFLOW 0x0008 +#define _MM_EXCEPT_UNDERFLOW 0x0010 +#define _MM_EXCEPT_INEXACT 0x0020 + +#define _MM_MASK_MASK 0x1f80 +#define _MM_MASK_INVALID 0x0080 +#define _MM_MASK_DENORM 0x0100 +#define _MM_MASK_DIV_ZERO 0x0200 +#define _MM_MASK_OVERFLOW 0x0400 +#define _MM_MASK_UNDERFLOW 0x0800 +#define _MM_MASK_INEXACT 0x1000 + +#define _MM_ROUND_MASK 0x6000 +#define _MM_ROUND_NEAREST 0x0000 +#define _MM_ROUND_DOWN 0x2000 +#define _MM_ROUND_UP 0x4000 +#define _MM_ROUND_TOWARD_ZERO 0x6000 + +#define _MM_FLUSH_ZERO_MASK 0x8000 +#define _MM_FLUSH_ZERO_ON 0x8000 +#define _MM_FLUSH_ZERO_OFF 0x0000 + +/* Create an undefined vector. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_undefined_ps (void) +{ + __m128 __Y = __Y; + return __Y; +} + +/* Create a vector of zeros. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_setzero_ps (void) +{ + return __extension__ (__m128){ 0.0f, 0.0f, 0.0f, 0.0f }; +} + +/* Perform the respective operation on the lower SPFP (single-precision + floating-point) values of A and B; the upper three SPFP values are + passed through from A. */ + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_add_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_addss ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sub_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_subss ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mul_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_mulss ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_div_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_divss ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sqrt_ss (__m128 __A) +{ + return (__m128) __builtin_ia32_sqrtss ((__v4sf)__A); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rcp_ss (__m128 __A) +{ + return (__m128) __builtin_ia32_rcpss ((__v4sf)__A); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rsqrt_ss (__m128 __A) +{ + return (__m128) __builtin_ia32_rsqrtss ((__v4sf)__A); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_min_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_minss ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_max_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_maxss ((__v4sf)__A, (__v4sf)__B); +} + +/* Perform the respective operation on the four SPFP values in A and B. */ + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_add_ps (__m128 __A, __m128 __B) +{ + return (__m128) ((__v4sf)__A + (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sub_ps (__m128 __A, __m128 __B) +{ + return (__m128) ((__v4sf)__A - (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mul_ps (__m128 __A, __m128 __B) +{ + return (__m128) ((__v4sf)__A * (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_div_ps (__m128 __A, __m128 __B) +{ + return (__m128) ((__v4sf)__A / (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sqrt_ps (__m128 __A) +{ + return (__m128) __builtin_ia32_sqrtps ((__v4sf)__A); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rcp_ps (__m128 __A) +{ + return (__m128) __builtin_ia32_rcpps ((__v4sf)__A); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rsqrt_ps (__m128 __A) +{ + return (__m128) __builtin_ia32_rsqrtps ((__v4sf)__A); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_min_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_minps ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_max_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_maxps ((__v4sf)__A, (__v4sf)__B); +} + +/* Perform logical bit-wise operations on 128-bit values. */ + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_and_ps (__m128 __A, __m128 __B) +{ + return __builtin_ia32_andps (__A, __B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_andnot_ps (__m128 __A, __m128 __B) +{ + return __builtin_ia32_andnps (__A, __B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_or_ps (__m128 __A, __m128 __B) +{ + return __builtin_ia32_orps (__A, __B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_xor_ps (__m128 __A, __m128 __B) +{ + return __builtin_ia32_xorps (__A, __B); +} + +/* Perform a comparison on the lower SPFP values of A and B. If the + comparison is true, place a mask of all ones in the result, otherwise a + mask of zeros. The upper three SPFP values are passed through from A. */ + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpeqss ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmplt_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpltss ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmple_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpless ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_movss ((__v4sf) __A, + (__v4sf) + __builtin_ia32_cmpltss ((__v4sf) __B, + (__v4sf) + __A)); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpge_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_movss ((__v4sf) __A, + (__v4sf) + __builtin_ia32_cmpless ((__v4sf) __B, + (__v4sf) + __A)); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpneq_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpneqss ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpnlt_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpnltss ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpnle_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpnless ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpngt_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_movss ((__v4sf) __A, + (__v4sf) + __builtin_ia32_cmpnltss ((__v4sf) __B, + (__v4sf) + __A)); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpnge_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_movss ((__v4sf) __A, + (__v4sf) + __builtin_ia32_cmpnless ((__v4sf) __B, + (__v4sf) + __A)); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpord_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpordss ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpunord_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpunordss ((__v4sf)__A, (__v4sf)__B); +} + +/* Perform a comparison on the four SPFP values of A and B. For each + element, if the comparison is true, place a mask of all ones in the + result, otherwise a mask of zeros. */ + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpeqps ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmplt_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpltps ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmple_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpleps ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpgtps ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpge_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpgeps ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpneq_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpneqps ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpnlt_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpnltps ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpnle_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpnleps ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpngt_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpngtps ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpnge_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpngeps ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpord_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpordps ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpunord_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpunordps ((__v4sf)__A, (__v4sf)__B); +} + +/* Compare the lower SPFP values of A and B and return 1 if true + and 0 if false. */ + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comieq_ss (__m128 __A, __m128 __B) +{ + return __builtin_ia32_comieq ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comilt_ss (__m128 __A, __m128 __B) +{ + return __builtin_ia32_comilt ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comile_ss (__m128 __A, __m128 __B) +{ + return __builtin_ia32_comile ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comigt_ss (__m128 __A, __m128 __B) +{ + return __builtin_ia32_comigt ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comige_ss (__m128 __A, __m128 __B) +{ + return __builtin_ia32_comige ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comineq_ss (__m128 __A, __m128 __B) +{ + return __builtin_ia32_comineq ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_ucomieq_ss (__m128 __A, __m128 __B) +{ + return __builtin_ia32_ucomieq ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_ucomilt_ss (__m128 __A, __m128 __B) +{ + return __builtin_ia32_ucomilt ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_ucomile_ss (__m128 __A, __m128 __B) +{ + return __builtin_ia32_ucomile ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_ucomigt_ss (__m128 __A, __m128 __B) +{ + return __builtin_ia32_ucomigt ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_ucomige_ss (__m128 __A, __m128 __B) +{ + return __builtin_ia32_ucomige ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_ucomineq_ss (__m128 __A, __m128 __B) +{ + return __builtin_ia32_ucomineq ((__v4sf)__A, (__v4sf)__B); +} + +/* Convert the lower SPFP value to a 32-bit integer according to the current + rounding mode. */ +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtss_si32 (__m128 __A) +{ + return __builtin_ia32_cvtss2si ((__v4sf) __A); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_ss2si (__m128 __A) +{ + return _mm_cvtss_si32 (__A); +} + +#ifdef __x86_64__ +/* Convert the lower SPFP value to a 32-bit integer according to the + current rounding mode. */ + +/* Intel intrinsic. */ +extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtss_si64 (__m128 __A) +{ + return __builtin_ia32_cvtss2si64 ((__v4sf) __A); +} + +/* Microsoft intrinsic. */ +extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtss_si64x (__m128 __A) +{ + return __builtin_ia32_cvtss2si64 ((__v4sf) __A); +} +#endif + +/* Convert the two lower SPFP values to 32-bit integers according to the + current rounding mode. Return the integers in packed form. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtps_pi32 (__m128 __A) +{ + return (__m64) __builtin_ia32_cvtps2pi ((__v4sf) __A); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_ps2pi (__m128 __A) +{ + return _mm_cvtps_pi32 (__A); +} + +/* Truncate the lower SPFP value to a 32-bit integer. */ +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttss_si32 (__m128 __A) +{ + return __builtin_ia32_cvttss2si ((__v4sf) __A); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtt_ss2si (__m128 __A) +{ + return _mm_cvttss_si32 (__A); +} + +#ifdef __x86_64__ +/* Truncate the lower SPFP value to a 32-bit integer. */ + +/* Intel intrinsic. */ +extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttss_si64 (__m128 __A) +{ + return __builtin_ia32_cvttss2si64 ((__v4sf) __A); +} + +/* Microsoft intrinsic. */ +extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttss_si64x (__m128 __A) +{ + return __builtin_ia32_cvttss2si64 ((__v4sf) __A); +} +#endif + +/* Truncate the two lower SPFP values to 32-bit integers. Return the + integers in packed form. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttps_pi32 (__m128 __A) +{ + return (__m64) __builtin_ia32_cvttps2pi ((__v4sf) __A); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtt_ps2pi (__m128 __A) +{ + return _mm_cvttps_pi32 (__A); +} + +/* Convert B to a SPFP value and insert it as element zero in A. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi32_ss (__m128 __A, int __B) +{ + return (__m128) __builtin_ia32_cvtsi2ss ((__v4sf) __A, __B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_si2ss (__m128 __A, int __B) +{ + return _mm_cvtsi32_ss (__A, __B); +} + +#ifdef __x86_64__ +/* Convert B to a SPFP value and insert it as element zero in A. */ + +/* Intel intrinsic. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi64_ss (__m128 __A, long long __B) +{ + return (__m128) __builtin_ia32_cvtsi642ss ((__v4sf) __A, __B); +} + +/* Microsoft intrinsic. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi64x_ss (__m128 __A, long long __B) +{ + return (__m128) __builtin_ia32_cvtsi642ss ((__v4sf) __A, __B); +} +#endif + +/* Convert the two 32-bit values in B to SPFP form and insert them + as the two lower elements in A. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtpi32_ps (__m128 __A, __m64 __B) +{ + return (__m128) __builtin_ia32_cvtpi2ps ((__v4sf) __A, (__v2si)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_pi2ps (__m128 __A, __m64 __B) +{ + return _mm_cvtpi32_ps (__A, __B); +} + +/* Convert the four signed 16-bit values in A to SPFP form. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtpi16_ps (__m64 __A) +{ + __v4hi __sign; + __v2si __hisi, __losi; + __v4sf __zero, __ra, __rb; + + /* This comparison against zero gives us a mask that can be used to + fill in the missing sign bits in the unpack operations below, so + that we get signed values after unpacking. */ + __sign = __builtin_ia32_pcmpgtw ((__v4hi)0LL, (__v4hi)__A); + + /* Convert the four words to doublewords. */ + __losi = (__v2si) __builtin_ia32_punpcklwd ((__v4hi)__A, __sign); + __hisi = (__v2si) __builtin_ia32_punpckhwd ((__v4hi)__A, __sign); + + /* Convert the doublewords to floating point two at a time. */ + __zero = (__v4sf) _mm_setzero_ps (); + __ra = __builtin_ia32_cvtpi2ps (__zero, __losi); + __rb = __builtin_ia32_cvtpi2ps (__ra, __hisi); + + return (__m128) __builtin_ia32_movlhps (__ra, __rb); +} + +/* Convert the four unsigned 16-bit values in A to SPFP form. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtpu16_ps (__m64 __A) +{ + __v2si __hisi, __losi; + __v4sf __zero, __ra, __rb; + + /* Convert the four words to doublewords. */ + __losi = (__v2si) __builtin_ia32_punpcklwd ((__v4hi)__A, (__v4hi)0LL); + __hisi = (__v2si) __builtin_ia32_punpckhwd ((__v4hi)__A, (__v4hi)0LL); + + /* Convert the doublewords to floating point two at a time. */ + __zero = (__v4sf) _mm_setzero_ps (); + __ra = __builtin_ia32_cvtpi2ps (__zero, __losi); + __rb = __builtin_ia32_cvtpi2ps (__ra, __hisi); + + return (__m128) __builtin_ia32_movlhps (__ra, __rb); +} + +/* Convert the low four signed 8-bit values in A to SPFP form. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtpi8_ps (__m64 __A) +{ + __v8qi __sign; + + /* This comparison against zero gives us a mask that can be used to + fill in the missing sign bits in the unpack operations below, so + that we get signed values after unpacking. */ + __sign = __builtin_ia32_pcmpgtb ((__v8qi)0LL, (__v8qi)__A); + + /* Convert the four low bytes to words. */ + __A = (__m64) __builtin_ia32_punpcklbw ((__v8qi)__A, __sign); + + return _mm_cvtpi16_ps(__A); +} + +/* Convert the low four unsigned 8-bit values in A to SPFP form. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtpu8_ps(__m64 __A) +{ + __A = (__m64) __builtin_ia32_punpcklbw ((__v8qi)__A, (__v8qi)0LL); + return _mm_cvtpu16_ps(__A); +} + +/* Convert the four signed 32-bit values in A and B to SPFP form. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtpi32x2_ps(__m64 __A, __m64 __B) +{ + __v4sf __zero = (__v4sf) _mm_setzero_ps (); + __v4sf __sfa = __builtin_ia32_cvtpi2ps (__zero, (__v2si)__A); + __v4sf __sfb = __builtin_ia32_cvtpi2ps (__sfa, (__v2si)__B); + return (__m128) __builtin_ia32_movlhps (__sfa, __sfb); +} + +/* Convert the four SPFP values in A to four signed 16-bit integers. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtps_pi16(__m128 __A) +{ + __v4sf __hisf = (__v4sf)__A; + __v4sf __losf = __builtin_ia32_movhlps (__hisf, __hisf); + __v2si __hisi = __builtin_ia32_cvtps2pi (__hisf); + __v2si __losi = __builtin_ia32_cvtps2pi (__losf); + return (__m64) __builtin_ia32_packssdw (__hisi, __losi); +} + +/* Convert the four SPFP values in A to four signed 8-bit integers. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtps_pi8(__m128 __A) +{ + __v4hi __tmp = (__v4hi) _mm_cvtps_pi16 (__A); + return (__m64) __builtin_ia32_packsswb (__tmp, (__v4hi)0LL); +} + +/* Selects four specific SPFP values from A and B based on MASK. */ +#ifdef __OPTIMIZE__ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_shuffle_ps (__m128 __A, __m128 __B, int const __mask) +{ + return (__m128) __builtin_ia32_shufps ((__v4sf)__A, (__v4sf)__B, __mask); +} +#else +#define _mm_shuffle_ps(A, B, MASK) \ + ((__m128) __builtin_ia32_shufps ((__v4sf)(__m128)(A), \ + (__v4sf)(__m128)(B), (int)(MASK))) +#endif + +/* Selects and interleaves the upper two SPFP values from A and B. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpackhi_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_unpckhps ((__v4sf)__A, (__v4sf)__B); +} + +/* Selects and interleaves the lower two SPFP values from A and B. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpacklo_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_unpcklps ((__v4sf)__A, (__v4sf)__B); +} + +/* Sets the upper two SPFP values with 64-bits of data loaded from P; + the lower two values are passed through from A. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_loadh_pi (__m128 __A, __m64 const *__P) +{ + return (__m128) __builtin_ia32_loadhps ((__v4sf)__A, (const __v2sf *)__P); +} + +/* Stores the upper two SPFP values of A into P. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_storeh_pi (__m64 *__P, __m128 __A) +{ + __builtin_ia32_storehps ((__v2sf *)__P, (__v4sf)__A); +} + +/* Moves the upper two values of B into the lower two values of A. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movehl_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_movhlps ((__v4sf)__A, (__v4sf)__B); +} + +/* Moves the lower two values of B into the upper two values of A. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movelh_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_movlhps ((__v4sf)__A, (__v4sf)__B); +} + +/* Sets the lower two SPFP values with 64-bits of data loaded from P; + the upper two values are passed through from A. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_loadl_pi (__m128 __A, __m64 const *__P) +{ + return (__m128) __builtin_ia32_loadlps ((__v4sf)__A, (const __v2sf *)__P); +} + +/* Stores the lower two SPFP values of A into P. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_storel_pi (__m64 *__P, __m128 __A) +{ + __builtin_ia32_storelps ((__v2sf *)__P, (__v4sf)__A); +} + +/* Creates a 4-bit mask from the most significant bits of the SPFP values. */ +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movemask_ps (__m128 __A) +{ + return __builtin_ia32_movmskps ((__v4sf)__A); +} + +/* Return the contents of the control register. */ +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_getcsr (void) +{ + return __builtin_ia32_stmxcsr (); +} + +/* Read exception bits from the control register. */ +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_MM_GET_EXCEPTION_STATE (void) +{ + return _mm_getcsr() & _MM_EXCEPT_MASK; +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_MM_GET_EXCEPTION_MASK (void) +{ + return _mm_getcsr() & _MM_MASK_MASK; +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_MM_GET_ROUNDING_MODE (void) +{ + return _mm_getcsr() & _MM_ROUND_MASK; +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_MM_GET_FLUSH_ZERO_MODE (void) +{ + return _mm_getcsr() & _MM_FLUSH_ZERO_MASK; +} + +/* Set the control register to I. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_setcsr (unsigned int __I) +{ + __builtin_ia32_ldmxcsr (__I); +} + +/* Set exception bits in the control register. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_MM_SET_EXCEPTION_STATE(unsigned int __mask) +{ + _mm_setcsr((_mm_getcsr() & ~_MM_EXCEPT_MASK) | __mask); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_MM_SET_EXCEPTION_MASK (unsigned int __mask) +{ + _mm_setcsr((_mm_getcsr() & ~_MM_MASK_MASK) | __mask); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_MM_SET_ROUNDING_MODE (unsigned int __mode) +{ + _mm_setcsr((_mm_getcsr() & ~_MM_ROUND_MASK) | __mode); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_MM_SET_FLUSH_ZERO_MODE (unsigned int __mode) +{ + _mm_setcsr((_mm_getcsr() & ~_MM_FLUSH_ZERO_MASK) | __mode); +} + +/* Create a vector with element 0 as F and the rest zero. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set_ss (float __F) +{ + return __extension__ (__m128)(__v4sf){ __F, 0.0f, 0.0f, 0.0f }; +} + +/* Create a vector with all four elements equal to F. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set1_ps (float __F) +{ + return __extension__ (__m128)(__v4sf){ __F, __F, __F, __F }; +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set_ps1 (float __F) +{ + return _mm_set1_ps (__F); +} + +/* Create a vector with element 0 as *P and the rest zero. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_load_ss (float const *__P) +{ + return _mm_set_ss (*__P); +} + +/* Create a vector with all four elements equal to *P. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_load1_ps (float const *__P) +{ + return _mm_set1_ps (*__P); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_load_ps1 (float const *__P) +{ + return _mm_load1_ps (__P); +} + +/* Load four SPFP values from P. The address must be 16-byte aligned. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_load_ps (float const *__P) +{ + return *(__m128 *)__P; +} + +/* Load four SPFP values from P. The address need not be 16-byte aligned. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_loadu_ps (float const *__P) +{ + return *(__m128_u *)__P; +} + +/* Load four SPFP values in reverse order. The address must be aligned. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_loadr_ps (float const *__P) +{ + __v4sf __tmp = *(__v4sf *)__P; + return (__m128) __builtin_ia32_shufps (__tmp, __tmp, _MM_SHUFFLE (0,1,2,3)); +} + +/* Create the vector [Z Y X W]. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set_ps (const float __Z, const float __Y, const float __X, const float __W) +{ + return __extension__ (__m128)(__v4sf){ __W, __X, __Y, __Z }; +} + +/* Create the vector [W X Y Z]. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_setr_ps (float __Z, float __Y, float __X, float __W) +{ + return __extension__ (__m128)(__v4sf){ __Z, __Y, __X, __W }; +} + +/* Stores the lower SPFP value. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_store_ss (float *__P, __m128 __A) +{ + *__P = ((__v4sf)__A)[0]; +} + +extern __inline float __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtss_f32 (__m128 __A) +{ + return ((__v4sf)__A)[0]; +} + +/* Store four SPFP values. The address must be 16-byte aligned. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_store_ps (float *__P, __m128 __A) +{ + *(__m128 *)__P = __A; +} + +/* Store four SPFP values. The address need not be 16-byte aligned. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_storeu_ps (float *__P, __m128 __A) +{ + *(__m128_u *)__P = __A; +} + +/* Store the lower SPFP value across four words. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_store1_ps (float *__P, __m128 __A) +{ + __v4sf __va = (__v4sf)__A; + __v4sf __tmp = __builtin_ia32_shufps (__va, __va, _MM_SHUFFLE (0,0,0,0)); + _mm_storeu_ps (__P, __tmp); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_store_ps1 (float *__P, __m128 __A) +{ + _mm_store1_ps (__P, __A); +} + +/* Store four SPFP values in reverse order. The address must be aligned. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_storer_ps (float *__P, __m128 __A) +{ + __v4sf __va = (__v4sf)__A; + __v4sf __tmp = __builtin_ia32_shufps (__va, __va, _MM_SHUFFLE (0,1,2,3)); + _mm_store_ps (__P, __tmp); +} + +/* Sets the low SPFP value of A from the low value of B. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_move_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_movss ((__v4sf)__A, (__v4sf)__B); +} + +/* Extracts one of the four words of A. The selector N must be immediate. */ +#ifdef __OPTIMIZE__ +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_extract_pi16 (__m64 const __A, int const __N) +{ + return __builtin_ia32_vec_ext_v4hi ((__v4hi)__A, __N); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pextrw (__m64 const __A, int const __N) +{ + return _mm_extract_pi16 (__A, __N); +} +#else +#define _mm_extract_pi16(A, N) \ + ((int) __builtin_ia32_vec_ext_v4hi ((__v4hi)(__m64)(A), (int)(N))) + +#define _m_pextrw(A, N) _mm_extract_pi16(A, N) +#endif + +/* Inserts word D into one of four words of A. The selector N must be + immediate. */ +#ifdef __OPTIMIZE__ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_insert_pi16 (__m64 const __A, int const __D, int const __N) +{ + return (__m64) __builtin_ia32_vec_set_v4hi ((__v4hi)__A, __D, __N); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pinsrw (__m64 const __A, int const __D, int const __N) +{ + return _mm_insert_pi16 (__A, __D, __N); +} +#else +#define _mm_insert_pi16(A, D, N) \ + ((__m64) __builtin_ia32_vec_set_v4hi ((__v4hi)(__m64)(A), \ + (int)(D), (int)(N))) + +#define _m_pinsrw(A, D, N) _mm_insert_pi16(A, D, N) +#endif + +/* Compute the element-wise maximum of signed 16-bit values. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_max_pi16 (__m64 __A, __m64 __B) +{ + return (__m64) __builtin_ia32_pmaxsw ((__v4hi)__A, (__v4hi)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pmaxsw (__m64 __A, __m64 __B) +{ + return _mm_max_pi16 (__A, __B); +} + +/* Compute the element-wise maximum of unsigned 8-bit values. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_max_pu8 (__m64 __A, __m64 __B) +{ + return (__m64) __builtin_ia32_pmaxub ((__v8qi)__A, (__v8qi)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pmaxub (__m64 __A, __m64 __B) +{ + return _mm_max_pu8 (__A, __B); +} + +/* Compute the element-wise minimum of signed 16-bit values. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_min_pi16 (__m64 __A, __m64 __B) +{ + return (__m64) __builtin_ia32_pminsw ((__v4hi)__A, (__v4hi)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pminsw (__m64 __A, __m64 __B) +{ + return _mm_min_pi16 (__A, __B); +} + +/* Compute the element-wise minimum of unsigned 8-bit values. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_min_pu8 (__m64 __A, __m64 __B) +{ + return (__m64) __builtin_ia32_pminub ((__v8qi)__A, (__v8qi)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pminub (__m64 __A, __m64 __B) +{ + return _mm_min_pu8 (__A, __B); +} + +/* Create an 8-bit mask of the signs of 8-bit values. */ +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movemask_pi8 (__m64 __A) +{ + return __builtin_ia32_pmovmskb ((__v8qi)__A); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pmovmskb (__m64 __A) +{ + return _mm_movemask_pi8 (__A); +} + +/* Multiply four unsigned 16-bit values in A by four unsigned 16-bit values + in B and produce the high 16 bits of the 32-bit results. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mulhi_pu16 (__m64 __A, __m64 __B) +{ + return (__m64) __builtin_ia32_pmulhuw ((__v4hi)__A, (__v4hi)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pmulhuw (__m64 __A, __m64 __B) +{ + return _mm_mulhi_pu16 (__A, __B); +} + +/* Return a combination of the four 16-bit values in A. The selector + must be an immediate. */ +#ifdef __OPTIMIZE__ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_shuffle_pi16 (__m64 __A, int const __N) +{ + return (__m64) __builtin_ia32_pshufw ((__v4hi)__A, __N); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pshufw (__m64 __A, int const __N) +{ + return _mm_shuffle_pi16 (__A, __N); +} +#else +#define _mm_shuffle_pi16(A, N) \ + ((__m64) __builtin_ia32_pshufw ((__v4hi)(__m64)(A), (int)(N))) + +#define _m_pshufw(A, N) _mm_shuffle_pi16 (A, N) +#endif + +/* Conditionally store byte elements of A into P. The high bit of each + byte in the selector N determines whether the corresponding byte from + A is stored. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskmove_si64 (__m64 __A, __m64 __N, char *__P) +{ + __builtin_ia32_maskmovq ((__v8qi)__A, (__v8qi)__N, __P); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_maskmovq (__m64 __A, __m64 __N, char *__P) +{ + _mm_maskmove_si64 (__A, __N, __P); +} + +/* Compute the rounded averages of the unsigned 8-bit values in A and B. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_avg_pu8 (__m64 __A, __m64 __B) +{ + return (__m64) __builtin_ia32_pavgb ((__v8qi)__A, (__v8qi)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pavgb (__m64 __A, __m64 __B) +{ + return _mm_avg_pu8 (__A, __B); +} + +/* Compute the rounded averages of the unsigned 16-bit values in A and B. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_avg_pu16 (__m64 __A, __m64 __B) +{ + return (__m64) __builtin_ia32_pavgw ((__v4hi)__A, (__v4hi)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pavgw (__m64 __A, __m64 __B) +{ + return _mm_avg_pu16 (__A, __B); +} + +/* Compute the sum of the absolute differences of the unsigned 8-bit + values in A and B. Return the value in the lower 16-bit word; the + upper words are cleared. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sad_pu8 (__m64 __A, __m64 __B) +{ + return (__m64) __builtin_ia32_psadbw ((__v8qi)__A, (__v8qi)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psadbw (__m64 __A, __m64 __B) +{ + return _mm_sad_pu8 (__A, __B); +} + +/* Stores the data in A to the address P without polluting the caches. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_stream_pi (__m64 *__P, __m64 __A) +{ + __builtin_ia32_movntq ((unsigned long long *)__P, (unsigned long long)__A); +} + +/* Likewise. The address must be 16-byte aligned. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_stream_ps (float *__P, __m128 __A) +{ + __builtin_ia32_movntps (__P, (__v4sf)__A); +} + +/* Guarantees that every preceding store is globally visible before + any subsequent store. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sfence (void) +{ + __builtin_ia32_sfence (); +} + +/* Transpose the 4x4 matrix composed of row[0-3]. */ +#define _MM_TRANSPOSE4_PS(row0, row1, row2, row3) \ +do { \ + __v4sf __r0 = (row0), __r1 = (row1), __r2 = (row2), __r3 = (row3); \ + __v4sf __t0 = __builtin_ia32_unpcklps (__r0, __r1); \ + __v4sf __t1 = __builtin_ia32_unpcklps (__r2, __r3); \ + __v4sf __t2 = __builtin_ia32_unpckhps (__r0, __r1); \ + __v4sf __t3 = __builtin_ia32_unpckhps (__r2, __r3); \ + (row0) = __builtin_ia32_movlhps (__t0, __t1); \ + (row1) = __builtin_ia32_movhlps (__t1, __t0); \ + (row2) = __builtin_ia32_movlhps (__t2, __t3); \ + (row3) = __builtin_ia32_movhlps (__t3, __t2); \ +} while (0) + +/* For backward source compatibility. */ +# include + +#ifdef __DISABLE_SSE__ +#undef __DISABLE_SSE__ +#pragma GCC pop_options +#endif /* __DISABLE_SSE__ */ + +/* The execution of the next instruction is delayed by an implementation + specific amount of time. The instruction does not modify the + architectural state. This is after the pop_options pragma because + it does not require SSE support in the processor--the encoding is a + nop on processors that do not support it. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_pause (void) +{ + __builtin_ia32_pause (); +} + +#endif /* _XMMINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/xopintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/xopintrin.h new file mode 100644 index 0000000000..3a83e24932 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/xopintrin.h @@ -0,0 +1,844 @@ +/* Copyright (C) 2007-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _X86INTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _XOPMMINTRIN_H_INCLUDED +#define _XOPMMINTRIN_H_INCLUDED + +#include + +#ifndef __XOP__ +#pragma GCC push_options +#pragma GCC target("xop") +#define __DISABLE_XOP__ +#endif /* __XOP__ */ + +/* Integer multiply/add instructions. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maccs_epi16(__m128i __A, __m128i __B, __m128i __C) +{ + return (__m128i) __builtin_ia32_vpmacssww ((__v8hi)__A,(__v8hi)__B, (__v8hi)__C); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_macc_epi16(__m128i __A, __m128i __B, __m128i __C) +{ + return (__m128i) __builtin_ia32_vpmacsww ((__v8hi)__A, (__v8hi)__B, (__v8hi)__C); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maccsd_epi16(__m128i __A, __m128i __B, __m128i __C) +{ + return (__m128i) __builtin_ia32_vpmacsswd ((__v8hi)__A, (__v8hi)__B, (__v4si)__C); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maccd_epi16(__m128i __A, __m128i __B, __m128i __C) +{ + return (__m128i) __builtin_ia32_vpmacswd ((__v8hi)__A, (__v8hi)__B, (__v4si)__C); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maccs_epi32(__m128i __A, __m128i __B, __m128i __C) +{ + return (__m128i) __builtin_ia32_vpmacssdd ((__v4si)__A, (__v4si)__B, (__v4si)__C); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_macc_epi32(__m128i __A, __m128i __B, __m128i __C) +{ + return (__m128i) __builtin_ia32_vpmacsdd ((__v4si)__A, (__v4si)__B, (__v4si)__C); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maccslo_epi32(__m128i __A, __m128i __B, __m128i __C) +{ + return (__m128i) __builtin_ia32_vpmacssdql ((__v4si)__A, (__v4si)__B, (__v2di)__C); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_macclo_epi32(__m128i __A, __m128i __B, __m128i __C) +{ + return (__m128i) __builtin_ia32_vpmacsdql ((__v4si)__A, (__v4si)__B, (__v2di)__C); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maccshi_epi32(__m128i __A, __m128i __B, __m128i __C) +{ + return (__m128i) __builtin_ia32_vpmacssdqh ((__v4si)__A, (__v4si)__B, (__v2di)__C); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_macchi_epi32(__m128i __A, __m128i __B, __m128i __C) +{ + return (__m128i) __builtin_ia32_vpmacsdqh ((__v4si)__A, (__v4si)__B, (__v2di)__C); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maddsd_epi16(__m128i __A, __m128i __B, __m128i __C) +{ + return (__m128i) __builtin_ia32_vpmadcsswd ((__v8hi)__A,(__v8hi)__B,(__v4si)__C); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maddd_epi16(__m128i __A, __m128i __B, __m128i __C) +{ + return (__m128i) __builtin_ia32_vpmadcswd ((__v8hi)__A,(__v8hi)__B,(__v4si)__C); +} + +/* Packed Integer Horizontal Add and Subtract */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_haddw_epi8(__m128i __A) +{ + return (__m128i) __builtin_ia32_vphaddbw ((__v16qi)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_haddd_epi8(__m128i __A) +{ + return (__m128i) __builtin_ia32_vphaddbd ((__v16qi)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_haddq_epi8(__m128i __A) +{ + return (__m128i) __builtin_ia32_vphaddbq ((__v16qi)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_haddd_epi16(__m128i __A) +{ + return (__m128i) __builtin_ia32_vphaddwd ((__v8hi)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_haddq_epi16(__m128i __A) +{ + return (__m128i) __builtin_ia32_vphaddwq ((__v8hi)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_haddq_epi32(__m128i __A) +{ + return (__m128i) __builtin_ia32_vphadddq ((__v4si)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_haddw_epu8(__m128i __A) +{ + return (__m128i) __builtin_ia32_vphaddubw ((__v16qi)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_haddd_epu8(__m128i __A) +{ + return (__m128i) __builtin_ia32_vphaddubd ((__v16qi)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_haddq_epu8(__m128i __A) +{ + return (__m128i) __builtin_ia32_vphaddubq ((__v16qi)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_haddd_epu16(__m128i __A) +{ + return (__m128i) __builtin_ia32_vphadduwd ((__v8hi)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_haddq_epu16(__m128i __A) +{ + return (__m128i) __builtin_ia32_vphadduwq ((__v8hi)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_haddq_epu32(__m128i __A) +{ + return (__m128i) __builtin_ia32_vphaddudq ((__v4si)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hsubw_epi8(__m128i __A) +{ + return (__m128i) __builtin_ia32_vphsubbw ((__v16qi)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hsubd_epi16(__m128i __A) +{ + return (__m128i) __builtin_ia32_vphsubwd ((__v8hi)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hsubq_epi32(__m128i __A) +{ + return (__m128i) __builtin_ia32_vphsubdq ((__v4si)__A); +} + +/* Vector conditional move and permute */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmov_si128(__m128i __A, __m128i __B, __m128i __C) +{ + return (__m128i) __builtin_ia32_vpcmov (__A, __B, __C); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_perm_epi8(__m128i __A, __m128i __B, __m128i __C) +{ + return (__m128i) __builtin_ia32_vpperm ((__v16qi)__A, (__v16qi)__B, (__v16qi)__C); +} + +/* Packed Integer Rotates and Shifts + Rotates - Non-Immediate form */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rot_epi8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vprotb ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rot_epi16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vprotw ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rot_epi32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vprotd ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rot_epi64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vprotq ((__v2di)__A, (__v2di)__B); +} + +/* Rotates - Immediate form */ + +#ifdef __OPTIMIZE__ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_roti_epi8(__m128i __A, const int __B) +{ + return (__m128i) __builtin_ia32_vprotbi ((__v16qi)__A, __B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_roti_epi16(__m128i __A, const int __B) +{ + return (__m128i) __builtin_ia32_vprotwi ((__v8hi)__A, __B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_roti_epi32(__m128i __A, const int __B) +{ + return (__m128i) __builtin_ia32_vprotdi ((__v4si)__A, __B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_roti_epi64(__m128i __A, const int __B) +{ + return (__m128i) __builtin_ia32_vprotqi ((__v2di)__A, __B); +} +#else +#define _mm_roti_epi8(A, N) \ + ((__m128i) __builtin_ia32_vprotbi ((__v16qi)(__m128i)(A), (int)(N))) +#define _mm_roti_epi16(A, N) \ + ((__m128i) __builtin_ia32_vprotwi ((__v8hi)(__m128i)(A), (int)(N))) +#define _mm_roti_epi32(A, N) \ + ((__m128i) __builtin_ia32_vprotdi ((__v4si)(__m128i)(A), (int)(N))) +#define _mm_roti_epi64(A, N) \ + ((__m128i) __builtin_ia32_vprotqi ((__v2di)(__m128i)(A), (int)(N))) +#endif + +/* Shifts */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_shl_epi8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpshlb ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_shl_epi16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpshlw ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_shl_epi32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpshld ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_shl_epi64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpshlq ((__v2di)__A, (__v2di)__B); +} + + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sha_epi8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpshab ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sha_epi16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpshaw ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sha_epi32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpshad ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sha_epi64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpshaq ((__v2di)__A, (__v2di)__B); +} + +/* Compare and Predicate Generation + pcom (integer, unsigned bytes) */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comlt_epu8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomltub ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comle_epu8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomleub ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comgt_epu8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomgtub ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comge_epu8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomgeub ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comeq_epu8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomequb ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comneq_epu8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomnequb ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comfalse_epu8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomfalseub ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comtrue_epu8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomtrueub ((__v16qi)__A, (__v16qi)__B); +} + +/*pcom (integer, unsigned words) */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comlt_epu16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomltuw ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comle_epu16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomleuw ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comgt_epu16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomgtuw ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comge_epu16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomgeuw ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comeq_epu16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomequw ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comneq_epu16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomnequw ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comfalse_epu16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomfalseuw ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comtrue_epu16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomtrueuw ((__v8hi)__A, (__v8hi)__B); +} + +/*pcom (integer, unsigned double words) */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comlt_epu32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomltud ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comle_epu32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomleud ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comgt_epu32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomgtud ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comge_epu32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomgeud ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comeq_epu32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomequd ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comneq_epu32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomnequd ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comfalse_epu32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomfalseud ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comtrue_epu32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomtrueud ((__v4si)__A, (__v4si)__B); +} + +/*pcom (integer, unsigned quad words) */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comlt_epu64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomltuq ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comle_epu64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomleuq ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comgt_epu64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomgtuq ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comge_epu64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomgeuq ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comeq_epu64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomequq ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comneq_epu64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomnequq ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comfalse_epu64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomfalseuq ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comtrue_epu64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomtrueuq ((__v2di)__A, (__v2di)__B); +} + +/*pcom (integer, signed bytes) */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comlt_epi8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomltb ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comle_epi8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomleb ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comgt_epi8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomgtb ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comge_epi8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomgeb ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comeq_epi8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomeqb ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comneq_epi8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomneqb ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comfalse_epi8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomfalseb ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comtrue_epi8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomtrueb ((__v16qi)__A, (__v16qi)__B); +} + +/*pcom (integer, signed words) */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comlt_epi16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomltw ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comle_epi16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomlew ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comgt_epi16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomgtw ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comge_epi16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomgew ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comeq_epi16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomeqw ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comneq_epi16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomneqw ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comfalse_epi16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomfalsew ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comtrue_epi16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomtruew ((__v8hi)__A, (__v8hi)__B); +} + +/*pcom (integer, signed double words) */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comlt_epi32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomltd ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comle_epi32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomled ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comgt_epi32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomgtd ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comge_epi32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomged ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comeq_epi32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomeqd ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comneq_epi32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomneqd ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comfalse_epi32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomfalsed ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comtrue_epi32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomtrued ((__v4si)__A, (__v4si)__B); +} + +/*pcom (integer, signed quad words) */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comlt_epi64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomltq ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comle_epi64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomleq ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comgt_epi64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomgtq ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comge_epi64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomgeq ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comeq_epi64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomeqq ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comneq_epi64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomneqq ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comfalse_epi64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomfalseq ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comtrue_epi64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomtrueq ((__v2di)__A, (__v2di)__B); +} + +/* FRCZ */ + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_frcz_ps (__m128 __A) +{ + return (__m128) __builtin_ia32_vfrczps ((__v4sf)__A); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_frcz_pd (__m128d __A) +{ + return (__m128d) __builtin_ia32_vfrczpd ((__v2df)__A); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_frcz_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_movss ((__v4sf)__A, + (__v4sf) + __builtin_ia32_vfrczss ((__v4sf)__B)); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_frcz_sd (__m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_movsd ((__v2df)__A, + (__v2df) + __builtin_ia32_vfrczsd ((__v2df)__B)); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_frcz_ps (__m256 __A) +{ + return (__m256) __builtin_ia32_vfrczps256 ((__v8sf)__A); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_frcz_pd (__m256d __A) +{ + return (__m256d) __builtin_ia32_vfrczpd256 ((__v4df)__A); +} + +/* PERMIL2 */ + +#ifdef __OPTIMIZE__ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permute2_pd (__m128d __X, __m128d __Y, __m128i __C, const int __I) +{ + return (__m128d) __builtin_ia32_vpermil2pd ((__v2df)__X, + (__v2df)__Y, + (__v2di)__C, + __I); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permute2_pd (__m256d __X, __m256d __Y, __m256i __C, const int __I) +{ + return (__m256d) __builtin_ia32_vpermil2pd256 ((__v4df)__X, + (__v4df)__Y, + (__v4di)__C, + __I); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permute2_ps (__m128 __X, __m128 __Y, __m128i __C, const int __I) +{ + return (__m128) __builtin_ia32_vpermil2ps ((__v4sf)__X, + (__v4sf)__Y, + (__v4si)__C, + __I); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permute2_ps (__m256 __X, __m256 __Y, __m256i __C, const int __I) +{ + return (__m256) __builtin_ia32_vpermil2ps256 ((__v8sf)__X, + (__v8sf)__Y, + (__v8si)__C, + __I); +} +#else +#define _mm_permute2_pd(X, Y, C, I) \ + ((__m128d) __builtin_ia32_vpermil2pd ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), \ + (__v2di)(__m128d)(C), \ + (int)(I))) + +#define _mm256_permute2_pd(X, Y, C, I) \ + ((__m256d) __builtin_ia32_vpermil2pd256 ((__v4df)(__m256d)(X), \ + (__v4df)(__m256d)(Y), \ + (__v4di)(__m256d)(C), \ + (int)(I))) + +#define _mm_permute2_ps(X, Y, C, I) \ + ((__m128) __builtin_ia32_vpermil2ps ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), \ + (__v4si)(__m128)(C), \ + (int)(I))) + +#define _mm256_permute2_ps(X, Y, C, I) \ + ((__m256) __builtin_ia32_vpermil2ps256 ((__v8sf)(__m256)(X), \ + (__v8sf)(__m256)(Y), \ + (__v8si)(__m256)(C), \ + (int)(I))) +#endif /* __OPTIMIZE__ */ + +#ifdef __DISABLE_XOP__ +#undef __DISABLE_XOP__ +#pragma GCC pop_options +#endif /* __DISABLE_XOP__ */ + +#endif /* _XOPMMINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/xsavecintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/xsavecintrin.h new file mode 100644 index 0000000000..a510a7e8e6 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/xsavecintrin.h @@ -0,0 +1,58 @@ +/* Copyright (C) 2014-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#if !defined _X86INTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _XSAVECINTRIN_H_INCLUDED +#define _XSAVECINTRIN_H_INCLUDED + +#ifndef __XSAVEC__ +#pragma GCC push_options +#pragma GCC target("xsavec") +#define __DISABLE_XSAVEC__ +#endif /* __XSAVEC__ */ + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_xsavec (void *__P, long long __M) +{ + __builtin_ia32_xsavec (__P, __M); +} + +#ifdef __x86_64__ +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_xsavec64 (void *__P, long long __M) +{ + __builtin_ia32_xsavec64 (__P, __M); +} +#endif + +#ifdef __DISABLE_XSAVEC__ +#undef __DISABLE_XSAVEC__ +#pragma GCC pop_options +#endif /* __DISABLE_XSAVEC__ */ + +#endif /* _XSAVECINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/xsaveintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/xsaveintrin.h new file mode 100644 index 0000000000..c136d30aa6 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/xsaveintrin.h @@ -0,0 +1,72 @@ +/* Copyright (C) 2012-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +/* #if !defined _X86INTRIN_H_INCLUDED && !defined _IMMINTRIN_H_INCLUDED */ +/* # error "Never use directly; include instead." */ +/* #endif */ + +#ifndef _XSAVEINTRIN_H_INCLUDED +#define _XSAVEINTRIN_H_INCLUDED + +#ifndef __XSAVE__ +#pragma GCC push_options +#pragma GCC target("xsave") +#define __DISABLE_XSAVE__ +#endif /* __XSAVE__ */ + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_xsave (void *__P, long long __M) +{ + __builtin_ia32_xsave (__P, __M); +} + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_xrstor (void *__P, long long __M) +{ + __builtin_ia32_xrstor (__P, __M); +} + +#ifdef __x86_64__ +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_xsave64 (void *__P, long long __M) +{ + __builtin_ia32_xsave64 (__P, __M); +} + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_xrstor64 (void *__P, long long __M) +{ + __builtin_ia32_xrstor64 (__P, __M); +} +#endif + +#ifdef __DISABLE_XSAVE__ +#undef __DISABLE_XSAVE__ +#pragma GCC pop_options +#endif /* __DISABLE_XSAVE__ */ + +#endif /* _XSAVEINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/xsaveoptintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/xsaveoptintrin.h new file mode 100644 index 0000000000..294fcee43a --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/xsaveoptintrin.h @@ -0,0 +1,58 @@ +/* Copyright (C) 2012-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +/* #if !defined _X86INTRIN_H_INCLUDED && !defined _IMMINTRIN_H_INCLUDED */ +/* # error "Never use directly; include instead." */ +/* #endif */ + +#ifndef _XSAVEOPTINTRIN_H_INCLUDED +#define _XSAVEOPTINTRIN_H_INCLUDED + +#ifndef __XSAVEOPT__ +#pragma GCC push_options +#pragma GCC target("xsaveopt") +#define __DISABLE_XSAVEOPT__ +#endif /* __XSAVEOPT__ */ + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_xsaveopt (void *__P, long long __M) +{ + __builtin_ia32_xsaveopt (__P, __M); +} + +#ifdef __x86_64__ +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_xsaveopt64 (void *__P, long long __M) +{ + __builtin_ia32_xsaveopt64 (__P, __M); +} +#endif + +#ifdef __DISABLE_XSAVEOPT__ +#undef __DISABLE_XSAVEOPT__ +#pragma GCC pop_options +#endif /* __DISABLE_XSAVEOPT__ */ + +#endif /* _XSAVEOPTINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/xsavesintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/xsavesintrin.h new file mode 100644 index 0000000000..b55a1834e2 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/xsavesintrin.h @@ -0,0 +1,72 @@ +/* Copyright (C) 2014-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#if !defined _X86INTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _XSAVESINTRIN_H_INCLUDED +#define _XSAVESINTRIN_H_INCLUDED + +#ifndef __XSAVES__ +#pragma GCC push_options +#pragma GCC target("xsaves") +#define __DISABLE_XSAVES__ +#endif /* __XSAVES__ */ + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_xsaves (void *__P, long long __M) +{ + __builtin_ia32_xsaves (__P, __M); +} + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_xrstors (void *__P, long long __M) +{ + __builtin_ia32_xrstors (__P, __M); +} + +#ifdef __x86_64__ +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_xrstors64 (void *__P, long long __M) +{ + __builtin_ia32_xrstors64 (__P, __M); +} + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_xsaves64 (void *__P, long long __M) +{ + __builtin_ia32_xsaves64 (__P, __M); +} +#endif + +#ifdef __DISABLE_XSAVES__ +#undef __DISABLE_XSAVES__ +#pragma GCC pop_options +#endif /* __DISABLE_XSAVES__ */ + +#endif /* _XSAVESINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/xtestintrin.h b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/xtestintrin.h new file mode 100644 index 0000000000..3c7425f868 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/include/xtestintrin.h @@ -0,0 +1,51 @@ +/* Copyright (C) 2012-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _XTESTINTRIN_H_INCLUDED +#define _XTESTINTRIN_H_INCLUDED + +#ifndef __RTM__ +#pragma GCC push_options +#pragma GCC target("rtm") +#define __DISABLE_RTM__ +#endif /* __RTM__ */ + +/* Return non-zero if the instruction executes inside an RTM or HLE code + region. Return zero otherwise. */ +extern __inline int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_xtest (void) +{ + return __builtin_ia32_xtest (); +} + +#ifdef __DISABLE_RTM__ +#undef __DISABLE_RTM__ +#pragma GCC pop_options +#endif /* __DISABLE_RTM__ */ + +#endif /* _XTESTINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z104_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z104_13_AVX512.c index a80f3fb280..6b140ae361 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z104_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z104_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z10_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z10_13_AVX512.c index b46f3f2e20..7f7e1087b5 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z10_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z10_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z112_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z112_13_AVX512.c index 14de356cc0..83fa2f21f9 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z112_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z112_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z11_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z11_13_AVX512.c index aeba3ddd1f..c8e62b97b7 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z11_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z11_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z120_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z120_13_AVX512.c index 71bc1db2bd..b9fd76c871 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z120_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z120_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z128_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z128_13_AVX512.c index 15edfb7647..2004322ffb 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z128_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z128_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z12_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z12_13_AVX512.c index 09610a25e7..65deb384f1 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z12_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z12_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z13_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z13_13_AVX512.c index 96e108ac27..7e58ff6a17 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z13_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z13_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z144_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z144_13_AVX512.c index 449becf031..4ea09fe992 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z144_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z144_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z14_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z14_13_AVX512.c index 0145645d66..34ddf44317 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z14_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z14_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z15_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z15_13_AVX512.c index c301777ab6..fb573992f8 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z15_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z15_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z160_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z160_13_AVX512.c index bd90abe1ed..dd92ec2676 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z160_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z160_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z16_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z16_13_AVX512.c index 4108cc350d..538104c629 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z16_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z16_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z176_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z176_13_AVX512.c index 2b8c34cfcd..d814e50b80 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z176_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z176_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z18_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z18_13_AVX512.c index 79fd7bb3fc..8926b2103e 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z18_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z18_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z192_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z192_13_AVX512.c index fa57e497b2..6395987931 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z192_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z192_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z208_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z208_13_AVX512.c index 6533824d04..8a221c9128 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z208_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z208_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z20_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z20_13_AVX512.c index b64ec809d7..a01cccf7ce 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z20_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z20_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z224_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z224_13_AVX512.c index 96bd8f328b..77f7a1bb6f 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z224_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z224_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z22_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z22_13_AVX512.c index 071aaa7acc..cdc2f559d9 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z22_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z22_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z240_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z240_13_AVX512.c index 53a9cdbdf4..f65c846db3 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z240_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z240_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z24_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z24_13_AVX512.c index a34658b5ec..e548ca3bbf 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z24_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z24_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z256_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z256_13_AVX512.c index c7a646abae..da32d87d67 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z256_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z256_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z26_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z26_13_AVX512.c index bd776e8af1..bfef1f9644 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z26_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z26_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z288_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z288_13_AVX512.c index 90553112b0..2d9560d8a5 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z288_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z288_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z28_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z28_13_AVX512.c index aca2ba32eb..dcd87ea8ce 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z28_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z28_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z2_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z2_13_AVX512.c index 92237b43f2..b7da8bc7de 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z2_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z2_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z30_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z30_13_AVX512.c index b08797b2c3..94e6f9834e 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z30_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z30_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z320_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z320_13_AVX512.c index 09f4b52d2f..21c02d68a4 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z320_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z320_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z32_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z32_13_AVX512.c index a09cfac995..15ba86b574 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z32_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z32_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z352_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z352_13_AVX512.c index d9e160a715..79387b3c66 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z352_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z352_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z36_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z36_13_AVX512.c index 25b58a5900..f25561e120 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z36_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z36_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z384_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z384_13_AVX512.c index 90743f4d94..a6cbeb7e92 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z384_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z384_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z3_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z3_13_AVX512.c index 68d316935c..daa6a9c2d8 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z3_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z3_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z40_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z40_13_AVX512.c index 39bd657832..2db9e6635a 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z40_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z40_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z44_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z44_13_AVX512.c index aa1e643497..3d1cfdf841 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z44_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z44_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z48_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z48_13_AVX512.c index 5638fe34d4..6c97d25bb4 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z48_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z48_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z4_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z4_13_AVX512.c index 268c1feda5..4744f6993c 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z4_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z4_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z52_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z52_13_AVX512.c index bbb23c68e1..8e2f726956 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z52_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z52_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z56_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z56_13_AVX512.c index 697578fb91..dfc5c4331b 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z56_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z56_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z5_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z5_13_AVX512.c index 9ecfe8253e..7428c76c24 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z5_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z5_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z60_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z60_13_AVX512.c index 7388cfc93c..2c4b54f2f4 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z60_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z60_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z64_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z64_13_AVX512.c index 344c581d07..f56cf3dced 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z64_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z64_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z6_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z6_13_AVX512.c index f36090f3c5..fb462ccff0 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z6_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z6_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z72_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z72_13_AVX512.c index 7cd843079a..07acbceec1 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z72_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z72_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z7_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z7_13_AVX512.c index b6b2217da5..e3ef69151a 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z7_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z7_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z80_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z80_13_AVX512.c index 4270e81f94..22498f909a 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z80_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z80_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z88_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z88_13_AVX512.c index e57da338bb..5776e20c83 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z88_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z88_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z8_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z8_13_AVX512.c index 0048bf497c..c81351f79e 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z8_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z8_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z96_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z96_13_AVX512.c index f04b440837..1d5df963b9 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z96_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z96_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z9_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z9_13_AVX512.c index 548ab5f8ab..24c4d7f51c 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z9_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG1_Z9_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z104_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z104_13_AVX512.c index 028fc8acd6..966334a68d 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z104_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z104_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z10_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z10_13_AVX512.c index 454c57bde2..1b3aead33f 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z10_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z10_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z112_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z112_13_AVX512.c index 471c209c13..b929b48d0e 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z112_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z112_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z11_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z11_13_AVX512.c index 7bdaecb163..83ec3eab87 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z11_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z11_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z120_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z120_13_AVX512.c index f27234e514..66b66d6c48 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z120_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z120_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z128_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z128_13_AVX512.c index ed82005362..4eef884767 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z128_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z128_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z12_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z12_13_AVX512.c index 4a6f4e84c7..5c8dbd18fd 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z12_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z12_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z13_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z13_13_AVX512.c index fddad47a3a..4b462304ba 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z13_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z13_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z144_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z144_13_AVX512.c index 53fc972914..0ba1baa27a 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z144_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z144_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z14_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z14_13_AVX512.c index c335d28ea0..000bb5bcb6 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z14_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z14_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z15_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z15_13_AVX512.c index f03c575141..89070e0962 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z15_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z15_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z160_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z160_13_AVX512.c index 6a4d58f191..2a1380845d 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z160_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z160_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z16_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z16_13_AVX512.c index bd10af445e..f142d25d1f 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z16_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z16_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z176_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z176_13_AVX512.c index 55e116b51f..b4ab3963ec 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z176_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z176_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z18_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z18_13_AVX512.c index 712ec87469..aec739240e 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z18_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z18_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z192_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z192_13_AVX512.c index eda5b6a463..501021f109 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z192_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z192_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z208_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z208_13_AVX512.c index 9a1081c951..835406c3fc 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z208_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z208_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z20_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z20_13_AVX512.c index 7a42b26ad1..df043f09da 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z20_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z20_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z224_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z224_13_AVX512.c index 1863faff1c..c88f4fd799 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z224_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z224_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z22_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z22_13_AVX512.c index fc8ec65f5b..3d8ad4087d 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z22_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z22_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z240_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z240_13_AVX512.c index 7c2b22ab84..e149354070 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z240_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z240_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z24_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z24_13_AVX512.c index a7e29917dc..87b84e20e3 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z24_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z24_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z256_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z256_13_AVX512.c index 8467ccc0ef..05625adeca 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z256_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z256_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z26_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z26_13_AVX512.c index dd9c501e23..8f1eef0d86 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z26_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z26_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z288_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z288_13_AVX512.c index 080ef69200..4d22f6cad3 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z288_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z288_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z28_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z28_13_AVX512.c index 04a81d12e9..0b0b19e37b 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z28_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z28_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z2_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z2_13_AVX512.c index 6fd089c473..e99c45d669 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z2_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z2_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z30_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z30_13_AVX512.c index 8a67896e39..25cc25772c 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z30_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z30_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z320_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z320_13_AVX512.c index 5e1acf0903..d1bb694f7e 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z320_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z320_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z32_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z32_13_AVX512.c index 14ce05ac0b..cfc73d3cae 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z32_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z32_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z352_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z352_13_AVX512.c index 35c1a730e8..1ec26ff03c 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z352_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z352_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z36_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z36_13_AVX512.c index be02e00e7f..2fefbad81c 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z36_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z36_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z384_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z384_13_AVX512.c index 22e55f4543..7910c9bb85 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z384_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z384_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z3_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z3_13_AVX512.c index f2752744e3..1bd4846f85 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z3_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z3_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z40_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z40_13_AVX512.c index 3d49278c3d..5592f8283f 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z40_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z40_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z44_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z44_13_AVX512.c index 9bff5a5d51..d0926082cf 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z44_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z44_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z48_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z48_13_AVX512.c index 73ff170732..cf5d4b986a 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z48_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z48_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z4_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z4_13_AVX512.c index 174556650b..65828bd949 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z4_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z4_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z52_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z52_13_AVX512.c index 4425d37550..641eb01505 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z52_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z52_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z56_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z56_13_AVX512.c index 34476031c1..ef9fb0faf6 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z56_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z56_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z5_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z5_13_AVX512.c index 4f780cfa67..25031943fd 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z5_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z5_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z60_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z60_13_AVX512.c index d102d9cda6..4b6d79d89c 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z60_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z60_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z64_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z64_13_AVX512.c index 649ce55fa0..a4abfb87e5 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z64_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z64_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z6_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z6_13_AVX512.c index 194c559442..ad1dc9e103 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z6_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z6_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z72_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z72_13_AVX512.c index fd82e34130..8e325bdfa4 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z72_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z72_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z7_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z7_13_AVX512.c index 0c6b156921..04e0ad4dbf 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z7_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z7_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z80_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z80_13_AVX512.c index 2d6c7477bd..2b2ca88268 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z80_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z80_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z88_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z88_13_AVX512.c index 6a639f9cb4..745249c196 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z88_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z88_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z8_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z8_13_AVX512.c index 8b3b9f9183..f59c40998e 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z8_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z8_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z96_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z96_13_AVX512.c index 63631b39b4..6854cd72a4 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z96_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z96_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z9_13_AVX512.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z9_13_AVX512.c index b19f2afe39..ce8ce8fd02 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z9_13_AVX512.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/ldpc_gen_files/nrLDPC_cnProc_BG2_Z9_13_AVX512.c @@ -1,5 +1,6 @@ #include #include +#include "../include/avx512fintrin.h" __m512i _mm512_sign_epi16(__m512i a, __m512i b){ b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/adxintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/adxintrin.h new file mode 100644 index 0000000000..7acdaf4ab6 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/adxintrin.h @@ -0,0 +1,81 @@ +/* Copyright (C) 2012-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#if !defined _X86INTRIN_H_INCLUDED && !defined _IMMINTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _ADXINTRIN_H_INCLUDED +#define _ADXINTRIN_H_INCLUDED + +extern __inline unsigned char +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_subborrow_u32 (unsigned char __CF, unsigned int __X, + unsigned int __Y, unsigned int *__P) +{ + return __builtin_ia32_sbb_u32 (__CF, __X, __Y, __P); +} + +extern __inline unsigned char +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_addcarry_u32 (unsigned char __CF, unsigned int __X, + unsigned int __Y, unsigned int *__P) +{ + return __builtin_ia32_addcarryx_u32 (__CF, __X, __Y, __P); +} + +extern __inline unsigned char +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_addcarryx_u32 (unsigned char __CF, unsigned int __X, + unsigned int __Y, unsigned int *__P) +{ + return __builtin_ia32_addcarryx_u32 (__CF, __X, __Y, __P); +} + +#ifdef __x86_64__ +extern __inline unsigned char +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_subborrow_u64 (unsigned char __CF, unsigned long long __X, + unsigned long long __Y, unsigned long long *__P) +{ + return __builtin_ia32_sbb_u64 (__CF, __X, __Y, __P); +} + +extern __inline unsigned char +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_addcarry_u64 (unsigned char __CF, unsigned long long __X, + unsigned long long __Y, unsigned long long *__P) +{ + return __builtin_ia32_addcarryx_u64 (__CF, __X, __Y, __P); +} + +extern __inline unsigned char +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_addcarryx_u64 (unsigned char __CF, unsigned long long __X, + unsigned long long __Y, unsigned long long *__P) +{ + return __builtin_ia32_addcarryx_u64 (__CF, __X, __Y, __P); +} +#endif + +#endif /* _ADXINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/ammintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/ammintrin.h new file mode 100644 index 0000000000..0093092781 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/ammintrin.h @@ -0,0 +1,93 @@ +/* Copyright (C) 2007-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +/* Implemented from the specification included in the AMD Programmers + Manual Update, version 2.x */ + +#ifndef _AMMINTRIN_H_INCLUDED +#define _AMMINTRIN_H_INCLUDED + +/* We need definitions from the SSE3, SSE2 and SSE header files*/ +#include + +#ifndef __SSE4A__ +#pragma GCC push_options +#pragma GCC target("sse4a") +#define __DISABLE_SSE4A__ +#endif /* __SSE4A__ */ + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_stream_sd (double * __P, __m128d __Y) +{ + __builtin_ia32_movntsd (__P, (__v2df) __Y); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_stream_ss (float * __P, __m128 __Y) +{ + __builtin_ia32_movntss (__P, (__v4sf) __Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_extract_si64 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_extrq ((__v2di) __X, (__v16qi) __Y); +} + +#ifdef __OPTIMIZE__ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_extracti_si64 (__m128i __X, unsigned const int __I, unsigned const int __L) +{ + return (__m128i) __builtin_ia32_extrqi ((__v2di) __X, __I, __L); +} +#else +#define _mm_extracti_si64(X, I, L) \ + ((__m128i) __builtin_ia32_extrqi ((__v2di)(__m128i)(X), \ + (unsigned int)(I), (unsigned int)(L))) +#endif + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_insert_si64 (__m128i __X,__m128i __Y) +{ + return (__m128i) __builtin_ia32_insertq ((__v2di)__X, (__v2di)__Y); +} + +#ifdef __OPTIMIZE__ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_inserti_si64(__m128i __X, __m128i __Y, unsigned const int __I, unsigned const int __L) +{ + return (__m128i) __builtin_ia32_insertqi ((__v2di)__X, (__v2di)__Y, __I, __L); +} +#else +#define _mm_inserti_si64(X, Y, I, L) \ + ((__m128i) __builtin_ia32_insertqi ((__v2di)(__m128i)(X), \ + (__v2di)(__m128i)(Y), \ + (unsigned int)(I), (unsigned int)(L))) +#endif + +#ifdef __DISABLE_SSE4A__ +#undef __DISABLE_SSE4A__ +#pragma GCC pop_options +#endif /* __DISABLE_SSE4A__ */ + +#endif /* _AMMINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx2intrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx2intrin.h new file mode 100644 index 0000000000..82f170a3d6 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx2intrin.h @@ -0,0 +1,1908 @@ +/* Copyright (C) 2011-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _AVX2INTRIN_H_INCLUDED +#define _AVX2INTRIN_H_INCLUDED + +#ifndef __AVX2__ +#pragma GCC push_options +#pragma GCC target("avx2") +#define __DISABLE_AVX2__ +#endif /* __AVX2__ */ + +/* Sum absolute 8-bit integer difference of adjacent groups of 4 + byte integers in the first 2 operands. Starting offsets within + operands are determined by the 3rd mask operand. */ +#ifdef __OPTIMIZE__ +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mpsadbw_epu8 (__m256i __X, __m256i __Y, const int __M) +{ + return (__m256i) __builtin_ia32_mpsadbw256 ((__v32qi)__X, + (__v32qi)__Y, __M); +} +#else +#define _mm256_mpsadbw_epu8(X, Y, M) \ + ((__m256i) __builtin_ia32_mpsadbw256 ((__v32qi)(__m256i)(X), \ + (__v32qi)(__m256i)(Y), (int)(M))) +#endif + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_abs_epi8 (__m256i __A) +{ + return (__m256i)__builtin_ia32_pabsb256 ((__v32qi)__A); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_abs_epi16 (__m256i __A) +{ + return (__m256i)__builtin_ia32_pabsw256 ((__v16hi)__A); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_abs_epi32 (__m256i __A) +{ + return (__m256i)__builtin_ia32_pabsd256 ((__v8si)__A); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_packs_epi32 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_packssdw256 ((__v8si)__A, (__v8si)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_packs_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_packsswb256 ((__v16hi)__A, (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_packus_epi32 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_packusdw256 ((__v8si)__A, (__v8si)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_packus_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_packuswb256 ((__v16hi)__A, (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_add_epi8 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v32qu)__A + (__v32qu)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_add_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v16hu)__A + (__v16hu)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_add_epi32 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v8su)__A + (__v8su)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_add_epi64 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v4du)__A + (__v4du)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_adds_epi8 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_paddsb256 ((__v32qi)__A, (__v32qi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_adds_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_paddsw256 ((__v16hi)__A, (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_adds_epu8 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_paddusb256 ((__v32qi)__A, (__v32qi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_adds_epu16 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_paddusw256 ((__v16hi)__A, (__v16hi)__B); +} + +#ifdef __OPTIMIZE__ +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_alignr_epi8 (__m256i __A, __m256i __B, const int __N) +{ + return (__m256i) __builtin_ia32_palignr256 ((__v4di)__A, + (__v4di)__B, + __N * 8); +} +#else +/* In that case (__N*8) will be in vreg, and insn will not be matched. */ +/* Use define instead */ +#define _mm256_alignr_epi8(A, B, N) \ + ((__m256i) __builtin_ia32_palignr256 ((__v4di)(__m256i)(A), \ + (__v4di)(__m256i)(B), \ + (int)(N) * 8)) +#endif + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_and_si256 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v4du)__A & (__v4du)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_andnot_si256 (__m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_andnotsi256 ((__v4di)__A, (__v4di)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_avg_epu8 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pavgb256 ((__v32qi)__A, (__v32qi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_avg_epu16 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pavgw256 ((__v16hi)__A, (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_blendv_epi8 (__m256i __X, __m256i __Y, __m256i __M) +{ + return (__m256i) __builtin_ia32_pblendvb256 ((__v32qi)__X, + (__v32qi)__Y, + (__v32qi)__M); +} + +#ifdef __OPTIMIZE__ +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_blend_epi16 (__m256i __X, __m256i __Y, const int __M) +{ + return (__m256i) __builtin_ia32_pblendw256 ((__v16hi)__X, + (__v16hi)__Y, + __M); +} +#else +#define _mm256_blend_epi16(X, Y, M) \ + ((__m256i) __builtin_ia32_pblendw256 ((__v16hi)(__m256i)(X), \ + (__v16hi)(__m256i)(Y), (int)(M))) +#endif + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpeq_epi8 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v32qi)__A == (__v32qi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpeq_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v16hi)__A == (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpeq_epi32 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v8si)__A == (__v8si)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpeq_epi64 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v4di)__A == (__v4di)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpgt_epi8 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v32qi)__A > (__v32qi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpgt_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v16hi)__A > (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpgt_epi32 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v8si)__A > (__v8si)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpgt_epi64 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v4di)__A > (__v4di)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_hadd_epi16 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_phaddw256 ((__v16hi)__X, + (__v16hi)__Y); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_hadd_epi32 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_phaddd256 ((__v8si)__X, (__v8si)__Y); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_hadds_epi16 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_phaddsw256 ((__v16hi)__X, + (__v16hi)__Y); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_hsub_epi16 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_phsubw256 ((__v16hi)__X, + (__v16hi)__Y); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_hsub_epi32 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_phsubd256 ((__v8si)__X, (__v8si)__Y); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_hsubs_epi16 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_phsubsw256 ((__v16hi)__X, + (__v16hi)__Y); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maddubs_epi16 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_pmaddubsw256 ((__v32qi)__X, + (__v32qi)__Y); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_madd_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pmaddwd256 ((__v16hi)__A, + (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_max_epi8 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pmaxsb256 ((__v32qi)__A, (__v32qi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_max_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pmaxsw256 ((__v16hi)__A, (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_max_epi32 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pmaxsd256 ((__v8si)__A, (__v8si)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_max_epu8 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pmaxub256 ((__v32qi)__A, (__v32qi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_max_epu16 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pmaxuw256 ((__v16hi)__A, (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_max_epu32 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pmaxud256 ((__v8si)__A, (__v8si)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_min_epi8 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pminsb256 ((__v32qi)__A, (__v32qi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_min_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pminsw256 ((__v16hi)__A, (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_min_epi32 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pminsd256 ((__v8si)__A, (__v8si)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_min_epu8 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pminub256 ((__v32qi)__A, (__v32qi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_min_epu16 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pminuw256 ((__v16hi)__A, (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_min_epu32 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pminud256 ((__v8si)__A, (__v8si)__B); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_movemask_epi8 (__m256i __A) +{ + return __builtin_ia32_pmovmskb256 ((__v32qi)__A); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi8_epi16 (__m128i __X) +{ + return (__m256i) __builtin_ia32_pmovsxbw256 ((__v16qi)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi8_epi32 (__m128i __X) +{ + return (__m256i) __builtin_ia32_pmovsxbd256 ((__v16qi)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi8_epi64 (__m128i __X) +{ + return (__m256i) __builtin_ia32_pmovsxbq256 ((__v16qi)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi16_epi32 (__m128i __X) +{ + return (__m256i) __builtin_ia32_pmovsxwd256 ((__v8hi)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi16_epi64 (__m128i __X) +{ + return (__m256i) __builtin_ia32_pmovsxwq256 ((__v8hi)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi32_epi64 (__m128i __X) +{ + return (__m256i) __builtin_ia32_pmovsxdq256 ((__v4si)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepu8_epi16 (__m128i __X) +{ + return (__m256i) __builtin_ia32_pmovzxbw256 ((__v16qi)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepu8_epi32 (__m128i __X) +{ + return (__m256i) __builtin_ia32_pmovzxbd256 ((__v16qi)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepu8_epi64 (__m128i __X) +{ + return (__m256i) __builtin_ia32_pmovzxbq256 ((__v16qi)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepu16_epi32 (__m128i __X) +{ + return (__m256i) __builtin_ia32_pmovzxwd256 ((__v8hi)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepu16_epi64 (__m128i __X) +{ + return (__m256i) __builtin_ia32_pmovzxwq256 ((__v8hi)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepu32_epi64 (__m128i __X) +{ + return (__m256i) __builtin_ia32_pmovzxdq256 ((__v4si)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mul_epi32 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_pmuldq256 ((__v8si)__X, (__v8si)__Y); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mulhrs_epi16 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_pmulhrsw256 ((__v16hi)__X, + (__v16hi)__Y); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mulhi_epu16 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pmulhuw256 ((__v16hi)__A, (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mulhi_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pmulhw256 ((__v16hi)__A, (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mullo_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v16hu)__A * (__v16hu)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mullo_epi32 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v8su)__A * (__v8su)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mul_epu32 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_pmuludq256 ((__v8si)__A, (__v8si)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_or_si256 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v4du)__A | (__v4du)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sad_epu8 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_psadbw256 ((__v32qi)__A, (__v32qi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_shuffle_epi8 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_pshufb256 ((__v32qi)__X, + (__v32qi)__Y); +} + +#ifdef __OPTIMIZE__ +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_shuffle_epi32 (__m256i __A, const int __mask) +{ + return (__m256i)__builtin_ia32_pshufd256 ((__v8si)__A, __mask); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_shufflehi_epi16 (__m256i __A, const int __mask) +{ + return (__m256i)__builtin_ia32_pshufhw256 ((__v16hi)__A, __mask); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_shufflelo_epi16 (__m256i __A, const int __mask) +{ + return (__m256i)__builtin_ia32_pshuflw256 ((__v16hi)__A, __mask); +} +#else +#define _mm256_shuffle_epi32(A, N) \ + ((__m256i)__builtin_ia32_pshufd256 ((__v8si)(__m256i)(A), (int)(N))) +#define _mm256_shufflehi_epi16(A, N) \ + ((__m256i)__builtin_ia32_pshufhw256 ((__v16hi)(__m256i)(A), (int)(N))) +#define _mm256_shufflelo_epi16(A, N) \ + ((__m256i)__builtin_ia32_pshuflw256 ((__v16hi)(__m256i)(A), (int)(N))) +#endif + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sign_epi8 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_psignb256 ((__v32qi)__X, (__v32qi)__Y); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sign_epi16 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_psignw256 ((__v16hi)__X, (__v16hi)__Y); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sign_epi32 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_psignd256 ((__v8si)__X, (__v8si)__Y); +} + +#ifdef __OPTIMIZE__ +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_bslli_epi128 (__m256i __A, const int __N) +{ + return (__m256i)__builtin_ia32_pslldqi256 (__A, __N * 8); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_slli_si256 (__m256i __A, const int __N) +{ + return (__m256i)__builtin_ia32_pslldqi256 (__A, __N * 8); +} +#else +#define _mm256_bslli_epi128(A, N) \ + ((__m256i)__builtin_ia32_pslldqi256 ((__m256i)(A), (int)(N) * 8)) +#define _mm256_slli_si256(A, N) \ + ((__m256i)__builtin_ia32_pslldqi256 ((__m256i)(A), (int)(N) * 8)) +#endif + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_slli_epi16 (__m256i __A, int __B) +{ + return (__m256i)__builtin_ia32_psllwi256 ((__v16hi)__A, __B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sll_epi16 (__m256i __A, __m128i __B) +{ + return (__m256i)__builtin_ia32_psllw256((__v16hi)__A, (__v8hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_slli_epi32 (__m256i __A, int __B) +{ + return (__m256i)__builtin_ia32_pslldi256 ((__v8si)__A, __B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sll_epi32 (__m256i __A, __m128i __B) +{ + return (__m256i)__builtin_ia32_pslld256((__v8si)__A, (__v4si)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_slli_epi64 (__m256i __A, int __B) +{ + return (__m256i)__builtin_ia32_psllqi256 ((__v4di)__A, __B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sll_epi64 (__m256i __A, __m128i __B) +{ + return (__m256i)__builtin_ia32_psllq256((__v4di)__A, (__v2di)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_srai_epi16 (__m256i __A, int __B) +{ + return (__m256i)__builtin_ia32_psrawi256 ((__v16hi)__A, __B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sra_epi16 (__m256i __A, __m128i __B) +{ + return (__m256i)__builtin_ia32_psraw256 ((__v16hi)__A, (__v8hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_srai_epi32 (__m256i __A, int __B) +{ + return (__m256i)__builtin_ia32_psradi256 ((__v8si)__A, __B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sra_epi32 (__m256i __A, __m128i __B) +{ + return (__m256i)__builtin_ia32_psrad256 ((__v8si)__A, (__v4si)__B); +} + +#ifdef __OPTIMIZE__ +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_bsrli_epi128 (__m256i __A, const int __N) +{ + return (__m256i)__builtin_ia32_psrldqi256 (__A, __N * 8); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_srli_si256 (__m256i __A, const int __N) +{ + return (__m256i)__builtin_ia32_psrldqi256 (__A, __N * 8); +} +#else +#define _mm256_bsrli_epi128(A, N) \ + ((__m256i)__builtin_ia32_psrldqi256 ((__m256i)(A), (int)(N) * 8)) +#define _mm256_srli_si256(A, N) \ + ((__m256i)__builtin_ia32_psrldqi256 ((__m256i)(A), (int)(N) * 8)) +#endif + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_srli_epi16 (__m256i __A, int __B) +{ + return (__m256i)__builtin_ia32_psrlwi256 ((__v16hi)__A, __B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_srl_epi16 (__m256i __A, __m128i __B) +{ + return (__m256i)__builtin_ia32_psrlw256((__v16hi)__A, (__v8hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_srli_epi32 (__m256i __A, int __B) +{ + return (__m256i)__builtin_ia32_psrldi256 ((__v8si)__A, __B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_srl_epi32 (__m256i __A, __m128i __B) +{ + return (__m256i)__builtin_ia32_psrld256((__v8si)__A, (__v4si)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_srli_epi64 (__m256i __A, int __B) +{ + return (__m256i)__builtin_ia32_psrlqi256 ((__v4di)__A, __B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_srl_epi64 (__m256i __A, __m128i __B) +{ + return (__m256i)__builtin_ia32_psrlq256((__v4di)__A, (__v2di)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sub_epi8 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v32qu)__A - (__v32qu)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sub_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v16hu)__A - (__v16hu)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sub_epi32 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v8su)__A - (__v8su)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sub_epi64 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v4du)__A - (__v4du)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_subs_epi8 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_psubsb256 ((__v32qi)__A, (__v32qi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_subs_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_psubsw256 ((__v16hi)__A, (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_subs_epu8 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_psubusb256 ((__v32qi)__A, (__v32qi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_subs_epu16 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_psubusw256 ((__v16hi)__A, (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_unpackhi_epi8 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_punpckhbw256 ((__v32qi)__A, (__v32qi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_unpackhi_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_punpckhwd256 ((__v16hi)__A, (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_unpackhi_epi32 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_punpckhdq256 ((__v8si)__A, (__v8si)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_unpackhi_epi64 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_punpckhqdq256 ((__v4di)__A, (__v4di)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_unpacklo_epi8 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_punpcklbw256 ((__v32qi)__A, (__v32qi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_unpacklo_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_punpcklwd256 ((__v16hi)__A, (__v16hi)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_unpacklo_epi32 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_punpckldq256 ((__v8si)__A, (__v8si)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_unpacklo_epi64 (__m256i __A, __m256i __B) +{ + return (__m256i)__builtin_ia32_punpcklqdq256 ((__v4di)__A, (__v4di)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_xor_si256 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v4du)__A ^ (__v4du)__B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_stream_load_si256 (__m256i const *__X) +{ + return (__m256i) __builtin_ia32_movntdqa256 ((__v4di *) __X); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_broadcastss_ps (__m128 __X) +{ + return (__m128) __builtin_ia32_vbroadcastss_ps ((__v4sf)__X); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcastss_ps (__m128 __X) +{ + return (__m256) __builtin_ia32_vbroadcastss_ps256 ((__v4sf)__X); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcastsd_pd (__m128d __X) +{ + return (__m256d) __builtin_ia32_vbroadcastsd_pd256 ((__v2df)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcastsi128_si256 (__m128i __X) +{ + return (__m256i) __builtin_ia32_vbroadcastsi256 ((__v2di)__X); +} + +#ifdef __OPTIMIZE__ +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_blend_epi32 (__m128i __X, __m128i __Y, const int __M) +{ + return (__m128i) __builtin_ia32_pblendd128 ((__v4si)__X, + (__v4si)__Y, + __M); +} +#else +#define _mm_blend_epi32(X, Y, M) \ + ((__m128i) __builtin_ia32_pblendd128 ((__v4si)(__m128i)(X), \ + (__v4si)(__m128i)(Y), (int)(M))) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_blend_epi32 (__m256i __X, __m256i __Y, const int __M) +{ + return (__m256i) __builtin_ia32_pblendd256 ((__v8si)__X, + (__v8si)__Y, + __M); +} +#else +#define _mm256_blend_epi32(X, Y, M) \ + ((__m256i) __builtin_ia32_pblendd256 ((__v8si)(__m256i)(X), \ + (__v8si)(__m256i)(Y), (int)(M))) +#endif + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcastb_epi8 (__m128i __X) +{ + return (__m256i) __builtin_ia32_pbroadcastb256 ((__v16qi)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcastw_epi16 (__m128i __X) +{ + return (__m256i) __builtin_ia32_pbroadcastw256 ((__v8hi)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcastd_epi32 (__m128i __X) +{ + return (__m256i) __builtin_ia32_pbroadcastd256 ((__v4si)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcastq_epi64 (__m128i __X) +{ + return (__m256i) __builtin_ia32_pbroadcastq256 ((__v2di)__X); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_broadcastb_epi8 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pbroadcastb128 ((__v16qi)__X); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_broadcastw_epi16 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pbroadcastw128 ((__v8hi)__X); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_broadcastd_epi32 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pbroadcastd128 ((__v4si)__X); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_broadcastq_epi64 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pbroadcastq128 ((__v2di)__X); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permutevar8x32_epi32 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_permvarsi256 ((__v8si)__X, (__v8si)__Y); +} + +#ifdef __OPTIMIZE__ +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permute4x64_pd (__m256d __X, const int __M) +{ + return (__m256d) __builtin_ia32_permdf256 ((__v4df)__X, __M); +} +#else +#define _mm256_permute4x64_pd(X, M) \ + ((__m256d) __builtin_ia32_permdf256 ((__v4df)(__m256d)(X), (int)(M))) +#endif + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permutevar8x32_ps (__m256 __X, __m256i __Y) +{ + return (__m256) __builtin_ia32_permvarsf256 ((__v8sf)__X, (__v8si)__Y); +} + +#ifdef __OPTIMIZE__ +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permute4x64_epi64 (__m256i __X, const int __M) +{ + return (__m256i) __builtin_ia32_permdi256 ((__v4di)__X, __M); +} +#else +#define _mm256_permute4x64_epi64(X, M) \ + ((__m256i) __builtin_ia32_permdi256 ((__v4di)(__m256i)(X), (int)(M))) +#endif + + +#ifdef __OPTIMIZE__ +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permute2x128_si256 (__m256i __X, __m256i __Y, const int __M) +{ + return (__m256i) __builtin_ia32_permti256 ((__v4di)__X, (__v4di)__Y, __M); +} +#else +#define _mm256_permute2x128_si256(X, Y, M) \ + ((__m256i) __builtin_ia32_permti256 ((__v4di)(__m256i)(X), (__v4di)(__m256i)(Y), (int)(M))) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_extracti128_si256 (__m256i __X, const int __M) +{ + return (__m128i) __builtin_ia32_extract128i256 ((__v4di)__X, __M); +} +#else +#define _mm256_extracti128_si256(X, M) \ + ((__m128i) __builtin_ia32_extract128i256 ((__v4di)(__m256i)(X), (int)(M))) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_inserti128_si256 (__m256i __X, __m128i __Y, const int __M) +{ + return (__m256i) __builtin_ia32_insert128i256 ((__v4di)__X, (__v2di)__Y, __M); +} +#else +#define _mm256_inserti128_si256(X, Y, M) \ + ((__m256i) __builtin_ia32_insert128i256 ((__v4di)(__m256i)(X), \ + (__v2di)(__m128i)(Y), \ + (int)(M))) +#endif + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskload_epi32 (int const *__X, __m256i __M ) +{ + return (__m256i) __builtin_ia32_maskloadd256 ((const __v8si *)__X, + (__v8si)__M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskload_epi64 (long long const *__X, __m256i __M ) +{ + return (__m256i) __builtin_ia32_maskloadq256 ((const __v4di *)__X, + (__v4di)__M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskload_epi32 (int const *__X, __m128i __M ) +{ + return (__m128i) __builtin_ia32_maskloadd ((const __v4si *)__X, + (__v4si)__M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskload_epi64 (long long const *__X, __m128i __M ) +{ + return (__m128i) __builtin_ia32_maskloadq ((const __v2di *)__X, + (__v2di)__M); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskstore_epi32 (int *__X, __m256i __M, __m256i __Y ) +{ + __builtin_ia32_maskstored256 ((__v8si *)__X, (__v8si)__M, (__v8si)__Y); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskstore_epi64 (long long *__X, __m256i __M, __m256i __Y ) +{ + __builtin_ia32_maskstoreq256 ((__v4di *)__X, (__v4di)__M, (__v4di)__Y); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskstore_epi32 (int *__X, __m128i __M, __m128i __Y ) +{ + __builtin_ia32_maskstored ((__v4si *)__X, (__v4si)__M, (__v4si)__Y); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskstore_epi64 (long long *__X, __m128i __M, __m128i __Y ) +{ + __builtin_ia32_maskstoreq (( __v2di *)__X, (__v2di)__M, (__v2di)__Y); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sllv_epi32 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_psllv8si ((__v8si)__X, (__v8si)__Y); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sllv_epi32 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_psllv4si ((__v4si)__X, (__v4si)__Y); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sllv_epi64 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_psllv4di ((__v4di)__X, (__v4di)__Y); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sllv_epi64 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_psllv2di ((__v2di)__X, (__v2di)__Y); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_srav_epi32 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_psrav8si ((__v8si)__X, (__v8si)__Y); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srav_epi32 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_psrav4si ((__v4si)__X, (__v4si)__Y); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_srlv_epi32 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_psrlv8si ((__v8si)__X, (__v8si)__Y); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srlv_epi32 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_psrlv4si ((__v4si)__X, (__v4si)__Y); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_srlv_epi64 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_psrlv4di ((__v4di)__X, (__v4di)__Y); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srlv_epi64 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_psrlv2di ((__v2di)__X, (__v2di)__Y); +} + +#ifdef __OPTIMIZE__ +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_i32gather_pd (double const *__base, __m128i __index, const int __scale) +{ + __v2df __zero = _mm_setzero_pd (); + __v2df __mask = _mm_cmpeq_pd (__zero, __zero); + + return (__m128d) __builtin_ia32_gathersiv2df (_mm_undefined_pd (), + __base, + (__v4si)__index, + __mask, + __scale); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_i32gather_pd (__m128d __src, double const *__base, __m128i __index, + __m128d __mask, const int __scale) +{ + return (__m128d) __builtin_ia32_gathersiv2df ((__v2df)__src, + __base, + (__v4si)__index, + (__v2df)__mask, + __scale); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_i32gather_pd (double const *__base, __m128i __index, const int __scale) +{ + __v4df __zero = _mm256_setzero_pd (); + __v4df __mask = _mm256_cmp_pd (__zero, __zero, _CMP_EQ_OQ); + + return (__m256d) __builtin_ia32_gathersiv4df (_mm256_undefined_pd (), + __base, + (__v4si)__index, + __mask, + __scale); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_i32gather_pd (__m256d __src, double const *__base, + __m128i __index, __m256d __mask, const int __scale) +{ + return (__m256d) __builtin_ia32_gathersiv4df ((__v4df)__src, + __base, + (__v4si)__index, + (__v4df)__mask, + __scale); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_i64gather_pd (double const *__base, __m128i __index, const int __scale) +{ + __v2df __src = _mm_setzero_pd (); + __v2df __mask = _mm_cmpeq_pd (__src, __src); + + return (__m128d) __builtin_ia32_gatherdiv2df (__src, + __base, + (__v2di)__index, + __mask, + __scale); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_i64gather_pd (__m128d __src, double const *__base, __m128i __index, + __m128d __mask, const int __scale) +{ + return (__m128d) __builtin_ia32_gatherdiv2df ((__v2df)__src, + __base, + (__v2di)__index, + (__v2df)__mask, + __scale); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_i64gather_pd (double const *__base, __m256i __index, const int __scale) +{ + __v4df __src = _mm256_setzero_pd (); + __v4df __mask = _mm256_cmp_pd (__src, __src, _CMP_EQ_OQ); + + return (__m256d) __builtin_ia32_gatherdiv4df (__src, + __base, + (__v4di)__index, + __mask, + __scale); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_i64gather_pd (__m256d __src, double const *__base, + __m256i __index, __m256d __mask, const int __scale) +{ + return (__m256d) __builtin_ia32_gatherdiv4df ((__v4df)__src, + __base, + (__v4di)__index, + (__v4df)__mask, + __scale); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_i32gather_ps (float const *__base, __m128i __index, const int __scale) +{ + __v4sf __src = _mm_setzero_ps (); + __v4sf __mask = _mm_cmpeq_ps (__src, __src); + + return (__m128) __builtin_ia32_gathersiv4sf (__src, + __base, + (__v4si)__index, + __mask, + __scale); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_i32gather_ps (__m128 __src, float const *__base, __m128i __index, + __m128 __mask, const int __scale) +{ + return (__m128) __builtin_ia32_gathersiv4sf ((__v4sf)__src, + __base, + (__v4si)__index, + (__v4sf)__mask, + __scale); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_i32gather_ps (float const *__base, __m256i __index, const int __scale) +{ + __v8sf __src = _mm256_setzero_ps (); + __v8sf __mask = _mm256_cmp_ps (__src, __src, _CMP_EQ_OQ); + + return (__m256) __builtin_ia32_gathersiv8sf (__src, + __base, + (__v8si)__index, + __mask, + __scale); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_i32gather_ps (__m256 __src, float const *__base, + __m256i __index, __m256 __mask, const int __scale) +{ + return (__m256) __builtin_ia32_gathersiv8sf ((__v8sf)__src, + __base, + (__v8si)__index, + (__v8sf)__mask, + __scale); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_i64gather_ps (float const *__base, __m128i __index, const int __scale) +{ + __v4sf __src = _mm_setzero_ps (); + __v4sf __mask = _mm_cmpeq_ps (__src, __src); + + return (__m128) __builtin_ia32_gatherdiv4sf (__src, + __base, + (__v2di)__index, + __mask, + __scale); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_i64gather_ps (__m128 __src, float const *__base, __m128i __index, + __m128 __mask, const int __scale) +{ + return (__m128) __builtin_ia32_gatherdiv4sf ((__v4sf)__src, + __base, + (__v2di)__index, + (__v4sf)__mask, + __scale); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_i64gather_ps (float const *__base, __m256i __index, const int __scale) +{ + __v4sf __src = _mm_setzero_ps (); + __v4sf __mask = _mm_cmpeq_ps (__src, __src); + + return (__m128) __builtin_ia32_gatherdiv4sf256 (__src, + __base, + (__v4di)__index, + __mask, + __scale); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_i64gather_ps (__m128 __src, float const *__base, + __m256i __index, __m128 __mask, const int __scale) +{ + return (__m128) __builtin_ia32_gatherdiv4sf256 ((__v4sf)__src, + __base, + (__v4di)__index, + (__v4sf)__mask, + __scale); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_i32gather_epi64 (long long int const *__base, + __m128i __index, const int __scale) +{ + __v2di __src = __extension__ (__v2di){ 0, 0 }; + __v2di __mask = __extension__ (__v2di){ ~0, ~0 }; + + return (__m128i) __builtin_ia32_gathersiv2di (__src, + __base, + (__v4si)__index, + __mask, + __scale); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_i32gather_epi64 (__m128i __src, long long int const *__base, + __m128i __index, __m128i __mask, const int __scale) +{ + return (__m128i) __builtin_ia32_gathersiv2di ((__v2di)__src, + __base, + (__v4si)__index, + (__v2di)__mask, + __scale); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_i32gather_epi64 (long long int const *__base, + __m128i __index, const int __scale) +{ + __v4di __src = __extension__ (__v4di){ 0, 0, 0, 0 }; + __v4di __mask = __extension__ (__v4di){ ~0, ~0, ~0, ~0 }; + + return (__m256i) __builtin_ia32_gathersiv4di (__src, + __base, + (__v4si)__index, + __mask, + __scale); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_i32gather_epi64 (__m256i __src, long long int const *__base, + __m128i __index, __m256i __mask, + const int __scale) +{ + return (__m256i) __builtin_ia32_gathersiv4di ((__v4di)__src, + __base, + (__v4si)__index, + (__v4di)__mask, + __scale); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_i64gather_epi64 (long long int const *__base, + __m128i __index, const int __scale) +{ + __v2di __src = __extension__ (__v2di){ 0, 0 }; + __v2di __mask = __extension__ (__v2di){ ~0, ~0 }; + + return (__m128i) __builtin_ia32_gatherdiv2di (__src, + __base, + (__v2di)__index, + __mask, + __scale); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_i64gather_epi64 (__m128i __src, long long int const *__base, + __m128i __index, __m128i __mask, const int __scale) +{ + return (__m128i) __builtin_ia32_gatherdiv2di ((__v2di)__src, + __base, + (__v2di)__index, + (__v2di)__mask, + __scale); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_i64gather_epi64 (long long int const *__base, + __m256i __index, const int __scale) +{ + __v4di __src = __extension__ (__v4di){ 0, 0, 0, 0 }; + __v4di __mask = __extension__ (__v4di){ ~0, ~0, ~0, ~0 }; + + return (__m256i) __builtin_ia32_gatherdiv4di (__src, + __base, + (__v4di)__index, + __mask, + __scale); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_i64gather_epi64 (__m256i __src, long long int const *__base, + __m256i __index, __m256i __mask, + const int __scale) +{ + return (__m256i) __builtin_ia32_gatherdiv4di ((__v4di)__src, + __base, + (__v4di)__index, + (__v4di)__mask, + __scale); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_i32gather_epi32 (int const *__base, __m128i __index, const int __scale) +{ + __v4si __src = __extension__ (__v4si){ 0, 0, 0, 0 }; + __v4si __mask = __extension__ (__v4si){ ~0, ~0, ~0, ~0 }; + + return (__m128i) __builtin_ia32_gathersiv4si (__src, + __base, + (__v4si)__index, + __mask, + __scale); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_i32gather_epi32 (__m128i __src, int const *__base, __m128i __index, + __m128i __mask, const int __scale) +{ + return (__m128i) __builtin_ia32_gathersiv4si ((__v4si)__src, + __base, + (__v4si)__index, + (__v4si)__mask, + __scale); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_i32gather_epi32 (int const *__base, __m256i __index, const int __scale) +{ + __v8si __src = __extension__ (__v8si){ 0, 0, 0, 0, 0, 0, 0, 0 }; + __v8si __mask = __extension__ (__v8si){ ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0 }; + + return (__m256i) __builtin_ia32_gathersiv8si (__src, + __base, + (__v8si)__index, + __mask, + __scale); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_i32gather_epi32 (__m256i __src, int const *__base, + __m256i __index, __m256i __mask, + const int __scale) +{ + return (__m256i) __builtin_ia32_gathersiv8si ((__v8si)__src, + __base, + (__v8si)__index, + (__v8si)__mask, + __scale); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_i64gather_epi32 (int const *__base, __m128i __index, const int __scale) +{ + __v4si __src = __extension__ (__v4si){ 0, 0, 0, 0 }; + __v4si __mask = __extension__ (__v4si){ ~0, ~0, ~0, ~0 }; + + return (__m128i) __builtin_ia32_gatherdiv4si (__src, + __base, + (__v2di)__index, + __mask, + __scale); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_i64gather_epi32 (__m128i __src, int const *__base, __m128i __index, + __m128i __mask, const int __scale) +{ + return (__m128i) __builtin_ia32_gatherdiv4si ((__v4si)__src, + __base, + (__v2di)__index, + (__v4si)__mask, + __scale); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_i64gather_epi32 (int const *__base, __m256i __index, const int __scale) +{ + __v4si __src = __extension__ (__v4si){ 0, 0, 0, 0 }; + __v4si __mask = __extension__ (__v4si){ ~0, ~0, ~0, ~0 }; + + return (__m128i) __builtin_ia32_gatherdiv4si256 (__src, + __base, + (__v4di)__index, + __mask, + __scale); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_i64gather_epi32 (__m128i __src, int const *__base, + __m256i __index, __m128i __mask, + const int __scale) +{ + return (__m128i) __builtin_ia32_gatherdiv4si256 ((__v4si)__src, + __base, + (__v4di)__index, + (__v4si)__mask, + __scale); +} +#else /* __OPTIMIZE__ */ +#define _mm_i32gather_pd(BASE, INDEX, SCALE) \ + (__m128d) __builtin_ia32_gathersiv2df ((__v2df) _mm_setzero_pd (), \ + (double const *)BASE, \ + (__v4si)(__m128i)INDEX, \ + (__v2df)_mm_set1_pd( \ + (double)(long long int) -1), \ + (int)SCALE) + +#define _mm_mask_i32gather_pd(SRC, BASE, INDEX, MASK, SCALE) \ + (__m128d) __builtin_ia32_gathersiv2df ((__v2df)(__m128d)SRC, \ + (double const *)BASE, \ + (__v4si)(__m128i)INDEX, \ + (__v2df)(__m128d)MASK, \ + (int)SCALE) + +#define _mm256_i32gather_pd(BASE, INDEX, SCALE) \ + (__m256d) __builtin_ia32_gathersiv4df ((__v4df) _mm256_setzero_pd (), \ + (double const *)BASE, \ + (__v4si)(__m128i)INDEX, \ + (__v4df)_mm256_set1_pd( \ + (double)(long long int) -1), \ + (int)SCALE) + +#define _mm256_mask_i32gather_pd(SRC, BASE, INDEX, MASK, SCALE) \ + (__m256d) __builtin_ia32_gathersiv4df ((__v4df)(__m256d)SRC, \ + (double const *)BASE, \ + (__v4si)(__m128i)INDEX, \ + (__v4df)(__m256d)MASK, \ + (int)SCALE) + +#define _mm_i64gather_pd(BASE, INDEX, SCALE) \ + (__m128d) __builtin_ia32_gatherdiv2df ((__v2df) _mm_setzero_pd (), \ + (double const *)BASE, \ + (__v2di)(__m128i)INDEX, \ + (__v2df)_mm_set1_pd( \ + (double)(long long int) -1), \ + (int)SCALE) + +#define _mm_mask_i64gather_pd(SRC, BASE, INDEX, MASK, SCALE) \ + (__m128d) __builtin_ia32_gatherdiv2df ((__v2df)(__m128d)SRC, \ + (double const *)BASE, \ + (__v2di)(__m128i)INDEX, \ + (__v2df)(__m128d)MASK, \ + (int)SCALE) + +#define _mm256_i64gather_pd(BASE, INDEX, SCALE) \ + (__m256d) __builtin_ia32_gatherdiv4df ((__v4df) _mm256_setzero_pd (), \ + (double const *)BASE, \ + (__v4di)(__m256i)INDEX, \ + (__v4df)_mm256_set1_pd( \ + (double)(long long int) -1), \ + (int)SCALE) + +#define _mm256_mask_i64gather_pd(SRC, BASE, INDEX, MASK, SCALE) \ + (__m256d) __builtin_ia32_gatherdiv4df ((__v4df)(__m256d)SRC, \ + (double const *)BASE, \ + (__v4di)(__m256i)INDEX, \ + (__v4df)(__m256d)MASK, \ + (int)SCALE) + +#define _mm_i32gather_ps(BASE, INDEX, SCALE) \ + (__m128) __builtin_ia32_gathersiv4sf ((__v4sf) _mm_setzero_ps (), \ + (float const *)BASE, \ + (__v4si)(__m128i)INDEX, \ + _mm_set1_ps ((float)(int) -1), \ + (int)SCALE) + +#define _mm_mask_i32gather_ps(SRC, BASE, INDEX, MASK, SCALE) \ + (__m128) __builtin_ia32_gathersiv4sf ((__v4sf)(__m128d)SRC, \ + (float const *)BASE, \ + (__v4si)(__m128i)INDEX, \ + (__v4sf)(__m128d)MASK, \ + (int)SCALE) + +#define _mm256_i32gather_ps(BASE, INDEX, SCALE) \ + (__m256) __builtin_ia32_gathersiv8sf ((__v8sf) _mm256_setzero_ps (), \ + (float const *)BASE, \ + (__v8si)(__m256i)INDEX, \ + (__v8sf)_mm256_set1_ps ( \ + (float)(int) -1), \ + (int)SCALE) + +#define _mm256_mask_i32gather_ps(SRC, BASE, INDEX, MASK, SCALE) \ + (__m256) __builtin_ia32_gathersiv8sf ((__v8sf)(__m256)SRC, \ + (float const *)BASE, \ + (__v8si)(__m256i)INDEX, \ + (__v8sf)(__m256d)MASK, \ + (int)SCALE) + +#define _mm_i64gather_ps(BASE, INDEX, SCALE) \ + (__m128) __builtin_ia32_gatherdiv4sf ((__v4sf) _mm_setzero_pd (), \ + (float const *)BASE, \ + (__v2di)(__m128i)INDEX, \ + (__v4sf)_mm_set1_ps ( \ + (float)(int) -1), \ + (int)SCALE) + +#define _mm_mask_i64gather_ps(SRC, BASE, INDEX, MASK, SCALE) \ + (__m128) __builtin_ia32_gatherdiv4sf ((__v4sf)(__m128)SRC, \ + (float const *)BASE, \ + (__v2di)(__m128i)INDEX, \ + (__v4sf)(__m128d)MASK, \ + (int)SCALE) + +#define _mm256_i64gather_ps(BASE, INDEX, SCALE) \ + (__m128) __builtin_ia32_gatherdiv4sf256 ((__v4sf) _mm_setzero_ps (), \ + (float const *)BASE, \ + (__v4di)(__m256i)INDEX, \ + (__v4sf)_mm_set1_ps( \ + (float)(int) -1), \ + (int)SCALE) + +#define _mm256_mask_i64gather_ps(SRC, BASE, INDEX, MASK, SCALE) \ + (__m128) __builtin_ia32_gatherdiv4sf256 ((__v4sf)(__m128)SRC, \ + (float const *)BASE, \ + (__v4di)(__m256i)INDEX, \ + (__v4sf)(__m128)MASK, \ + (int)SCALE) + +#define _mm_i32gather_epi64(BASE, INDEX, SCALE) \ + (__m128i) __builtin_ia32_gathersiv2di ((__v2di) _mm_setzero_si128 (), \ + (long long const *)BASE, \ + (__v4si)(__m128i)INDEX, \ + (__v2di)_mm_set1_epi64x (-1), \ + (int)SCALE) + +#define _mm_mask_i32gather_epi64(SRC, BASE, INDEX, MASK, SCALE) \ + (__m128i) __builtin_ia32_gathersiv2di ((__v2di)(__m128i)SRC, \ + (long long const *)BASE, \ + (__v4si)(__m128i)INDEX, \ + (__v2di)(__m128i)MASK, \ + (int)SCALE) + +#define _mm256_i32gather_epi64(BASE, INDEX, SCALE) \ + (__m256i) __builtin_ia32_gathersiv4di ((__v4di) _mm256_setzero_si256 (), \ + (long long const *)BASE, \ + (__v4si)(__m128i)INDEX, \ + (__v4di)_mm256_set1_epi64x (-1), \ + (int)SCALE) + +#define _mm256_mask_i32gather_epi64(SRC, BASE, INDEX, MASK, SCALE) \ + (__m256i) __builtin_ia32_gathersiv4di ((__v4di)(__m256i)SRC, \ + (long long const *)BASE, \ + (__v4si)(__m128i)INDEX, \ + (__v4di)(__m256i)MASK, \ + (int)SCALE) + +#define _mm_i64gather_epi64(BASE, INDEX, SCALE) \ + (__m128i) __builtin_ia32_gatherdiv2di ((__v2di) _mm_setzero_si128 (), \ + (long long const *)BASE, \ + (__v2di)(__m128i)INDEX, \ + (__v2di)_mm_set1_epi64x (-1), \ + (int)SCALE) + +#define _mm_mask_i64gather_epi64(SRC, BASE, INDEX, MASK, SCALE) \ + (__m128i) __builtin_ia32_gatherdiv2di ((__v2di)(__m128i)SRC, \ + (long long const *)BASE, \ + (__v2di)(__m128i)INDEX, \ + (__v2di)(__m128i)MASK, \ + (int)SCALE) + +#define _mm256_i64gather_epi64(BASE, INDEX, SCALE) \ + (__m256i) __builtin_ia32_gatherdiv4di ((__v4di) _mm256_setzero_si256 (), \ + (long long const *)BASE, \ + (__v4di)(__m256i)INDEX, \ + (__v4di)_mm256_set1_epi64x (-1), \ + (int)SCALE) + +#define _mm256_mask_i64gather_epi64(SRC, BASE, INDEX, MASK, SCALE) \ + (__m256i) __builtin_ia32_gatherdiv4di ((__v4di)(__m256i)SRC, \ + (long long const *)BASE, \ + (__v4di)(__m256i)INDEX, \ + (__v4di)(__m256i)MASK, \ + (int)SCALE) + +#define _mm_i32gather_epi32(BASE, INDEX, SCALE) \ + (__m128i) __builtin_ia32_gathersiv4si ((__v4si) _mm_setzero_si128 (), \ + (int const *)BASE, \ + (__v4si)(__m128i)INDEX, \ + (__v4si)_mm_set1_epi32 (-1), \ + (int)SCALE) + +#define _mm_mask_i32gather_epi32(SRC, BASE, INDEX, MASK, SCALE) \ + (__m128i) __builtin_ia32_gathersiv4si ((__v4si)(__m128i)SRC, \ + (int const *)BASE, \ + (__v4si)(__m128i)INDEX, \ + (__v4si)(__m128i)MASK, \ + (int)SCALE) + +#define _mm256_i32gather_epi32(BASE, INDEX, SCALE) \ + (__m256i) __builtin_ia32_gathersiv8si ((__v8si) _mm256_setzero_si256 (), \ + (int const *)BASE, \ + (__v8si)(__m256i)INDEX, \ + (__v8si)_mm256_set1_epi32 (-1), \ + (int)SCALE) + +#define _mm256_mask_i32gather_epi32(SRC, BASE, INDEX, MASK, SCALE) \ + (__m256i) __builtin_ia32_gathersiv8si ((__v8si)(__m256i)SRC, \ + (int const *)BASE, \ + (__v8si)(__m256i)INDEX, \ + (__v8si)(__m256i)MASK, \ + (int)SCALE) + +#define _mm_i64gather_epi32(BASE, INDEX, SCALE) \ + (__m128i) __builtin_ia32_gatherdiv4si ((__v4si) _mm_setzero_si128 (), \ + (int const *)BASE, \ + (__v2di)(__m128i)INDEX, \ + (__v4si)_mm_set1_epi32 (-1), \ + (int)SCALE) + +#define _mm_mask_i64gather_epi32(SRC, BASE, INDEX, MASK, SCALE) \ + (__m128i) __builtin_ia32_gatherdiv4si ((__v4si)(__m128i)SRC, \ + (int const *)BASE, \ + (__v2di)(__m128i)INDEX, \ + (__v4si)(__m128i)MASK, \ + (int)SCALE) + +#define _mm256_i64gather_epi32(BASE, INDEX, SCALE) \ + (__m128i) __builtin_ia32_gatherdiv4si256 ((__v4si) _mm_setzero_si128 (), \ + (int const *)BASE, \ + (__v4di)(__m256i)INDEX, \ + (__v4si)_mm_set1_epi32(-1), \ + (int)SCALE) + +#define _mm256_mask_i64gather_epi32(SRC, BASE, INDEX, MASK, SCALE) \ + (__m128i) __builtin_ia32_gatherdiv4si256 ((__v4si)(__m128i)SRC, \ + (int const *)BASE, \ + (__v4di)(__m256i)INDEX, \ + (__v4si)(__m128i)MASK, \ + (int)SCALE) +#endif /* __OPTIMIZE__ */ + +#ifdef __DISABLE_AVX2__ +#undef __DISABLE_AVX2__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX2__ */ + +#endif /* _AVX2INTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx5124fmapsintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx5124fmapsintrin.h new file mode 100644 index 0000000000..45ac0d4f5d --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx5124fmapsintrin.h @@ -0,0 +1,216 @@ +/* Copyright (C) 2015-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#if !defined _IMMINTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _AVX5124FMAPSINTRIN_H_INCLUDED +#define _AVX5124FMAPSINTRIN_H_INCLUDED + +#ifndef __AVX5124FMAPS__ +#pragma GCC push_options +//#pragma GCC target("avx5124fmaps") +#define __DISABLE_AVX5124FMAPS__ +#endif /* __AVX5124FMAPS__ */ + +extern __inline __m512 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_4fmadd_ps (__m512 __A, __m512 __B, __m512 __C, + __m512 __D, __m512 __E, __m128 *__F) +{ + return (__m512) __builtin_ia32_4fmaddps ((__v16sf) __B, + (__v16sf) __C, + (__v16sf) __D, + (__v16sf) __E, + (__v16sf) __A, + (const __v4sf *) __F); +} + +extern __inline __m512 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_4fmadd_ps (__m512 __A, __mmask16 __U, __m512 __B, + __m512 __C, __m512 __D, __m512 __E, __m128 *__F) +{ + return (__m512) __builtin_ia32_4fmaddps_mask ((__v16sf) __B, + (__v16sf) __C, + (__v16sf) __D, + (__v16sf) __E, + (__v16sf) __A, + (const __v4sf *) __F, + (__v16sf) __A, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_4fmadd_ps (__mmask16 __U, + __m512 __A, __m512 __B, __m512 __C, + __m512 __D, __m512 __E, __m128 *__F) +{ + return (__m512) __builtin_ia32_4fmaddps_mask ((__v16sf) __B, + (__v16sf) __C, + (__v16sf) __D, + (__v16sf) __E, + (__v16sf) __A, + (const __v4sf *) __F, + (__v16sf) _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m128 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_4fmadd_ss (__m128 __A, __m128 __B, __m128 __C, + __m128 __D, __m128 __E, __m128 *__F) +{ + return (__m128) __builtin_ia32_4fmaddss ((__v4sf) __B, + (__v4sf) __C, + (__v4sf) __D, + (__v4sf) __E, + (__v4sf) __A, + (const __v4sf *) __F); +} + +extern __inline __m128 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_4fmadd_ss (__m128 __A, __mmask8 __U, __m128 __B, __m128 __C, + __m128 __D, __m128 __E, __m128 *__F) +{ + return (__m128) __builtin_ia32_4fmaddss_mask ((__v4sf) __B, + (__v4sf) __C, + (__v4sf) __D, + (__v4sf) __E, + (__v4sf) __A, + (const __v4sf *) __F, + (__v4sf) __A, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_4fmadd_ss (__mmask8 __U, __m128 __A, __m128 __B, __m128 __C, + __m128 __D, __m128 __E, __m128 *__F) +{ + return (__m128) __builtin_ia32_4fmaddss_mask ((__v4sf) __B, + (__v4sf) __C, + (__v4sf) __D, + (__v4sf) __E, + (__v4sf) __A, + (const __v4sf *) __F, + (__v4sf) _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_4fnmadd_ps (__m512 __A, __m512 __B, __m512 __C, + __m512 __D, __m512 __E, __m128 *__F) +{ + return (__m512) __builtin_ia32_4fnmaddps ((__v16sf) __B, + (__v16sf) __C, + (__v16sf) __D, + (__v16sf) __E, + (__v16sf) __A, + (const __v4sf *) __F); +} + +extern __inline __m512 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_4fnmadd_ps (__m512 __A, __mmask16 __U, __m512 __B, + __m512 __C, __m512 __D, __m512 __E, __m128 *__F) +{ + return (__m512) __builtin_ia32_4fnmaddps_mask ((__v16sf) __B, + (__v16sf) __C, + (__v16sf) __D, + (__v16sf) __E, + (__v16sf) __A, + (const __v4sf *) __F, + (__v16sf) __A, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_4fnmadd_ps (__mmask16 __U, + __m512 __A, __m512 __B, __m512 __C, + __m512 __D, __m512 __E, __m128 *__F) +{ + return (__m512) __builtin_ia32_4fnmaddps_mask ((__v16sf) __B, + (__v16sf) __C, + (__v16sf) __D, + (__v16sf) __E, + (__v16sf) __A, + (const __v4sf *) __F, + (__v16sf) _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m128 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_4fnmadd_ss (__m128 __A, __m128 __B, __m128 __C, + __m128 __D, __m128 __E, __m128 *__F) +{ + return (__m128) __builtin_ia32_4fnmaddss ((__v4sf) __B, + (__v4sf) __C, + (__v4sf) __D, + (__v4sf) __E, + (__v4sf) __A, + (const __v4sf *) __F); +} + +extern __inline __m128 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_4fnmadd_ss (__m128 __A, __mmask8 __U, __m128 __B, __m128 __C, + __m128 __D, __m128 __E, __m128 *__F) +{ + return (__m128) __builtin_ia32_4fnmaddss_mask ((__v4sf) __B, + (__v4sf) __C, + (__v4sf) __D, + (__v4sf) __E, + (__v4sf) __A, + (const __v4sf *) __F, + (__v4sf) __A, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_4fnmadd_ss (__mmask8 __U, __m128 __A, __m128 __B, __m128 __C, + __m128 __D, __m128 __E, __m128 *__F) +{ + return (__m128) __builtin_ia32_4fnmaddss_mask ((__v4sf) __B, + (__v4sf) __C, + (__v4sf) __D, + (__v4sf) __E, + (__v4sf) __A, + (const __v4sf *) __F, + (__v4sf) _mm_setzero_ps (), + (__mmask8) __U); +} + +#ifdef __DISABLE_AVX5124FMAPS__ +#undef __DISABLE_AVX5124FMAPS__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX5124FMAPS__ */ + +#endif /* _AVX5124FMAPSINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx5124vnniwintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx5124vnniwintrin.h new file mode 100644 index 0000000000..483096ec26 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx5124vnniwintrin.h @@ -0,0 +1,132 @@ +/* Copyright (C) 2015-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#if !defined _IMMINTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _AVX5124VNNIWINTRIN_H_INCLUDED +#define _AVX5124VNNIWINTRIN_H_INCLUDED + +#ifndef __AVX5124VNNIW__ +#pragma GCC push_options +//#pragma GCC target("avx5124vnniw") +#define __DISABLE_AVX5124VNNIW__ +#endif /* __AVX5124VNNIW__ */ + +extern __inline __m512i +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_4dpwssd_epi32 (__m512i __A, __m512i __B, __m512i __C, + __m512i __D, __m512i __E, __m128i *__F) +{ + return (__m512i) __builtin_ia32_vp4dpwssd ((__v16si) __B, + (__v16si) __C, + (__v16si) __D, + (__v16si) __E, + (__v16si) __A, + (const __v4si *) __F); +} + +extern __inline __m512i +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_4dpwssd_epi32 (__m512i __A, __mmask16 __U, __m512i __B, + __m512i __C, __m512i __D, __m512i __E, + __m128i *__F) +{ + return (__m512i) __builtin_ia32_vp4dpwssd_mask ((__v16si) __B, + (__v16si) __C, + (__v16si) __D, + (__v16si) __E, + (__v16si) __A, + (const __v4si *) __F, + (__v16si) __A, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_4dpwssd_epi32 (__mmask16 __U, __m512i __A, __m512i __B, + __m512i __C, __m512i __D, __m512i __E, + __m128i *__F) +{ + return (__m512i) __builtin_ia32_vp4dpwssd_mask ((__v16si) __B, + (__v16si) __C, + (__v16si) __D, + (__v16si) __E, + (__v16si) __A, + (const __v4si *) __F, + (__v16si) _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_4dpwssds_epi32 (__m512i __A, __m512i __B, __m512i __C, + __m512i __D, __m512i __E, __m128i *__F) +{ + return (__m512i) __builtin_ia32_vp4dpwssds ((__v16si) __B, + (__v16si) __C, + (__v16si) __D, + (__v16si) __E, + (__v16si) __A, + (const __v4si *) __F); +} + +extern __inline __m512i +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_4dpwssds_epi32 (__m512i __A, __mmask16 __U, __m512i __B, + __m512i __C, __m512i __D, __m512i __E, + __m128i *__F) +{ + return (__m512i) __builtin_ia32_vp4dpwssds_mask ((__v16si) __B, + (__v16si) __C, + (__v16si) __D, + (__v16si) __E, + (__v16si) __A, + (const __v4si *) __F, + (__v16si) __A, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_4dpwssds_epi32 (__mmask16 __U, __m512i __A, __m512i __B, + __m512i __C, __m512i __D, __m512i __E, + __m128i *__F) +{ + return (__m512i) __builtin_ia32_vp4dpwssds_mask ((__v16si) __B, + (__v16si) __C, + (__v16si) __D, + (__v16si) __E, + (__v16si) __A, + (const __v4si *) __F, + (__v16si) _mm512_setzero_ps (), + (__mmask16) __U); +} + +#ifdef __DISABLE_AVX5124VNNIW__ +#undef __DISABLE_AVX5124VNNIW__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX5124VNNIW__ */ + +#endif /* _AVX5124VNNIWINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512bwintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512bwintrin.h new file mode 100644 index 0000000000..bf316cf48e --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512bwintrin.h @@ -0,0 +1,3280 @@ +/* Copyright (C) 2014-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +#error "Never use directly; include instead." +#endif + +#ifndef _AVX512BWINTRIN_H_INCLUDED +#define _AVX512BWINTRIN_H_INCLUDED + +#ifndef __AVX512BW__ +#pragma GCC push_options +//#pragma GCC target("avx512bw") +#define __DISABLE_AVX512BW__ +#endif /* __AVX512BW__ */ + +/* Internal data types for implementing the intrinsics. */ +typedef short __v32hi __attribute__ ((__vector_size__ (64))); +typedef char __v64qi __attribute__ ((__vector_size__ (64))); + +typedef unsigned long long __mmask64; + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_ktest_mask32_u8 (__mmask32 __A, __mmask32 __B, unsigned char *__CF) +{ + *__CF = (unsigned char) __builtin_ia32_ktestcsi (__A, __B); + return (unsigned char) __builtin_ia32_ktestzsi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_ktest_mask64_u8 (__mmask64 __A, __mmask64 __B, unsigned char *__CF) +{ + *__CF = (unsigned char) __builtin_ia32_ktestcdi (__A, __B); + return (unsigned char) __builtin_ia32_ktestzdi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_ktestz_mask32_u8 (__mmask32 __A, __mmask32 __B) +{ + return (unsigned char) __builtin_ia32_ktestzsi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_ktestz_mask64_u8 (__mmask64 __A, __mmask64 __B) +{ + return (unsigned char) __builtin_ia32_ktestzdi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_ktestc_mask32_u8 (__mmask32 __A, __mmask32 __B) +{ + return (unsigned char) __builtin_ia32_ktestcsi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_ktestc_mask64_u8 (__mmask64 __A, __mmask64 __B) +{ + return (unsigned char) __builtin_ia32_ktestcdi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kortest_mask32_u8 (__mmask32 __A, __mmask32 __B, unsigned char *__CF) +{ + *__CF = (unsigned char) __builtin_ia32_kortestcsi (__A, __B); + return (unsigned char) __builtin_ia32_kortestzsi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kortest_mask64_u8 (__mmask64 __A, __mmask64 __B, unsigned char *__CF) +{ + *__CF = (unsigned char) __builtin_ia32_kortestcdi (__A, __B); + return (unsigned char) __builtin_ia32_kortestzdi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kortestz_mask32_u8 (__mmask32 __A, __mmask32 __B) +{ + return (unsigned char) __builtin_ia32_kortestzsi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kortestz_mask64_u8 (__mmask64 __A, __mmask64 __B) +{ + return (unsigned char) __builtin_ia32_kortestzdi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kortestc_mask32_u8 (__mmask32 __A, __mmask32 __B) +{ + return (unsigned char) __builtin_ia32_kortestcsi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kortestc_mask64_u8 (__mmask64 __A, __mmask64 __B) +{ + return (unsigned char) __builtin_ia32_kortestcdi (__A, __B); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kadd_mask32 (__mmask32 __A, __mmask32 __B) +{ + return (__mmask32) __builtin_ia32_kaddsi ((__mmask32) __A, (__mmask32) __B); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kadd_mask64 (__mmask64 __A, __mmask64 __B) +{ + return (__mmask64) __builtin_ia32_kadddi ((__mmask64) __A, (__mmask64) __B); +} + +extern __inline unsigned int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_cvtmask32_u32 (__mmask32 __A) +{ + return (unsigned int) __builtin_ia32_kmovd ((__mmask32) __A); +} + +extern __inline unsigned long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_cvtmask64_u64 (__mmask64 __A) +{ + return (unsigned long long) __builtin_ia32_kmovq ((__mmask64) __A); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_cvtu32_mask32 (unsigned int __A) +{ + return (__mmask32) __builtin_ia32_kmovd ((__mmask32) __A); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_cvtu64_mask64 (unsigned long long __A) +{ + return (__mmask64) __builtin_ia32_kmovq ((__mmask64) __A); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_load_mask32 (__mmask32 *__A) +{ + return (__mmask32) __builtin_ia32_kmovd (*__A); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_load_mask64 (__mmask64 *__A) +{ + return (__mmask64) __builtin_ia32_kmovq (*(__mmask64 *) __A); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_store_mask32 (__mmask32 *__A, __mmask32 __B) +{ + *(__mmask32 *) __A = __builtin_ia32_kmovd (__B); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_store_mask64 (__mmask64 *__A, __mmask64 __B) +{ + *(__mmask64 *) __A = __builtin_ia32_kmovq (__B); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_knot_mask32 (__mmask32 __A) +{ + return (__mmask32) __builtin_ia32_knotsi ((__mmask32) __A); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_knot_mask64 (__mmask64 __A) +{ + return (__mmask64) __builtin_ia32_knotdi ((__mmask64) __A); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kor_mask32 (__mmask32 __A, __mmask32 __B) +{ + return (__mmask32) __builtin_ia32_korsi ((__mmask32) __A, (__mmask32) __B); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kor_mask64 (__mmask64 __A, __mmask64 __B) +{ + return (__mmask64) __builtin_ia32_kordi ((__mmask64) __A, (__mmask64) __B); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kxnor_mask32 (__mmask32 __A, __mmask32 __B) +{ + return (__mmask32) __builtin_ia32_kxnorsi ((__mmask32) __A, (__mmask32) __B); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kxnor_mask64 (__mmask64 __A, __mmask64 __B) +{ + return (__mmask64) __builtin_ia32_kxnordi ((__mmask64) __A, (__mmask64) __B); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kxor_mask32 (__mmask32 __A, __mmask32 __B) +{ + return (__mmask32) __builtin_ia32_kxorsi ((__mmask32) __A, (__mmask32) __B); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kxor_mask64 (__mmask64 __A, __mmask64 __B) +{ + return (__mmask64) __builtin_ia32_kxordi ((__mmask64) __A, (__mmask64) __B); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kand_mask32 (__mmask32 __A, __mmask32 __B) +{ + return (__mmask32) __builtin_ia32_kandsi ((__mmask32) __A, (__mmask32) __B); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kand_mask64 (__mmask64 __A, __mmask64 __B) +{ + return (__mmask64) __builtin_ia32_kanddi ((__mmask64) __A, (__mmask64) __B); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kandn_mask32 (__mmask32 __A, __mmask32 __B) +{ + return (__mmask32) __builtin_ia32_kandnsi ((__mmask32) __A, (__mmask32) __B); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kandn_mask64 (__mmask64 __A, __mmask64 __B) +{ + return (__mmask64) __builtin_ia32_kandndi ((__mmask64) __A, (__mmask64) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mov_epi16 (__m512i __W, __mmask32 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_movdquhi512_mask ((__v32hi) __A, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mov_epi16 (__mmask32 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_movdquhi512_mask ((__v32hi) __A, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_loadu_epi16 (__m512i __W, __mmask32 __U, void const *__P) +{ + return (__m512i) __builtin_ia32_loaddquhi512_mask ((const short *) __P, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_loadu_epi16 (__mmask32 __U, void const *__P) +{ + return (__m512i) __builtin_ia32_loaddquhi512_mask ((const short *) __P, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_storeu_epi16 (void *__P, __mmask32 __U, __m512i __A) +{ + __builtin_ia32_storedquhi512_mask ((short *) __P, + (__v32hi) __A, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mov_epi8 (__m512i __W, __mmask64 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_movdquqi512_mask ((__v64qi) __A, + (__v64qi) __W, + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mov_epi8 (__mmask64 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_movdquqi512_mask ((__v64qi) __A, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __U); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_kunpackw (__mmask32 __A, __mmask32 __B) +{ + return (__mmask32) __builtin_ia32_kunpcksi ((__mmask32) __A, + (__mmask32) __B); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kunpackw_mask32 (__mmask16 __A, __mmask16 __B) +{ + return (__mmask32) __builtin_ia32_kunpcksi ((__mmask32) __A, + (__mmask32) __B); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_kunpackd (__mmask64 __A, __mmask64 __B) +{ + return (__mmask64) __builtin_ia32_kunpckdi ((__mmask64) __A, + (__mmask64) __B); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kunpackd_mask64 (__mmask32 __A, __mmask32 __B) +{ + return (__mmask64) __builtin_ia32_kunpckdi ((__mmask64) __A, + (__mmask64) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_loadu_epi8 (__m512i __W, __mmask64 __U, void const *__P) +{ + return (__m512i) __builtin_ia32_loaddquqi512_mask ((const char *) __P, + (__v64qi) __W, + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_loadu_epi8 (__mmask64 __U, void const *__P) +{ + return (__m512i) __builtin_ia32_loaddquqi512_mask ((const char *) __P, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_storeu_epi8 (void *__P, __mmask64 __U, __m512i __A) +{ + __builtin_ia32_storedquqi512_mask ((char *) __P, + (__v64qi) __A, + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sad_epu8 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psadbw512 ((__v64qi) __A, + (__v64qi) __B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepi16_epi8 (__m512i __A) +{ + return (__m256i) __builtin_ia32_pmovwb512_mask ((__v32hi) __A, + (__v32qi) _mm256_undefined_si256(), + (__mmask32) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi16_epi8 (__m256i __O, __mmask32 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovwb512_mask ((__v32hi) __A, + (__v32qi) __O, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepi16_epi8 (__mmask32 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovwb512_mask ((__v32hi) __A, + (__v32qi) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtsepi16_epi8 (__m512i __A) +{ + return (__m256i) __builtin_ia32_pmovswb512_mask ((__v32hi) __A, + (__v32qi)_mm256_undefined_si256(), + (__mmask32) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtsepi16_epi8 (__m256i __O, __mmask32 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovswb512_mask ((__v32hi) __A, + (__v32qi)__O, + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtsepi16_epi8 (__mmask32 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovswb512_mask ((__v32hi) __A, + (__v32qi) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtusepi16_epi8 (__m512i __A) +{ + return (__m256i) __builtin_ia32_pmovuswb512_mask ((__v32hi) __A, + (__v32qi)_mm256_undefined_si256(), + (__mmask32) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtusepi16_epi8 (__m256i __O, __mmask32 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovuswb512_mask ((__v32hi) __A, + (__v32qi) __O, + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtusepi16_epi8 (__mmask32 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovuswb512_mask ((__v32hi) __A, + (__v32qi) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcastb_epi8 (__m128i __A) +{ + return (__m512i) __builtin_ia32_pbroadcastb512_mask ((__v16qi) __A, + (__v64qi)_mm512_undefined_epi32(), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_broadcastb_epi8 (__m512i __O, __mmask64 __M, __m128i __A) +{ + return (__m512i) __builtin_ia32_pbroadcastb512_mask ((__v16qi) __A, + (__v64qi) __O, + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_broadcastb_epi8 (__mmask64 __M, __m128i __A) +{ + return (__m512i) __builtin_ia32_pbroadcastb512_mask ((__v16qi) __A, + (__v64qi) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_set1_epi8 (__m512i __O, __mmask64 __M, char __A) +{ + return (__m512i) __builtin_ia32_pbroadcastb512_gpr_mask (__A, + (__v64qi) __O, + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_set1_epi8 (__mmask64 __M, char __A) +{ + return (__m512i) + __builtin_ia32_pbroadcastb512_gpr_mask (__A, + (__v64qi) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcastw_epi16 (__m128i __A) +{ + return (__m512i) __builtin_ia32_pbroadcastw512_mask ((__v8hi) __A, + (__v32hi)_mm512_undefined_epi32(), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_broadcastw_epi16 (__m512i __O, __mmask32 __M, __m128i __A) +{ + return (__m512i) __builtin_ia32_pbroadcastw512_mask ((__v8hi) __A, + (__v32hi) __O, + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_broadcastw_epi16 (__mmask32 __M, __m128i __A) +{ + return (__m512i) __builtin_ia32_pbroadcastw512_mask ((__v8hi) __A, + (__v32hi) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_set1_epi16 (__m512i __O, __mmask32 __M, short __A) +{ + return (__m512i) __builtin_ia32_pbroadcastw512_gpr_mask (__A, + (__v32hi) __O, + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_set1_epi16 (__mmask32 __M, short __A) +{ + return (__m512i) + __builtin_ia32_pbroadcastw512_gpr_mask (__A, + (__v32hi) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mulhrs_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmulhrsw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mulhrs_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pmulhrsw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mulhrs_epi16 (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmulhrsw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mulhi_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmulhw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mulhi_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pmulhw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mulhi_epi16 (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmulhw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mulhi_epu16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmulhuw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mulhi_epu16 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pmulhuw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mulhi_epu16 (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmulhuw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mullo_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v32hu) __A * (__v32hu) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mullo_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pmullw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mullo_epi16 (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmullw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepi8_epi16 (__m256i __A) +{ + return (__m512i) __builtin_ia32_pmovsxbw512_mask ((__v32qi) __A, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi8_epi16 (__m512i __W, __mmask32 __U, __m256i __A) +{ + return (__m512i) __builtin_ia32_pmovsxbw512_mask ((__v32qi) __A, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepi8_epi16 (__mmask32 __U, __m256i __A) +{ + return (__m512i) __builtin_ia32_pmovsxbw512_mask ((__v32qi) __A, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepu8_epi16 (__m256i __A) +{ + return (__m512i) __builtin_ia32_pmovzxbw512_mask ((__v32qi) __A, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepu8_epi16 (__m512i __W, __mmask32 __U, __m256i __A) +{ + return (__m512i) __builtin_ia32_pmovzxbw512_mask ((__v32qi) __A, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepu8_epi16 (__mmask32 __U, __m256i __A) +{ + return (__m512i) __builtin_ia32_pmovzxbw512_mask ((__v32qi) __A, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permutexvar_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_permvarhi512_mask ((__v32hi) __B, + (__v32hi) __A, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permutexvar_epi16 (__mmask32 __M, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_permvarhi512_mask ((__v32hi) __B, + (__v32hi) __A, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permutexvar_epi16 (__m512i __W, __mmask32 __M, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_permvarhi512_mask ((__v32hi) __B, + (__v32hi) __A, + (__v32hi) __W, + (__mmask32) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permutex2var_epi16 (__m512i __A, __m512i __I, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpermt2varhi512_mask ((__v32hi) __I + /* idx */ , + (__v32hi) __A, + (__v32hi) __B, + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permutex2var_epi16 (__m512i __A, __mmask32 __U, + __m512i __I, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpermt2varhi512_mask ((__v32hi) __I + /* idx */ , + (__v32hi) __A, + (__v32hi) __B, + (__mmask32) + __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask2_permutex2var_epi16 (__m512i __A, __m512i __I, + __mmask32 __U, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpermi2varhi512_mask ((__v32hi) __A, + (__v32hi) __I + /* idx */ , + (__v32hi) __B, + (__mmask32) + __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permutex2var_epi16 (__mmask32 __U, __m512i __A, + __m512i __I, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpermt2varhi512_maskz ((__v32hi) __I + /* idx */ , + (__v32hi) __A, + (__v32hi) __B, + (__mmask32) + __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_avg_epu8 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pavgb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_avg_epu8 (__m512i __W, __mmask64 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pavgb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) __W, + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_avg_epu8 (__mmask64 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pavgb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_add_epi8 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v64qu) __A + (__v64qu) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_add_epi8 (__m512i __W, __mmask64 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_paddb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) __W, + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_add_epi8 (__mmask64 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_paddb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sub_epi8 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v64qu) __A - (__v64qu) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sub_epi8 (__m512i __W, __mmask64 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_psubb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) __W, + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sub_epi8 (__mmask64 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psubb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_avg_epu16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pavgw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_avg_epu16 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pavgw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_avg_epu16 (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pavgw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_subs_epi8 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psubsb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_subs_epi8 (__m512i __W, __mmask64 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_psubsb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) __W, + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_subs_epi8 (__mmask64 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psubsb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_subs_epu8 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psubusb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_subs_epu8 (__m512i __W, __mmask64 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_psubusb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) __W, + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_subs_epu8 (__mmask64 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psubusb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_adds_epi8 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_paddsb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_adds_epi8 (__m512i __W, __mmask64 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_paddsb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) __W, + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_adds_epi8 (__mmask64 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_paddsb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_adds_epu8 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_paddusb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_adds_epu8 (__m512i __W, __mmask64 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_paddusb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) __W, + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_adds_epu8 (__mmask64 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_paddusb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sub_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v32hu) __A - (__v32hu) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sub_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_psubw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sub_epi16 (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psubw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_subs_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psubsw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_subs_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_psubsw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_subs_epi16 (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psubsw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_subs_epu16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psubusw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_subs_epu16 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_psubusw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_subs_epu16 (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psubusw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_add_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v32hu) __A + (__v32hu) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_add_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_paddw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_add_epi16 (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_paddw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_adds_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_paddsw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_adds_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_paddsw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_adds_epi16 (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_paddsw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_adds_epu16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_paddusw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_adds_epu16 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_paddusw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_adds_epu16 (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_paddusw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_srl_epi16 (__m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psrlw512_mask ((__v32hi) __A, + (__v8hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_srl_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + __m128i __B) +{ + return (__m512i) __builtin_ia32_psrlw512_mask ((__v32hi) __A, + (__v8hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_srl_epi16 (__mmask32 __U, __m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psrlw512_mask ((__v32hi) __A, + (__v8hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_packs_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_packsswb512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sll_epi16 (__m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psllw512_mask ((__v32hi) __A, + (__v8hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sll_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + __m128i __B) +{ + return (__m512i) __builtin_ia32_psllw512_mask ((__v32hi) __A, + (__v8hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sll_epi16 (__mmask32 __U, __m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psllw512_mask ((__v32hi) __A, + (__v8hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maddubs_epi16 (__m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_pmaddubsw512_mask ((__v64qi) __X, + (__v64qi) __Y, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_maddubs_epi16 (__m512i __W, __mmask32 __U, __m512i __X, + __m512i __Y) +{ + return (__m512i) __builtin_ia32_pmaddubsw512_mask ((__v64qi) __X, + (__v64qi) __Y, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_maddubs_epi16 (__mmask32 __U, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_pmaddubsw512_mask ((__v64qi) __X, + (__v64qi) __Y, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_madd_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaddwd512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_madd_epi16 (__m512i __W, __mmask16 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaddwd512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_madd_epi16 (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaddwd512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_unpackhi_epi8 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpckhbw512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_unpackhi_epi8 (__m512i __W, __mmask64 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_punpckhbw512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) __W, + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_unpackhi_epi8 (__mmask64 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpckhbw512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_unpackhi_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpckhwd512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_unpackhi_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_punpckhwd512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_unpackhi_epi16 (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpckhwd512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_unpacklo_epi8 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpcklbw512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_unpacklo_epi8 (__m512i __W, __mmask64 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_punpcklbw512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) __W, + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_unpacklo_epi8 (__mmask64 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpcklbw512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_unpacklo_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpcklwd512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_unpacklo_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_punpcklwd512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_unpacklo_epi16 (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpcklwd512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpeq_epu8_mask (__m512i __A, __m512i __B) +{ + return (__mmask64) __builtin_ia32_ucmpb512_mask ((__v64qi) __A, + (__v64qi) __B, 0, + (__mmask64) -1); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpeq_epi8_mask (__m512i __A, __m512i __B) +{ + return (__mmask64) __builtin_ia32_pcmpeqb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__mmask64) -1); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpeq_epu8_mask (__mmask64 __U, __m512i __A, __m512i __B) +{ + return (__mmask64) __builtin_ia32_ucmpb512_mask ((__v64qi) __A, + (__v64qi) __B, 0, + __U); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpeq_epi8_mask (__mmask64 __U, __m512i __A, __m512i __B) +{ + return (__mmask64) __builtin_ia32_pcmpeqb512_mask ((__v64qi) __A, + (__v64qi) __B, + __U); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpeq_epu16_mask (__m512i __A, __m512i __B) +{ + return (__mmask32) __builtin_ia32_ucmpw512_mask ((__v32hi) __A, + (__v32hi) __B, 0, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpeq_epi16_mask (__m512i __A, __m512i __B) +{ + return (__mmask32) __builtin_ia32_pcmpeqw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpeq_epu16_mask (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__mmask32) __builtin_ia32_ucmpw512_mask ((__v32hi) __A, + (__v32hi) __B, 0, + __U); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpeq_epi16_mask (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__mmask32) __builtin_ia32_pcmpeqw512_mask ((__v32hi) __A, + (__v32hi) __B, + __U); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpgt_epu8_mask (__m512i __A, __m512i __B) +{ + return (__mmask64) __builtin_ia32_ucmpb512_mask ((__v64qi) __A, + (__v64qi) __B, 6, + (__mmask64) -1); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpgt_epi8_mask (__m512i __A, __m512i __B) +{ + return (__mmask64) __builtin_ia32_pcmpgtb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__mmask64) -1); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpgt_epu8_mask (__mmask64 __U, __m512i __A, __m512i __B) +{ + return (__mmask64) __builtin_ia32_ucmpb512_mask ((__v64qi) __A, + (__v64qi) __B, 6, + __U); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpgt_epi8_mask (__mmask64 __U, __m512i __A, __m512i __B) +{ + return (__mmask64) __builtin_ia32_pcmpgtb512_mask ((__v64qi) __A, + (__v64qi) __B, + __U); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpgt_epu16_mask (__m512i __A, __m512i __B) +{ + return (__mmask32) __builtin_ia32_ucmpw512_mask ((__v32hi) __A, + (__v32hi) __B, 6, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpgt_epi16_mask (__m512i __A, __m512i __B) +{ + return (__mmask32) __builtin_ia32_pcmpgtw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpgt_epu16_mask (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__mmask32) __builtin_ia32_ucmpw512_mask ((__v32hi) __A, + (__v32hi) __B, 6, + __U); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpgt_epi16_mask (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__mmask32) __builtin_ia32_pcmpgtw512_mask ((__v32hi) __A, + (__v32hi) __B, + __U); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_movepi8_mask (__m512i __A) +{ + return (__mmask64) __builtin_ia32_cvtb2mask512 ((__v64qi) __A); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_movepi16_mask (__m512i __A) +{ + return (__mmask32) __builtin_ia32_cvtw2mask512 ((__v32hi) __A); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_movm_epi8 (__mmask64 __A) +{ + return (__m512i) __builtin_ia32_cvtmask2b512 (__A); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_movm_epi16 (__mmask32 __A) +{ + return (__m512i) __builtin_ia32_cvtmask2w512 (__A); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_test_epi8_mask (__m512i __A, __m512i __B) +{ + return (__mmask64) __builtin_ia32_ptestmb512 ((__v64qi) __A, + (__v64qi) __B, + (__mmask64) -1); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_test_epi8_mask (__mmask64 __U, __m512i __A, __m512i __B) +{ + return (__mmask64) __builtin_ia32_ptestmb512 ((__v64qi) __A, + (__v64qi) __B, __U); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_test_epi16_mask (__m512i __A, __m512i __B) +{ + return (__mmask32) __builtin_ia32_ptestmw512 ((__v32hi) __A, + (__v32hi) __B, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_test_epi16_mask (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__mmask32) __builtin_ia32_ptestmw512 ((__v32hi) __A, + (__v32hi) __B, __U); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_testn_epi8_mask (__m512i __A, __m512i __B) +{ + return (__mmask64) __builtin_ia32_ptestnmb512 ((__v64qi) __A, + (__v64qi) __B, + (__mmask64) -1); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_testn_epi8_mask (__mmask64 __U, __m512i __A, __m512i __B) +{ + return (__mmask64) __builtin_ia32_ptestnmb512 ((__v64qi) __A, + (__v64qi) __B, __U); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_testn_epi16_mask (__m512i __A, __m512i __B) +{ + return (__mmask32) __builtin_ia32_ptestnmw512 ((__v32hi) __A, + (__v32hi) __B, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_testn_epi16_mask (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__mmask32) __builtin_ia32_ptestnmw512 ((__v32hi) __A, + (__v32hi) __B, __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_shuffle_epi8 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pshufb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_shuffle_epi8 (__m512i __W, __mmask64 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pshufb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) __W, + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_shuffle_epi8 (__mmask64 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pshufb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_min_epu16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminuw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_min_epu16 (__mmask32 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminuw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_min_epu16 (__m512i __W, __mmask32 __M, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pminuw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_min_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminsw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_min_epi16 (__mmask32 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminsw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_min_epi16 (__m512i __W, __mmask32 __M, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pminsw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_max_epu8 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxub512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_max_epu8 (__mmask64 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxub512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_max_epu8 (__m512i __W, __mmask64 __M, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxub512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) __W, + (__mmask64) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_max_epi8 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxsb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_max_epi8 (__mmask64 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxsb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_max_epi8 (__m512i __W, __mmask64 __M, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxsb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) __W, + (__mmask64) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_min_epu8 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminub512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_min_epu8 (__mmask64 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminub512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_min_epu8 (__m512i __W, __mmask64 __M, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pminub512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) __W, + (__mmask64) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_min_epi8 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminsb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_min_epi8 (__mmask64 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminsb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_min_epi8 (__m512i __W, __mmask64 __M, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pminsb512_mask ((__v64qi) __A, + (__v64qi) __B, + (__v64qi) __W, + (__mmask64) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_max_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxsw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_max_epi16 (__mmask32 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxsw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_max_epi16 (__m512i __W, __mmask32 __M, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxsw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_max_epu16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxuw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_max_epu16 (__mmask32 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxuw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_max_epu16 (__m512i __W, __mmask32 __M, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxuw512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sra_epi16 (__m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psraw512_mask ((__v32hi) __A, + (__v8hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sra_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + __m128i __B) +{ + return (__m512i) __builtin_ia32_psraw512_mask ((__v32hi) __A, + (__v8hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sra_epi16 (__mmask32 __U, __m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psraw512_mask ((__v32hi) __A, + (__v8hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_srav_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psrav32hi_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_srav_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_psrav32hi_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_srav_epi16 (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psrav32hi_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_srlv_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psrlv32hi_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_srlv_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_psrlv32hi_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_srlv_epi16 (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psrlv32hi_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sllv_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psllv32hi_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sllv_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_psllv32hi_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sllv_epi16 (__mmask32 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psllv32hi_mask ((__v32hi) __A, + (__v32hi) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_packs_epi16 (__m512i __W, __mmask64 __M, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_packsswb512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v64qi) __W, + (__mmask64) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_packs_epi16 (__mmask64 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_packsswb512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v64qi) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_packus_epi16 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_packuswb512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_packus_epi16 (__m512i __W, __mmask64 __M, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_packuswb512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v64qi) __W, + (__mmask64) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_packus_epi16 (__mmask64 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_packuswb512_mask ((__v32hi) __A, + (__v32hi) __B, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_abs_epi8 (__m512i __A) +{ + return (__m512i) __builtin_ia32_pabsb512_mask ((__v64qi) __A, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_abs_epi8 (__m512i __W, __mmask64 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_pabsb512_mask ((__v64qi) __A, + (__v64qi) __W, + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_abs_epi8 (__mmask64 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_pabsb512_mask ((__v64qi) __A, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_abs_epi16 (__m512i __A) +{ + return (__m512i) __builtin_ia32_pabsw512_mask ((__v32hi) __A, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_abs_epi16 (__m512i __W, __mmask32 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_pabsw512_mask ((__v32hi) __A, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_abs_epi16 (__mmask32 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_pabsw512_mask ((__v32hi) __A, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpneq_epu8_mask (__mmask64 __M, __m512i __X, __m512i __Y) +{ + return (__mmask64) __builtin_ia32_ucmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, 4, + (__mmask64) __M); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmplt_epu8_mask (__mmask64 __M, __m512i __X, __m512i __Y) +{ + return (__mmask64) __builtin_ia32_ucmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, 1, + (__mmask64) __M); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpge_epu8_mask (__mmask64 __M, __m512i __X, __m512i __Y) +{ + return (__mmask64) __builtin_ia32_ucmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, 5, + (__mmask64) __M); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmple_epu8_mask (__mmask64 __M, __m512i __X, __m512i __Y) +{ + return (__mmask64) __builtin_ia32_ucmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, 2, + (__mmask64) __M); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpneq_epu16_mask (__mmask32 __M, __m512i __X, __m512i __Y) +{ + return (__mmask32) __builtin_ia32_ucmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, 4, + (__mmask32) __M); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmplt_epu16_mask (__mmask32 __M, __m512i __X, __m512i __Y) +{ + return (__mmask32) __builtin_ia32_ucmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, 1, + (__mmask32) __M); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpge_epu16_mask (__mmask32 __M, __m512i __X, __m512i __Y) +{ + return (__mmask32) __builtin_ia32_ucmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, 5, + (__mmask32) __M); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmple_epu16_mask (__mmask32 __M, __m512i __X, __m512i __Y) +{ + return (__mmask32) __builtin_ia32_ucmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, 2, + (__mmask32) __M); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpneq_epi8_mask (__mmask64 __M, __m512i __X, __m512i __Y) +{ + return (__mmask64) __builtin_ia32_cmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, 4, + (__mmask64) __M); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmplt_epi8_mask (__mmask64 __M, __m512i __X, __m512i __Y) +{ + return (__mmask64) __builtin_ia32_cmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, 1, + (__mmask64) __M); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpge_epi8_mask (__mmask64 __M, __m512i __X, __m512i __Y) +{ + return (__mmask64) __builtin_ia32_cmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, 5, + (__mmask64) __M); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmple_epi8_mask (__mmask64 __M, __m512i __X, __m512i __Y) +{ + return (__mmask64) __builtin_ia32_cmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, 2, + (__mmask64) __M); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpneq_epi16_mask (__mmask32 __M, __m512i __X, __m512i __Y) +{ + return (__mmask32) __builtin_ia32_cmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, 4, + (__mmask32) __M); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmplt_epi16_mask (__mmask32 __M, __m512i __X, __m512i __Y) +{ + return (__mmask32) __builtin_ia32_cmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, 1, + (__mmask32) __M); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpge_epi16_mask (__mmask32 __M, __m512i __X, __m512i __Y) +{ + return (__mmask32) __builtin_ia32_cmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, 5, + (__mmask32) __M); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmple_epi16_mask (__mmask32 __M, __m512i __X, __m512i __Y) +{ + return (__mmask32) __builtin_ia32_cmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, 2, + (__mmask32) __M); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpneq_epu8_mask (__m512i __X, __m512i __Y) +{ + return (__mmask64) __builtin_ia32_ucmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, 4, + (__mmask64) -1); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmplt_epu8_mask (__m512i __X, __m512i __Y) +{ + return (__mmask64) __builtin_ia32_ucmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, 1, + (__mmask64) -1); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpge_epu8_mask (__m512i __X, __m512i __Y) +{ + return (__mmask64) __builtin_ia32_ucmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, 5, + (__mmask64) -1); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmple_epu8_mask (__m512i __X, __m512i __Y) +{ + return (__mmask64) __builtin_ia32_ucmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, 2, + (__mmask64) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpneq_epu16_mask (__m512i __X, __m512i __Y) +{ + return (__mmask32) __builtin_ia32_ucmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, 4, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmplt_epu16_mask (__m512i __X, __m512i __Y) +{ + return (__mmask32) __builtin_ia32_ucmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, 1, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpge_epu16_mask (__m512i __X, __m512i __Y) +{ + return (__mmask32) __builtin_ia32_ucmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, 5, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmple_epu16_mask (__m512i __X, __m512i __Y) +{ + return (__mmask32) __builtin_ia32_ucmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, 2, + (__mmask32) -1); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpneq_epi8_mask (__m512i __X, __m512i __Y) +{ + return (__mmask64) __builtin_ia32_cmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, 4, + (__mmask64) -1); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmplt_epi8_mask (__m512i __X, __m512i __Y) +{ + return (__mmask64) __builtin_ia32_cmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, 1, + (__mmask64) -1); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpge_epi8_mask (__m512i __X, __m512i __Y) +{ + return (__mmask64) __builtin_ia32_cmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, 5, + (__mmask64) -1); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmple_epi8_mask (__m512i __X, __m512i __Y) +{ + return (__mmask64) __builtin_ia32_cmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, 2, + (__mmask64) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpneq_epi16_mask (__m512i __X, __m512i __Y) +{ + return (__mmask32) __builtin_ia32_cmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, 4, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmplt_epi16_mask (__m512i __X, __m512i __Y) +{ + return (__mmask32) __builtin_ia32_cmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, 1, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpge_epi16_mask (__m512i __X, __m512i __Y) +{ + return (__mmask32) __builtin_ia32_cmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, 5, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmple_epi16_mask (__m512i __X, __m512i __Y) +{ + return (__mmask32) __builtin_ia32_cmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, 2, + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_packs_epi32 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_packssdw512_mask ((__v16si) __A, + (__v16si) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_packs_epi32 (__mmask32 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_packssdw512_mask ((__v16si) __A, + (__v16si) __B, + (__v32hi) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_packs_epi32 (__m512i __W, __mmask32 __M, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_packssdw512_mask ((__v16si) __A, + (__v16si) __B, + (__v32hi) __W, + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_packus_epi32 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_packusdw512_mask ((__v16si) __A, + (__v16si) __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_packus_epi32 (__mmask32 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_packusdw512_mask ((__v16si) __A, + (__v16si) __B, + (__v32hi) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_packus_epi32 (__m512i __W, __mmask32 __M, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_packusdw512_mask ((__v16si) __A, + (__v16si) __B, + (__v32hi) __W, + __M); +} + +#ifdef __OPTIMIZE__ +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kshiftli_mask32 (__mmask32 __A, unsigned int __B) +{ + return (__mmask32) __builtin_ia32_kshiftlisi ((__mmask32) __A, + (__mmask8) __B); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kshiftli_mask64 (__mmask64 __A, unsigned int __B) +{ + return (__mmask64) __builtin_ia32_kshiftlidi ((__mmask64) __A, + (__mmask8) __B); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kshiftri_mask32 (__mmask32 __A, unsigned int __B) +{ + return (__mmask32) __builtin_ia32_kshiftrisi ((__mmask32) __A, + (__mmask8) __B); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kshiftri_mask64 (__mmask64 __A, unsigned int __B) +{ + return (__mmask64) __builtin_ia32_kshiftridi ((__mmask64) __A, + (__mmask8) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_alignr_epi8 (__m512i __A, __m512i __B, const int __N) +{ + return (__m512i) __builtin_ia32_palignr512 ((__v8di) __A, + (__v8di) __B, __N * 8); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_alignr_epi8 (__m512i __W, __mmask64 __U, __m512i __A, + __m512i __B, const int __N) +{ + return (__m512i) __builtin_ia32_palignr512_mask ((__v8di) __A, + (__v8di) __B, + __N * 8, + (__v8di) __W, + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_alignr_epi8 (__mmask64 __U, __m512i __A, __m512i __B, + const int __N) +{ + return (__m512i) __builtin_ia32_palignr512_mask ((__v8di) __A, + (__v8di) __B, + __N * 8, + (__v8di) + _mm512_setzero_si512 (), + (__mmask64) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_dbsad_epu8 (__m512i __A, __m512i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_dbpsadbw512_mask ((__v64qi) __A, + (__v64qi) __B, + __imm, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_dbsad_epu8 (__m512i __W, __mmask32 __U, __m512i __A, + __m512i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_dbpsadbw512_mask ((__v64qi) __A, + (__v64qi) __B, + __imm, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_dbsad_epu8 (__mmask32 __U, __m512i __A, __m512i __B, + const int __imm) +{ + return (__m512i) __builtin_ia32_dbpsadbw512_mask ((__v64qi) __A, + (__v64qi) __B, + __imm, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_srli_epi16 (__m512i __A, const int __imm) +{ + return (__m512i) __builtin_ia32_psrlwi512_mask ((__v32hi) __A, __imm, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_srli_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + const int __imm) +{ + return (__m512i) __builtin_ia32_psrlwi512_mask ((__v32hi) __A, __imm, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_srli_epi16 (__mmask32 __U, __m512i __A, const int __imm) +{ + return (__m512i) __builtin_ia32_psrlwi512_mask ((__v32hi) __A, __imm, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_slli_epi16 (__m512i __A, const int __B) +{ + return (__m512i) __builtin_ia32_psllwi512_mask ((__v32hi) __A, __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_slli_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + const int __B) +{ + return (__m512i) __builtin_ia32_psllwi512_mask ((__v32hi) __A, __B, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_slli_epi16 (__mmask32 __U, __m512i __A, const int __B) +{ + return (__m512i) __builtin_ia32_psllwi512_mask ((__v32hi) __A, __B, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_shufflehi_epi16 (__m512i __A, const int __imm) +{ + return (__m512i) __builtin_ia32_pshufhw512_mask ((__v32hi) __A, + __imm, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_shufflehi_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + const int __imm) +{ + return (__m512i) __builtin_ia32_pshufhw512_mask ((__v32hi) __A, + __imm, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_shufflehi_epi16 (__mmask32 __U, __m512i __A, + const int __imm) +{ + return (__m512i) __builtin_ia32_pshufhw512_mask ((__v32hi) __A, + __imm, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_shufflelo_epi16 (__m512i __A, const int __imm) +{ + return (__m512i) __builtin_ia32_pshuflw512_mask ((__v32hi) __A, + __imm, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_shufflelo_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + const int __imm) +{ + return (__m512i) __builtin_ia32_pshuflw512_mask ((__v32hi) __A, + __imm, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_shufflelo_epi16 (__mmask32 __U, __m512i __A, + const int __imm) +{ + return (__m512i) __builtin_ia32_pshuflw512_mask ((__v32hi) __A, + __imm, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_srai_epi16 (__m512i __A, const int __imm) +{ + return (__m512i) __builtin_ia32_psrawi512_mask ((__v32hi) __A, __imm, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_srai_epi16 (__m512i __W, __mmask32 __U, __m512i __A, + const int __imm) +{ + return (__m512i) __builtin_ia32_psrawi512_mask ((__v32hi) __A, __imm, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_srai_epi16 (__mmask32 __U, __m512i __A, const int __imm) +{ + return (__m512i) __builtin_ia32_psrawi512_mask ((__v32hi) __A, __imm, + (__v32hi) + _mm512_setzero_si512 (), + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_blend_epi16 (__mmask32 __U, __m512i __A, __m512i __W) +{ + return (__m512i) __builtin_ia32_blendmw_512_mask ((__v32hi) __A, + (__v32hi) __W, + (__mmask32) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_blend_epi8 (__mmask64 __U, __m512i __A, __m512i __W) +{ + return (__m512i) __builtin_ia32_blendmb_512_mask ((__v64qi) __A, + (__v64qi) __W, + (__mmask64) __U); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmp_epi16_mask (__mmask32 __U, __m512i __X, __m512i __Y, + const int __P) +{ + return (__mmask32) __builtin_ia32_cmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, __P, + (__mmask32) __U); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmp_epi16_mask (__m512i __X, __m512i __Y, const int __P) +{ + return (__mmask32) __builtin_ia32_cmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, __P, + (__mmask32) -1); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmp_epi8_mask (__mmask32 __U, __m512i __X, __m512i __Y, + const int __P) +{ + return (__mmask64) __builtin_ia32_cmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, __P, + (__mmask64) __U); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmp_epi8_mask (__m512i __X, __m512i __Y, const int __P) +{ + return (__mmask64) __builtin_ia32_cmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, __P, + (__mmask64) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmp_epu16_mask (__mmask32 __U, __m512i __X, __m512i __Y, + const int __P) +{ + return (__mmask32) __builtin_ia32_ucmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, __P, + (__mmask32) __U); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmp_epu16_mask (__m512i __X, __m512i __Y, const int __P) +{ + return (__mmask32) __builtin_ia32_ucmpw512_mask ((__v32hi) __X, + (__v32hi) __Y, __P, + (__mmask32) -1); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmp_epu8_mask (__mmask32 __U, __m512i __X, __m512i __Y, + const int __P) +{ + return (__mmask64) __builtin_ia32_ucmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, __P, + (__mmask64) __U); +} + +extern __inline __mmask64 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmp_epu8_mask (__m512i __X, __m512i __Y, const int __P) +{ + return (__mmask64) __builtin_ia32_ucmpb512_mask ((__v64qi) __X, + (__v64qi) __Y, __P, + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_bslli_epi128 (__m512i __A, const int __N) +{ + return (__m512i) __builtin_ia32_pslldq512 (__A, __N * 8); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_bsrli_epi128 (__m512i __A, const int __N) +{ + return (__m512i) __builtin_ia32_psrldq512 (__A, __N * 8); +} + +#else +#define _kshiftli_mask32(X, Y) \ + ((__mmask32) __builtin_ia32_kshiftlisi ((__mmask32)(X), (__mmask8)(Y))) + +#define _kshiftli_mask64(X, Y) \ + ((__mmask64) __builtin_ia32_kshiftlidi ((__mmask64)(X), (__mmask8)(Y))) + +#define _kshiftri_mask32(X, Y) \ + ((__mmask32) __builtin_ia32_kshiftrisi ((__mmask32)(X), (__mmask8)(Y))) + +#define _kshiftri_mask64(X, Y) \ + ((__mmask64) __builtin_ia32_kshiftridi ((__mmask64)(X), (__mmask8)(Y))) + +#define _mm512_alignr_epi8(X, Y, N) \ + ((__m512i) __builtin_ia32_palignr512 ((__v8di)(__m512i)(X), \ + (__v8di)(__m512i)(Y), \ + (int)(N * 8))) + +#define _mm512_mask_alignr_epi8(W, U, X, Y, N) \ + ((__m512i) __builtin_ia32_palignr512_mask ((__v8di)(__m512i)(X), \ + (__v8di)(__m512i)(Y), (int)(N * 8), \ + (__v8di)(__m512i)(W), (__mmask64)(U))) + +#define _mm512_maskz_alignr_epi8(U, X, Y, N) \ + ((__m512i) __builtin_ia32_palignr512_mask ((__v8di)(__m512i)(X), \ + (__v8di)(__m512i)(Y), (int)(N * 8), \ + (__v8di)(__m512i) \ + _mm512_setzero_si512 (), \ + (__mmask64)(U))) + +#define _mm512_dbsad_epu8(X, Y, C) \ + ((__m512i) __builtin_ia32_dbpsadbw512_mask ((__v64qi)(__m512i) (X), \ + (__v64qi)(__m512i) (Y), (int) (C), \ + (__v32hi)(__m512i) \ + _mm512_setzero_si512 (), \ + (__mmask32)-1)) + +#define _mm512_mask_dbsad_epu8(W, U, X, Y, C) \ + ((__m512i) __builtin_ia32_dbpsadbw512_mask ((__v64qi)(__m512i) (X), \ + (__v64qi)(__m512i) (Y), (int) (C), \ + (__v32hi)(__m512i)(W), \ + (__mmask32)(U))) + +#define _mm512_maskz_dbsad_epu8(U, X, Y, C) \ + ((__m512i) __builtin_ia32_dbpsadbw512_mask ((__v64qi)(__m512i) (X), \ + (__v64qi)(__m512i) (Y), (int) (C), \ + (__v32hi)(__m512i) \ + _mm512_setzero_si512 (), \ + (__mmask32)(U))) + +#define _mm512_srli_epi16(A, B) \ + ((__m512i) __builtin_ia32_psrlwi512_mask ((__v32hi)(__m512i)(A), \ + (int)(B), (__v32hi)_mm512_setzero_si512 (), (__mmask32)-1)) + +#define _mm512_mask_srli_epi16(W, U, A, B) \ + ((__m512i) __builtin_ia32_psrlwi512_mask ((__v32hi)(__m512i)(A), \ + (int)(B), (__v32hi)(__m512i)(W), (__mmask32)(U))) + +#define _mm512_maskz_srli_epi16(U, A, B) \ + ((__m512i) __builtin_ia32_psrlwi512_mask ((__v32hi)(__m512i)(A), \ + (int)(B), (__v32hi)_mm512_setzero_si512 (), (__mmask32)(U))) + +#define _mm512_slli_epi16(X, C) \ + ((__m512i)__builtin_ia32_psllwi512_mask ((__v32hi)(__m512i)(X), (int)(C),\ + (__v32hi)(__m512i)_mm512_setzero_si512 (), \ + (__mmask32)-1)) + +#define _mm512_mask_slli_epi16(W, U, X, C) \ + ((__m512i)__builtin_ia32_psllwi512_mask ((__v32hi)(__m512i)(X), (int)(C),\ + (__v32hi)(__m512i)(W),\ + (__mmask32)(U))) + +#define _mm512_maskz_slli_epi16(U, X, C) \ + ((__m512i)__builtin_ia32_psllwi512_mask ((__v32hi)(__m512i)(X), (int)(C),\ + (__v32hi)(__m512i)_mm512_setzero_si512 (), \ + (__mmask32)(U))) + +#define _mm512_shufflehi_epi16(A, B) \ + ((__m512i) __builtin_ia32_pshufhw512_mask ((__v32hi)(__m512i)(A), (int)(B), \ + (__v32hi)(__m512i) \ + _mm512_setzero_si512 (), \ + (__mmask32)-1)) + +#define _mm512_mask_shufflehi_epi16(W, U, A, B) \ + ((__m512i) __builtin_ia32_pshufhw512_mask ((__v32hi)(__m512i)(A), (int)(B), \ + (__v32hi)(__m512i)(W), \ + (__mmask32)(U))) + +#define _mm512_maskz_shufflehi_epi16(U, A, B) \ + ((__m512i) __builtin_ia32_pshufhw512_mask ((__v32hi)(__m512i)(A), (int)(B), \ + (__v32hi)(__m512i) \ + _mm512_setzero_si512 (), \ + (__mmask32)(U))) + +#define _mm512_shufflelo_epi16(A, B) \ + ((__m512i) __builtin_ia32_pshuflw512_mask ((__v32hi)(__m512i)(A), (int)(B), \ + (__v32hi)(__m512i) \ + _mm512_setzero_si512 (), \ + (__mmask32)-1)) + +#define _mm512_mask_shufflelo_epi16(W, U, A, B) \ + ((__m512i) __builtin_ia32_pshuflw512_mask ((__v32hi)(__m512i)(A), (int)(B), \ + (__v32hi)(__m512i)(W), \ + (__mmask32)(U))) + +#define _mm512_maskz_shufflelo_epi16(U, A, B) \ + ((__m512i) __builtin_ia32_pshuflw512_mask ((__v32hi)(__m512i)(A), (int)(B), \ + (__v32hi)(__m512i) \ + _mm512_setzero_si512 (), \ + (__mmask32)(U))) + +#define _mm512_srai_epi16(A, B) \ + ((__m512i) __builtin_ia32_psrawi512_mask ((__v32hi)(__m512i)(A), \ + (int)(B), (__v32hi)_mm512_setzero_si512 (), (__mmask32)-1)) + +#define _mm512_mask_srai_epi16(W, U, A, B) \ + ((__m512i) __builtin_ia32_psrawi512_mask ((__v32hi)(__m512i)(A), \ + (int)(B), (__v32hi)(__m512i)(W), (__mmask32)(U))) + +#define _mm512_maskz_srai_epi16(U, A, B) \ + ((__m512i) __builtin_ia32_psrawi512_mask ((__v32hi)(__m512i)(A), \ + (int)(B), (__v32hi)_mm512_setzero_si512 (), (__mmask32)(U))) + +#define _mm512_mask_blend_epi16(__U, __A, __W) \ + ((__m512i) __builtin_ia32_blendmw_512_mask ((__v32hi) (__A), \ + (__v32hi) (__W), \ + (__mmask32) (__U))) + +#define _mm512_mask_blend_epi8(__U, __A, __W) \ + ((__m512i) __builtin_ia32_blendmb_512_mask ((__v64qi) (__A), \ + (__v64qi) (__W), \ + (__mmask64) (__U))) + +#define _mm512_cmp_epi16_mask(X, Y, P) \ + ((__mmask32) __builtin_ia32_cmpw512_mask ((__v32hi)(__m512i)(X), \ + (__v32hi)(__m512i)(Y), (int)(P),\ + (__mmask32)(-1))) + +#define _mm512_cmp_epi8_mask(X, Y, P) \ + ((__mmask64) __builtin_ia32_cmpb512_mask ((__v64qi)(__m512i)(X), \ + (__v64qi)(__m512i)(Y), (int)(P),\ + (__mmask64)(-1))) + +#define _mm512_cmp_epu16_mask(X, Y, P) \ + ((__mmask32) __builtin_ia32_ucmpw512_mask ((__v32hi)(__m512i)(X), \ + (__v32hi)(__m512i)(Y), (int)(P),\ + (__mmask32)(-1))) + +#define _mm512_cmp_epu8_mask(X, Y, P) \ + ((__mmask64) __builtin_ia32_ucmpb512_mask ((__v64qi)(__m512i)(X), \ + (__v64qi)(__m512i)(Y), (int)(P),\ + (__mmask64)(-1))) + +#define _mm512_mask_cmp_epi16_mask(M, X, Y, P) \ + ((__mmask32) __builtin_ia32_cmpw512_mask ((__v32hi)(__m512i)(X), \ + (__v32hi)(__m512i)(Y), (int)(P),\ + (__mmask32)(M))) + +#define _mm512_mask_cmp_epi8_mask(M, X, Y, P) \ + ((__mmask64) __builtin_ia32_cmpb512_mask ((__v64qi)(__m512i)(X), \ + (__v64qi)(__m512i)(Y), (int)(P),\ + (__mmask64)(M))) + +#define _mm512_mask_cmp_epu16_mask(M, X, Y, P) \ + ((__mmask32) __builtin_ia32_ucmpw512_mask ((__v32hi)(__m512i)(X), \ + (__v32hi)(__m512i)(Y), (int)(P),\ + (__mmask32)(M))) + +#define _mm512_mask_cmp_epu8_mask(M, X, Y, P) \ + ((__mmask64) __builtin_ia32_ucmpb512_mask ((__v64qi)(__m512i)(X), \ + (__v64qi)(__m512i)(Y), (int)(P),\ + (__mmask64)(M))) + +#define _mm512_bslli_epi128(A, N) \ + ((__m512i)__builtin_ia32_pslldq512 ((__m512i)(A), (int)(N) * 8)) + +#define _mm512_bsrli_epi128(A, N) \ + ((__m512i)__builtin_ia32_psrldq512 ((__m512i)(A), (int)(N) * 8)) + +#endif + +#ifdef __DISABLE_AVX512BW__ +#undef __DISABLE_AVX512BW__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX512BW__ */ + +#endif /* _AVX512BWINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512cdintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512cdintrin.h new file mode 100644 index 0000000000..7fc5851cf1 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512cdintrin.h @@ -0,0 +1,184 @@ +/* Copyright (C) 2013-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +#error "Never use directly; include instead." +#endif + +#ifndef _AVX512CDINTRIN_H_INCLUDED +#define _AVX512CDINTRIN_H_INCLUDED + +#ifndef __AVX512CD__ +#pragma GCC push_options +//#pragma GCC target("avx512cd") +#define __DISABLE_AVX512CD__ +#endif /* __AVX512CD__ */ + +/* Internal data types for implementing the intrinsics. */ +typedef long long __v8di __attribute__ ((__vector_size__ (64))); +typedef int __v16si __attribute__ ((__vector_size__ (64))); + +/* The Intel API is flexible enough that we must allow aliasing with other + vector types, and their scalar components. */ +typedef long long __m512i __attribute__ ((__vector_size__ (64), __may_alias__)); +typedef double __m512d __attribute__ ((__vector_size__ (64), __may_alias__)); + +typedef unsigned char __mmask8; +typedef unsigned short __mmask16; + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_conflict_epi32 (__m512i __A) +{ + return (__m512i) + __builtin_ia32_vpconflictsi_512_mask ((__v16si) __A, + (__v16si) _mm512_setzero_si512 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_conflict_epi32 (__m512i __W, __mmask16 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_vpconflictsi_512_mask ((__v16si) __A, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_conflict_epi32 (__mmask16 __U, __m512i __A) +{ + return (__m512i) + __builtin_ia32_vpconflictsi_512_mask ((__v16si) __A, + (__v16si) _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_conflict_epi64 (__m512i __A) +{ + return (__m512i) + __builtin_ia32_vpconflictdi_512_mask ((__v8di) __A, + (__v8di) _mm512_setzero_si512 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_conflict_epi64 (__m512i __W, __mmask8 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_vpconflictdi_512_mask ((__v8di) __A, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_conflict_epi64 (__mmask8 __U, __m512i __A) +{ + return (__m512i) + __builtin_ia32_vpconflictdi_512_mask ((__v8di) __A, + (__v8di) _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_lzcnt_epi64 (__m512i __A) +{ + return (__m512i) + __builtin_ia32_vplzcntq_512_mask ((__v8di) __A, + (__v8di) _mm512_setzero_si512 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_lzcnt_epi64 (__m512i __W, __mmask8 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_vplzcntq_512_mask ((__v8di) __A, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_lzcnt_epi64 (__mmask8 __U, __m512i __A) +{ + return (__m512i) + __builtin_ia32_vplzcntq_512_mask ((__v8di) __A, + (__v8di) _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_lzcnt_epi32 (__m512i __A) +{ + return (__m512i) + __builtin_ia32_vplzcntd_512_mask ((__v16si) __A, + (__v16si) _mm512_setzero_si512 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_lzcnt_epi32 (__m512i __W, __mmask16 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_vplzcntd_512_mask ((__v16si) __A, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_lzcnt_epi32 (__mmask16 __U, __m512i __A) +{ + return (__m512i) + __builtin_ia32_vplzcntd_512_mask ((__v16si) __A, + (__v16si) _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcastmb_epi64 (__mmask8 __A) +{ + return (__m512i) __builtin_ia32_broadcastmb512 (__A); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcastmw_epi32 (__mmask16 __A) +{ + return (__m512i) __builtin_ia32_broadcastmw512 (__A); +} + +#ifdef __DISABLE_AVX512CD__ +#undef __DISABLE_AVX512CD__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX512CD__ */ + +#endif /* _AVX512CDINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512dqintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512dqintrin.h new file mode 100644 index 0000000000..8da9478f5f --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512dqintrin.h @@ -0,0 +1,2466 @@ +/* Copyright (C) 2014-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +#error "Never use directly; include instead." +#endif + +#ifndef _AVX512DQINTRIN_H_INCLUDED +#define _AVX512DQINTRIN_H_INCLUDED + +#ifndef __AVX512DQ__ +#pragma GCC push_options +//#pragma GCC target("avx512dq") +#define __DISABLE_AVX512DQ__ +#endif /* __AVX512DQ__ */ + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_ktest_mask8_u8 (__mmask8 __A, __mmask8 __B, unsigned char *__CF) +{ + *__CF = (unsigned char) __builtin_ia32_ktestcqi (__A, __B); + return (unsigned char) __builtin_ia32_ktestzqi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_ktestz_mask8_u8 (__mmask8 __A, __mmask8 __B) +{ + return (unsigned char) __builtin_ia32_ktestzqi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_ktestc_mask8_u8 (__mmask8 __A, __mmask8 __B) +{ + return (unsigned char) __builtin_ia32_ktestcqi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_ktest_mask16_u8 (__mmask16 __A, __mmask16 __B, unsigned char *__CF) +{ + *__CF = (unsigned char) __builtin_ia32_ktestchi (__A, __B); + return (unsigned char) __builtin_ia32_ktestzhi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_ktestz_mask16_u8 (__mmask16 __A, __mmask16 __B) +{ + return (unsigned char) __builtin_ia32_ktestzhi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_ktestc_mask16_u8 (__mmask16 __A, __mmask16 __B) +{ + return (unsigned char) __builtin_ia32_ktestchi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kortest_mask8_u8 (__mmask8 __A, __mmask8 __B, unsigned char *__CF) +{ + *__CF = (unsigned char) __builtin_ia32_kortestcqi (__A, __B); + return (unsigned char) __builtin_ia32_kortestzqi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kortestz_mask8_u8 (__mmask8 __A, __mmask8 __B) +{ + return (unsigned char) __builtin_ia32_kortestzqi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kortestc_mask8_u8 (__mmask8 __A, __mmask8 __B) +{ + return (unsigned char) __builtin_ia32_kortestcqi (__A, __B); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kadd_mask8 (__mmask8 __A, __mmask8 __B) +{ + return (__mmask8) __builtin_ia32_kaddqi ((__mmask8) __A, (__mmask8) __B); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kadd_mask16 (__mmask16 __A, __mmask16 __B) +{ + return (__mmask16) __builtin_ia32_kaddhi ((__mmask16) __A, (__mmask16) __B); +} + +extern __inline unsigned int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_cvtmask8_u32 (__mmask8 __A) +{ + return (unsigned int) __builtin_ia32_kmovb ((__mmask8 ) __A); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_cvtu32_mask8 (unsigned int __A) +{ + return (__mmask8) __builtin_ia32_kmovb ((__mmask8) __A); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_load_mask8 (__mmask8 *__A) +{ + return (__mmask8) __builtin_ia32_kmovb (*(__mmask8 *) __A); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_store_mask8 (__mmask8 *__A, __mmask8 __B) +{ + *(__mmask8 *) __A = __builtin_ia32_kmovb (__B); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_knot_mask8 (__mmask8 __A) +{ + return (__mmask8) __builtin_ia32_knotqi ((__mmask8) __A); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kor_mask8 (__mmask8 __A, __mmask8 __B) +{ + return (__mmask8) __builtin_ia32_korqi ((__mmask8) __A, (__mmask8) __B); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kxnor_mask8 (__mmask8 __A, __mmask8 __B) +{ + return (__mmask8) __builtin_ia32_kxnorqi ((__mmask8) __A, (__mmask8) __B); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kxor_mask8 (__mmask8 __A, __mmask8 __B) +{ + return (__mmask8) __builtin_ia32_kxorqi ((__mmask8) __A, (__mmask8) __B); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kand_mask8 (__mmask8 __A, __mmask8 __B) +{ + return (__mmask8) __builtin_ia32_kandqi ((__mmask8) __A, (__mmask8) __B); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kandn_mask8 (__mmask8 __A, __mmask8 __B) +{ + return (__mmask8) __builtin_ia32_kandnqi ((__mmask8) __A, (__mmask8) __B); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcast_f64x2 (__m128d __A) +{ + return (__m512d) + __builtin_ia32_broadcastf64x2_512_mask ((__v2df) __A, + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_broadcast_f64x2 (__m512d __O, __mmask8 __M, __m128d __A) +{ + return (__m512d) __builtin_ia32_broadcastf64x2_512_mask ((__v2df) + __A, + (__v8df) + __O, __M); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_broadcast_f64x2 (__mmask8 __M, __m128d __A) +{ + return (__m512d) __builtin_ia32_broadcastf64x2_512_mask ((__v2df) + __A, + (__v8df) + _mm512_setzero_ps (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcast_i64x2 (__m128i __A) +{ + return (__m512i) + __builtin_ia32_broadcasti64x2_512_mask ((__v2di) __A, + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_broadcast_i64x2 (__m512i __O, __mmask8 __M, __m128i __A) +{ + return (__m512i) __builtin_ia32_broadcasti64x2_512_mask ((__v2di) + __A, + (__v8di) + __O, __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_broadcast_i64x2 (__mmask8 __M, __m128i __A) +{ + return (__m512i) __builtin_ia32_broadcasti64x2_512_mask ((__v2di) + __A, + (__v8di) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcast_f32x2 (__m128 __A) +{ + return (__m512) + __builtin_ia32_broadcastf32x2_512_mask ((__v4sf) __A, + (__v16sf)_mm512_undefined_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_broadcast_f32x2 (__m512 __O, __mmask16 __M, __m128 __A) +{ + return (__m512) __builtin_ia32_broadcastf32x2_512_mask ((__v4sf) __A, + (__v16sf) + __O, __M); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_broadcast_f32x2 (__mmask16 __M, __m128 __A) +{ + return (__m512) __builtin_ia32_broadcastf32x2_512_mask ((__v4sf) __A, + (__v16sf) + _mm512_setzero_ps (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcast_i32x2 (__m128i __A) +{ + return (__m512i) + __builtin_ia32_broadcasti32x2_512_mask ((__v4si) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_broadcast_i32x2 (__m512i __O, __mmask16 __M, __m128i __A) +{ + return (__m512i) __builtin_ia32_broadcasti32x2_512_mask ((__v4si) + __A, + (__v16si) + __O, __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_broadcast_i32x2 (__mmask16 __M, __m128i __A) +{ + return (__m512i) __builtin_ia32_broadcasti32x2_512_mask ((__v4si) + __A, + (__v16si) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcast_f32x8 (__m256 __A) +{ + return (__m512) + __builtin_ia32_broadcastf32x8_512_mask ((__v8sf) __A, + _mm512_undefined_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_broadcast_f32x8 (__m512 __O, __mmask16 __M, __m256 __A) +{ + return (__m512) __builtin_ia32_broadcastf32x8_512_mask ((__v8sf) __A, + (__v16sf)__O, + __M); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_broadcast_f32x8 (__mmask16 __M, __m256 __A) +{ + return (__m512) __builtin_ia32_broadcastf32x8_512_mask ((__v8sf) __A, + (__v16sf) + _mm512_setzero_ps (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcast_i32x8 (__m256i __A) +{ + return (__m512i) + __builtin_ia32_broadcasti32x8_512_mask ((__v8si) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_broadcast_i32x8 (__m512i __O, __mmask16 __M, __m256i __A) +{ + return (__m512i) __builtin_ia32_broadcasti32x8_512_mask ((__v8si) + __A, + (__v16si)__O, + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_broadcast_i32x8 (__mmask16 __M, __m256i __A) +{ + return (__m512i) __builtin_ia32_broadcasti32x8_512_mask ((__v8si) + __A, + (__v16si) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mullo_epi64 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v8du) __A * (__v8du) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mullo_epi64 (__m512i __W, __mmask8 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_pmullq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mullo_epi64 (__mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmullq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_xor_pd (__m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_xorpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_xor_pd (__m512d __W, __mmask8 __U, __m512d __A, + __m512d __B) +{ + return (__m512d) __builtin_ia32_xorpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_xor_pd (__mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_xorpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_xor_ps (__m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_xorps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_xor_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_xorps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_xor_ps (__mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_xorps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_or_pd (__m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_orpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_or_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_orpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_or_pd (__mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_orpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_or_ps (__m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_orps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_or_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_orps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_or_ps (__mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_orps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_and_pd (__m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_andpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_and_pd (__m512d __W, __mmask8 __U, __m512d __A, + __m512d __B) +{ + return (__m512d) __builtin_ia32_andpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_and_pd (__mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_andpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_and_ps (__m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_andps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_and_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_andps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_and_ps (__mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_andps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_andnot_pd (__m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_andnpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_andnot_pd (__m512d __W, __mmask8 __U, __m512d __A, + __m512d __B) +{ + return (__m512d) __builtin_ia32_andnpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_andnot_pd (__mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_andnpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_andnot_ps (__m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_andnps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_andnot_ps (__m512 __W, __mmask16 __U, __m512 __A, + __m512 __B) +{ + return (__m512) __builtin_ia32_andnps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_andnot_ps (__mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_andnps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_movepi32_mask (__m512i __A) +{ + return (__mmask16) __builtin_ia32_cvtd2mask512 ((__v16si) __A); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_movepi64_mask (__m512i __A) +{ + return (__mmask8) __builtin_ia32_cvtq2mask512 ((__v8di) __A); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_movm_epi32 (__mmask16 __A) +{ + return (__m512i) __builtin_ia32_cvtmask2d512 (__A); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_movm_epi64 (__mmask8 __A) +{ + return (__m512i) __builtin_ia32_cvtmask2q512 (__A); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvttpd_epi64 (__m512d __A) +{ + return (__m512i) __builtin_ia32_cvttpd2qq512_mask ((__v8df) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvttpd_epi64 (__m512i __W, __mmask8 __U, __m512d __A) +{ + return (__m512i) __builtin_ia32_cvttpd2qq512_mask ((__v8df) __A, + (__v8di) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvttpd_epi64 (__mmask8 __U, __m512d __A) +{ + return (__m512i) __builtin_ia32_cvttpd2qq512_mask ((__v8df) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvttpd_epu64 (__m512d __A) +{ + return (__m512i) __builtin_ia32_cvttpd2uqq512_mask ((__v8df) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvttpd_epu64 (__m512i __W, __mmask8 __U, __m512d __A) +{ + return (__m512i) __builtin_ia32_cvttpd2uqq512_mask ((__v8df) __A, + (__v8di) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvttpd_epu64 (__mmask8 __U, __m512d __A) +{ + return (__m512i) __builtin_ia32_cvttpd2uqq512_mask ((__v8df) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvttps_epi64 (__m256 __A) +{ + return (__m512i) __builtin_ia32_cvttps2qq512_mask ((__v8sf) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvttps_epi64 (__m512i __W, __mmask8 __U, __m256 __A) +{ + return (__m512i) __builtin_ia32_cvttps2qq512_mask ((__v8sf) __A, + (__v8di) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvttps_epi64 (__mmask8 __U, __m256 __A) +{ + return (__m512i) __builtin_ia32_cvttps2qq512_mask ((__v8sf) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvttps_epu64 (__m256 __A) +{ + return (__m512i) __builtin_ia32_cvttps2uqq512_mask ((__v8sf) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvttps_epu64 (__m512i __W, __mmask8 __U, __m256 __A) +{ + return (__m512i) __builtin_ia32_cvttps2uqq512_mask ((__v8sf) __A, + (__v8di) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvttps_epu64 (__mmask8 __U, __m256 __A) +{ + return (__m512i) __builtin_ia32_cvttps2uqq512_mask ((__v8sf) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtpd_epi64 (__m512d __A) +{ + return (__m512i) __builtin_ia32_cvtpd2qq512_mask ((__v8df) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtpd_epi64 (__m512i __W, __mmask8 __U, __m512d __A) +{ + return (__m512i) __builtin_ia32_cvtpd2qq512_mask ((__v8df) __A, + (__v8di) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtpd_epi64 (__mmask8 __U, __m512d __A) +{ + return (__m512i) __builtin_ia32_cvtpd2qq512_mask ((__v8df) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtpd_epu64 (__m512d __A) +{ + return (__m512i) __builtin_ia32_cvtpd2uqq512_mask ((__v8df) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtpd_epu64 (__m512i __W, __mmask8 __U, __m512d __A) +{ + return (__m512i) __builtin_ia32_cvtpd2uqq512_mask ((__v8df) __A, + (__v8di) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtpd_epu64 (__mmask8 __U, __m512d __A) +{ + return (__m512i) __builtin_ia32_cvtpd2uqq512_mask ((__v8df) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtps_epi64 (__m256 __A) +{ + return (__m512i) __builtin_ia32_cvtps2qq512_mask ((__v8sf) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtps_epi64 (__m512i __W, __mmask8 __U, __m256 __A) +{ + return (__m512i) __builtin_ia32_cvtps2qq512_mask ((__v8sf) __A, + (__v8di) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtps_epi64 (__mmask8 __U, __m256 __A) +{ + return (__m512i) __builtin_ia32_cvtps2qq512_mask ((__v8sf) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtps_epu64 (__m256 __A) +{ + return (__m512i) __builtin_ia32_cvtps2uqq512_mask ((__v8sf) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtps_epu64 (__m512i __W, __mmask8 __U, __m256 __A) +{ + return (__m512i) __builtin_ia32_cvtps2uqq512_mask ((__v8sf) __A, + (__v8di) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtps_epu64 (__mmask8 __U, __m256 __A) +{ + return (__m512i) __builtin_ia32_cvtps2uqq512_mask ((__v8sf) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepi64_ps (__m512i __A) +{ + return (__m256) __builtin_ia32_cvtqq2ps512_mask ((__v8di) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi64_ps (__m256 __W, __mmask8 __U, __m512i __A) +{ + return (__m256) __builtin_ia32_cvtqq2ps512_mask ((__v8di) __A, + (__v8sf) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepi64_ps (__mmask8 __U, __m512i __A) +{ + return (__m256) __builtin_ia32_cvtqq2ps512_mask ((__v8di) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepu64_ps (__m512i __A) +{ + return (__m256) __builtin_ia32_cvtuqq2ps512_mask ((__v8di) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepu64_ps (__m256 __W, __mmask8 __U, __m512i __A) +{ + return (__m256) __builtin_ia32_cvtuqq2ps512_mask ((__v8di) __A, + (__v8sf) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepu64_ps (__mmask8 __U, __m512i __A) +{ + return (__m256) __builtin_ia32_cvtuqq2ps512_mask ((__v8di) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepi64_pd (__m512i __A) +{ + return (__m512d) __builtin_ia32_cvtqq2pd512_mask ((__v8di) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi64_pd (__m512d __W, __mmask8 __U, __m512i __A) +{ + return (__m512d) __builtin_ia32_cvtqq2pd512_mask ((__v8di) __A, + (__v8df) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepi64_pd (__mmask8 __U, __m512i __A) +{ + return (__m512d) __builtin_ia32_cvtqq2pd512_mask ((__v8di) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepu64_pd (__m512i __A) +{ + return (__m512d) __builtin_ia32_cvtuqq2pd512_mask ((__v8di) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepu64_pd (__m512d __W, __mmask8 __U, __m512i __A) +{ + return (__m512d) __builtin_ia32_cvtuqq2pd512_mask ((__v8di) __A, + (__v8df) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepu64_pd (__mmask8 __U, __m512i __A) +{ + return (__m512d) __builtin_ia32_cvtuqq2pd512_mask ((__v8di) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +#ifdef __OPTIMIZE__ +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kshiftli_mask8 (__mmask8 __A, unsigned int __B) +{ + return (__mmask8) __builtin_ia32_kshiftliqi ((__mmask8) __A, (__mmask8) __B); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kshiftri_mask8 (__mmask8 __A, unsigned int __B) +{ + return (__mmask8) __builtin_ia32_kshiftriqi ((__mmask8) __A, (__mmask8) __B); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_range_pd (__m512d __A, __m512d __B, int __C) +{ + return (__m512d) __builtin_ia32_rangepd512_mask ((__v8df) __A, + (__v8df) __B, __C, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_range_pd (__m512d __W, __mmask8 __U, + __m512d __A, __m512d __B, int __C) +{ + return (__m512d) __builtin_ia32_rangepd512_mask ((__v8df) __A, + (__v8df) __B, __C, + (__v8df) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_range_pd (__mmask8 __U, __m512d __A, __m512d __B, int __C) +{ + return (__m512d) __builtin_ia32_rangepd512_mask ((__v8df) __A, + (__v8df) __B, __C, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_range_ps (__m512 __A, __m512 __B, int __C) +{ + return (__m512) __builtin_ia32_rangeps512_mask ((__v16sf) __A, + (__v16sf) __B, __C, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_range_ps (__m512 __W, __mmask16 __U, + __m512 __A, __m512 __B, int __C) +{ + return (__m512) __builtin_ia32_rangeps512_mask ((__v16sf) __A, + (__v16sf) __B, __C, + (__v16sf) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_range_ps (__mmask16 __U, __m512 __A, __m512 __B, int __C) +{ + return (__m512) __builtin_ia32_rangeps512_mask ((__v16sf) __A, + (__v16sf) __B, __C, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_reduce_sd (__m128d __A, __m128d __B, int __C) +{ + return (__m128d) __builtin_ia32_reducesd ((__v2df) __A, + (__v2df) __B, __C); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_reduce_ss (__m128 __A, __m128 __B, int __C) +{ + return (__m128) __builtin_ia32_reducess ((__v4sf) __A, + (__v4sf) __B, __C); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_range_sd (__m128d __A, __m128d __B, int __C) +{ + return (__m128d) __builtin_ia32_rangesd128_round ((__v2df) __A, + (__v2df) __B, __C, + _MM_FROUND_CUR_DIRECTION); +} + + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_range_ss (__m128 __A, __m128 __B, int __C) +{ + return (__m128) __builtin_ia32_rangess128_round ((__v4sf) __A, + (__v4sf) __B, __C, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_range_round_sd (__m128d __A, __m128d __B, int __C, const int __R) +{ + return (__m128d) __builtin_ia32_rangesd128_round ((__v2df) __A, + (__v2df) __B, __C, + __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_range_round_ss (__m128 __A, __m128 __B, int __C, const int __R) +{ + return (__m128) __builtin_ia32_rangess128_round ((__v4sf) __A, + (__v4sf) __B, __C, + __R); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fpclass_ss_mask (__m128 __A, const int __imm) +{ + return (__mmask8) __builtin_ia32_fpclassss ((__v4sf) __A, __imm); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fpclass_sd_mask (__m128d __A, const int __imm) +{ + return (__mmask8) __builtin_ia32_fpclasssd ((__v2df) __A, __imm); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtt_roundpd_epi64 (__m512d __A, const int __R) +{ + return (__m512i) __builtin_ia32_cvttpd2qq512_mask ((__v8df) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtt_roundpd_epi64 (__m512i __W, __mmask8 __U, __m512d __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvttpd2qq512_mask ((__v8df) __A, + (__v8di) __W, + (__mmask8) __U, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtt_roundpd_epi64 (__mmask8 __U, __m512d __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvttpd2qq512_mask ((__v8df) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtt_roundpd_epu64 (__m512d __A, const int __R) +{ + return (__m512i) __builtin_ia32_cvttpd2uqq512_mask ((__v8df) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtt_roundpd_epu64 (__m512i __W, __mmask8 __U, __m512d __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvttpd2uqq512_mask ((__v8df) __A, + (__v8di) __W, + (__mmask8) __U, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtt_roundpd_epu64 (__mmask8 __U, __m512d __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvttpd2uqq512_mask ((__v8df) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtt_roundps_epi64 (__m256 __A, const int __R) +{ + return (__m512i) __builtin_ia32_cvttps2qq512_mask ((__v8sf) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtt_roundps_epi64 (__m512i __W, __mmask8 __U, __m256 __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvttps2qq512_mask ((__v8sf) __A, + (__v8di) __W, + (__mmask8) __U, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtt_roundps_epi64 (__mmask8 __U, __m256 __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvttps2qq512_mask ((__v8sf) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtt_roundps_epu64 (__m256 __A, const int __R) +{ + return (__m512i) __builtin_ia32_cvttps2uqq512_mask ((__v8sf) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtt_roundps_epu64 (__m512i __W, __mmask8 __U, __m256 __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvttps2uqq512_mask ((__v8sf) __A, + (__v8di) __W, + (__mmask8) __U, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtt_roundps_epu64 (__mmask8 __U, __m256 __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvttps2uqq512_mask ((__v8sf) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundpd_epi64 (__m512d __A, const int __R) +{ + return (__m512i) __builtin_ia32_cvtpd2qq512_mask ((__v8df) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundpd_epi64 (__m512i __W, __mmask8 __U, __m512d __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvtpd2qq512_mask ((__v8df) __A, + (__v8di) __W, + (__mmask8) __U, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundpd_epi64 (__mmask8 __U, __m512d __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvtpd2qq512_mask ((__v8df) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundpd_epu64 (__m512d __A, const int __R) +{ + return (__m512i) __builtin_ia32_cvtpd2uqq512_mask ((__v8df) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundpd_epu64 (__m512i __W, __mmask8 __U, __m512d __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvtpd2uqq512_mask ((__v8df) __A, + (__v8di) __W, + (__mmask8) __U, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundpd_epu64 (__mmask8 __U, __m512d __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvtpd2uqq512_mask ((__v8df) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundps_epi64 (__m256 __A, const int __R) +{ + return (__m512i) __builtin_ia32_cvtps2qq512_mask ((__v8sf) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundps_epi64 (__m512i __W, __mmask8 __U, __m256 __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvtps2qq512_mask ((__v8sf) __A, + (__v8di) __W, + (__mmask8) __U, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundps_epi64 (__mmask8 __U, __m256 __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvtps2qq512_mask ((__v8sf) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundps_epu64 (__m256 __A, const int __R) +{ + return (__m512i) __builtin_ia32_cvtps2uqq512_mask ((__v8sf) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundps_epu64 (__m512i __W, __mmask8 __U, __m256 __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvtps2uqq512_mask ((__v8sf) __A, + (__v8di) __W, + (__mmask8) __U, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundps_epu64 (__mmask8 __U, __m256 __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvtps2uqq512_mask ((__v8sf) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U, + __R); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundepi64_ps (__m512i __A, const int __R) +{ + return (__m256) __builtin_ia32_cvtqq2ps512_mask ((__v8di) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) -1, + __R); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundepi64_ps (__m256 __W, __mmask8 __U, __m512i __A, + const int __R) +{ + return (__m256) __builtin_ia32_cvtqq2ps512_mask ((__v8di) __A, + (__v8sf) __W, + (__mmask8) __U, + __R); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundepi64_ps (__mmask8 __U, __m512i __A, + const int __R) +{ + return (__m256) __builtin_ia32_cvtqq2ps512_mask ((__v8di) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U, + __R); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundepu64_ps (__m512i __A, const int __R) +{ + return (__m256) __builtin_ia32_cvtuqq2ps512_mask ((__v8di) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) -1, + __R); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundepu64_ps (__m256 __W, __mmask8 __U, __m512i __A, + const int __R) +{ + return (__m256) __builtin_ia32_cvtuqq2ps512_mask ((__v8di) __A, + (__v8sf) __W, + (__mmask8) __U, + __R); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundepu64_ps (__mmask8 __U, __m512i __A, + const int __R) +{ + return (__m256) __builtin_ia32_cvtuqq2ps512_mask ((__v8di) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U, + __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundepi64_pd (__m512i __A, const int __R) +{ + return (__m512d) __builtin_ia32_cvtqq2pd512_mask ((__v8di) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) -1, + __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundepi64_pd (__m512d __W, __mmask8 __U, __m512i __A, + const int __R) +{ + return (__m512d) __builtin_ia32_cvtqq2pd512_mask ((__v8di) __A, + (__v8df) __W, + (__mmask8) __U, + __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundepi64_pd (__mmask8 __U, __m512i __A, + const int __R) +{ + return (__m512d) __builtin_ia32_cvtqq2pd512_mask ((__v8di) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, + __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundepu64_pd (__m512i __A, const int __R) +{ + return (__m512d) __builtin_ia32_cvtuqq2pd512_mask ((__v8di) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) -1, + __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundepu64_pd (__m512d __W, __mmask8 __U, __m512i __A, + const int __R) +{ + return (__m512d) __builtin_ia32_cvtuqq2pd512_mask ((__v8di) __A, + (__v8df) __W, + (__mmask8) __U, + __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundepu64_pd (__mmask8 __U, __m512i __A, + const int __R) +{ + return (__m512d) __builtin_ia32_cvtuqq2pd512_mask ((__v8di) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, + __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_pd (__m512d __A, int __B) +{ + return (__m512d) __builtin_ia32_reducepd512_mask ((__v8df) __A, __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_pd (__m512d __W, __mmask8 __U, __m512d __A, int __B) +{ + return (__m512d) __builtin_ia32_reducepd512_mask ((__v8df) __A, __B, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_reduce_pd (__mmask8 __U, __m512d __A, int __B) +{ + return (__m512d) __builtin_ia32_reducepd512_mask ((__v8df) __A, __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_ps (__m512 __A, int __B) +{ + return (__m512) __builtin_ia32_reduceps512_mask ((__v16sf) __A, __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_ps (__m512 __W, __mmask16 __U, __m512 __A, int __B) +{ + return (__m512) __builtin_ia32_reduceps512_mask ((__v16sf) __A, __B, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_reduce_ps (__mmask16 __U, __m512 __A, int __B) +{ + return (__m512) __builtin_ia32_reduceps512_mask ((__v16sf) __A, __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_extractf32x8_ps (__m512 __A, const int __imm) +{ + return (__m256) __builtin_ia32_extractf32x8_mask ((__v16sf) __A, + __imm, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_extractf32x8_ps (__m256 __W, __mmask8 __U, __m512 __A, + const int __imm) +{ + return (__m256) __builtin_ia32_extractf32x8_mask ((__v16sf) __A, + __imm, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_extractf32x8_ps (__mmask8 __U, __m512 __A, + const int __imm) +{ + return (__m256) __builtin_ia32_extractf32x8_mask ((__v16sf) __A, + __imm, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_extractf64x2_pd (__m512d __A, const int __imm) +{ + return (__m128d) __builtin_ia32_extractf64x2_512_mask ((__v8df) __A, + __imm, + (__v2df) + _mm_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_extractf64x2_pd (__m128d __W, __mmask8 __U, __m512d __A, + const int __imm) +{ + return (__m128d) __builtin_ia32_extractf64x2_512_mask ((__v8df) __A, + __imm, + (__v2df) __W, + (__mmask8) + __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_extractf64x2_pd (__mmask8 __U, __m512d __A, + const int __imm) +{ + return (__m128d) __builtin_ia32_extractf64x2_512_mask ((__v8df) __A, + __imm, + (__v2df) + _mm_setzero_pd (), + (__mmask8) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_extracti32x8_epi32 (__m512i __A, const int __imm) +{ + return (__m256i) __builtin_ia32_extracti32x8_mask ((__v16si) __A, + __imm, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_extracti32x8_epi32 (__m256i __W, __mmask8 __U, __m512i __A, + const int __imm) +{ + return (__m256i) __builtin_ia32_extracti32x8_mask ((__v16si) __A, + __imm, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_extracti32x8_epi32 (__mmask8 __U, __m512i __A, + const int __imm) +{ + return (__m256i) __builtin_ia32_extracti32x8_mask ((__v16si) __A, + __imm, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_extracti64x2_epi64 (__m512i __A, const int __imm) +{ + return (__m128i) __builtin_ia32_extracti64x2_512_mask ((__v8di) __A, + __imm, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_extracti64x2_epi64 (__m128i __W, __mmask8 __U, __m512i __A, + const int __imm) +{ + return (__m128i) __builtin_ia32_extracti64x2_512_mask ((__v8di) __A, + __imm, + (__v2di) __W, + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_extracti64x2_epi64 (__mmask8 __U, __m512i __A, + const int __imm) +{ + return (__m128i) __builtin_ia32_extracti64x2_512_mask ((__v8di) __A, + __imm, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) + __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_range_round_pd (__m512d __A, __m512d __B, int __C, + const int __R) +{ + return (__m512d) __builtin_ia32_rangepd512_mask ((__v8df) __A, + (__v8df) __B, __C, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) -1, + __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_range_round_pd (__m512d __W, __mmask8 __U, + __m512d __A, __m512d __B, int __C, + const int __R) +{ + return (__m512d) __builtin_ia32_rangepd512_mask ((__v8df) __A, + (__v8df) __B, __C, + (__v8df) __W, + (__mmask8) __U, + __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_range_round_pd (__mmask8 __U, __m512d __A, __m512d __B, + int __C, const int __R) +{ + return (__m512d) __builtin_ia32_rangepd512_mask ((__v8df) __A, + (__v8df) __B, __C, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, + __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_range_round_ps (__m512 __A, __m512 __B, int __C, const int __R) +{ + return (__m512) __builtin_ia32_rangeps512_mask ((__v16sf) __A, + (__v16sf) __B, __C, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) -1, + __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_range_round_ps (__m512 __W, __mmask16 __U, + __m512 __A, __m512 __B, int __C, + const int __R) +{ + return (__m512) __builtin_ia32_rangeps512_mask ((__v16sf) __A, + (__v16sf) __B, __C, + (__v16sf) __W, + (__mmask16) __U, + __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_range_round_ps (__mmask16 __U, __m512 __A, __m512 __B, + int __C, const int __R) +{ + return (__m512) __builtin_ia32_rangeps512_mask ((__v16sf) __A, + (__v16sf) __B, __C, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, + __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_inserti32x8 (__m512i __A, __m256i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_inserti32x8_mask ((__v16si) __A, + (__v8si) __B, + __imm, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_inserti32x8 (__m512i __W, __mmask16 __U, __m512i __A, + __m256i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_inserti32x8_mask ((__v16si) __A, + (__v8si) __B, + __imm, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_inserti32x8 (__mmask16 __U, __m512i __A, __m256i __B, + const int __imm) +{ + return (__m512i) __builtin_ia32_inserti32x8_mask ((__v16si) __A, + (__v8si) __B, + __imm, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_insertf32x8 (__m512 __A, __m256 __B, const int __imm) +{ + return (__m512) __builtin_ia32_insertf32x8_mask ((__v16sf) __A, + (__v8sf) __B, + __imm, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_insertf32x8 (__m512 __W, __mmask16 __U, __m512 __A, + __m256 __B, const int __imm) +{ + return (__m512) __builtin_ia32_insertf32x8_mask ((__v16sf) __A, + (__v8sf) __B, + __imm, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_insertf32x8 (__mmask16 __U, __m512 __A, __m256 __B, + const int __imm) +{ + return (__m512) __builtin_ia32_insertf32x8_mask ((__v16sf) __A, + (__v8sf) __B, + __imm, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_inserti64x2 (__m512i __A, __m128i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_inserti64x2_512_mask ((__v8di) __A, + (__v2di) __B, + __imm, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_inserti64x2 (__m512i __W, __mmask8 __U, __m512i __A, + __m128i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_inserti64x2_512_mask ((__v8di) __A, + (__v2di) __B, + __imm, + (__v8di) __W, + (__mmask8) + __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_inserti64x2 (__mmask8 __U, __m512i __A, __m128i __B, + const int __imm) +{ + return (__m512i) __builtin_ia32_inserti64x2_512_mask ((__v8di) __A, + (__v2di) __B, + __imm, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) + __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_insertf64x2 (__m512d __A, __m128d __B, const int __imm) +{ + return (__m512d) __builtin_ia32_insertf64x2_512_mask ((__v8df) __A, + (__v2df) __B, + __imm, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_insertf64x2 (__m512d __W, __mmask8 __U, __m512d __A, + __m128d __B, const int __imm) +{ + return (__m512d) __builtin_ia32_insertf64x2_512_mask ((__v8df) __A, + (__v2df) __B, + __imm, + (__v8df) __W, + (__mmask8) + __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_insertf64x2 (__mmask8 __U, __m512d __A, __m128d __B, + const int __imm) +{ + return (__m512d) __builtin_ia32_insertf64x2_512_mask ((__v8df) __A, + (__v2df) __B, + __imm, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) + __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fpclass_pd_mask (__mmask8 __U, __m512d __A, + const int __imm) +{ + return (__mmask8) __builtin_ia32_fpclasspd512_mask ((__v8df) __A, + __imm, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fpclass_pd_mask (__m512d __A, const int __imm) +{ + return (__mmask8) __builtin_ia32_fpclasspd512_mask ((__v8df) __A, + __imm, + (__mmask8) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fpclass_ps_mask (__mmask16 __U, __m512 __A, + const int __imm) +{ + return (__mmask16) __builtin_ia32_fpclassps512_mask ((__v16sf) __A, + __imm, __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fpclass_ps_mask (__m512 __A, const int __imm) +{ + return (__mmask16) __builtin_ia32_fpclassps512_mask ((__v16sf) __A, + __imm, + (__mmask16) -1); +} + +#else +#define _kshiftli_mask8(X, Y) \ + ((__mmask8) __builtin_ia32_kshiftliqi ((__mmask8)(X), (__mmask8)(Y))) + +#define _kshiftri_mask8(X, Y) \ + ((__mmask8) __builtin_ia32_kshiftriqi ((__mmask8)(X), (__mmask8)(Y))) + +#define _mm_range_sd(A, B, C) \ + ((__m128d) __builtin_ia32_rangesd128_round ((__v2df)(__m128d)(A), \ + (__v2df)(__m128d)(B), (int)(C), \ + _MM_FROUND_CUR_DIRECTION)) + +#define _mm_range_ss(A, B, C) \ + ((__m128) __builtin_ia32_rangess128_round ((__v4sf)(__m128)(A), \ + (__v4sf)(__m128)(B), (int)(C), \ + _MM_FROUND_CUR_DIRECTION)) + +#define _mm_range_round_sd(A, B, C, R) \ + ((__m128d) __builtin_ia32_rangesd128_round ((__v2df)(__m128d)(A), \ + (__v2df)(__m128d)(B), (int)(C), (R))) + +#define _mm_range_round_ss(A, B, C, R) \ + ((__m128) __builtin_ia32_rangess128_round ((__v4sf)(__m128)(A), \ + (__v4sf)(__m128)(B), (int)(C), (R))) + +#define _mm512_cvtt_roundpd_epi64(A, B) \ + ((__m512i)__builtin_ia32_cvttpd2qq512_mask ((A), (__v8di) \ + _mm512_setzero_si512 (), \ + -1, (B))) + +#define _mm512_mask_cvtt_roundpd_epi64(W, U, A, B) \ + ((__m512i)__builtin_ia32_cvttpd2qq512_mask ((A), (__v8di)(W), (U), (B))) + +#define _mm512_maskz_cvtt_roundpd_epi64(U, A, B) \ + ((__m512i)__builtin_ia32_cvttpd2qq512_mask ((A), (__v8di)_mm512_setzero_si512 (), (U), (B))) + +#define _mm512_cvtt_roundpd_epu64(A, B) \ + ((__m512i)__builtin_ia32_cvttpd2uqq512_mask ((A), (__v8di)_mm512_setzero_si512 (), -1, (B))) + +#define _mm512_mask_cvtt_roundpd_epu64(W, U, A, B) \ + ((__m512i)__builtin_ia32_cvttpd2uqq512_mask ((A), (__v8di)(W), (U), (B))) + +#define _mm512_maskz_cvtt_roundpd_epu64(U, A, B) \ + ((__m512i)__builtin_ia32_cvttpd2uqq512_mask ((A), (__v8di)_mm512_setzero_si512 (), (U), (B))) + +#define _mm512_cvtt_roundps_epi64(A, B) \ + ((__m512i)__builtin_ia32_cvttps2qq512_mask ((A), (__v8di)_mm512_setzero_si512 (), -1, (B))) + +#define _mm512_mask_cvtt_roundps_epi64(W, U, A, B) \ + ((__m512i)__builtin_ia32_cvttps2qq512_mask ((A), (__v8di)(W), (U), (B))) + +#define _mm512_maskz_cvtt_roundps_epi64(U, A, B) \ + ((__m512i)__builtin_ia32_cvttps2qq512_mask ((A), (__v8di)_mm512_setzero_si512 (), (U), (B))) + +#define _mm512_cvtt_roundps_epu64(A, B) \ + ((__m512i)__builtin_ia32_cvttps2uqq512_mask ((A), (__v8di)_mm512_setzero_si512 (), -1, (B))) + +#define _mm512_mask_cvtt_roundps_epu64(W, U, A, B) \ + ((__m512i)__builtin_ia32_cvttps2uqq512_mask ((A), (__v8di)(W), (U), (B))) + +#define _mm512_maskz_cvtt_roundps_epu64(U, A, B) \ + ((__m512i)__builtin_ia32_cvttps2uqq512_mask ((A), (__v8di)_mm512_setzero_si512 (), (U), (B))) + +#define _mm512_cvt_roundpd_epi64(A, B) \ + ((__m512i)__builtin_ia32_cvtpd2qq512_mask ((A), (__v8di)_mm512_setzero_si512 (), -1, (B))) + +#define _mm512_mask_cvt_roundpd_epi64(W, U, A, B) \ + ((__m512i)__builtin_ia32_cvtpd2qq512_mask ((A), (__v8di)(W), (U), (B))) + +#define _mm512_maskz_cvt_roundpd_epi64(U, A, B) \ + ((__m512i)__builtin_ia32_cvtpd2qq512_mask ((A), (__v8di)_mm512_setzero_si512 (), (U), (B))) + +#define _mm512_cvt_roundpd_epu64(A, B) \ + ((__m512i)__builtin_ia32_cvtpd2uqq512_mask ((A), (__v8di)_mm512_setzero_si512 (), -1, (B))) + +#define _mm512_mask_cvt_roundpd_epu64(W, U, A, B) \ + ((__m512i)__builtin_ia32_cvtpd2uqq512_mask ((A), (__v8di)(W), (U), (B))) + +#define _mm512_maskz_cvt_roundpd_epu64(U, A, B) \ + ((__m512i)__builtin_ia32_cvtpd2uqq512_mask ((A), (__v8di)_mm512_setzero_si512 (), (U), (B))) + +#define _mm512_cvt_roundps_epi64(A, B) \ + ((__m512i)__builtin_ia32_cvtps2qq512_mask ((A), (__v8di)_mm512_setzero_si512 (), -1, (B))) + +#define _mm512_mask_cvt_roundps_epi64(W, U, A, B) \ + ((__m512i)__builtin_ia32_cvtps2qq512_mask ((A), (__v8di)(W), (U), (B))) + +#define _mm512_maskz_cvt_roundps_epi64(U, A, B) \ + ((__m512i)__builtin_ia32_cvtps2qq512_mask ((A), (__v8di)_mm512_setzero_si512 (), (U), (B))) + +#define _mm512_cvt_roundps_epu64(A, B) \ + ((__m512i)__builtin_ia32_cvtps2uqq512_mask ((A), (__v8di)_mm512_setzero_si512 (), -1, (B))) + +#define _mm512_mask_cvt_roundps_epu64(W, U, A, B) \ + ((__m512i)__builtin_ia32_cvtps2uqq512_mask ((A), (__v8di)(W), (U), (B))) + +#define _mm512_maskz_cvt_roundps_epu64(U, A, B) \ + ((__m512i)__builtin_ia32_cvtps2uqq512_mask ((A), (__v8di)_mm512_setzero_si512 (), (U), (B))) + +#define _mm512_cvt_roundepi64_ps(A, B) \ + ((__m256)__builtin_ia32_cvtqq2ps512_mask ((__v8di)(A), (__v8sf)_mm256_setzero_ps (), -1, (B))) + +#define _mm512_mask_cvt_roundepi64_ps(W, U, A, B) \ + ((__m256)__builtin_ia32_cvtqq2ps512_mask ((__v8di)(A), (W), (U), (B))) + +#define _mm512_maskz_cvt_roundepi64_ps(U, A, B) \ + ((__m256)__builtin_ia32_cvtqq2ps512_mask ((__v8di)(A), (__v8sf)_mm256_setzero_ps (), (U), (B))) + +#define _mm512_cvt_roundepu64_ps(A, B) \ + ((__m256)__builtin_ia32_cvtuqq2ps512_mask ((__v8di)(A), (__v8sf)_mm256_setzero_ps (), -1, (B))) + +#define _mm512_mask_cvt_roundepu64_ps(W, U, A, B) \ + ((__m256)__builtin_ia32_cvtuqq2ps512_mask ((__v8di)(A), (W), (U), (B))) + +#define _mm512_maskz_cvt_roundepu64_ps(U, A, B) \ + ((__m256)__builtin_ia32_cvtuqq2ps512_mask ((__v8di)(A), (__v8sf)_mm256_setzero_ps (), (U), (B))) + +#define _mm512_cvt_roundepi64_pd(A, B) \ + ((__m512d)__builtin_ia32_cvtqq2pd512_mask ((__v8di)(A), (__v8df)_mm512_setzero_pd (), -1, (B))) + +#define _mm512_mask_cvt_roundepi64_pd(W, U, A, B) \ + ((__m512d)__builtin_ia32_cvtqq2pd512_mask ((__v8di)(A), (W), (U), (B))) + +#define _mm512_maskz_cvt_roundepi64_pd(U, A, B) \ + ((__m512d)__builtin_ia32_cvtqq2pd512_mask ((__v8di)(A), (__v8df)_mm512_setzero_pd (), (U), (B))) + +#define _mm512_cvt_roundepu64_pd(A, B) \ + ((__m512d)__builtin_ia32_cvtuqq2pd512_mask ((__v8di)(A), (__v8df)_mm512_setzero_pd (), -1, (B))) + +#define _mm512_mask_cvt_roundepu64_pd(W, U, A, B) \ + ((__m512d)__builtin_ia32_cvtuqq2pd512_mask ((__v8di)(A), (W), (U), (B))) + +#define _mm512_maskz_cvt_roundepu64_pd(U, A, B) \ + ((__m512d)__builtin_ia32_cvtuqq2pd512_mask ((__v8di)(A), (__v8df)_mm512_setzero_pd (), (U), (B))) + +#define _mm512_reduce_pd(A, B) \ + ((__m512d) __builtin_ia32_reducepd512_mask ((__v8df)(__m512d)(A), \ + (int)(B), (__v8df)_mm512_setzero_pd (), (__mmask8)-1)) + +#define _mm512_mask_reduce_pd(W, U, A, B) \ + ((__m512d) __builtin_ia32_reducepd512_mask ((__v8df)(__m512d)(A), \ + (int)(B), (__v8df)(__m512d)(W), (__mmask8)(U))) + +#define _mm512_maskz_reduce_pd(U, A, B) \ + ((__m512d) __builtin_ia32_reducepd512_mask ((__v8df)(__m512d)(A), \ + (int)(B), (__v8df)_mm512_setzero_pd (), (__mmask8)(U))) + +#define _mm512_reduce_ps(A, B) \ + ((__m512) __builtin_ia32_reduceps512_mask ((__v16sf)(__m512)(A), \ + (int)(B), (__v16sf)_mm512_setzero_ps (), (__mmask16)-1)) + +#define _mm512_mask_reduce_ps(W, U, A, B) \ + ((__m512) __builtin_ia32_reduceps512_mask ((__v16sf)(__m512)(A), \ + (int)(B), (__v16sf)(__m512)(W), (__mmask16)(U))) + +#define _mm512_maskz_reduce_ps(U, A, B) \ + ((__m512) __builtin_ia32_reduceps512_mask ((__v16sf)(__m512)(A), \ + (int)(B), (__v16sf)_mm512_setzero_ps (), (__mmask16)(U))) + +#define _mm512_extractf32x8_ps(X, C) \ + ((__m256) __builtin_ia32_extractf32x8_mask ((__v16sf)(__m512) (X), \ + (int) (C), (__v8sf)(__m256) _mm256_setzero_ps (), (__mmask8)-1)) + +#define _mm512_mask_extractf32x8_ps(W, U, X, C) \ + ((__m256) __builtin_ia32_extractf32x8_mask ((__v16sf)(__m512) (X), \ + (int) (C), (__v8sf)(__m256) (W), (__mmask8) (U))) + +#define _mm512_maskz_extractf32x8_ps(U, X, C) \ + ((__m256) __builtin_ia32_extractf32x8_mask ((__v16sf)(__m512) (X), \ + (int) (C), (__v8sf)(__m256) _mm256_setzero_ps (), (__mmask8) (U))) + +#define _mm512_extractf64x2_pd(X, C) \ + ((__m128d) __builtin_ia32_extractf64x2_512_mask ((__v8df)(__m512d) (X),\ + (int) (C), (__v2df)(__m128d) _mm_setzero_pd (), (__mmask8)-1)) + +#define _mm512_mask_extractf64x2_pd(W, U, X, C) \ + ((__m128d) __builtin_ia32_extractf64x2_512_mask ((__v8df)(__m512d) (X),\ + (int) (C), (__v2df)(__m128d) (W), (__mmask8) (U))) + +#define _mm512_maskz_extractf64x2_pd(U, X, C) \ + ((__m128d) __builtin_ia32_extractf64x2_512_mask ((__v8df)(__m512d) (X),\ + (int) (C), (__v2df)(__m128d) _mm_setzero_pd (), (__mmask8) (U))) + +#define _mm512_extracti32x8_epi32(X, C) \ + ((__m256i) __builtin_ia32_extracti32x8_mask ((__v16si)(__m512i) (X), \ + (int) (C), (__v8si)(__m256i) _mm256_setzero_si256 (), (__mmask8)-1)) + +#define _mm512_mask_extracti32x8_epi32(W, U, X, C) \ + ((__m256i) __builtin_ia32_extracti32x8_mask ((__v16si)(__m512i) (X), \ + (int) (C), (__v8si)(__m256i) (W), (__mmask8) (U))) + +#define _mm512_maskz_extracti32x8_epi32(U, X, C) \ + ((__m256i) __builtin_ia32_extracti32x8_mask ((__v16si)(__m512i) (X), \ + (int) (C), (__v8si)(__m256i) _mm256_setzero_si256 (), (__mmask8) (U))) + +#define _mm512_extracti64x2_epi64(X, C) \ + ((__m128i) __builtin_ia32_extracti64x2_512_mask ((__v8di)(__m512i) (X),\ + (int) (C), (__v2di)(__m128i) _mm_setzero_si128 (), (__mmask8)-1)) + +#define _mm512_mask_extracti64x2_epi64(W, U, X, C) \ + ((__m128i) __builtin_ia32_extracti64x2_512_mask ((__v8di)(__m512i) (X),\ + (int) (C), (__v2di)(__m128i) (W), (__mmask8) (U))) + +#define _mm512_maskz_extracti64x2_epi64(U, X, C) \ + ((__m128i) __builtin_ia32_extracti64x2_512_mask ((__v8di)(__m512i) (X),\ + (int) (C), (__v2di)(__m128i) _mm_setzero_si128 (), (__mmask8) (U))) + +#define _mm512_range_pd(A, B, C) \ + ((__m512d) __builtin_ia32_rangepd512_mask ((__v8df)(__m512d)(A), \ + (__v8df)(__m512d)(B), (int)(C), \ + (__v8df)_mm512_setzero_pd (), (__mmask8)-1, _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_mask_range_pd(W, U, A, B, C) \ + ((__m512d) __builtin_ia32_rangepd512_mask ((__v8df)(__m512d)(A), \ + (__v8df)(__m512d)(B), (int)(C), \ + (__v8df)(__m512d)(W), (__mmask8)(U), _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_maskz_range_pd(U, A, B, C) \ + ((__m512d) __builtin_ia32_rangepd512_mask ((__v8df)(__m512d)(A), \ + (__v8df)(__m512d)(B), (int)(C), \ + (__v8df)_mm512_setzero_pd (), (__mmask8)(U), _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_range_ps(A, B, C) \ + ((__m512) __builtin_ia32_rangeps512_mask ((__v16sf)(__m512)(A), \ + (__v16sf)(__m512)(B), (int)(C), \ + (__v16sf)_mm512_setzero_ps (), (__mmask16)-1, _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_mask_range_ps(W, U, A, B, C) \ + ((__m512) __builtin_ia32_rangeps512_mask ((__v16sf)(__m512)(A), \ + (__v16sf)(__m512)(B), (int)(C), \ + (__v16sf)(__m512)(W), (__mmask16)(U), _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_maskz_range_ps(U, A, B, C) \ + ((__m512) __builtin_ia32_rangeps512_mask ((__v16sf)(__m512)(A), \ + (__v16sf)(__m512)(B), (int)(C), \ + (__v16sf)_mm512_setzero_ps (), (__mmask16)(U), _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_range_round_pd(A, B, C, R) \ + ((__m512d) __builtin_ia32_rangepd512_mask ((__v8df)(__m512d)(A), \ + (__v8df)(__m512d)(B), (int)(C), \ + (__v8df)_mm512_setzero_pd (), (__mmask8)-1, (R))) + +#define _mm512_mask_range_round_pd(W, U, A, B, C, R) \ + ((__m512d) __builtin_ia32_rangepd512_mask ((__v8df)(__m512d)(A), \ + (__v8df)(__m512d)(B), (int)(C), \ + (__v8df)(__m512d)(W), (__mmask8)(U), (R))) + +#define _mm512_maskz_range_round_pd(U, A, B, C, R) \ + ((__m512d) __builtin_ia32_rangepd512_mask ((__v8df)(__m512d)(A), \ + (__v8df)(__m512d)(B), (int)(C), \ + (__v8df)_mm512_setzero_pd (), (__mmask8)(U), (R))) + +#define _mm512_range_round_ps(A, B, C, R) \ + ((__m512) __builtin_ia32_rangeps512_mask ((__v16sf)(__m512)(A), \ + (__v16sf)(__m512)(B), (int)(C), \ + (__v16sf)_mm512_setzero_ps (), (__mmask16)-1, (R))) + +#define _mm512_mask_range_round_ps(W, U, A, B, C, R) \ + ((__m512) __builtin_ia32_rangeps512_mask ((__v16sf)(__m512)(A), \ + (__v16sf)(__m512)(B), (int)(C), \ + (__v16sf)(__m512)(W), (__mmask16)(U), (R))) + +#define _mm512_maskz_range_round_ps(U, A, B, C, R) \ + ((__m512) __builtin_ia32_rangeps512_mask ((__v16sf)(__m512)(A), \ + (__v16sf)(__m512)(B), (int)(C), \ + (__v16sf)_mm512_setzero_ps (), (__mmask16)(U), (R))) + +#define _mm512_insertf64x2(X, Y, C) \ + ((__m512d) __builtin_ia32_insertf64x2_512_mask ((__v8df)(__m512d) (X),\ + (__v2df)(__m128d) (Y), (int) (C), (__v8df)(__m512d) (X), \ + (__mmask8)-1)) + +#define _mm512_mask_insertf64x2(W, U, X, Y, C) \ + ((__m512d) __builtin_ia32_insertf64x2_512_mask ((__v8df)(__m512d) (X),\ + (__v2df)(__m128d) (Y), (int) (C), (__v8df)(__m512d) (W), \ + (__mmask8) (U))) + +#define _mm512_maskz_insertf64x2(U, X, Y, C) \ + ((__m512d) __builtin_ia32_insertf64x2_512_mask ((__v8df)(__m512d) (X),\ + (__v2df)(__m128d) (Y), (int) (C), \ + (__v8df)(__m512d) _mm512_setzero_pd (), (__mmask8) (U))) + +#define _mm512_inserti64x2(X, Y, C) \ + ((__m512i) __builtin_ia32_inserti64x2_512_mask ((__v8di)(__m512i) (X),\ + (__v2di)(__m128i) (Y), (int) (C), (__v8di)(__m512i) (X), (__mmask8)-1)) + +#define _mm512_mask_inserti64x2(W, U, X, Y, C) \ + ((__m512i) __builtin_ia32_inserti64x2_512_mask ((__v8di)(__m512i) (X),\ + (__v2di)(__m128i) (Y), (int) (C), (__v8di)(__m512i) (W), \ + (__mmask8) (U))) + +#define _mm512_maskz_inserti64x2(U, X, Y, C) \ + ((__m512i) __builtin_ia32_inserti64x2_512_mask ((__v8di)(__m512i) (X),\ + (__v2di)(__m128i) (Y), (int) (C), \ + (__v8di)(__m512i) _mm512_setzero_si512 (), (__mmask8) (U))) + +#define _mm512_insertf32x8(X, Y, C) \ + ((__m512) __builtin_ia32_insertf32x8_mask ((__v16sf)(__m512) (X), \ + (__v8sf)(__m256) (Y), (int) (C),\ + (__v16sf)(__m512)_mm512_setzero_ps (),\ + (__mmask16)-1)) + +#define _mm512_mask_insertf32x8(W, U, X, Y, C) \ + ((__m512) __builtin_ia32_insertf32x8_mask ((__v16sf)(__m512) (X), \ + (__v8sf)(__m256) (Y), (int) (C),\ + (__v16sf)(__m512)(W),\ + (__mmask16)(U))) + +#define _mm512_maskz_insertf32x8(U, X, Y, C) \ + ((__m512) __builtin_ia32_insertf32x8_mask ((__v16sf)(__m512) (X), \ + (__v8sf)(__m256) (Y), (int) (C),\ + (__v16sf)(__m512)_mm512_setzero_ps (),\ + (__mmask16)(U))) + +#define _mm512_inserti32x8(X, Y, C) \ + ((__m512i) __builtin_ia32_inserti32x8_mask ((__v16si)(__m512i) (X), \ + (__v8si)(__m256i) (Y), (int) (C),\ + (__v16si)(__m512i)_mm512_setzero_si512 (),\ + (__mmask16)-1)) + +#define _mm512_mask_inserti32x8(W, U, X, Y, C) \ + ((__m512i) __builtin_ia32_inserti32x8_mask ((__v16si)(__m512i) (X), \ + (__v8si)(__m256i) (Y), (int) (C),\ + (__v16si)(__m512i)(W),\ + (__mmask16)(U))) + +#define _mm512_maskz_inserti32x8(U, X, Y, C) \ + ((__m512i) __builtin_ia32_inserti32x8_mask ((__v16si)(__m512i) (X), \ + (__v8si)(__m256i) (Y), (int) (C),\ + (__v16si)(__m512i)_mm512_setzero_si512 (),\ + (__mmask16)(U))) + +#define _mm_fpclass_ss_mask(X, C) \ + ((__mmask8) __builtin_ia32_fpclassss ((__v4sf) (__m128) (X), (int) (C))) \ + +#define _mm_fpclass_sd_mask(X, C) \ + ((__mmask8) __builtin_ia32_fpclasssd ((__v2df) (__m128d) (X), (int) (C))) \ + +#define _mm512_mask_fpclass_pd_mask(u, X, C) \ + ((__mmask8) __builtin_ia32_fpclasspd512_mask ((__v8df) (__m512d) (X), \ + (int) (C), (__mmask8)(u))) + +#define _mm512_mask_fpclass_ps_mask(u, x, c) \ + ((__mmask16) __builtin_ia32_fpclassps512_mask ((__v16sf) (__m512) (x),\ + (int) (c),(__mmask8)(u))) + +#define _mm512_fpclass_pd_mask(X, C) \ + ((__mmask8) __builtin_ia32_fpclasspd512_mask ((__v8df) (__m512d) (X), \ + (int) (C), (__mmask8)-1)) + +#define _mm512_fpclass_ps_mask(x, c) \ + ((__mmask16) __builtin_ia32_fpclassps512_mask ((__v16sf) (__m512) (x),\ + (int) (c),(__mmask8)-1)) + +#define _mm_reduce_sd(A, B, C) \ + ((__m128d) __builtin_ia32_reducesd ((__v2df)(__m128d)(A), \ + (__v2df)(__m128d)(B), (int)(C))) \ + +#define _mm_reduce_ss(A, B, C) \ + ((__m128) __builtin_ia32_reducess ((__v4sf)(__m128)(A), \ + (__v4sf)(__m128)(A), (int)(C))) \ + +#endif + +#ifdef __DISABLE_AVX512DQ__ +#undef __DISABLE_AVX512DQ__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX512DQ__ */ + +#endif /* _AVX512DQINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512erintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512erintrin.h new file mode 100644 index 0000000000..b46ec62aa4 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512erintrin.h @@ -0,0 +1,394 @@ +/* Copyright (C) 2013-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +#error "Never use directly; include instead." +#endif + +#ifndef _AVX512ERINTRIN_H_INCLUDED +#define _AVX512ERINTRIN_H_INCLUDED + +#ifndef __AVX512ER__ +//#pragma GCC push_options +#pragma GCC target("avx512er") +#define __DISABLE_AVX512ER__ +#endif /* __AVX512ER__ */ + +/* Internal data types for implementing the intrinsics. */ +typedef double __v8df __attribute__ ((__vector_size__ (64))); +typedef float __v16sf __attribute__ ((__vector_size__ (64))); + +/* The Intel API is flexible enough that we must allow aliasing with other + vector types, and their scalar components. */ +typedef float __m512 __attribute__ ((__vector_size__ (64), __may_alias__)); +typedef double __m512d __attribute__ ((__vector_size__ (64), __may_alias__)); + +typedef unsigned char __mmask8; +typedef unsigned short __mmask16; + +#ifdef __OPTIMIZE__ +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_exp2a23_round_pd (__m512d __A, int __R) +{ + __m512d __W; + return (__m512d) __builtin_ia32_exp2pd_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_exp2a23_round_pd (__m512d __W, __mmask8 __U, __m512d __A, int __R) +{ + return (__m512d) __builtin_ia32_exp2pd_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_exp2a23_round_pd (__mmask8 __U, __m512d __A, int __R) +{ + return (__m512d) __builtin_ia32_exp2pd_mask ((__v8df) __A, + (__v8df) _mm512_setzero_pd (), + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_exp2a23_round_ps (__m512 __A, int __R) +{ + __m512 __W; + return (__m512) __builtin_ia32_exp2ps_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_exp2a23_round_ps (__m512 __W, __mmask16 __U, __m512 __A, int __R) +{ + return (__m512) __builtin_ia32_exp2ps_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_exp2a23_round_ps (__mmask16 __U, __m512 __A, int __R) +{ + return (__m512) __builtin_ia32_exp2ps_mask ((__v16sf) __A, + (__v16sf) _mm512_setzero_ps (), + (__mmask16) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_rcp28_round_pd (__m512d __A, int __R) +{ + __m512d __W; + return (__m512d) __builtin_ia32_rcp28pd_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_rcp28_round_pd (__m512d __W, __mmask8 __U, __m512d __A, int __R) +{ + return (__m512d) __builtin_ia32_rcp28pd_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_rcp28_round_pd (__mmask8 __U, __m512d __A, int __R) +{ + return (__m512d) __builtin_ia32_rcp28pd_mask ((__v8df) __A, + (__v8df) _mm512_setzero_pd (), + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_rcp28_round_ps (__m512 __A, int __R) +{ + __m512 __W; + return (__m512) __builtin_ia32_rcp28ps_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_rcp28_round_ps (__m512 __W, __mmask16 __U, __m512 __A, int __R) +{ + return (__m512) __builtin_ia32_rcp28ps_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_rcp28_round_ps (__mmask16 __U, __m512 __A, int __R) +{ + return (__m512) __builtin_ia32_rcp28ps_mask ((__v16sf) __A, + (__v16sf) _mm512_setzero_ps (), + (__mmask16) __U, __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rcp28_round_sd (__m128d __A, __m128d __B, int __R) +{ + return (__m128d) __builtin_ia32_rcp28sd_round ((__v2df) __B, + (__v2df) __A, + __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rcp28_round_ss (__m128 __A, __m128 __B, int __R) +{ + return (__m128) __builtin_ia32_rcp28ss_round ((__v4sf) __B, + (__v4sf) __A, + __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_rsqrt28_round_pd (__m512d __A, int __R) +{ + __m512d __W; + return (__m512d) __builtin_ia32_rsqrt28pd_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_rsqrt28_round_pd (__m512d __W, __mmask8 __U, __m512d __A, int __R) +{ + return (__m512d) __builtin_ia32_rsqrt28pd_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_rsqrt28_round_pd (__mmask8 __U, __m512d __A, int __R) +{ + return (__m512d) __builtin_ia32_rsqrt28pd_mask ((__v8df) __A, + (__v8df) _mm512_setzero_pd (), + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_rsqrt28_round_ps (__m512 __A, int __R) +{ + __m512 __W; + return (__m512) __builtin_ia32_rsqrt28ps_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_rsqrt28_round_ps (__m512 __W, __mmask16 __U, __m512 __A, int __R) +{ + return (__m512) __builtin_ia32_rsqrt28ps_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_rsqrt28_round_ps (__mmask16 __U, __m512 __A, int __R) +{ + return (__m512) __builtin_ia32_rsqrt28ps_mask ((__v16sf) __A, + (__v16sf) _mm512_setzero_ps (), + (__mmask16) __U, __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rsqrt28_round_sd (__m128d __A, __m128d __B, int __R) +{ + return (__m128d) __builtin_ia32_rsqrt28sd_round ((__v2df) __B, + (__v2df) __A, + __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rsqrt28_round_ss (__m128 __A, __m128 __B, int __R) +{ + return (__m128) __builtin_ia32_rsqrt28ss_round ((__v4sf) __B, + (__v4sf) __A, + __R); +} + +#else +#define _mm512_exp2a23_round_pd(A, C) \ + __builtin_ia32_exp2pd_mask(A, (__v8df)_mm512_setzero_pd(), -1, C) + +#define _mm512_mask_exp2a23_round_pd(W, U, A, C) \ + __builtin_ia32_exp2pd_mask(A, W, U, C) + +#define _mm512_maskz_exp2a23_round_pd(U, A, C) \ + __builtin_ia32_exp2pd_mask(A, (__v8df)_mm512_setzero_pd(), U, C) + +#define _mm512_exp2a23_round_ps(A, C) \ + __builtin_ia32_exp2ps_mask(A, (__v16sf)_mm512_setzero_ps(), -1, C) + +#define _mm512_mask_exp2a23_round_ps(W, U, A, C) \ + __builtin_ia32_exp2ps_mask(A, W, U, C) + +#define _mm512_maskz_exp2a23_round_ps(U, A, C) \ + __builtin_ia32_exp2ps_mask(A, (__v16sf)_mm512_setzero_ps(), U, C) + +#define _mm512_rcp28_round_pd(A, C) \ + __builtin_ia32_rcp28pd_mask(A, (__v8df)_mm512_setzero_pd(), -1, C) + +#define _mm512_mask_rcp28_round_pd(W, U, A, C) \ + __builtin_ia32_rcp28pd_mask(A, W, U, C) + +#define _mm512_maskz_rcp28_round_pd(U, A, C) \ + __builtin_ia32_rcp28pd_mask(A, (__v8df)_mm512_setzero_pd(), U, C) + +#define _mm512_rcp28_round_ps(A, C) \ + __builtin_ia32_rcp28ps_mask(A, (__v16sf)_mm512_setzero_ps(), -1, C) + +#define _mm512_mask_rcp28_round_ps(W, U, A, C) \ + __builtin_ia32_rcp28ps_mask(A, W, U, C) + +#define _mm512_maskz_rcp28_round_ps(U, A, C) \ + __builtin_ia32_rcp28ps_mask(A, (__v16sf)_mm512_setzero_ps(), U, C) + +#define _mm512_rsqrt28_round_pd(A, C) \ + __builtin_ia32_rsqrt28pd_mask(A, (__v8df)_mm512_setzero_pd(), -1, C) + +#define _mm512_mask_rsqrt28_round_pd(W, U, A, C) \ + __builtin_ia32_rsqrt28pd_mask(A, W, U, C) + +#define _mm512_maskz_rsqrt28_round_pd(U, A, C) \ + __builtin_ia32_rsqrt28pd_mask(A, (__v8df)_mm512_setzero_pd(), U, C) + +#define _mm512_rsqrt28_round_ps(A, C) \ + __builtin_ia32_rsqrt28ps_mask(A, (__v16sf)_mm512_setzero_ps(), -1, C) + +#define _mm512_mask_rsqrt28_round_ps(W, U, A, C) \ + __builtin_ia32_rsqrt28ps_mask(A, W, U, C) + +#define _mm512_maskz_rsqrt28_round_ps(U, A, C) \ + __builtin_ia32_rsqrt28ps_mask(A, (__v16sf)_mm512_setzero_ps(), U, C) + +#define _mm_rcp28_round_sd(A, B, R) \ + __builtin_ia32_rcp28sd_round(A, B, R) + +#define _mm_rcp28_round_ss(A, B, R) \ + __builtin_ia32_rcp28ss_round(A, B, R) + +#define _mm_rsqrt28_round_sd(A, B, R) \ + __builtin_ia32_rsqrt28sd_round(A, B, R) + +#define _mm_rsqrt28_round_ss(A, B, R) \ + __builtin_ia32_rsqrt28ss_round(A, B, R) + +#endif + +#define _mm512_exp2a23_pd(A) \ + _mm512_exp2a23_round_pd(A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_mask_exp2a23_pd(W, U, A) \ + _mm512_mask_exp2a23_round_pd(W, U, A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_maskz_exp2a23_pd(U, A) \ + _mm512_maskz_exp2a23_round_pd(U, A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_exp2a23_ps(A) \ + _mm512_exp2a23_round_ps(A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_mask_exp2a23_ps(W, U, A) \ + _mm512_mask_exp2a23_round_ps(W, U, A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_maskz_exp2a23_ps(U, A) \ + _mm512_maskz_exp2a23_round_ps(U, A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_rcp28_pd(A) \ + _mm512_rcp28_round_pd(A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_mask_rcp28_pd(W, U, A) \ + _mm512_mask_rcp28_round_pd(W, U, A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_maskz_rcp28_pd(U, A) \ + _mm512_maskz_rcp28_round_pd(U, A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_rcp28_ps(A) \ + _mm512_rcp28_round_ps(A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_mask_rcp28_ps(W, U, A) \ + _mm512_mask_rcp28_round_ps(W, U, A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_maskz_rcp28_ps(U, A) \ + _mm512_maskz_rcp28_round_ps(U, A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_rsqrt28_pd(A) \ + _mm512_rsqrt28_round_pd(A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_mask_rsqrt28_pd(W, U, A) \ + _mm512_mask_rsqrt28_round_pd(W, U, A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_maskz_rsqrt28_pd(U, A) \ + _mm512_maskz_rsqrt28_round_pd(U, A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_rsqrt28_ps(A) \ + _mm512_rsqrt28_round_ps(A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_mask_rsqrt28_ps(W, U, A) \ + _mm512_mask_rsqrt28_round_ps(W, U, A, _MM_FROUND_CUR_DIRECTION) + +#define _mm512_maskz_rsqrt28_ps(U, A) \ + _mm512_maskz_rsqrt28_round_ps(U, A, _MM_FROUND_CUR_DIRECTION) + +#define _mm_rcp28_sd(A, B) \ + __builtin_ia32_rcp28sd_round(B, A, _MM_FROUND_CUR_DIRECTION) + +#define _mm_rcp28_ss(A, B) \ + __builtin_ia32_rcp28ss_round(B, A, _MM_FROUND_CUR_DIRECTION) + +#define _mm_rsqrt28_sd(A, B) \ + __builtin_ia32_rsqrt28sd_round(B, A, _MM_FROUND_CUR_DIRECTION) + +#define _mm_rsqrt28_ss(A, B) \ + __builtin_ia32_rsqrt28ss_round(B, A, _MM_FROUND_CUR_DIRECTION) + +#ifdef __DISABLE_AVX512ER__ +#undef __DISABLE_AVX512ER__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX512ER__ */ + +#endif /* _AVX512ERINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512fintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512fintrin.h new file mode 100644 index 0000000000..15e3b0ba94 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512fintrin.h @@ -0,0 +1,13754 @@ +/* Copyright (C) 2013-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +#error "Never use directly; include instead." +#endif + +#ifndef _AVX512FINTRIN_H_INCLUDED +#define _AVX512FINTRIN_H_INCLUDED + +#ifndef __AVX512F__ +#pragma GCC push_options +//#pragma GCC target("avx512f") +#define __DISABLE_AVX512F__ +#endif /* __AVX512F__ */ + +/* Internal data types for implementing the intrinsics. */ +typedef double __v8df __attribute__ ((__vector_size__ (64))); +typedef float __v16sf __attribute__ ((__vector_size__ (64))); +typedef long long __v8di __attribute__ ((__vector_size__ (64))); +typedef unsigned long long __v8du __attribute__ ((__vector_size__ (64))); +typedef int __v16si __attribute__ ((__vector_size__ (64))); +typedef unsigned int __v16su __attribute__ ((__vector_size__ (64))); +typedef short __v32hi __attribute__ ((__vector_size__ (64))); +typedef unsigned short __v32hu __attribute__ ((__vector_size__ (64))); +typedef char __v64qi __attribute__ ((__vector_size__ (64))); +typedef unsigned char __v64qu __attribute__ ((__vector_size__ (64))); + +/* The Intel API is flexible enough that we must allow aliasing with other + vector types, and their scalar components. */ +typedef float __m512 __attribute__ ((__vector_size__ (64), __may_alias__)); +typedef long long __m512i __attribute__ ((__vector_size__ (64), __may_alias__)); +typedef double __m512d __attribute__ ((__vector_size__ (64), __may_alias__)); + +/* Unaligned version of the same type. */ +typedef float __m512_u __attribute__ ((__vector_size__ (64), __may_alias__, __aligned__ (1))); +typedef long long __m512i_u __attribute__ ((__vector_size__ (64), __may_alias__, __aligned__ (1))); +typedef double __m512d_u __attribute__ ((__vector_size__ (64), __may_alias__, __aligned__ (1))); + +typedef unsigned char __mmask8; +typedef unsigned short __mmask16; + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_int2mask (int __M) +{ + return (__mmask16) __M; +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask2int (__mmask16 __M) +{ + return (int) __M; +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_set_epi64 (long long __A, long long __B, long long __C, + long long __D, long long __E, long long __F, + long long __G, long long __H) +{ + return __extension__ (__m512i) (__v8di) + { __H, __G, __F, __E, __D, __C, __B, __A }; +} + +/* Create the vector [A B C D E F G H I J K L M N O P]. */ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_set_epi32 (int __A, int __B, int __C, int __D, + int __E, int __F, int __G, int __H, + int __I, int __J, int __K, int __L, + int __M, int __N, int __O, int __P) +{ + return __extension__ (__m512i)(__v16si) + { __P, __O, __N, __M, __L, __K, __J, __I, + __H, __G, __F, __E, __D, __C, __B, __A }; +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_set_pd (double __A, double __B, double __C, double __D, + double __E, double __F, double __G, double __H) +{ + return __extension__ (__m512d) + { __H, __G, __F, __E, __D, __C, __B, __A }; +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_set_ps (float __A, float __B, float __C, float __D, + float __E, float __F, float __G, float __H, + float __I, float __J, float __K, float __L, + float __M, float __N, float __O, float __P) +{ + return __extension__ (__m512) + { __P, __O, __N, __M, __L, __K, __J, __I, + __H, __G, __F, __E, __D, __C, __B, __A }; +} + +#define _mm512_setr_epi64(e0,e1,e2,e3,e4,e5,e6,e7) \ + _mm512_set_epi64(e7,e6,e5,e4,e3,e2,e1,e0) + +#define _mm512_setr_epi32(e0,e1,e2,e3,e4,e5,e6,e7, \ + e8,e9,e10,e11,e12,e13,e14,e15) \ + _mm512_set_epi32(e15,e14,e13,e12,e11,e10,e9,e8,e7,e6,e5,e4,e3,e2,e1,e0) + +#define _mm512_setr_pd(e0,e1,e2,e3,e4,e5,e6,e7) \ + _mm512_set_pd(e7,e6,e5,e4,e3,e2,e1,e0) + +#define _mm512_setr_ps(e0,e1,e2,e3,e4,e5,e6,e7,e8,e9,e10,e11,e12,e13,e14,e15) \ + _mm512_set_ps(e15,e14,e13,e12,e11,e10,e9,e8,e7,e6,e5,e4,e3,e2,e1,e0) + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_undefined_ps (void) +{ + __m512 __Y = __Y; + return __Y; +} + +#define _mm512_undefined _mm512_undefined_ps + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_undefined_pd (void) +{ + __m512d __Y = __Y; + return __Y; +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_undefined_epi32 (void) +{ + __m512i __Y = __Y; + return __Y; +} + +#define _mm512_undefined_si512 _mm512_undefined_epi32 + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_set1_epi8 (char __A) +{ + return __extension__ (__m512i)(__v64qi) + { __A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A }; +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_set1_epi16 (short __A) +{ + return __extension__ (__m512i)(__v32hi) + { __A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A }; +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_set1_pd (double __A) +{ + return (__m512d) __builtin_ia32_broadcastsd512 (__extension__ + (__v2df) { __A, }, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_set1_ps (float __A) +{ + return (__m512) __builtin_ia32_broadcastss512 (__extension__ + (__v4sf) { __A, }, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1); +} + +/* Create the vector [A B C D A B C D A B C D A B C D]. */ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_set4_epi32 (int __A, int __B, int __C, int __D) +{ + return __extension__ (__m512i)(__v16si) + { __D, __C, __B, __A, __D, __C, __B, __A, + __D, __C, __B, __A, __D, __C, __B, __A }; +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_set4_epi64 (long long __A, long long __B, long long __C, + long long __D) +{ + return __extension__ (__m512i) (__v8di) + { __D, __C, __B, __A, __D, __C, __B, __A }; +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_set4_pd (double __A, double __B, double __C, double __D) +{ + return __extension__ (__m512d) + { __D, __C, __B, __A, __D, __C, __B, __A }; +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_set4_ps (float __A, float __B, float __C, float __D) +{ + return __extension__ (__m512) + { __D, __C, __B, __A, __D, __C, __B, __A, + __D, __C, __B, __A, __D, __C, __B, __A }; +} + +#define _mm512_setr4_epi64(e0,e1,e2,e3) \ + _mm512_set4_epi64(e3,e2,e1,e0) + +#define _mm512_setr4_epi32(e0,e1,e2,e3) \ + _mm512_set4_epi32(e3,e2,e1,e0) + +#define _mm512_setr4_pd(e0,e1,e2,e3) \ + _mm512_set4_pd(e3,e2,e1,e0) + +#define _mm512_setr4_ps(e0,e1,e2,e3) \ + _mm512_set4_ps(e3,e2,e1,e0) + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_setzero_ps (void) +{ + return __extension__ (__m512){ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 }; +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_setzero_pd (void) +{ + return __extension__ (__m512d) { 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 }; +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_setzero_epi32 (void) +{ + return __extension__ (__m512i)(__v8di){ 0, 0, 0, 0, 0, 0, 0, 0 }; +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_setzero_si512 (void) +{ + return __extension__ (__m512i)(__v8di){ 0, 0, 0, 0, 0, 0, 0, 0 }; +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mov_pd (__m512d __W, __mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_movapd512_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mov_pd (__mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_movapd512_mask ((__v8df) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mov_ps (__m512 __W, __mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_movaps512_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mov_ps (__mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_movaps512_mask ((__v16sf) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_load_pd (void const *__P) +{ + return *(__m512d *) __P; +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_load_pd (__m512d __W, __mmask8 __U, void const *__P) +{ + return (__m512d) __builtin_ia32_loadapd512_mask ((const __v8df *) __P, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_load_pd (__mmask8 __U, void const *__P) +{ + return (__m512d) __builtin_ia32_loadapd512_mask ((const __v8df *) __P, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_store_pd (void *__P, __m512d __A) +{ + *(__m512d *) __P = __A; +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_store_pd (void *__P, __mmask8 __U, __m512d __A) +{ + __builtin_ia32_storeapd512_mask ((__v8df *) __P, (__v8df) __A, + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_load_ps (void const *__P) +{ + return *(__m512 *) __P; +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_load_ps (__m512 __W, __mmask16 __U, void const *__P) +{ + return (__m512) __builtin_ia32_loadaps512_mask ((const __v16sf *) __P, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_load_ps (__mmask16 __U, void const *__P) +{ + return (__m512) __builtin_ia32_loadaps512_mask ((const __v16sf *) __P, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_store_ps (void *__P, __m512 __A) +{ + *(__m512 *) __P = __A; +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_store_ps (void *__P, __mmask16 __U, __m512 __A) +{ + __builtin_ia32_storeaps512_mask ((__v16sf *) __P, (__v16sf) __A, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mov_epi64 (__m512i __W, __mmask8 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_movdqa64_512_mask ((__v8di) __A, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mov_epi64 (__mmask8 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_movdqa64_512_mask ((__v8di) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_load_epi64 (void const *__P) +{ + return *(__m512i *) __P; +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_load_epi64 (__m512i __W, __mmask8 __U, void const *__P) +{ + return (__m512i) __builtin_ia32_movdqa64load512_mask ((const __v8di *) __P, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_load_epi64 (__mmask8 __U, void const *__P) +{ + return (__m512i) __builtin_ia32_movdqa64load512_mask ((const __v8di *) __P, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_store_epi64 (void *__P, __m512i __A) +{ + *(__m512i *) __P = __A; +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_store_epi64 (void *__P, __mmask8 __U, __m512i __A) +{ + __builtin_ia32_movdqa64store512_mask ((__v8di *) __P, (__v8di) __A, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mov_epi32 (__m512i __W, __mmask16 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_movdqa32_512_mask ((__v16si) __A, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mov_epi32 (__mmask16 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_movdqa32_512_mask ((__v16si) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_load_si512 (void const *__P) +{ + return *(__m512i *) __P; +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_load_epi32 (void const *__P) +{ + return *(__m512i *) __P; +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_load_epi32 (__m512i __W, __mmask16 __U, void const *__P) +{ + return (__m512i) __builtin_ia32_movdqa32load512_mask ((const __v16si *) __P, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_load_epi32 (__mmask16 __U, void const *__P) +{ + return (__m512i) __builtin_ia32_movdqa32load512_mask ((const __v16si *) __P, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_store_si512 (void *__P, __m512i __A) +{ + *(__m512i *) __P = __A; +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_store_epi32 (void *__P, __m512i __A) +{ + *(__m512i *) __P = __A; +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_store_epi32 (void *__P, __mmask16 __U, __m512i __A) +{ + __builtin_ia32_movdqa32store512_mask ((__v16si *) __P, (__v16si) __A, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mullo_epi32 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v16su) __A * (__v16su) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mullo_epi32 (__mmask16 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmulld512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mullo_epi32 (__m512i __W, __mmask16 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmulld512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __W, __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sllv_epi32 (__m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psllv16si_mask ((__v16si) __X, + (__v16si) __Y, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sllv_epi32 (__m512i __W, __mmask16 __U, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psllv16si_mask ((__v16si) __X, + (__v16si) __Y, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sllv_epi32 (__mmask16 __U, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psllv16si_mask ((__v16si) __X, + (__v16si) __Y, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_srav_epi32 (__m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psrav16si_mask ((__v16si) __X, + (__v16si) __Y, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_srav_epi32 (__m512i __W, __mmask16 __U, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psrav16si_mask ((__v16si) __X, + (__v16si) __Y, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_srav_epi32 (__mmask16 __U, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psrav16si_mask ((__v16si) __X, + (__v16si) __Y, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_srlv_epi32 (__m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psrlv16si_mask ((__v16si) __X, + (__v16si) __Y, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_srlv_epi32 (__m512i __W, __mmask16 __U, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psrlv16si_mask ((__v16si) __X, + (__v16si) __Y, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_srlv_epi32 (__mmask16 __U, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psrlv16si_mask ((__v16si) __X, + (__v16si) __Y, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_add_epi64 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v8du) __A + (__v8du) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_add_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_paddq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_add_epi64 (__mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_paddq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sub_epi64 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v8du) __A - (__v8du) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sub_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psubq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sub_epi64 (__mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psubq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sllv_epi64 (__m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psllv8di_mask ((__v8di) __X, + (__v8di) __Y, + (__v8di) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sllv_epi64 (__m512i __W, __mmask8 __U, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psllv8di_mask ((__v8di) __X, + (__v8di) __Y, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sllv_epi64 (__mmask8 __U, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psllv8di_mask ((__v8di) __X, + (__v8di) __Y, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_srav_epi64 (__m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psrav8di_mask ((__v8di) __X, + (__v8di) __Y, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_srav_epi64 (__m512i __W, __mmask8 __U, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psrav8di_mask ((__v8di) __X, + (__v8di) __Y, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_srav_epi64 (__mmask8 __U, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psrav8di_mask ((__v8di) __X, + (__v8di) __Y, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_srlv_epi64 (__m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psrlv8di_mask ((__v8di) __X, + (__v8di) __Y, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_srlv_epi64 (__m512i __W, __mmask8 __U, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psrlv8di_mask ((__v8di) __X, + (__v8di) __Y, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_srlv_epi64 (__mmask8 __U, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_psrlv8di_mask ((__v8di) __X, + (__v8di) __Y, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_add_epi32 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v16su) __A + (__v16su) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_add_epi32 (__m512i __W, __mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_paddd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_add_epi32 (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_paddd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mul_epi32 (__m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_pmuldq512_mask ((__v16si) __X, + (__v16si) __Y, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mul_epi32 (__m512i __W, __mmask8 __M, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_pmuldq512_mask ((__v16si) __X, + (__v16si) __Y, + (__v8di) __W, __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mul_epi32 (__mmask8 __M, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_pmuldq512_mask ((__v16si) __X, + (__v16si) __Y, + (__v8di) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sub_epi32 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v16su) __A - (__v16su) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sub_epi32 (__m512i __W, __mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psubd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sub_epi32 (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_psubd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mul_epu32 (__m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_pmuludq512_mask ((__v16si) __X, + (__v16si) __Y, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mul_epu32 (__m512i __W, __mmask8 __M, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_pmuludq512_mask ((__v16si) __X, + (__v16si) __Y, + (__v8di) __W, __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mul_epu32 (__mmask8 __M, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_pmuludq512_mask ((__v16si) __X, + (__v16si) __Y, + (__v8di) + _mm512_setzero_si512 (), + __M); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_slli_epi64 (__m512i __A, unsigned int __B) +{ + return (__m512i) __builtin_ia32_psllqi512_mask ((__v8di) __A, __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_slli_epi64 (__m512i __W, __mmask8 __U, __m512i __A, + unsigned int __B) +{ + return (__m512i) __builtin_ia32_psllqi512_mask ((__v8di) __A, __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_slli_epi64 (__mmask8 __U, __m512i __A, unsigned int __B) +{ + return (__m512i) __builtin_ia32_psllqi512_mask ((__v8di) __A, __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} +#else +#define _mm512_slli_epi64(X, C) \ + ((__m512i) __builtin_ia32_psllqi512_mask ((__v8di)(__m512i)(X), (int)(C),\ + (__v8di)(__m512i)_mm512_undefined_epi32 (),\ + (__mmask8)-1)) + +#define _mm512_mask_slli_epi64(W, U, X, C) \ + ((__m512i) __builtin_ia32_psllqi512_mask ((__v8di)(__m512i)(X), (int)(C),\ + (__v8di)(__m512i)(W),\ + (__mmask8)(U))) + +#define _mm512_maskz_slli_epi64(U, X, C) \ + ((__m512i) __builtin_ia32_psllqi512_mask ((__v8di)(__m512i)(X), (int)(C),\ + (__v8di)(__m512i)_mm512_setzero_si512 (),\ + (__mmask8)(U))) +#endif + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sll_epi64 (__m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psllq512_mask ((__v8di) __A, + (__v2di) __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sll_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psllq512_mask ((__v8di) __A, + (__v2di) __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sll_epi64 (__mmask8 __U, __m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psllq512_mask ((__v8di) __A, + (__v2di) __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_srli_epi64 (__m512i __A, unsigned int __B) +{ + return (__m512i) __builtin_ia32_psrlqi512_mask ((__v8di) __A, __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_srli_epi64 (__m512i __W, __mmask8 __U, + __m512i __A, unsigned int __B) +{ + return (__m512i) __builtin_ia32_psrlqi512_mask ((__v8di) __A, __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_srli_epi64 (__mmask8 __U, __m512i __A, unsigned int __B) +{ + return (__m512i) __builtin_ia32_psrlqi512_mask ((__v8di) __A, __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} +#else +#define _mm512_srli_epi64(X, C) \ + ((__m512i) __builtin_ia32_psrlqi512_mask ((__v8di)(__m512i)(X), (int)(C),\ + (__v8di)(__m512i)_mm512_undefined_epi32 (),\ + (__mmask8)-1)) + +#define _mm512_mask_srli_epi64(W, U, X, C) \ + ((__m512i) __builtin_ia32_psrlqi512_mask ((__v8di)(__m512i)(X), (int)(C),\ + (__v8di)(__m512i)(W),\ + (__mmask8)(U))) + +#define _mm512_maskz_srli_epi64(U, X, C) \ + ((__m512i) __builtin_ia32_psrlqi512_mask ((__v8di)(__m512i)(X), (int)(C),\ + (__v8di)(__m512i)_mm512_setzero_si512 (),\ + (__mmask8)(U))) +#endif + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_srl_epi64 (__m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psrlq512_mask ((__v8di) __A, + (__v2di) __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_srl_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psrlq512_mask ((__v8di) __A, + (__v2di) __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_srl_epi64 (__mmask8 __U, __m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psrlq512_mask ((__v8di) __A, + (__v2di) __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_srai_epi64 (__m512i __A, unsigned int __B) +{ + return (__m512i) __builtin_ia32_psraqi512_mask ((__v8di) __A, __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_srai_epi64 (__m512i __W, __mmask8 __U, __m512i __A, + unsigned int __B) +{ + return (__m512i) __builtin_ia32_psraqi512_mask ((__v8di) __A, __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_srai_epi64 (__mmask8 __U, __m512i __A, unsigned int __B) +{ + return (__m512i) __builtin_ia32_psraqi512_mask ((__v8di) __A, __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} +#else +#define _mm512_srai_epi64(X, C) \ + ((__m512i) __builtin_ia32_psraqi512_mask ((__v8di)(__m512i)(X), (int)(C),\ + (__v8di)(__m512i)_mm512_undefined_epi32 (),\ + (__mmask8)-1)) + +#define _mm512_mask_srai_epi64(W, U, X, C) \ + ((__m512i) __builtin_ia32_psraqi512_mask ((__v8di)(__m512i)(X), (int)(C),\ + (__v8di)(__m512i)(W),\ + (__mmask8)(U))) + +#define _mm512_maskz_srai_epi64(U, X, C) \ + ((__m512i) __builtin_ia32_psraqi512_mask ((__v8di)(__m512i)(X), (int)(C),\ + (__v8di)(__m512i)_mm512_setzero_si512 (),\ + (__mmask8)(U))) +#endif + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sra_epi64 (__m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psraq512_mask ((__v8di) __A, + (__v2di) __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sra_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psraq512_mask ((__v8di) __A, + (__v2di) __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sra_epi64 (__mmask8 __U, __m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psraq512_mask ((__v8di) __A, + (__v2di) __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_slli_epi32 (__m512i __A, unsigned int __B) +{ + return (__m512i) __builtin_ia32_pslldi512_mask ((__v16si) __A, __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_slli_epi32 (__m512i __W, __mmask16 __U, __m512i __A, + unsigned int __B) +{ + return (__m512i) __builtin_ia32_pslldi512_mask ((__v16si) __A, __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_slli_epi32 (__mmask16 __U, __m512i __A, unsigned int __B) +{ + return (__m512i) __builtin_ia32_pslldi512_mask ((__v16si) __A, __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} +#else +#define _mm512_slli_epi32(X, C) \ + ((__m512i) __builtin_ia32_pslldi512_mask ((__v16si)(__m512i)(X), (int)(C),\ + (__v16si)(__m512i)_mm512_undefined_epi32 (),\ + (__mmask16)-1)) + +#define _mm512_mask_slli_epi32(W, U, X, C) \ + ((__m512i) __builtin_ia32_pslldi512_mask ((__v16si)(__m512i)(X), (int)(C),\ + (__v16si)(__m512i)(W),\ + (__mmask16)(U))) + +#define _mm512_maskz_slli_epi32(U, X, C) \ + ((__m512i) __builtin_ia32_pslldi512_mask ((__v16si)(__m512i)(X), (int)(C),\ + (__v16si)(__m512i)_mm512_setzero_si512 (),\ + (__mmask16)(U))) +#endif + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sll_epi32 (__m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_pslld512_mask ((__v16si) __A, + (__v4si) __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sll_epi32 (__m512i __W, __mmask16 __U, __m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_pslld512_mask ((__v16si) __A, + (__v4si) __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sll_epi32 (__mmask16 __U, __m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_pslld512_mask ((__v16si) __A, + (__v4si) __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_srli_epi32 (__m512i __A, unsigned int __B) +{ + return (__m512i) __builtin_ia32_psrldi512_mask ((__v16si) __A, __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_srli_epi32 (__m512i __W, __mmask16 __U, + __m512i __A, unsigned int __B) +{ + return (__m512i) __builtin_ia32_psrldi512_mask ((__v16si) __A, __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_srli_epi32 (__mmask16 __U, __m512i __A, unsigned int __B) +{ + return (__m512i) __builtin_ia32_psrldi512_mask ((__v16si) __A, __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} +#else +#define _mm512_srli_epi32(X, C) \ + ((__m512i) __builtin_ia32_psrldi512_mask ((__v16si)(__m512i)(X), (int)(C),\ + (__v16si)(__m512i)_mm512_undefined_epi32 (),\ + (__mmask16)-1)) + +#define _mm512_mask_srli_epi32(W, U, X, C) \ + ((__m512i) __builtin_ia32_psrldi512_mask ((__v16si)(__m512i)(X), (int)(C),\ + (__v16si)(__m512i)(W),\ + (__mmask16)(U))) + +#define _mm512_maskz_srli_epi32(U, X, C) \ + ((__m512i) __builtin_ia32_psrldi512_mask ((__v16si)(__m512i)(X), (int)(C),\ + (__v16si)(__m512i)_mm512_setzero_si512 (),\ + (__mmask16)(U))) +#endif + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_srl_epi32 (__m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psrld512_mask ((__v16si) __A, + (__v4si) __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_srl_epi32 (__m512i __W, __mmask16 __U, __m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psrld512_mask ((__v16si) __A, + (__v4si) __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_srl_epi32 (__mmask16 __U, __m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psrld512_mask ((__v16si) __A, + (__v4si) __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_srai_epi32 (__m512i __A, unsigned int __B) +{ + return (__m512i) __builtin_ia32_psradi512_mask ((__v16si) __A, __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_srai_epi32 (__m512i __W, __mmask16 __U, __m512i __A, + unsigned int __B) +{ + return (__m512i) __builtin_ia32_psradi512_mask ((__v16si) __A, __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_srai_epi32 (__mmask16 __U, __m512i __A, unsigned int __B) +{ + return (__m512i) __builtin_ia32_psradi512_mask ((__v16si) __A, __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} +#else +#define _mm512_srai_epi32(X, C) \ + ((__m512i) __builtin_ia32_psradi512_mask ((__v16si)(__m512i)(X), (int)(C),\ + (__v16si)(__m512i)_mm512_undefined_epi32 (),\ + (__mmask16)-1)) + +#define _mm512_mask_srai_epi32(W, U, X, C) \ + ((__m512i) __builtin_ia32_psradi512_mask ((__v16si)(__m512i)(X), (int)(C),\ + (__v16si)(__m512i)(W),\ + (__mmask16)(U))) + +#define _mm512_maskz_srai_epi32(U, X, C) \ + ((__m512i) __builtin_ia32_psradi512_mask ((__v16si)(__m512i)(X), (int)(C),\ + (__v16si)(__m512i)_mm512_setzero_si512 (),\ + (__mmask16)(U))) +#endif + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sra_epi32 (__m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psrad512_mask ((__v16si) __A, + (__v4si) __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sra_epi32 (__m512i __W, __mmask16 __U, __m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psrad512_mask ((__v16si) __A, + (__v4si) __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sra_epi32 (__mmask16 __U, __m512i __A, __m128i __B) +{ + return (__m512i) __builtin_ia32_psrad512_mask ((__v16si) __A, + (__v4si) __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +#ifdef __OPTIMIZE__ +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_add_round_sd (__m128d __A, __m128d __B, const int __R) +{ + return (__m128d) __builtin_ia32_addsd_round ((__v2df) __A, + (__v2df) __B, + __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_add_round_ss (__m128 __A, __m128 __B, const int __R) +{ + return (__m128) __builtin_ia32_addss_round ((__v4sf) __A, + (__v4sf) __B, + __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sub_round_sd (__m128d __A, __m128d __B, const int __R) +{ + return (__m128d) __builtin_ia32_subsd_round ((__v2df) __A, + (__v2df) __B, + __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sub_round_ss (__m128 __A, __m128 __B, const int __R) +{ + return (__m128) __builtin_ia32_subss_round ((__v4sf) __A, + (__v4sf) __B, + __R); +} + +#else +#define _mm_add_round_sd(A, B, C) \ + (__m128d)__builtin_ia32_addsd_round(A, B, C) + +#define _mm_add_round_ss(A, B, C) \ + (__m128)__builtin_ia32_addss_round(A, B, C) + +#define _mm_sub_round_sd(A, B, C) \ + (__m128d)__builtin_ia32_subsd_round(A, B, C) + +#define _mm_sub_round_ss(A, B, C) \ + (__m128)__builtin_ia32_subss_round(A, B, C) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_ternarylogic_epi64 (__m512i __A, __m512i __B, __m512i __C, + const int __imm) +{ + return (__m512i) __builtin_ia32_pternlogq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __C, __imm, + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_ternarylogic_epi64 (__m512i __A, __mmask8 __U, __m512i __B, + __m512i __C, const int __imm) +{ + return (__m512i) __builtin_ia32_pternlogq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __C, __imm, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_ternarylogic_epi64 (__mmask8 __U, __m512i __A, __m512i __B, + __m512i __C, const int __imm) +{ + return (__m512i) __builtin_ia32_pternlogq512_maskz ((__v8di) __A, + (__v8di) __B, + (__v8di) __C, + __imm, (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_ternarylogic_epi32 (__m512i __A, __m512i __B, __m512i __C, + const int __imm) +{ + return (__m512i) __builtin_ia32_pternlogd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __C, + __imm, (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_ternarylogic_epi32 (__m512i __A, __mmask16 __U, __m512i __B, + __m512i __C, const int __imm) +{ + return (__m512i) __builtin_ia32_pternlogd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __C, + __imm, (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_ternarylogic_epi32 (__mmask16 __U, __m512i __A, __m512i __B, + __m512i __C, const int __imm) +{ + return (__m512i) __builtin_ia32_pternlogd512_maskz ((__v16si) __A, + (__v16si) __B, + (__v16si) __C, + __imm, (__mmask16) __U); +} +#else +#define _mm512_ternarylogic_epi64(A, B, C, I) \ + ((__m512i) __builtin_ia32_pternlogq512_mask ((__v8di)(__m512i)(A), \ + (__v8di)(__m512i)(B), (__v8di)(__m512i)(C), (int)(I), (__mmask8)-1)) +#define _mm512_mask_ternarylogic_epi64(A, U, B, C, I) \ + ((__m512i) __builtin_ia32_pternlogq512_mask ((__v8di)(__m512i)(A), \ + (__v8di)(__m512i)(B), (__v8di)(__m512i)(C), (int)(I), (__mmask8)(U))) +#define _mm512_maskz_ternarylogic_epi64(U, A, B, C, I) \ + ((__m512i) __builtin_ia32_pternlogq512_maskz ((__v8di)(__m512i)(A), \ + (__v8di)(__m512i)(B), (__v8di)(__m512i)(C), (int)(I), (__mmask8)(U))) +#define _mm512_ternarylogic_epi32(A, B, C, I) \ + ((__m512i) __builtin_ia32_pternlogd512_mask ((__v16si)(__m512i)(A), \ + (__v16si)(__m512i)(B), (__v16si)(__m512i)(C), (int)(I), \ + (__mmask16)-1)) +#define _mm512_mask_ternarylogic_epi32(A, U, B, C, I) \ + ((__m512i) __builtin_ia32_pternlogd512_mask ((__v16si)(__m512i)(A), \ + (__v16si)(__m512i)(B), (__v16si)(__m512i)(C), (int)(I), \ + (__mmask16)(U))) +#define _mm512_maskz_ternarylogic_epi32(U, A, B, C, I) \ + ((__m512i) __builtin_ia32_pternlogd512_maskz ((__v16si)(__m512i)(A), \ + (__v16si)(__m512i)(B), (__v16si)(__m512i)(C), (int)(I), \ + (__mmask16)(U))) +#endif + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_rcp14_pd (__m512d __A) +{ + return (__m512d) __builtin_ia32_rcp14pd512_mask ((__v8df) __A, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_rcp14_pd (__m512d __W, __mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_rcp14pd512_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_rcp14_pd (__mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_rcp14pd512_mask ((__v8df) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_rcp14_ps (__m512 __A) +{ + return (__m512) __builtin_ia32_rcp14ps512_mask ((__v16sf) __A, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_rcp14_ps (__m512 __W, __mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_rcp14ps512_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_rcp14_ps (__mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_rcp14ps512_mask ((__v16sf) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rcp14_sd (__m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_rcp14sd ((__v2df) __B, + (__v2df) __A); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rcp14_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_rcp14ss ((__v4sf) __B, + (__v4sf) __A); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_rsqrt14_pd (__m512d __A) +{ + return (__m512d) __builtin_ia32_rsqrt14pd512_mask ((__v8df) __A, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_rsqrt14_pd (__m512d __W, __mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_rsqrt14pd512_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_rsqrt14_pd (__mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_rsqrt14pd512_mask ((__v8df) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_rsqrt14_ps (__m512 __A) +{ + return (__m512) __builtin_ia32_rsqrt14ps512_mask ((__v16sf) __A, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_rsqrt14_ps (__m512 __W, __mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_rsqrt14ps512_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_rsqrt14_ps (__mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_rsqrt14ps512_mask ((__v16sf) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rsqrt14_sd (__m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_rsqrt14sd ((__v2df) __B, + (__v2df) __A); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rsqrt14_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_rsqrt14ss ((__v4sf) __B, + (__v4sf) __A); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sqrt_round_pd (__m512d __A, const int __R) +{ + return (__m512d) __builtin_ia32_sqrtpd512_mask ((__v8df) __A, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sqrt_round_pd (__m512d __W, __mmask8 __U, __m512d __A, + const int __R) +{ + return (__m512d) __builtin_ia32_sqrtpd512_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sqrt_round_pd (__mmask8 __U, __m512d __A, const int __R) +{ + return (__m512d) __builtin_ia32_sqrtpd512_mask ((__v8df) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sqrt_round_ps (__m512 __A, const int __R) +{ + return (__m512) __builtin_ia32_sqrtps512_mask ((__v16sf) __A, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sqrt_round_ps (__m512 __W, __mmask16 __U, __m512 __A, const int __R) +{ + return (__m512) __builtin_ia32_sqrtps512_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sqrt_round_ps (__mmask16 __U, __m512 __A, const int __R) +{ + return (__m512) __builtin_ia32_sqrtps512_mask ((__v16sf) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sqrt_round_sd (__m128d __A, __m128d __B, const int __R) +{ + return (__m128d) __builtin_ia32_sqrtsd_round ((__v2df) __B, + (__v2df) __A, + __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sqrt_round_ss (__m128 __A, __m128 __B, const int __R) +{ + return (__m128) __builtin_ia32_sqrtss_round ((__v4sf) __B, + (__v4sf) __A, + __R); +} +#else +#define _mm512_sqrt_round_pd(A, C) \ + (__m512d)__builtin_ia32_sqrtpd512_mask(A, (__v8df)_mm512_undefined_pd(), -1, C) + +#define _mm512_mask_sqrt_round_pd(W, U, A, C) \ + (__m512d)__builtin_ia32_sqrtpd512_mask(A, W, U, C) + +#define _mm512_maskz_sqrt_round_pd(U, A, C) \ + (__m512d)__builtin_ia32_sqrtpd512_mask(A, (__v8df)_mm512_setzero_pd(), U, C) + +#define _mm512_sqrt_round_ps(A, C) \ + (__m512)__builtin_ia32_sqrtps512_mask(A, (__v16sf)_mm512_undefined_ps(), -1, C) + +#define _mm512_mask_sqrt_round_ps(W, U, A, C) \ + (__m512)__builtin_ia32_sqrtps512_mask(A, W, U, C) + +#define _mm512_maskz_sqrt_round_ps(U, A, C) \ + (__m512)__builtin_ia32_sqrtps512_mask(A, (__v16sf)_mm512_setzero_ps(), U, C) + +#define _mm_sqrt_round_sd(A, B, C) \ + (__m128d)__builtin_ia32_sqrtsd_round(A, B, C) + +#define _mm_sqrt_round_ss(A, B, C) \ + (__m128)__builtin_ia32_sqrtss_round(A, B, C) +#endif + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepi8_epi32 (__m128i __A) +{ + return (__m512i) __builtin_ia32_pmovsxbd512_mask ((__v16qi) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi8_epi32 (__m512i __W, __mmask16 __U, __m128i __A) +{ + return (__m512i) __builtin_ia32_pmovsxbd512_mask ((__v16qi) __A, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepi8_epi32 (__mmask16 __U, __m128i __A) +{ + return (__m512i) __builtin_ia32_pmovsxbd512_mask ((__v16qi) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepi8_epi64 (__m128i __A) +{ + return (__m512i) __builtin_ia32_pmovsxbq512_mask ((__v16qi) __A, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi8_epi64 (__m512i __W, __mmask8 __U, __m128i __A) +{ + return (__m512i) __builtin_ia32_pmovsxbq512_mask ((__v16qi) __A, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepi8_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m512i) __builtin_ia32_pmovsxbq512_mask ((__v16qi) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepi16_epi32 (__m256i __A) +{ + return (__m512i) __builtin_ia32_pmovsxwd512_mask ((__v16hi) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi16_epi32 (__m512i __W, __mmask16 __U, __m256i __A) +{ + return (__m512i) __builtin_ia32_pmovsxwd512_mask ((__v16hi) __A, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepi16_epi32 (__mmask16 __U, __m256i __A) +{ + return (__m512i) __builtin_ia32_pmovsxwd512_mask ((__v16hi) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepi16_epi64 (__m128i __A) +{ + return (__m512i) __builtin_ia32_pmovsxwq512_mask ((__v8hi) __A, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi16_epi64 (__m512i __W, __mmask8 __U, __m128i __A) +{ + return (__m512i) __builtin_ia32_pmovsxwq512_mask ((__v8hi) __A, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepi16_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m512i) __builtin_ia32_pmovsxwq512_mask ((__v8hi) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepi32_epi64 (__m256i __X) +{ + return (__m512i) __builtin_ia32_pmovsxdq512_mask ((__v8si) __X, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi32_epi64 (__m512i __W, __mmask8 __U, __m256i __X) +{ + return (__m512i) __builtin_ia32_pmovsxdq512_mask ((__v8si) __X, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepi32_epi64 (__mmask8 __U, __m256i __X) +{ + return (__m512i) __builtin_ia32_pmovsxdq512_mask ((__v8si) __X, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepu8_epi32 (__m128i __A) +{ + return (__m512i) __builtin_ia32_pmovzxbd512_mask ((__v16qi) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepu8_epi32 (__m512i __W, __mmask16 __U, __m128i __A) +{ + return (__m512i) __builtin_ia32_pmovzxbd512_mask ((__v16qi) __A, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepu8_epi32 (__mmask16 __U, __m128i __A) +{ + return (__m512i) __builtin_ia32_pmovzxbd512_mask ((__v16qi) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepu8_epi64 (__m128i __A) +{ + return (__m512i) __builtin_ia32_pmovzxbq512_mask ((__v16qi) __A, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepu8_epi64 (__m512i __W, __mmask8 __U, __m128i __A) +{ + return (__m512i) __builtin_ia32_pmovzxbq512_mask ((__v16qi) __A, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepu8_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m512i) __builtin_ia32_pmovzxbq512_mask ((__v16qi) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepu16_epi32 (__m256i __A) +{ + return (__m512i) __builtin_ia32_pmovzxwd512_mask ((__v16hi) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepu16_epi32 (__m512i __W, __mmask16 __U, __m256i __A) +{ + return (__m512i) __builtin_ia32_pmovzxwd512_mask ((__v16hi) __A, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepu16_epi32 (__mmask16 __U, __m256i __A) +{ + return (__m512i) __builtin_ia32_pmovzxwd512_mask ((__v16hi) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepu16_epi64 (__m128i __A) +{ + return (__m512i) __builtin_ia32_pmovzxwq512_mask ((__v8hi) __A, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepu16_epi64 (__m512i __W, __mmask8 __U, __m128i __A) +{ + return (__m512i) __builtin_ia32_pmovzxwq512_mask ((__v8hi) __A, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepu16_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m512i) __builtin_ia32_pmovzxwq512_mask ((__v8hi) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepu32_epi64 (__m256i __X) +{ + return (__m512i) __builtin_ia32_pmovzxdq512_mask ((__v8si) __X, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepu32_epi64 (__m512i __W, __mmask8 __U, __m256i __X) +{ + return (__m512i) __builtin_ia32_pmovzxdq512_mask ((__v8si) __X, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepu32_epi64 (__mmask8 __U, __m256i __X) +{ + return (__m512i) __builtin_ia32_pmovzxdq512_mask ((__v8si) __X, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_add_round_pd (__m512d __A, __m512d __B, const int __R) +{ + return (__m512d) __builtin_ia32_addpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_add_round_pd (__m512d __W, __mmask8 __U, __m512d __A, + __m512d __B, const int __R) +{ + return (__m512d) __builtin_ia32_addpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_add_round_pd (__mmask8 __U, __m512d __A, __m512d __B, + const int __R) +{ + return (__m512d) __builtin_ia32_addpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_add_round_ps (__m512 __A, __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_addps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_add_round_ps (__m512 __W, __mmask16 __U, __m512 __A, + __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_addps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_add_round_ps (__mmask16 __U, __m512 __A, __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_addps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sub_round_pd (__m512d __A, __m512d __B, const int __R) +{ + return (__m512d) __builtin_ia32_subpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sub_round_pd (__m512d __W, __mmask8 __U, __m512d __A, + __m512d __B, const int __R) +{ + return (__m512d) __builtin_ia32_subpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sub_round_pd (__mmask8 __U, __m512d __A, __m512d __B, + const int __R) +{ + return (__m512d) __builtin_ia32_subpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sub_round_ps (__m512 __A, __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_subps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sub_round_ps (__m512 __W, __mmask16 __U, __m512 __A, + __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_subps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sub_round_ps (__mmask16 __U, __m512 __A, __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_subps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, __R); +} +#else +#define _mm512_add_round_pd(A, B, C) \ + (__m512d)__builtin_ia32_addpd512_mask(A, B, (__v8df)_mm512_undefined_pd(), -1, C) + +#define _mm512_mask_add_round_pd(W, U, A, B, C) \ + (__m512d)__builtin_ia32_addpd512_mask(A, B, W, U, C) + +#define _mm512_maskz_add_round_pd(U, A, B, C) \ + (__m512d)__builtin_ia32_addpd512_mask(A, B, (__v8df)_mm512_setzero_pd(), U, C) + +#define _mm512_add_round_ps(A, B, C) \ + (__m512)__builtin_ia32_addps512_mask(A, B, (__v16sf)_mm512_undefined_ps(), -1, C) + +#define _mm512_mask_add_round_ps(W, U, A, B, C) \ + (__m512)__builtin_ia32_addps512_mask(A, B, W, U, C) + +#define _mm512_maskz_add_round_ps(U, A, B, C) \ + (__m512)__builtin_ia32_addps512_mask(A, B, (__v16sf)_mm512_setzero_ps(), U, C) + +#define _mm512_sub_round_pd(A, B, C) \ + (__m512d)__builtin_ia32_subpd512_mask(A, B, (__v8df)_mm512_undefined_pd(), -1, C) + +#define _mm512_mask_sub_round_pd(W, U, A, B, C) \ + (__m512d)__builtin_ia32_subpd512_mask(A, B, W, U, C) + +#define _mm512_maskz_sub_round_pd(U, A, B, C) \ + (__m512d)__builtin_ia32_subpd512_mask(A, B, (__v8df)_mm512_setzero_pd(), U, C) + +#define _mm512_sub_round_ps(A, B, C) \ + (__m512)__builtin_ia32_subps512_mask(A, B, (__v16sf)_mm512_undefined_ps(), -1, C) + +#define _mm512_mask_sub_round_ps(W, U, A, B, C) \ + (__m512)__builtin_ia32_subps512_mask(A, B, W, U, C) + +#define _mm512_maskz_sub_round_ps(U, A, B, C) \ + (__m512)__builtin_ia32_subps512_mask(A, B, (__v16sf)_mm512_setzero_ps(), U, C) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mul_round_pd (__m512d __A, __m512d __B, const int __R) +{ + return (__m512d) __builtin_ia32_mulpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mul_round_pd (__m512d __W, __mmask8 __U, __m512d __A, + __m512d __B, const int __R) +{ + return (__m512d) __builtin_ia32_mulpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mul_round_pd (__mmask8 __U, __m512d __A, __m512d __B, + const int __R) +{ + return (__m512d) __builtin_ia32_mulpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mul_round_ps (__m512 __A, __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_mulps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mul_round_ps (__m512 __W, __mmask16 __U, __m512 __A, + __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_mulps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mul_round_ps (__mmask16 __U, __m512 __A, __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_mulps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_div_round_pd (__m512d __M, __m512d __V, const int __R) +{ + return (__m512d) __builtin_ia32_divpd512_mask ((__v8df) __M, + (__v8df) __V, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_div_round_pd (__m512d __W, __mmask8 __U, __m512d __M, + __m512d __V, const int __R) +{ + return (__m512d) __builtin_ia32_divpd512_mask ((__v8df) __M, + (__v8df) __V, + (__v8df) __W, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_div_round_pd (__mmask8 __U, __m512d __M, __m512d __V, + const int __R) +{ + return (__m512d) __builtin_ia32_divpd512_mask ((__v8df) __M, + (__v8df) __V, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_div_round_ps (__m512 __A, __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_divps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_div_round_ps (__m512 __W, __mmask16 __U, __m512 __A, + __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_divps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_div_round_ps (__mmask16 __U, __m512 __A, __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_divps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mul_round_sd (__m128d __A, __m128d __B, const int __R) +{ + return (__m128d) __builtin_ia32_mulsd_round ((__v2df) __A, + (__v2df) __B, + __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mul_round_ss (__m128 __A, __m128 __B, const int __R) +{ + return (__m128) __builtin_ia32_mulss_round ((__v4sf) __A, + (__v4sf) __B, + __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_div_round_sd (__m128d __A, __m128d __B, const int __R) +{ + return (__m128d) __builtin_ia32_divsd_round ((__v2df) __A, + (__v2df) __B, + __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_div_round_ss (__m128 __A, __m128 __B, const int __R) +{ + return (__m128) __builtin_ia32_divss_round ((__v4sf) __A, + (__v4sf) __B, + __R); +} + +#else +#define _mm512_mul_round_pd(A, B, C) \ + (__m512d)__builtin_ia32_mulpd512_mask(A, B, (__v8df)_mm512_undefined_pd(), -1, C) + +#define _mm512_mask_mul_round_pd(W, U, A, B, C) \ + (__m512d)__builtin_ia32_mulpd512_mask(A, B, W, U, C) + +#define _mm512_maskz_mul_round_pd(U, A, B, C) \ + (__m512d)__builtin_ia32_mulpd512_mask(A, B, (__v8df)_mm512_setzero_pd(), U, C) + +#define _mm512_mul_round_ps(A, B, C) \ + (__m512)__builtin_ia32_mulps512_mask(A, B, (__v16sf)_mm512_undefined_ps(), -1, C) + +#define _mm512_mask_mul_round_ps(W, U, A, B, C) \ + (__m512)__builtin_ia32_mulps512_mask(A, B, W, U, C) + +#define _mm512_maskz_mul_round_ps(U, A, B, C) \ + (__m512)__builtin_ia32_mulps512_mask(A, B, (__v16sf)_mm512_setzero_ps(), U, C) + +#define _mm512_div_round_pd(A, B, C) \ + (__m512d)__builtin_ia32_divpd512_mask(A, B, (__v8df)_mm512_undefined_pd(), -1, C) + +#define _mm512_mask_div_round_pd(W, U, A, B, C) \ + (__m512d)__builtin_ia32_divpd512_mask(A, B, W, U, C) + +#define _mm512_maskz_div_round_pd(U, A, B, C) \ + (__m512d)__builtin_ia32_divpd512_mask(A, B, (__v8df)_mm512_setzero_pd(), U, C) + +#define _mm512_div_round_ps(A, B, C) \ + (__m512)__builtin_ia32_divps512_mask(A, B, (__v16sf)_mm512_undefined_ps(), -1, C) + +#define _mm512_mask_div_round_ps(W, U, A, B, C) \ + (__m512)__builtin_ia32_divps512_mask(A, B, W, U, C) + +#define _mm512_maskz_div_round_ps(U, A, B, C) \ + (__m512)__builtin_ia32_divps512_mask(A, B, (__v16sf)_mm512_setzero_ps(), U, C) + +#define _mm_mul_round_sd(A, B, C) \ + (__m128d)__builtin_ia32_mulsd_round(A, B, C) + +#define _mm_mul_round_ss(A, B, C) \ + (__m128)__builtin_ia32_mulss_round(A, B, C) + +#define _mm_div_round_sd(A, B, C) \ + (__m128d)__builtin_ia32_divsd_round(A, B, C) + +#define _mm_div_round_ss(A, B, C) \ + (__m128)__builtin_ia32_divss_round(A, B, C) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_max_round_pd (__m512d __A, __m512d __B, const int __R) +{ + return (__m512d) __builtin_ia32_maxpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_max_round_pd (__m512d __W, __mmask8 __U, __m512d __A, + __m512d __B, const int __R) +{ + return (__m512d) __builtin_ia32_maxpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_max_round_pd (__mmask8 __U, __m512d __A, __m512d __B, + const int __R) +{ + return (__m512d) __builtin_ia32_maxpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_max_round_ps (__m512 __A, __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_maxps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_max_round_ps (__m512 __W, __mmask16 __U, __m512 __A, + __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_maxps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_max_round_ps (__mmask16 __U, __m512 __A, __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_maxps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_min_round_pd (__m512d __A, __m512d __B, const int __R) +{ + return (__m512d) __builtin_ia32_minpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_min_round_pd (__m512d __W, __mmask8 __U, __m512d __A, + __m512d __B, const int __R) +{ + return (__m512d) __builtin_ia32_minpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_min_round_pd (__mmask8 __U, __m512d __A, __m512d __B, + const int __R) +{ + return (__m512d) __builtin_ia32_minpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_min_round_ps (__m512 __A, __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_minps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_min_round_ps (__m512 __W, __mmask16 __U, __m512 __A, + __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_minps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_min_round_ps (__mmask16 __U, __m512 __A, __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_minps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, __R); +} +#else +#define _mm512_max_round_pd(A, B, R) \ + (__m512d)__builtin_ia32_maxpd512_mask(A, B, (__v8df)_mm512_undefined_pd(), -1, R) + +#define _mm512_mask_max_round_pd(W, U, A, B, R) \ + (__m512d)__builtin_ia32_maxpd512_mask(A, B, W, U, R) + +#define _mm512_maskz_max_round_pd(U, A, B, R) \ + (__m512d)__builtin_ia32_maxpd512_mask(A, B, (__v8df)_mm512_setzero_pd(), U, R) + +#define _mm512_max_round_ps(A, B, R) \ + (__m512)__builtin_ia32_maxps512_mask(A, B, (__v16sf)_mm512_undefined_pd(), -1, R) + +#define _mm512_mask_max_round_ps(W, U, A, B, R) \ + (__m512)__builtin_ia32_maxps512_mask(A, B, W, U, R) + +#define _mm512_maskz_max_round_ps(U, A, B, R) \ + (__m512)__builtin_ia32_maxps512_mask(A, B, (__v16sf)_mm512_setzero_ps(), U, R) + +#define _mm512_min_round_pd(A, B, R) \ + (__m512d)__builtin_ia32_minpd512_mask(A, B, (__v8df)_mm512_undefined_pd(), -1, R) + +#define _mm512_mask_min_round_pd(W, U, A, B, R) \ + (__m512d)__builtin_ia32_minpd512_mask(A, B, W, U, R) + +#define _mm512_maskz_min_round_pd(U, A, B, R) \ + (__m512d)__builtin_ia32_minpd512_mask(A, B, (__v8df)_mm512_setzero_pd(), U, R) + +#define _mm512_min_round_ps(A, B, R) \ + (__m512)__builtin_ia32_minps512_mask(A, B, (__v16sf)_mm512_undefined_ps(), -1, R) + +#define _mm512_mask_min_round_ps(W, U, A, B, R) \ + (__m512)__builtin_ia32_minps512_mask(A, B, W, U, R) + +#define _mm512_maskz_min_round_ps(U, A, B, R) \ + (__m512)__builtin_ia32_minps512_mask(A, B, (__v16sf)_mm512_setzero_ps(), U, R) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_scalef_round_pd (__m512d __A, __m512d __B, const int __R) +{ + return (__m512d) __builtin_ia32_scalefpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_scalef_round_pd (__m512d __W, __mmask8 __U, __m512d __A, + __m512d __B, const int __R) +{ + return (__m512d) __builtin_ia32_scalefpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_scalef_round_pd (__mmask8 __U, __m512d __A, __m512d __B, + const int __R) +{ + return (__m512d) __builtin_ia32_scalefpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_scalef_round_ps (__m512 __A, __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_scalefps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_scalef_round_ps (__m512 __W, __mmask16 __U, __m512 __A, + __m512 __B, const int __R) +{ + return (__m512) __builtin_ia32_scalefps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_scalef_round_ps (__mmask16 __U, __m512 __A, __m512 __B, + const int __R) +{ + return (__m512) __builtin_ia32_scalefps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_scalef_round_sd (__m128d __A, __m128d __B, const int __R) +{ + return (__m128d) __builtin_ia32_scalefsd_round ((__v2df) __A, + (__v2df) __B, + __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_scalef_round_ss (__m128 __A, __m128 __B, const int __R) +{ + return (__m128) __builtin_ia32_scalefss_round ((__v4sf) __A, + (__v4sf) __B, + __R); +} +#else +#define _mm512_scalef_round_pd(A, B, C) \ + (__m512d)__builtin_ia32_scalefpd512_mask(A, B, (__v8df)_mm512_undefined_pd(), -1, C) + +#define _mm512_mask_scalef_round_pd(W, U, A, B, C) \ + (__m512d)__builtin_ia32_scalefpd512_mask(A, B, W, U, C) + +#define _mm512_maskz_scalef_round_pd(U, A, B, C) \ + (__m512d)__builtin_ia32_scalefpd512_mask(A, B, (__v8df)_mm512_setzero_pd(), U, C) + +#define _mm512_scalef_round_ps(A, B, C) \ + (__m512)__builtin_ia32_scalefps512_mask(A, B, (__v16sf)_mm512_undefined_ps(), -1, C) + +#define _mm512_mask_scalef_round_ps(W, U, A, B, C) \ + (__m512)__builtin_ia32_scalefps512_mask(A, B, W, U, C) + +#define _mm512_maskz_scalef_round_ps(U, A, B, C) \ + (__m512)__builtin_ia32_scalefps512_mask(A, B, (__v16sf)_mm512_setzero_ps(), U, C) + +#define _mm_scalef_round_sd(A, B, C) \ + (__m128d)__builtin_ia32_scalefsd_round(A, B, C) + +#define _mm_scalef_round_ss(A, B, C) \ + (__m128)__builtin_ia32_scalefss_round(A, B, C) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fmadd_round_pd (__m512d __A, __m512d __B, __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fmadd_round_pd (__m512d __A, __mmask8 __U, __m512d __B, + __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fmadd_round_pd (__m512d __A, __m512d __B, __m512d __C, + __mmask8 __U, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_mask3 ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fmadd_round_pd (__mmask8 __U, __m512d __A, __m512d __B, + __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_maskz ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fmadd_round_ps (__m512 __A, __m512 __B, __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fmadd_round_ps (__m512 __A, __mmask16 __U, __m512 __B, + __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fmadd_round_ps (__m512 __A, __m512 __B, __m512 __C, + __mmask16 __U, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddps512_mask3 ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fmadd_round_ps (__mmask16 __U, __m512 __A, __m512 __B, + __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddps512_maskz ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fmsub_round_pd (__m512d __A, __m512d __B, __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A, + (__v8df) __B, + -(__v8df) __C, + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fmsub_round_pd (__m512d __A, __mmask8 __U, __m512d __B, + __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A, + (__v8df) __B, + -(__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fmsub_round_pd (__m512d __A, __m512d __B, __m512d __C, + __mmask8 __U, const int __R) +{ + return (__m512d) __builtin_ia32_vfmsubpd512_mask3 ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fmsub_round_pd (__mmask8 __U, __m512d __A, __m512d __B, + __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_maskz ((__v8df) __A, + (__v8df) __B, + -(__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fmsub_round_ps (__m512 __A, __m512 __B, __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A, + (__v16sf) __B, + -(__v16sf) __C, + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fmsub_round_ps (__m512 __A, __mmask16 __U, __m512 __B, + __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A, + (__v16sf) __B, + -(__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fmsub_round_ps (__m512 __A, __m512 __B, __m512 __C, + __mmask16 __U, const int __R) +{ + return (__m512) __builtin_ia32_vfmsubps512_mask3 ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fmsub_round_ps (__mmask16 __U, __m512 __A, __m512 __B, + __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddps512_maskz ((__v16sf) __A, + (__v16sf) __B, + -(__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fmaddsub_round_pd (__m512d __A, __m512d __B, __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddsubpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fmaddsub_round_pd (__m512d __A, __mmask8 __U, __m512d __B, + __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddsubpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fmaddsub_round_pd (__m512d __A, __m512d __B, __m512d __C, + __mmask8 __U, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddsubpd512_mask3 ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fmaddsub_round_pd (__mmask8 __U, __m512d __A, __m512d __B, + __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddsubpd512_maskz ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fmaddsub_round_ps (__m512 __A, __m512 __B, __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddsubps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fmaddsub_round_ps (__m512 __A, __mmask16 __U, __m512 __B, + __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddsubps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fmaddsub_round_ps (__m512 __A, __m512 __B, __m512 __C, + __mmask16 __U, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddsubps512_mask3 ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fmaddsub_round_ps (__mmask16 __U, __m512 __A, __m512 __B, + __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddsubps512_maskz ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fmsubadd_round_pd (__m512d __A, __m512d __B, __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddsubpd512_mask ((__v8df) __A, + (__v8df) __B, + -(__v8df) __C, + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fmsubadd_round_pd (__m512d __A, __mmask8 __U, __m512d __B, + __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddsubpd512_mask ((__v8df) __A, + (__v8df) __B, + -(__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fmsubadd_round_pd (__m512d __A, __m512d __B, __m512d __C, + __mmask8 __U, const int __R) +{ + return (__m512d) __builtin_ia32_vfmsubaddpd512_mask3 ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fmsubadd_round_pd (__mmask8 __U, __m512d __A, __m512d __B, + __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddsubpd512_maskz ((__v8df) __A, + (__v8df) __B, + -(__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fmsubadd_round_ps (__m512 __A, __m512 __B, __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddsubps512_mask ((__v16sf) __A, + (__v16sf) __B, + -(__v16sf) __C, + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fmsubadd_round_ps (__m512 __A, __mmask16 __U, __m512 __B, + __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddsubps512_mask ((__v16sf) __A, + (__v16sf) __B, + -(__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fmsubadd_round_ps (__m512 __A, __m512 __B, __m512 __C, + __mmask16 __U, const int __R) +{ + return (__m512) __builtin_ia32_vfmsubaddps512_mask3 ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fmsubadd_round_ps (__mmask16 __U, __m512 __A, __m512 __B, + __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddsubps512_maskz ((__v16sf) __A, + (__v16sf) __B, + -(__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fnmadd_round_pd (__m512d __A, __m512d __B, __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_mask (-(__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fnmadd_round_pd (__m512d __A, __mmask8 __U, __m512d __B, + __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfnmaddpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fnmadd_round_pd (__m512d __A, __m512d __B, __m512d __C, + __mmask8 __U, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_mask3 (-(__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fnmadd_round_pd (__mmask8 __U, __m512d __A, __m512d __B, + __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_maskz (-(__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fnmadd_round_ps (__m512 __A, __m512 __B, __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddps512_mask (-(__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fnmadd_round_ps (__m512 __A, __mmask16 __U, __m512 __B, + __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfnmaddps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fnmadd_round_ps (__m512 __A, __m512 __B, __m512 __C, + __mmask16 __U, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddps512_mask3 (-(__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fnmadd_round_ps (__mmask16 __U, __m512 __A, __m512 __B, + __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddps512_maskz (-(__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fnmsub_round_pd (__m512d __A, __m512d __B, __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_mask (-(__v8df) __A, + (__v8df) __B, + -(__v8df) __C, + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fnmsub_round_pd (__m512d __A, __mmask8 __U, __m512d __B, + __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfnmsubpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fnmsub_round_pd (__m512d __A, __m512d __B, __m512d __C, + __mmask8 __U, const int __R) +{ + return (__m512d) __builtin_ia32_vfnmsubpd512_mask3 ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fnmsub_round_pd (__mmask8 __U, __m512d __A, __m512d __B, + __m512d __C, const int __R) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_maskz (-(__v8df) __A, + (__v8df) __B, + -(__v8df) __C, + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fnmsub_round_ps (__m512 __A, __m512 __B, __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddps512_mask (-(__v16sf) __A, + (__v16sf) __B, + -(__v16sf) __C, + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fnmsub_round_ps (__m512 __A, __mmask16 __U, __m512 __B, + __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfnmsubps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fnmsub_round_ps (__m512 __A, __m512 __B, __m512 __C, + __mmask16 __U, const int __R) +{ + return (__m512) __builtin_ia32_vfnmsubps512_mask3 ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fnmsub_round_ps (__mmask16 __U, __m512 __A, __m512 __B, + __m512 __C, const int __R) +{ + return (__m512) __builtin_ia32_vfmaddps512_maskz (-(__v16sf) __A, + (__v16sf) __B, + -(__v16sf) __C, + (__mmask16) __U, __R); +} +#else +#define _mm512_fmadd_round_pd(A, B, C, R) \ + (__m512d)__builtin_ia32_vfmaddpd512_mask(A, B, C, -1, R) + +#define _mm512_mask_fmadd_round_pd(A, U, B, C, R) \ + (__m512d)__builtin_ia32_vfmaddpd512_mask(A, B, C, U, R) + +#define _mm512_mask3_fmadd_round_pd(A, B, C, U, R) \ + (__m512d)__builtin_ia32_vfmaddpd512_mask3(A, B, C, U, R) + +#define _mm512_maskz_fmadd_round_pd(U, A, B, C, R) \ + (__m512d)__builtin_ia32_vfmaddpd512_maskz(A, B, C, U, R) + +#define _mm512_fmadd_round_ps(A, B, C, R) \ + (__m512)__builtin_ia32_vfmaddps512_mask(A, B, C, -1, R) + +#define _mm512_mask_fmadd_round_ps(A, U, B, C, R) \ + (__m512)__builtin_ia32_vfmaddps512_mask(A, B, C, U, R) + +#define _mm512_mask3_fmadd_round_ps(A, B, C, U, R) \ + (__m512)__builtin_ia32_vfmaddps512_mask3(A, B, C, U, R) + +#define _mm512_maskz_fmadd_round_ps(U, A, B, C, R) \ + (__m512)__builtin_ia32_vfmaddps512_maskz(A, B, C, U, R) + +#define _mm512_fmsub_round_pd(A, B, C, R) \ + (__m512d)__builtin_ia32_vfmaddpd512_mask(A, B, -(C), -1, R) + +#define _mm512_mask_fmsub_round_pd(A, U, B, C, R) \ + (__m512d)__builtin_ia32_vfmaddpd512_mask(A, B, -(C), U, R) + +#define _mm512_mask3_fmsub_round_pd(A, B, C, U, R) \ + (__m512d)__builtin_ia32_vfmsubpd512_mask3(A, B, C, U, R) + +#define _mm512_maskz_fmsub_round_pd(U, A, B, C, R) \ + (__m512d)__builtin_ia32_vfmaddpd512_maskz(A, B, -(C), U, R) + +#define _mm512_fmsub_round_ps(A, B, C, R) \ + (__m512)__builtin_ia32_vfmaddps512_mask(A, B, -(C), -1, R) + +#define _mm512_mask_fmsub_round_ps(A, U, B, C, R) \ + (__m512)__builtin_ia32_vfmaddps512_mask(A, B, -(C), U, R) + +#define _mm512_mask3_fmsub_round_ps(A, B, C, U, R) \ + (__m512)__builtin_ia32_vfmsubps512_mask3(A, B, C, U, R) + +#define _mm512_maskz_fmsub_round_ps(U, A, B, C, R) \ + (__m512)__builtin_ia32_vfmaddps512_maskz(A, B, -(C), U, R) + +#define _mm512_fmaddsub_round_pd(A, B, C, R) \ + (__m512d)__builtin_ia32_vfmaddsubpd512_mask(A, B, C, -1, R) + +#define _mm512_mask_fmaddsub_round_pd(A, U, B, C, R) \ + (__m512d)__builtin_ia32_vfmaddpd512_mask(A, B, C, U, R) + +#define _mm512_mask3_fmaddsub_round_pd(A, B, C, U, R) \ + (__m512d)__builtin_ia32_vfmaddsubpd512_mask3(A, B, C, U, R) + +#define _mm512_maskz_fmaddsub_round_pd(U, A, B, C, R) \ + (__m512d)__builtin_ia32_vfmaddsubpd512_maskz(A, B, C, U, R) + +#define _mm512_fmaddsub_round_ps(A, B, C, R) \ + (__m512)__builtin_ia32_vfmaddsubps512_mask(A, B, C, -1, R) + +#define _mm512_mask_fmaddsub_round_ps(A, U, B, C, R) \ + (__m512)__builtin_ia32_vfmaddsubps512_mask(A, B, C, U, R) + +#define _mm512_mask3_fmaddsub_round_ps(A, B, C, U, R) \ + (__m512)__builtin_ia32_vfmaddsubps512_mask3(A, B, C, U, R) + +#define _mm512_maskz_fmaddsub_round_ps(U, A, B, C, R) \ + (__m512)__builtin_ia32_vfmaddsubps512_maskz(A, B, C, U, R) + +#define _mm512_fmsubadd_round_pd(A, B, C, R) \ + (__m512d)__builtin_ia32_vfmaddsubpd512_mask(A, B, -(C), -1, R) + +#define _mm512_mask_fmsubadd_round_pd(A, U, B, C, R) \ + (__m512d)__builtin_ia32_vfmaddsubpd512_mask(A, B, -(C), U, R) + +#define _mm512_mask3_fmsubadd_round_pd(A, B, C, U, R) \ + (__m512d)__builtin_ia32_vfmsubaddpd512_mask3(A, B, C, U, R) + +#define _mm512_maskz_fmsubadd_round_pd(U, A, B, C, R) \ + (__m512d)__builtin_ia32_vfmaddsubpd512_maskz(A, B, -(C), U, R) + +#define _mm512_fmsubadd_round_ps(A, B, C, R) \ + (__m512)__builtin_ia32_vfmaddsubps512_mask(A, B, -(C), -1, R) + +#define _mm512_mask_fmsubadd_round_ps(A, U, B, C, R) \ + (__m512)__builtin_ia32_vfmaddsubps512_mask(A, B, -(C), U, R) + +#define _mm512_mask3_fmsubadd_round_ps(A, B, C, U, R) \ + (__m512)__builtin_ia32_vfmsubaddps512_mask3(A, B, C, U, R) + +#define _mm512_maskz_fmsubadd_round_ps(U, A, B, C, R) \ + (__m512)__builtin_ia32_vfmaddsubps512_maskz(A, B, -(C), U, R) + +#define _mm512_fnmadd_round_pd(A, B, C, R) \ + (__m512d)__builtin_ia32_vfmaddpd512_mask(-(A), B, C, -1, R) + +#define _mm512_mask_fnmadd_round_pd(A, U, B, C, R) \ + (__m512d)__builtin_ia32_vfnmaddpd512_mask(-(A), B, C, U, R) + +#define _mm512_mask3_fnmadd_round_pd(A, B, C, U, R) \ + (__m512d)__builtin_ia32_vfmaddpd512_mask3(-(A), B, C, U, R) + +#define _mm512_maskz_fnmadd_round_pd(U, A, B, C, R) \ + (__m512d)__builtin_ia32_vfmaddpd512_maskz(-(A), B, C, U, R) + +#define _mm512_fnmadd_round_ps(A, B, C, R) \ + (__m512)__builtin_ia32_vfmaddps512_mask(-(A), B, C, -1, R) + +#define _mm512_mask_fnmadd_round_ps(A, U, B, C, R) \ + (__m512)__builtin_ia32_vfnmaddps512_mask(-(A), B, C, U, R) + +#define _mm512_mask3_fnmadd_round_ps(A, B, C, U, R) \ + (__m512)__builtin_ia32_vfmaddps512_mask3(-(A), B, C, U, R) + +#define _mm512_maskz_fnmadd_round_ps(U, A, B, C, R) \ + (__m512)__builtin_ia32_vfmaddps512_maskz(-(A), B, C, U, R) + +#define _mm512_fnmsub_round_pd(A, B, C, R) \ + (__m512d)__builtin_ia32_vfmaddpd512_mask(-(A), B, -(C), -1, R) + +#define _mm512_mask_fnmsub_round_pd(A, U, B, C, R) \ + (__m512d)__builtin_ia32_vfnmsubpd512_mask(A, B, C, U, R) + +#define _mm512_mask3_fnmsub_round_pd(A, B, C, U, R) \ + (__m512d)__builtin_ia32_vfnmsubpd512_mask3(A, B, C, U, R) + +#define _mm512_maskz_fnmsub_round_pd(U, A, B, C, R) \ + (__m512d)__builtin_ia32_vfmaddpd512_maskz(-(A), B, -(C), U, R) + +#define _mm512_fnmsub_round_ps(A, B, C, R) \ + (__m512)__builtin_ia32_vfmaddps512_mask(-(A), B, -(C), -1, R) + +#define _mm512_mask_fnmsub_round_ps(A, U, B, C, R) \ + (__m512)__builtin_ia32_vfnmsubps512_mask(A, B, C, U, R) + +#define _mm512_mask3_fnmsub_round_ps(A, B, C, U, R) \ + (__m512)__builtin_ia32_vfnmsubps512_mask3(A, B, C, U, R) + +#define _mm512_maskz_fnmsub_round_ps(U, A, B, C, R) \ + (__m512)__builtin_ia32_vfmaddps512_maskz(-(A), B, -(C), U, R) +#endif + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_abs_epi64 (__m512i __A) +{ + return (__m512i) __builtin_ia32_pabsq512_mask ((__v8di) __A, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_abs_epi64 (__m512i __W, __mmask8 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_pabsq512_mask ((__v8di) __A, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_abs_epi64 (__mmask8 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_pabsq512_mask ((__v8di) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_abs_epi32 (__m512i __A) +{ + return (__m512i) __builtin_ia32_pabsd512_mask ((__v16si) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_abs_epi32 (__m512i __W, __mmask16 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_pabsd512_mask ((__v16si) __A, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_abs_epi32 (__mmask16 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_pabsd512_mask ((__v16si) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcastss_ps (__m128 __A) +{ + return (__m512) __builtin_ia32_broadcastss512 ((__v4sf) __A, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_broadcastss_ps (__m512 __O, __mmask16 __M, __m128 __A) +{ + return (__m512) __builtin_ia32_broadcastss512 ((__v4sf) __A, + (__v16sf) __O, __M); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_broadcastss_ps (__mmask16 __M, __m128 __A) +{ + return (__m512) __builtin_ia32_broadcastss512 ((__v4sf) __A, + (__v16sf) + _mm512_setzero_ps (), + __M); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcastsd_pd (__m128d __A) +{ + return (__m512d) __builtin_ia32_broadcastsd512 ((__v2df) __A, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_broadcastsd_pd (__m512d __O, __mmask8 __M, __m128d __A) +{ + return (__m512d) __builtin_ia32_broadcastsd512 ((__v2df) __A, + (__v8df) __O, __M); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_broadcastsd_pd (__mmask8 __M, __m128d __A) +{ + return (__m512d) __builtin_ia32_broadcastsd512 ((__v2df) __A, + (__v8df) + _mm512_setzero_pd (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcastd_epi32 (__m128i __A) +{ + return (__m512i) __builtin_ia32_pbroadcastd512 ((__v4si) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_broadcastd_epi32 (__m512i __O, __mmask16 __M, __m128i __A) +{ + return (__m512i) __builtin_ia32_pbroadcastd512 ((__v4si) __A, + (__v16si) __O, __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_broadcastd_epi32 (__mmask16 __M, __m128i __A) +{ + return (__m512i) __builtin_ia32_pbroadcastd512 ((__v4si) __A, + (__v16si) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_set1_epi32 (int __A) +{ + return (__m512i) __builtin_ia32_pbroadcastd512_gpr_mask (__A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16)(-1)); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_set1_epi32 (__m512i __O, __mmask16 __M, int __A) +{ + return (__m512i) __builtin_ia32_pbroadcastd512_gpr_mask (__A, (__v16si) __O, + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_set1_epi32 (__mmask16 __M, int __A) +{ + return (__m512i) + __builtin_ia32_pbroadcastd512_gpr_mask (__A, + (__v16si) _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcastq_epi64 (__m128i __A) +{ + return (__m512i) __builtin_ia32_pbroadcastq512 ((__v2di) __A, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_broadcastq_epi64 (__m512i __O, __mmask8 __M, __m128i __A) +{ + return (__m512i) __builtin_ia32_pbroadcastq512 ((__v2di) __A, + (__v8di) __O, __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_broadcastq_epi64 (__mmask8 __M, __m128i __A) +{ + return (__m512i) __builtin_ia32_pbroadcastq512 ((__v2di) __A, + (__v8di) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_set1_epi64 (long long __A) +{ + return (__m512i) __builtin_ia32_pbroadcastq512_gpr_mask (__A, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8)(-1)); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_set1_epi64 (__m512i __O, __mmask8 __M, long long __A) +{ + return (__m512i) __builtin_ia32_pbroadcastq512_gpr_mask (__A, (__v8di) __O, + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_set1_epi64 (__mmask8 __M, long long __A) +{ + return (__m512i) + __builtin_ia32_pbroadcastq512_gpr_mask (__A, + (__v8di) _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcast_f32x4 (__m128 __A) +{ + return (__m512) __builtin_ia32_broadcastf32x4_512 ((__v4sf) __A, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_broadcast_f32x4 (__m512 __O, __mmask16 __M, __m128 __A) +{ + return (__m512) __builtin_ia32_broadcastf32x4_512 ((__v4sf) __A, + (__v16sf) __O, + __M); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_broadcast_f32x4 (__mmask16 __M, __m128 __A) +{ + return (__m512) __builtin_ia32_broadcastf32x4_512 ((__v4sf) __A, + (__v16sf) + _mm512_setzero_ps (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcast_i32x4 (__m128i __A) +{ + return (__m512i) __builtin_ia32_broadcasti32x4_512 ((__v4si) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_broadcast_i32x4 (__m512i __O, __mmask16 __M, __m128i __A) +{ + return (__m512i) __builtin_ia32_broadcasti32x4_512 ((__v4si) __A, + (__v16si) __O, + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_broadcast_i32x4 (__mmask16 __M, __m128i __A) +{ + return (__m512i) __builtin_ia32_broadcasti32x4_512 ((__v4si) __A, + (__v16si) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcast_f64x4 (__m256d __A) +{ + return (__m512d) __builtin_ia32_broadcastf64x4_512 ((__v4df) __A, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_broadcast_f64x4 (__m512d __O, __mmask8 __M, __m256d __A) +{ + return (__m512d) __builtin_ia32_broadcastf64x4_512 ((__v4df) __A, + (__v8df) __O, + __M); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_broadcast_f64x4 (__mmask8 __M, __m256d __A) +{ + return (__m512d) __builtin_ia32_broadcastf64x4_512 ((__v4df) __A, + (__v8df) + _mm512_setzero_pd (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_broadcast_i64x4 (__m256i __A) +{ + return (__m512i) __builtin_ia32_broadcasti64x4_512 ((__v4di) __A, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_broadcast_i64x4 (__m512i __O, __mmask8 __M, __m256i __A) +{ + return (__m512i) __builtin_ia32_broadcasti64x4_512 ((__v4di) __A, + (__v8di) __O, + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_broadcast_i64x4 (__mmask8 __M, __m256i __A) +{ + return (__m512i) __builtin_ia32_broadcasti64x4_512 ((__v4di) __A, + (__v8di) + _mm512_setzero_si512 (), + __M); +} + +typedef enum +{ + _MM_PERM_AAAA = 0x00, _MM_PERM_AAAB = 0x01, _MM_PERM_AAAC = 0x02, + _MM_PERM_AAAD = 0x03, _MM_PERM_AABA = 0x04, _MM_PERM_AABB = 0x05, + _MM_PERM_AABC = 0x06, _MM_PERM_AABD = 0x07, _MM_PERM_AACA = 0x08, + _MM_PERM_AACB = 0x09, _MM_PERM_AACC = 0x0A, _MM_PERM_AACD = 0x0B, + _MM_PERM_AADA = 0x0C, _MM_PERM_AADB = 0x0D, _MM_PERM_AADC = 0x0E, + _MM_PERM_AADD = 0x0F, _MM_PERM_ABAA = 0x10, _MM_PERM_ABAB = 0x11, + _MM_PERM_ABAC = 0x12, _MM_PERM_ABAD = 0x13, _MM_PERM_ABBA = 0x14, + _MM_PERM_ABBB = 0x15, _MM_PERM_ABBC = 0x16, _MM_PERM_ABBD = 0x17, + _MM_PERM_ABCA = 0x18, _MM_PERM_ABCB = 0x19, _MM_PERM_ABCC = 0x1A, + _MM_PERM_ABCD = 0x1B, _MM_PERM_ABDA = 0x1C, _MM_PERM_ABDB = 0x1D, + _MM_PERM_ABDC = 0x1E, _MM_PERM_ABDD = 0x1F, _MM_PERM_ACAA = 0x20, + _MM_PERM_ACAB = 0x21, _MM_PERM_ACAC = 0x22, _MM_PERM_ACAD = 0x23, + _MM_PERM_ACBA = 0x24, _MM_PERM_ACBB = 0x25, _MM_PERM_ACBC = 0x26, + _MM_PERM_ACBD = 0x27, _MM_PERM_ACCA = 0x28, _MM_PERM_ACCB = 0x29, + _MM_PERM_ACCC = 0x2A, _MM_PERM_ACCD = 0x2B, _MM_PERM_ACDA = 0x2C, + _MM_PERM_ACDB = 0x2D, _MM_PERM_ACDC = 0x2E, _MM_PERM_ACDD = 0x2F, + _MM_PERM_ADAA = 0x30, _MM_PERM_ADAB = 0x31, _MM_PERM_ADAC = 0x32, + _MM_PERM_ADAD = 0x33, _MM_PERM_ADBA = 0x34, _MM_PERM_ADBB = 0x35, + _MM_PERM_ADBC = 0x36, _MM_PERM_ADBD = 0x37, _MM_PERM_ADCA = 0x38, + _MM_PERM_ADCB = 0x39, _MM_PERM_ADCC = 0x3A, _MM_PERM_ADCD = 0x3B, + _MM_PERM_ADDA = 0x3C, _MM_PERM_ADDB = 0x3D, _MM_PERM_ADDC = 0x3E, + _MM_PERM_ADDD = 0x3F, _MM_PERM_BAAA = 0x40, _MM_PERM_BAAB = 0x41, + _MM_PERM_BAAC = 0x42, _MM_PERM_BAAD = 0x43, _MM_PERM_BABA = 0x44, + _MM_PERM_BABB = 0x45, _MM_PERM_BABC = 0x46, _MM_PERM_BABD = 0x47, + _MM_PERM_BACA = 0x48, _MM_PERM_BACB = 0x49, _MM_PERM_BACC = 0x4A, + _MM_PERM_BACD = 0x4B, _MM_PERM_BADA = 0x4C, _MM_PERM_BADB = 0x4D, + _MM_PERM_BADC = 0x4E, _MM_PERM_BADD = 0x4F, _MM_PERM_BBAA = 0x50, + _MM_PERM_BBAB = 0x51, _MM_PERM_BBAC = 0x52, _MM_PERM_BBAD = 0x53, + _MM_PERM_BBBA = 0x54, _MM_PERM_BBBB = 0x55, _MM_PERM_BBBC = 0x56, + _MM_PERM_BBBD = 0x57, _MM_PERM_BBCA = 0x58, _MM_PERM_BBCB = 0x59, + _MM_PERM_BBCC = 0x5A, _MM_PERM_BBCD = 0x5B, _MM_PERM_BBDA = 0x5C, + _MM_PERM_BBDB = 0x5D, _MM_PERM_BBDC = 0x5E, _MM_PERM_BBDD = 0x5F, + _MM_PERM_BCAA = 0x60, _MM_PERM_BCAB = 0x61, _MM_PERM_BCAC = 0x62, + _MM_PERM_BCAD = 0x63, _MM_PERM_BCBA = 0x64, _MM_PERM_BCBB = 0x65, + _MM_PERM_BCBC = 0x66, _MM_PERM_BCBD = 0x67, _MM_PERM_BCCA = 0x68, + _MM_PERM_BCCB = 0x69, _MM_PERM_BCCC = 0x6A, _MM_PERM_BCCD = 0x6B, + _MM_PERM_BCDA = 0x6C, _MM_PERM_BCDB = 0x6D, _MM_PERM_BCDC = 0x6E, + _MM_PERM_BCDD = 0x6F, _MM_PERM_BDAA = 0x70, _MM_PERM_BDAB = 0x71, + _MM_PERM_BDAC = 0x72, _MM_PERM_BDAD = 0x73, _MM_PERM_BDBA = 0x74, + _MM_PERM_BDBB = 0x75, _MM_PERM_BDBC = 0x76, _MM_PERM_BDBD = 0x77, + _MM_PERM_BDCA = 0x78, _MM_PERM_BDCB = 0x79, _MM_PERM_BDCC = 0x7A, + _MM_PERM_BDCD = 0x7B, _MM_PERM_BDDA = 0x7C, _MM_PERM_BDDB = 0x7D, + _MM_PERM_BDDC = 0x7E, _MM_PERM_BDDD = 0x7F, _MM_PERM_CAAA = 0x80, + _MM_PERM_CAAB = 0x81, _MM_PERM_CAAC = 0x82, _MM_PERM_CAAD = 0x83, + _MM_PERM_CABA = 0x84, _MM_PERM_CABB = 0x85, _MM_PERM_CABC = 0x86, + _MM_PERM_CABD = 0x87, _MM_PERM_CACA = 0x88, _MM_PERM_CACB = 0x89, + _MM_PERM_CACC = 0x8A, _MM_PERM_CACD = 0x8B, _MM_PERM_CADA = 0x8C, + _MM_PERM_CADB = 0x8D, _MM_PERM_CADC = 0x8E, _MM_PERM_CADD = 0x8F, + _MM_PERM_CBAA = 0x90, _MM_PERM_CBAB = 0x91, _MM_PERM_CBAC = 0x92, + _MM_PERM_CBAD = 0x93, _MM_PERM_CBBA = 0x94, _MM_PERM_CBBB = 0x95, + _MM_PERM_CBBC = 0x96, _MM_PERM_CBBD = 0x97, _MM_PERM_CBCA = 0x98, + _MM_PERM_CBCB = 0x99, _MM_PERM_CBCC = 0x9A, _MM_PERM_CBCD = 0x9B, + _MM_PERM_CBDA = 0x9C, _MM_PERM_CBDB = 0x9D, _MM_PERM_CBDC = 0x9E, + _MM_PERM_CBDD = 0x9F, _MM_PERM_CCAA = 0xA0, _MM_PERM_CCAB = 0xA1, + _MM_PERM_CCAC = 0xA2, _MM_PERM_CCAD = 0xA3, _MM_PERM_CCBA = 0xA4, + _MM_PERM_CCBB = 0xA5, _MM_PERM_CCBC = 0xA6, _MM_PERM_CCBD = 0xA7, + _MM_PERM_CCCA = 0xA8, _MM_PERM_CCCB = 0xA9, _MM_PERM_CCCC = 0xAA, + _MM_PERM_CCCD = 0xAB, _MM_PERM_CCDA = 0xAC, _MM_PERM_CCDB = 0xAD, + _MM_PERM_CCDC = 0xAE, _MM_PERM_CCDD = 0xAF, _MM_PERM_CDAA = 0xB0, + _MM_PERM_CDAB = 0xB1, _MM_PERM_CDAC = 0xB2, _MM_PERM_CDAD = 0xB3, + _MM_PERM_CDBA = 0xB4, _MM_PERM_CDBB = 0xB5, _MM_PERM_CDBC = 0xB6, + _MM_PERM_CDBD = 0xB7, _MM_PERM_CDCA = 0xB8, _MM_PERM_CDCB = 0xB9, + _MM_PERM_CDCC = 0xBA, _MM_PERM_CDCD = 0xBB, _MM_PERM_CDDA = 0xBC, + _MM_PERM_CDDB = 0xBD, _MM_PERM_CDDC = 0xBE, _MM_PERM_CDDD = 0xBF, + _MM_PERM_DAAA = 0xC0, _MM_PERM_DAAB = 0xC1, _MM_PERM_DAAC = 0xC2, + _MM_PERM_DAAD = 0xC3, _MM_PERM_DABA = 0xC4, _MM_PERM_DABB = 0xC5, + _MM_PERM_DABC = 0xC6, _MM_PERM_DABD = 0xC7, _MM_PERM_DACA = 0xC8, + _MM_PERM_DACB = 0xC9, _MM_PERM_DACC = 0xCA, _MM_PERM_DACD = 0xCB, + _MM_PERM_DADA = 0xCC, _MM_PERM_DADB = 0xCD, _MM_PERM_DADC = 0xCE, + _MM_PERM_DADD = 0xCF, _MM_PERM_DBAA = 0xD0, _MM_PERM_DBAB = 0xD1, + _MM_PERM_DBAC = 0xD2, _MM_PERM_DBAD = 0xD3, _MM_PERM_DBBA = 0xD4, + _MM_PERM_DBBB = 0xD5, _MM_PERM_DBBC = 0xD6, _MM_PERM_DBBD = 0xD7, + _MM_PERM_DBCA = 0xD8, _MM_PERM_DBCB = 0xD9, _MM_PERM_DBCC = 0xDA, + _MM_PERM_DBCD = 0xDB, _MM_PERM_DBDA = 0xDC, _MM_PERM_DBDB = 0xDD, + _MM_PERM_DBDC = 0xDE, _MM_PERM_DBDD = 0xDF, _MM_PERM_DCAA = 0xE0, + _MM_PERM_DCAB = 0xE1, _MM_PERM_DCAC = 0xE2, _MM_PERM_DCAD = 0xE3, + _MM_PERM_DCBA = 0xE4, _MM_PERM_DCBB = 0xE5, _MM_PERM_DCBC = 0xE6, + _MM_PERM_DCBD = 0xE7, _MM_PERM_DCCA = 0xE8, _MM_PERM_DCCB = 0xE9, + _MM_PERM_DCCC = 0xEA, _MM_PERM_DCCD = 0xEB, _MM_PERM_DCDA = 0xEC, + _MM_PERM_DCDB = 0xED, _MM_PERM_DCDC = 0xEE, _MM_PERM_DCDD = 0xEF, + _MM_PERM_DDAA = 0xF0, _MM_PERM_DDAB = 0xF1, _MM_PERM_DDAC = 0xF2, + _MM_PERM_DDAD = 0xF3, _MM_PERM_DDBA = 0xF4, _MM_PERM_DDBB = 0xF5, + _MM_PERM_DDBC = 0xF6, _MM_PERM_DDBD = 0xF7, _MM_PERM_DDCA = 0xF8, + _MM_PERM_DDCB = 0xF9, _MM_PERM_DDCC = 0xFA, _MM_PERM_DDCD = 0xFB, + _MM_PERM_DDDA = 0xFC, _MM_PERM_DDDB = 0xFD, _MM_PERM_DDDC = 0xFE, + _MM_PERM_DDDD = 0xFF +} _MM_PERM_ENUM; + +#ifdef __OPTIMIZE__ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_shuffle_epi32 (__m512i __A, _MM_PERM_ENUM __mask) +{ + return (__m512i) __builtin_ia32_pshufd512_mask ((__v16si) __A, + __mask, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_shuffle_epi32 (__m512i __W, __mmask16 __U, __m512i __A, + _MM_PERM_ENUM __mask) +{ + return (__m512i) __builtin_ia32_pshufd512_mask ((__v16si) __A, + __mask, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_shuffle_epi32 (__mmask16 __U, __m512i __A, _MM_PERM_ENUM __mask) +{ + return (__m512i) __builtin_ia32_pshufd512_mask ((__v16si) __A, + __mask, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_shuffle_i64x2 (__m512i __A, __m512i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_shuf_i64x2_mask ((__v8di) __A, + (__v8di) __B, __imm, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_shuffle_i64x2 (__m512i __W, __mmask8 __U, __m512i __A, + __m512i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_shuf_i64x2_mask ((__v8di) __A, + (__v8di) __B, __imm, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_shuffle_i64x2 (__mmask8 __U, __m512i __A, __m512i __B, + const int __imm) +{ + return (__m512i) __builtin_ia32_shuf_i64x2_mask ((__v8di) __A, + (__v8di) __B, __imm, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_shuffle_i32x4 (__m512i __A, __m512i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_shuf_i32x4_mask ((__v16si) __A, + (__v16si) __B, + __imm, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_shuffle_i32x4 (__m512i __W, __mmask16 __U, __m512i __A, + __m512i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_shuf_i32x4_mask ((__v16si) __A, + (__v16si) __B, + __imm, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_shuffle_i32x4 (__mmask16 __U, __m512i __A, __m512i __B, + const int __imm) +{ + return (__m512i) __builtin_ia32_shuf_i32x4_mask ((__v16si) __A, + (__v16si) __B, + __imm, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_shuffle_f64x2 (__m512d __A, __m512d __B, const int __imm) +{ + return (__m512d) __builtin_ia32_shuf_f64x2_mask ((__v8df) __A, + (__v8df) __B, __imm, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_shuffle_f64x2 (__m512d __W, __mmask8 __U, __m512d __A, + __m512d __B, const int __imm) +{ + return (__m512d) __builtin_ia32_shuf_f64x2_mask ((__v8df) __A, + (__v8df) __B, __imm, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_shuffle_f64x2 (__mmask8 __U, __m512d __A, __m512d __B, + const int __imm) +{ + return (__m512d) __builtin_ia32_shuf_f64x2_mask ((__v8df) __A, + (__v8df) __B, __imm, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_shuffle_f32x4 (__m512 __A, __m512 __B, const int __imm) +{ + return (__m512) __builtin_ia32_shuf_f32x4_mask ((__v16sf) __A, + (__v16sf) __B, __imm, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_shuffle_f32x4 (__m512 __W, __mmask16 __U, __m512 __A, + __m512 __B, const int __imm) +{ + return (__m512) __builtin_ia32_shuf_f32x4_mask ((__v16sf) __A, + (__v16sf) __B, __imm, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_shuffle_f32x4 (__mmask16 __U, __m512 __A, __m512 __B, + const int __imm) +{ + return (__m512) __builtin_ia32_shuf_f32x4_mask ((__v16sf) __A, + (__v16sf) __B, __imm, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +#else +#define _mm512_shuffle_epi32(X, C) \ + ((__m512i) __builtin_ia32_pshufd512_mask ((__v16si)(__m512i)(X), (int)(C),\ + (__v16si)(__m512i)_mm512_undefined_epi32 (),\ + (__mmask16)-1)) + +#define _mm512_mask_shuffle_epi32(W, U, X, C) \ + ((__m512i) __builtin_ia32_pshufd512_mask ((__v16si)(__m512i)(X), (int)(C),\ + (__v16si)(__m512i)(W),\ + (__mmask16)(U))) + +#define _mm512_maskz_shuffle_epi32(U, X, C) \ + ((__m512i) __builtin_ia32_pshufd512_mask ((__v16si)(__m512i)(X), (int)(C),\ + (__v16si)(__m512i)_mm512_setzero_si512 (),\ + (__mmask16)(U))) + +#define _mm512_shuffle_i64x2(X, Y, C) \ + ((__m512i) __builtin_ia32_shuf_i64x2_mask ((__v8di)(__m512i)(X), \ + (__v8di)(__m512i)(Y), (int)(C),\ + (__v8di)(__m512i)_mm512_undefined_epi32 (),\ + (__mmask8)-1)) + +#define _mm512_mask_shuffle_i64x2(W, U, X, Y, C) \ + ((__m512i) __builtin_ia32_shuf_i64x2_mask ((__v8di)(__m512i)(X), \ + (__v8di)(__m512i)(Y), (int)(C),\ + (__v8di)(__m512i)(W),\ + (__mmask8)(U))) + +#define _mm512_maskz_shuffle_i64x2(U, X, Y, C) \ + ((__m512i) __builtin_ia32_shuf_i64x2_mask ((__v8di)(__m512i)(X), \ + (__v8di)(__m512i)(Y), (int)(C),\ + (__v8di)(__m512i)_mm512_setzero_si512 (),\ + (__mmask8)(U))) + +#define _mm512_shuffle_i32x4(X, Y, C) \ + ((__m512i) __builtin_ia32_shuf_i32x4_mask ((__v16si)(__m512i)(X), \ + (__v16si)(__m512i)(Y), (int)(C),\ + (__v16si)(__m512i)_mm512_undefined_epi32 (),\ + (__mmask16)-1)) + +#define _mm512_mask_shuffle_i32x4(W, U, X, Y, C) \ + ((__m512i) __builtin_ia32_shuf_i32x4_mask ((__v16si)(__m512i)(X), \ + (__v16si)(__m512i)(Y), (int)(C),\ + (__v16si)(__m512i)(W),\ + (__mmask16)(U))) + +#define _mm512_maskz_shuffle_i32x4(U, X, Y, C) \ + ((__m512i) __builtin_ia32_shuf_i32x4_mask ((__v16si)(__m512i)(X), \ + (__v16si)(__m512i)(Y), (int)(C),\ + (__v16si)(__m512i)_mm512_setzero_si512 (),\ + (__mmask16)(U))) + +#define _mm512_shuffle_f64x2(X, Y, C) \ + ((__m512d) __builtin_ia32_shuf_f64x2_mask ((__v8df)(__m512d)(X), \ + (__v8df)(__m512d)(Y), (int)(C),\ + (__v8df)(__m512d)_mm512_undefined_pd(),\ + (__mmask8)-1)) + +#define _mm512_mask_shuffle_f64x2(W, U, X, Y, C) \ + ((__m512d) __builtin_ia32_shuf_f64x2_mask ((__v8df)(__m512d)(X), \ + (__v8df)(__m512d)(Y), (int)(C),\ + (__v8df)(__m512d)(W),\ + (__mmask8)(U))) + +#define _mm512_maskz_shuffle_f64x2(U, X, Y, C) \ + ((__m512d) __builtin_ia32_shuf_f64x2_mask ((__v8df)(__m512d)(X), \ + (__v8df)(__m512d)(Y), (int)(C),\ + (__v8df)(__m512d)_mm512_setzero_pd(),\ + (__mmask8)(U))) + +#define _mm512_shuffle_f32x4(X, Y, C) \ + ((__m512) __builtin_ia32_shuf_f32x4_mask ((__v16sf)(__m512)(X), \ + (__v16sf)(__m512)(Y), (int)(C),\ + (__v16sf)(__m512)_mm512_undefined_ps(),\ + (__mmask16)-1)) + +#define _mm512_mask_shuffle_f32x4(W, U, X, Y, C) \ + ((__m512) __builtin_ia32_shuf_f32x4_mask ((__v16sf)(__m512)(X), \ + (__v16sf)(__m512)(Y), (int)(C),\ + (__v16sf)(__m512)(W),\ + (__mmask16)(U))) + +#define _mm512_maskz_shuffle_f32x4(U, X, Y, C) \ + ((__m512) __builtin_ia32_shuf_f32x4_mask ((__v16sf)(__m512)(X), \ + (__v16sf)(__m512)(Y), (int)(C),\ + (__v16sf)(__m512)_mm512_setzero_ps(),\ + (__mmask16)(U))) +#endif + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_rolv_epi32 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_prolvd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_rolv_epi32 (__m512i __W, __mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_prolvd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_rolv_epi32 (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_prolvd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_rorv_epi32 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_prorvd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_rorv_epi32 (__m512i __W, __mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_prorvd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_rorv_epi32 (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_prorvd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_rolv_epi64 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_prolvq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_rolv_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_prolvq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_rolv_epi64 (__mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_prolvq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_rorv_epi64 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_prorvq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_rorv_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_prorvq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_rorv_epi64 (__mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_prorvq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +#ifdef __OPTIMIZE__ +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtt_roundpd_epi32 (__m512d __A, const int __R) +{ + return (__m256i) __builtin_ia32_cvttpd2dq512_mask ((__v8df) __A, + (__v8si) + _mm256_undefined_si256 (), + (__mmask8) -1, __R); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtt_roundpd_epi32 (__m256i __W, __mmask8 __U, __m512d __A, + const int __R) +{ + return (__m256i) __builtin_ia32_cvttpd2dq512_mask ((__v8df) __A, + (__v8si) __W, + (__mmask8) __U, __R); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtt_roundpd_epi32 (__mmask8 __U, __m512d __A, const int __R) +{ + return (__m256i) __builtin_ia32_cvttpd2dq512_mask ((__v8df) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U, __R); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtt_roundpd_epu32 (__m512d __A, const int __R) +{ + return (__m256i) __builtin_ia32_cvttpd2udq512_mask ((__v8df) __A, + (__v8si) + _mm256_undefined_si256 (), + (__mmask8) -1, __R); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtt_roundpd_epu32 (__m256i __W, __mmask8 __U, __m512d __A, + const int __R) +{ + return (__m256i) __builtin_ia32_cvttpd2udq512_mask ((__v8df) __A, + (__v8si) __W, + (__mmask8) __U, __R); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtt_roundpd_epu32 (__mmask8 __U, __m512d __A, const int __R) +{ + return (__m256i) __builtin_ia32_cvttpd2udq512_mask ((__v8df) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U, __R); +} +#else +#define _mm512_cvtt_roundpd_epi32(A, B) \ + ((__m256i)__builtin_ia32_cvttpd2dq512_mask(A, (__v8si)_mm256_undefined_si256(), -1, B)) + +#define _mm512_mask_cvtt_roundpd_epi32(W, U, A, B) \ + ((__m256i)__builtin_ia32_cvttpd2dq512_mask(A, (__v8si)(W), U, B)) + +#define _mm512_maskz_cvtt_roundpd_epi32(U, A, B) \ + ((__m256i)__builtin_ia32_cvttpd2dq512_mask(A, (__v8si)_mm256_setzero_si256(), U, B)) + +#define _mm512_cvtt_roundpd_epu32(A, B) \ + ((__m256i)__builtin_ia32_cvttpd2udq512_mask(A, (__v8si)_mm256_undefined_si256(), -1, B)) + +#define _mm512_mask_cvtt_roundpd_epu32(W, U, A, B) \ + ((__m256i)__builtin_ia32_cvttpd2udq512_mask(A, (__v8si)(W), U, B)) + +#define _mm512_maskz_cvtt_roundpd_epu32(U, A, B) \ + ((__m256i)__builtin_ia32_cvttpd2udq512_mask(A, (__v8si)_mm256_setzero_si256(), U, B)) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundpd_epi32 (__m512d __A, const int __R) +{ + return (__m256i) __builtin_ia32_cvtpd2dq512_mask ((__v8df) __A, + (__v8si) + _mm256_undefined_si256 (), + (__mmask8) -1, __R); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundpd_epi32 (__m256i __W, __mmask8 __U, __m512d __A, + const int __R) +{ + return (__m256i) __builtin_ia32_cvtpd2dq512_mask ((__v8df) __A, + (__v8si) __W, + (__mmask8) __U, __R); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundpd_epi32 (__mmask8 __U, __m512d __A, const int __R) +{ + return (__m256i) __builtin_ia32_cvtpd2dq512_mask ((__v8df) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U, __R); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundpd_epu32 (__m512d __A, const int __R) +{ + return (__m256i) __builtin_ia32_cvtpd2udq512_mask ((__v8df) __A, + (__v8si) + _mm256_undefined_si256 (), + (__mmask8) -1, __R); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundpd_epu32 (__m256i __W, __mmask8 __U, __m512d __A, + const int __R) +{ + return (__m256i) __builtin_ia32_cvtpd2udq512_mask ((__v8df) __A, + (__v8si) __W, + (__mmask8) __U, __R); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundpd_epu32 (__mmask8 __U, __m512d __A, const int __R) +{ + return (__m256i) __builtin_ia32_cvtpd2udq512_mask ((__v8df) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U, __R); +} +#else +#define _mm512_cvt_roundpd_epi32(A, B) \ + ((__m256i)__builtin_ia32_cvtpd2dq512_mask(A, (__v8si)_mm256_undefined_si256(), -1, B)) + +#define _mm512_mask_cvt_roundpd_epi32(W, U, A, B) \ + ((__m256i)__builtin_ia32_cvtpd2dq512_mask(A, (__v8si)(W), U, B)) + +#define _mm512_maskz_cvt_roundpd_epi32(U, A, B) \ + ((__m256i)__builtin_ia32_cvtpd2dq512_mask(A, (__v8si)_mm256_setzero_si256(), U, B)) + +#define _mm512_cvt_roundpd_epu32(A, B) \ + ((__m256i)__builtin_ia32_cvtpd2udq512_mask(A, (__v8si)_mm256_undefined_si256(), -1, B)) + +#define _mm512_mask_cvt_roundpd_epu32(W, U, A, B) \ + ((__m256i)__builtin_ia32_cvtpd2udq512_mask(A, (__v8si)(W), U, B)) + +#define _mm512_maskz_cvt_roundpd_epu32(U, A, B) \ + ((__m256i)__builtin_ia32_cvtpd2udq512_mask(A, (__v8si)_mm256_setzero_si256(), U, B)) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtt_roundps_epi32 (__m512 __A, const int __R) +{ + return (__m512i) __builtin_ia32_cvttps2dq512_mask ((__v16sf) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1, __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtt_roundps_epi32 (__m512i __W, __mmask16 __U, __m512 __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvttps2dq512_mask ((__v16sf) __A, + (__v16si) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtt_roundps_epi32 (__mmask16 __U, __m512 __A, const int __R) +{ + return (__m512i) __builtin_ia32_cvttps2dq512_mask ((__v16sf) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U, __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtt_roundps_epu32 (__m512 __A, const int __R) +{ + return (__m512i) __builtin_ia32_cvttps2udq512_mask ((__v16sf) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1, __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtt_roundps_epu32 (__m512i __W, __mmask16 __U, __m512 __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvttps2udq512_mask ((__v16sf) __A, + (__v16si) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtt_roundps_epu32 (__mmask16 __U, __m512 __A, const int __R) +{ + return (__m512i) __builtin_ia32_cvttps2udq512_mask ((__v16sf) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U, __R); +} +#else +#define _mm512_cvtt_roundps_epi32(A, B) \ + ((__m512i)__builtin_ia32_cvttps2dq512_mask(A, (__v16si)_mm512_undefined_epi32 (), -1, B)) + +#define _mm512_mask_cvtt_roundps_epi32(W, U, A, B) \ + ((__m512i)__builtin_ia32_cvttps2dq512_mask(A, (__v16si)(W), U, B)) + +#define _mm512_maskz_cvtt_roundps_epi32(U, A, B) \ + ((__m512i)__builtin_ia32_cvttps2dq512_mask(A, (__v16si)_mm512_setzero_si512 (), U, B)) + +#define _mm512_cvtt_roundps_epu32(A, B) \ + ((__m512i)__builtin_ia32_cvttps2udq512_mask(A, (__v16si)_mm512_undefined_epi32 (), -1, B)) + +#define _mm512_mask_cvtt_roundps_epu32(W, U, A, B) \ + ((__m512i)__builtin_ia32_cvttps2udq512_mask(A, (__v16si)(W), U, B)) + +#define _mm512_maskz_cvtt_roundps_epu32(U, A, B) \ + ((__m512i)__builtin_ia32_cvttps2udq512_mask(A, (__v16si)_mm512_setzero_si512 (), U, B)) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundps_epi32 (__m512 __A, const int __R) +{ + return (__m512i) __builtin_ia32_cvtps2dq512_mask ((__v16sf) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1, __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundps_epi32 (__m512i __W, __mmask16 __U, __m512 __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvtps2dq512_mask ((__v16sf) __A, + (__v16si) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundps_epi32 (__mmask16 __U, __m512 __A, const int __R) +{ + return (__m512i) __builtin_ia32_cvtps2dq512_mask ((__v16sf) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U, __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundps_epu32 (__m512 __A, const int __R) +{ + return (__m512i) __builtin_ia32_cvtps2udq512_mask ((__v16sf) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1, __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundps_epu32 (__m512i __W, __mmask16 __U, __m512 __A, + const int __R) +{ + return (__m512i) __builtin_ia32_cvtps2udq512_mask ((__v16sf) __A, + (__v16si) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundps_epu32 (__mmask16 __U, __m512 __A, const int __R) +{ + return (__m512i) __builtin_ia32_cvtps2udq512_mask ((__v16sf) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U, __R); +} +#else +#define _mm512_cvt_roundps_epi32(A, B) \ + ((__m512i)__builtin_ia32_cvtps2dq512_mask(A, (__v16si)_mm512_undefined_epi32 (), -1, B)) + +#define _mm512_mask_cvt_roundps_epi32(W, U, A, B) \ + ((__m512i)__builtin_ia32_cvtps2dq512_mask(A, (__v16si)(W), U, B)) + +#define _mm512_maskz_cvt_roundps_epi32(U, A, B) \ + ((__m512i)__builtin_ia32_cvtps2dq512_mask(A, (__v16si)_mm512_setzero_si512 (), U, B)) + +#define _mm512_cvt_roundps_epu32(A, B) \ + ((__m512i)__builtin_ia32_cvtps2udq512_mask(A, (__v16si)_mm512_undefined_epi32 (), -1, B)) + +#define _mm512_mask_cvt_roundps_epu32(W, U, A, B) \ + ((__m512i)__builtin_ia32_cvtps2udq512_mask(A, (__v16si)(W), U, B)) + +#define _mm512_maskz_cvt_roundps_epu32(U, A, B) \ + ((__m512i)__builtin_ia32_cvtps2udq512_mask(A, (__v16si)_mm512_setzero_si512 (), U, B)) +#endif + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtu32_sd (__m128d __A, unsigned __B) +{ + return (__m128d) __builtin_ia32_cvtusi2sd32 ((__v2df) __A, __B); +} + +#ifdef __x86_64__ +#ifdef __OPTIMIZE__ +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundu64_sd (__m128d __A, unsigned long long __B, const int __R) +{ + return (__m128d) __builtin_ia32_cvtusi2sd64 ((__v2df) __A, __B, __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundi64_sd (__m128d __A, long long __B, const int __R) +{ + return (__m128d) __builtin_ia32_cvtsi2sd64 ((__v2df) __A, __B, __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundsi64_sd (__m128d __A, long long __B, const int __R) +{ + return (__m128d) __builtin_ia32_cvtsi2sd64 ((__v2df) __A, __B, __R); +} +#else +#define _mm_cvt_roundu64_sd(A, B, C) \ + (__m128d)__builtin_ia32_cvtusi2sd64(A, B, C) + +#define _mm_cvt_roundi64_sd(A, B, C) \ + (__m128d)__builtin_ia32_cvtsi2sd64(A, B, C) + +#define _mm_cvt_roundsi64_sd(A, B, C) \ + (__m128d)__builtin_ia32_cvtsi2sd64(A, B, C) +#endif + +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundu32_ss (__m128 __A, unsigned __B, const int __R) +{ + return (__m128) __builtin_ia32_cvtusi2ss32 ((__v4sf) __A, __B, __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundsi32_ss (__m128 __A, int __B, const int __R) +{ + return (__m128) __builtin_ia32_cvtsi2ss32 ((__v4sf) __A, __B, __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundi32_ss (__m128 __A, int __B, const int __R) +{ + return (__m128) __builtin_ia32_cvtsi2ss32 ((__v4sf) __A, __B, __R); +} +#else +#define _mm_cvt_roundu32_ss(A, B, C) \ + (__m128)__builtin_ia32_cvtusi2ss32(A, B, C) + +#define _mm_cvt_roundi32_ss(A, B, C) \ + (__m128)__builtin_ia32_cvtsi2ss32(A, B, C) + +#define _mm_cvt_roundsi32_ss(A, B, C) \ + (__m128)__builtin_ia32_cvtsi2ss32(A, B, C) +#endif + +#ifdef __x86_64__ +#ifdef __OPTIMIZE__ +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundu64_ss (__m128 __A, unsigned long long __B, const int __R) +{ + return (__m128) __builtin_ia32_cvtusi2ss64 ((__v4sf) __A, __B, __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundsi64_ss (__m128 __A, long long __B, const int __R) +{ + return (__m128) __builtin_ia32_cvtsi2ss64 ((__v4sf) __A, __B, __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundi64_ss (__m128 __A, long long __B, const int __R) +{ + return (__m128) __builtin_ia32_cvtsi2ss64 ((__v4sf) __A, __B, __R); +} +#else +#define _mm_cvt_roundu64_ss(A, B, C) \ + (__m128)__builtin_ia32_cvtusi2ss64(A, B, C) + +#define _mm_cvt_roundi64_ss(A, B, C) \ + (__m128)__builtin_ia32_cvtsi2ss64(A, B, C) + +#define _mm_cvt_roundsi64_ss(A, B, C) \ + (__m128)__builtin_ia32_cvtsi2ss64(A, B, C) +#endif + +#endif + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepi32_epi8 (__m512i __A) +{ + return (__m128i) __builtin_ia32_pmovdb512_mask ((__v16si) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask16) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi32_storeu_epi8 (void * __P, __mmask16 __M, __m512i __A) +{ + __builtin_ia32_pmovdb512mem_mask ((__v16qi *) __P, (__v16si) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi32_epi8 (__m128i __O, __mmask16 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovdb512_mask ((__v16si) __A, + (__v16qi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepi32_epi8 (__mmask16 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovdb512_mask ((__v16si) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtsepi32_epi8 (__m512i __A) +{ + return (__m128i) __builtin_ia32_pmovsdb512_mask ((__v16si) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask16) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtsepi32_storeu_epi8 (void * __P, __mmask16 __M, __m512i __A) +{ + __builtin_ia32_pmovsdb512mem_mask ((__v16qi *) __P, (__v16si) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtsepi32_epi8 (__m128i __O, __mmask16 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovsdb512_mask ((__v16si) __A, + (__v16qi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtsepi32_epi8 (__mmask16 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovsdb512_mask ((__v16si) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtusepi32_epi8 (__m512i __A) +{ + return (__m128i) __builtin_ia32_pmovusdb512_mask ((__v16si) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask16) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtusepi32_storeu_epi8 (void * __P, __mmask16 __M, __m512i __A) +{ + __builtin_ia32_pmovusdb512mem_mask ((__v16qi *) __P, (__v16si) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtusepi32_epi8 (__m128i __O, __mmask16 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovusdb512_mask ((__v16si) __A, + (__v16qi) __O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtusepi32_epi8 (__mmask16 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovusdb512_mask ((__v16si) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepi32_epi16 (__m512i __A) +{ + return (__m256i) __builtin_ia32_pmovdw512_mask ((__v16si) __A, + (__v16hi) + _mm256_undefined_si256 (), + (__mmask16) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi32_storeu_epi16 (void * __P, __mmask16 __M, __m512i __A) +{ + __builtin_ia32_pmovdw512mem_mask ((__v16hi *) __P, (__v16si) __A, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi32_epi16 (__m256i __O, __mmask16 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovdw512_mask ((__v16si) __A, + (__v16hi) __O, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepi32_epi16 (__mmask16 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovdw512_mask ((__v16si) __A, + (__v16hi) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtsepi32_epi16 (__m512i __A) +{ + return (__m256i) __builtin_ia32_pmovsdw512_mask ((__v16si) __A, + (__v16hi) + _mm256_undefined_si256 (), + (__mmask16) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtsepi32_storeu_epi16 (void *__P, __mmask16 __M, __m512i __A) +{ + __builtin_ia32_pmovsdw512mem_mask ((__v16hi*) __P, (__v16si) __A, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtsepi32_epi16 (__m256i __O, __mmask16 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovsdw512_mask ((__v16si) __A, + (__v16hi) __O, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtsepi32_epi16 (__mmask16 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovsdw512_mask ((__v16si) __A, + (__v16hi) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtusepi32_epi16 (__m512i __A) +{ + return (__m256i) __builtin_ia32_pmovusdw512_mask ((__v16si) __A, + (__v16hi) + _mm256_undefined_si256 (), + (__mmask16) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtusepi32_storeu_epi16 (void *__P, __mmask16 __M, __m512i __A) +{ + __builtin_ia32_pmovusdw512mem_mask ((__v16hi*) __P, (__v16si) __A, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtusepi32_epi16 (__m256i __O, __mmask16 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovusdw512_mask ((__v16si) __A, + (__v16hi) __O, + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtusepi32_epi16 (__mmask16 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovusdw512_mask ((__v16si) __A, + (__v16hi) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepi64_epi32 (__m512i __A) +{ + return (__m256i) __builtin_ia32_pmovqd512_mask ((__v8di) __A, + (__v8si) + _mm256_undefined_si256 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi64_storeu_epi32 (void* __P, __mmask8 __M, __m512i __A) +{ + __builtin_ia32_pmovqd512mem_mask ((__v8si *) __P, (__v8di) __A, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi64_epi32 (__m256i __O, __mmask8 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovqd512_mask ((__v8di) __A, + (__v8si) __O, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepi64_epi32 (__mmask8 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovqd512_mask ((__v8di) __A, + (__v8si) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtsepi64_epi32 (__m512i __A) +{ + return (__m256i) __builtin_ia32_pmovsqd512_mask ((__v8di) __A, + (__v8si) + _mm256_undefined_si256 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtsepi64_storeu_epi32 (void *__P, __mmask8 __M, __m512i __A) +{ + __builtin_ia32_pmovsqd512mem_mask ((__v8si *) __P, (__v8di) __A, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtsepi64_epi32 (__m256i __O, __mmask8 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovsqd512_mask ((__v8di) __A, + (__v8si) __O, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtsepi64_epi32 (__mmask8 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovsqd512_mask ((__v8di) __A, + (__v8si) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtusepi64_epi32 (__m512i __A) +{ + return (__m256i) __builtin_ia32_pmovusqd512_mask ((__v8di) __A, + (__v8si) + _mm256_undefined_si256 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtusepi64_storeu_epi32 (void* __P, __mmask8 __M, __m512i __A) +{ + __builtin_ia32_pmovusqd512mem_mask ((__v8si*) __P, (__v8di) __A, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtusepi64_epi32 (__m256i __O, __mmask8 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovusqd512_mask ((__v8di) __A, + (__v8si) __O, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtusepi64_epi32 (__mmask8 __M, __m512i __A) +{ + return (__m256i) __builtin_ia32_pmovusqd512_mask ((__v8di) __A, + (__v8si) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepi64_epi16 (__m512i __A) +{ + return (__m128i) __builtin_ia32_pmovqw512_mask ((__v8di) __A, + (__v8hi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi64_storeu_epi16 (void *__P, __mmask8 __M, __m512i __A) +{ + __builtin_ia32_pmovqw512mem_mask ((__v8hi *) __P, (__v8di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi64_epi16 (__m128i __O, __mmask8 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovqw512_mask ((__v8di) __A, + (__v8hi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepi64_epi16 (__mmask8 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovqw512_mask ((__v8di) __A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtsepi64_epi16 (__m512i __A) +{ + return (__m128i) __builtin_ia32_pmovsqw512_mask ((__v8di) __A, + (__v8hi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtsepi64_storeu_epi16 (void * __P, __mmask8 __M, __m512i __A) +{ + __builtin_ia32_pmovsqw512mem_mask ((__v8hi *) __P, (__v8di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtsepi64_epi16 (__m128i __O, __mmask8 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovsqw512_mask ((__v8di) __A, + (__v8hi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtsepi64_epi16 (__mmask8 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovsqw512_mask ((__v8di) __A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtusepi64_epi16 (__m512i __A) +{ + return (__m128i) __builtin_ia32_pmovusqw512_mask ((__v8di) __A, + (__v8hi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtusepi64_storeu_epi16 (void *__P, __mmask8 __M, __m512i __A) +{ + __builtin_ia32_pmovusqw512mem_mask ((__v8hi*) __P, (__v8di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtusepi64_epi16 (__m128i __O, __mmask8 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovusqw512_mask ((__v8di) __A, + (__v8hi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtusepi64_epi16 (__mmask8 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovusqw512_mask ((__v8di) __A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepi64_epi8 (__m512i __A) +{ + return (__m128i) __builtin_ia32_pmovqb512_mask ((__v8di) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi64_storeu_epi8 (void * __P, __mmask8 __M, __m512i __A) +{ + __builtin_ia32_pmovqb512mem_mask ((__v16qi *) __P, (__v8di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi64_epi8 (__m128i __O, __mmask8 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovqb512_mask ((__v8di) __A, + (__v16qi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepi64_epi8 (__mmask8 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovqb512_mask ((__v8di) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtsepi64_epi8 (__m512i __A) +{ + return (__m128i) __builtin_ia32_pmovsqb512_mask ((__v8di) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtsepi64_storeu_epi8 (void * __P, __mmask8 __M, __m512i __A) +{ + __builtin_ia32_pmovsqb512mem_mask ((__v16qi *) __P, (__v8di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtsepi64_epi8 (__m128i __O, __mmask8 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovsqb512_mask ((__v8di) __A, + (__v16qi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtsepi64_epi8 (__mmask8 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovsqb512_mask ((__v8di) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtusepi64_epi8 (__m512i __A) +{ + return (__m128i) __builtin_ia32_pmovusqb512_mask ((__v8di) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtusepi64_storeu_epi8 (void * __P, __mmask8 __M, __m512i __A) +{ + __builtin_ia32_pmovusqb512mem_mask ((__v16qi *) __P, (__v8di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtusepi64_epi8 (__m128i __O, __mmask8 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovusqb512_mask ((__v8di) __A, + (__v16qi) __O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtusepi64_epi8 (__mmask8 __M, __m512i __A) +{ + return (__m128i) __builtin_ia32_pmovusqb512_mask ((__v8di) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepi32_pd (__m256i __A) +{ + return (__m512d) __builtin_ia32_cvtdq2pd512_mask ((__v8si) __A, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi32_pd (__m512d __W, __mmask8 __U, __m256i __A) +{ + return (__m512d) __builtin_ia32_cvtdq2pd512_mask ((__v8si) __A, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepi32_pd (__mmask8 __U, __m256i __A) +{ + return (__m512d) __builtin_ia32_cvtdq2pd512_mask ((__v8si) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepu32_pd (__m256i __A) +{ + return (__m512d) __builtin_ia32_cvtudq2pd512_mask ((__v8si) __A, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepu32_pd (__m512d __W, __mmask8 __U, __m256i __A) +{ + return (__m512d) __builtin_ia32_cvtudq2pd512_mask ((__v8si) __A, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepu32_pd (__mmask8 __U, __m256i __A) +{ + return (__m512d) __builtin_ia32_cvtudq2pd512_mask ((__v8si) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundepi32_ps (__m512i __A, const int __R) +{ + return (__m512) __builtin_ia32_cvtdq2ps512_mask ((__v16si) __A, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundepi32_ps (__m512 __W, __mmask16 __U, __m512i __A, + const int __R) +{ + return (__m512) __builtin_ia32_cvtdq2ps512_mask ((__v16si) __A, + (__v16sf) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundepi32_ps (__mmask16 __U, __m512i __A, const int __R) +{ + return (__m512) __builtin_ia32_cvtdq2ps512_mask ((__v16si) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundepu32_ps (__m512i __A, const int __R) +{ + return (__m512) __builtin_ia32_cvtudq2ps512_mask ((__v16si) __A, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundepu32_ps (__m512 __W, __mmask16 __U, __m512i __A, + const int __R) +{ + return (__m512) __builtin_ia32_cvtudq2ps512_mask ((__v16si) __A, + (__v16sf) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundepu32_ps (__mmask16 __U, __m512i __A, const int __R) +{ + return (__m512) __builtin_ia32_cvtudq2ps512_mask ((__v16si) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, __R); +} + +#else +#define _mm512_cvt_roundepi32_ps(A, B) \ + (__m512)__builtin_ia32_cvtdq2ps512_mask((__v16si)(A), (__v16sf)_mm512_undefined_ps(), -1, B) + +#define _mm512_mask_cvt_roundepi32_ps(W, U, A, B) \ + (__m512)__builtin_ia32_cvtdq2ps512_mask((__v16si)(A), W, U, B) + +#define _mm512_maskz_cvt_roundepi32_ps(U, A, B) \ + (__m512)__builtin_ia32_cvtdq2ps512_mask((__v16si)(A), (__v16sf)_mm512_setzero_ps(), U, B) + +#define _mm512_cvt_roundepu32_ps(A, B) \ + (__m512)__builtin_ia32_cvtudq2ps512_mask((__v16si)(A), (__v16sf)_mm512_undefined_ps(), -1, B) + +#define _mm512_mask_cvt_roundepu32_ps(W, U, A, B) \ + (__m512)__builtin_ia32_cvtudq2ps512_mask((__v16si)(A), W, U, B) + +#define _mm512_maskz_cvt_roundepu32_ps(U, A, B) \ + (__m512)__builtin_ia32_cvtudq2ps512_mask((__v16si)(A), (__v16sf)_mm512_setzero_ps(), U, B) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_extractf64x4_pd (__m512d __A, const int __imm) +{ + return (__m256d) __builtin_ia32_extractf64x4_mask ((__v8df) __A, + __imm, + (__v4df) + _mm256_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_extractf64x4_pd (__m256d __W, __mmask8 __U, __m512d __A, + const int __imm) +{ + return (__m256d) __builtin_ia32_extractf64x4_mask ((__v8df) __A, + __imm, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_extractf64x4_pd (__mmask8 __U, __m512d __A, const int __imm) +{ + return (__m256d) __builtin_ia32_extractf64x4_mask ((__v8df) __A, + __imm, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_extractf32x4_ps (__m512 __A, const int __imm) +{ + return (__m128) __builtin_ia32_extractf32x4_mask ((__v16sf) __A, + __imm, + (__v4sf) + _mm_undefined_ps (), + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_extractf32x4_ps (__m128 __W, __mmask8 __U, __m512 __A, + const int __imm) +{ + return (__m128) __builtin_ia32_extractf32x4_mask ((__v16sf) __A, + __imm, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_extractf32x4_ps (__mmask8 __U, __m512 __A, const int __imm) +{ + return (__m128) __builtin_ia32_extractf32x4_mask ((__v16sf) __A, + __imm, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_extracti64x4_epi64 (__m512i __A, const int __imm) +{ + return (__m256i) __builtin_ia32_extracti64x4_mask ((__v8di) __A, + __imm, + (__v4di) + _mm256_undefined_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_extracti64x4_epi64 (__m256i __W, __mmask8 __U, __m512i __A, + const int __imm) +{ + return (__m256i) __builtin_ia32_extracti64x4_mask ((__v8di) __A, + __imm, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_extracti64x4_epi64 (__mmask8 __U, __m512i __A, const int __imm) +{ + return (__m256i) __builtin_ia32_extracti64x4_mask ((__v8di) __A, + __imm, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_extracti32x4_epi32 (__m512i __A, const int __imm) +{ + return (__m128i) __builtin_ia32_extracti32x4_mask ((__v16si) __A, + __imm, + (__v4si) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_extracti32x4_epi32 (__m128i __W, __mmask8 __U, __m512i __A, + const int __imm) +{ + return (__m128i) __builtin_ia32_extracti32x4_mask ((__v16si) __A, + __imm, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_extracti32x4_epi32 (__mmask8 __U, __m512i __A, const int __imm) +{ + return (__m128i) __builtin_ia32_extracti32x4_mask ((__v16si) __A, + __imm, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} +#else + +#define _mm512_extractf64x4_pd(X, C) \ + ((__m256d) __builtin_ia32_extractf64x4_mask ((__v8df)(__m512d) (X), \ + (int) (C),\ + (__v4df)(__m256d)_mm256_undefined_pd(),\ + (__mmask8)-1)) + +#define _mm512_mask_extractf64x4_pd(W, U, X, C) \ + ((__m256d) __builtin_ia32_extractf64x4_mask ((__v8df)(__m512d) (X), \ + (int) (C),\ + (__v4df)(__m256d)(W),\ + (__mmask8)(U))) + +#define _mm512_maskz_extractf64x4_pd(U, X, C) \ + ((__m256d) __builtin_ia32_extractf64x4_mask ((__v8df)(__m512d) (X), \ + (int) (C),\ + (__v4df)(__m256d)_mm256_setzero_pd(),\ + (__mmask8)(U))) + +#define _mm512_extractf32x4_ps(X, C) \ + ((__m128) __builtin_ia32_extractf32x4_mask ((__v16sf)(__m512) (X), \ + (int) (C),\ + (__v4sf)(__m128)_mm_undefined_ps(),\ + (__mmask8)-1)) + +#define _mm512_mask_extractf32x4_ps(W, U, X, C) \ + ((__m128) __builtin_ia32_extractf32x4_mask ((__v16sf)(__m512) (X), \ + (int) (C),\ + (__v4sf)(__m128)(W),\ + (__mmask8)(U))) + +#define _mm512_maskz_extractf32x4_ps(U, X, C) \ + ((__m128) __builtin_ia32_extractf32x4_mask ((__v16sf)(__m512) (X), \ + (int) (C),\ + (__v4sf)(__m128)_mm_setzero_ps(),\ + (__mmask8)(U))) + +#define _mm512_extracti64x4_epi64(X, C) \ + ((__m256i) __builtin_ia32_extracti64x4_mask ((__v8di)(__m512i) (X), \ + (int) (C),\ + (__v4di)(__m256i)_mm256_undefined_si256 (),\ + (__mmask8)-1)) + +#define _mm512_mask_extracti64x4_epi64(W, U, X, C) \ + ((__m256i) __builtin_ia32_extracti64x4_mask ((__v8di)(__m512i) (X), \ + (int) (C),\ + (__v4di)(__m256i)(W),\ + (__mmask8)(U))) + +#define _mm512_maskz_extracti64x4_epi64(U, X, C) \ + ((__m256i) __builtin_ia32_extracti64x4_mask ((__v8di)(__m512i) (X), \ + (int) (C),\ + (__v4di)(__m256i)_mm256_setzero_si256 (),\ + (__mmask8)(U))) + +#define _mm512_extracti32x4_epi32(X, C) \ + ((__m128i) __builtin_ia32_extracti32x4_mask ((__v16si)(__m512i) (X), \ + (int) (C),\ + (__v4si)(__m128i)_mm_undefined_si128 (),\ + (__mmask8)-1)) + +#define _mm512_mask_extracti32x4_epi32(W, U, X, C) \ + ((__m128i) __builtin_ia32_extracti32x4_mask ((__v16si)(__m512i) (X), \ + (int) (C),\ + (__v4si)(__m128i)(W),\ + (__mmask8)(U))) + +#define _mm512_maskz_extracti32x4_epi32(U, X, C) \ + ((__m128i) __builtin_ia32_extracti32x4_mask ((__v16si)(__m512i) (X), \ + (int) (C),\ + (__v4si)(__m128i)_mm_setzero_si128 (),\ + (__mmask8)(U))) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_inserti32x4 (__m512i __A, __m128i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_inserti32x4_mask ((__v16si) __A, + (__v4si) __B, + __imm, + (__v16si) __A, -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_insertf32x4 (__m512 __A, __m128 __B, const int __imm) +{ + return (__m512) __builtin_ia32_insertf32x4_mask ((__v16sf) __A, + (__v4sf) __B, + __imm, + (__v16sf) __A, -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_inserti64x4 (__m512i __A, __m256i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_inserti64x4_mask ((__v8di) __A, + (__v4di) __B, + __imm, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_inserti64x4 (__m512i __W, __mmask8 __U, __m512i __A, + __m256i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_inserti64x4_mask ((__v8di) __A, + (__v4di) __B, + __imm, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_inserti64x4 (__mmask8 __U, __m512i __A, __m256i __B, + const int __imm) +{ + return (__m512i) __builtin_ia32_inserti64x4_mask ((__v8di) __A, + (__v4di) __B, + __imm, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_insertf64x4 (__m512d __A, __m256d __B, const int __imm) +{ + return (__m512d) __builtin_ia32_insertf64x4_mask ((__v8df) __A, + (__v4df) __B, + __imm, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_insertf64x4 (__m512d __W, __mmask8 __U, __m512d __A, + __m256d __B, const int __imm) +{ + return (__m512d) __builtin_ia32_insertf64x4_mask ((__v8df) __A, + (__v4df) __B, + __imm, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_insertf64x4 (__mmask8 __U, __m512d __A, __m256d __B, + const int __imm) +{ + return (__m512d) __builtin_ia32_insertf64x4_mask ((__v8df) __A, + (__v4df) __B, + __imm, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} +#else +#define _mm512_insertf32x4(X, Y, C) \ + ((__m512) __builtin_ia32_insertf32x4_mask ((__v16sf)(__m512) (X), \ + (__v4sf)(__m128) (Y), (int) (C), (__v16sf)(__m512) (X), (__mmask16)(-1))) + +#define _mm512_inserti32x4(X, Y, C) \ + ((__m512i) __builtin_ia32_inserti32x4_mask ((__v16si)(__m512i) (X), \ + (__v4si)(__m128i) (Y), (int) (C), (__v16si)(__m512i) (X), (__mmask16)(-1))) + +#define _mm512_insertf64x4(X, Y, C) \ + ((__m512d) __builtin_ia32_insertf64x4_mask ((__v8df)(__m512d) (X), \ + (__v4df)(__m256d) (Y), (int) (C), \ + (__v8df)(__m512d)_mm512_undefined_pd(), \ + (__mmask8)-1)) + +#define _mm512_mask_insertf64x4(W, U, X, Y, C) \ + ((__m512d) __builtin_ia32_insertf64x4_mask ((__v8df)(__m512d) (X), \ + (__v4df)(__m256d) (Y), (int) (C), \ + (__v8df)(__m512d)(W), \ + (__mmask8)(U))) + +#define _mm512_maskz_insertf64x4(U, X, Y, C) \ + ((__m512d) __builtin_ia32_insertf64x4_mask ((__v8df)(__m512d) (X), \ + (__v4df)(__m256d) (Y), (int) (C), \ + (__v8df)(__m512d)_mm512_setzero_pd(), \ + (__mmask8)(U))) + +#define _mm512_inserti64x4(X, Y, C) \ + ((__m512i) __builtin_ia32_inserti64x4_mask ((__v8di)(__m512i) (X), \ + (__v4di)(__m256i) (Y), (int) (C), \ + (__v8di)(__m512i)_mm512_undefined_epi32 (), \ + (__mmask8)-1)) + +#define _mm512_mask_inserti64x4(W, U, X, Y, C) \ + ((__m512i) __builtin_ia32_inserti64x4_mask ((__v8di)(__m512i) (X), \ + (__v4di)(__m256i) (Y), (int) (C),\ + (__v8di)(__m512i)(W),\ + (__mmask8)(U))) + +#define _mm512_maskz_inserti64x4(U, X, Y, C) \ + ((__m512i) __builtin_ia32_inserti64x4_mask ((__v8di)(__m512i) (X), \ + (__v4di)(__m256i) (Y), (int) (C), \ + (__v8di)(__m512i)_mm512_setzero_si512 (), \ + (__mmask8)(U))) +#endif + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_loadu_pd (void const *__P) +{ + return *(__m512d_u *)__P; +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_loadu_pd (__m512d __W, __mmask8 __U, void const *__P) +{ + return (__m512d) __builtin_ia32_loadupd512_mask ((const double *) __P, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_loadu_pd (__mmask8 __U, void const *__P) +{ + return (__m512d) __builtin_ia32_loadupd512_mask ((const double *) __P, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_storeu_pd (void *__P, __m512d __A) +{ + *(__m512d_u *)__P = __A; +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_storeu_pd (void *__P, __mmask8 __U, __m512d __A) +{ + __builtin_ia32_storeupd512_mask ((double *) __P, (__v8df) __A, + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_loadu_ps (void const *__P) +{ + return *(__m512_u *)__P; +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_loadu_ps (__m512 __W, __mmask16 __U, void const *__P) +{ + return (__m512) __builtin_ia32_loadups512_mask ((const float *) __P, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_loadu_ps (__mmask16 __U, void const *__P) +{ + return (__m512) __builtin_ia32_loadups512_mask ((const float *) __P, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_storeu_ps (void *__P, __m512 __A) +{ + *(__m512_u *)__P = __A; +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_storeu_ps (void *__P, __mmask16 __U, __m512 __A) +{ + __builtin_ia32_storeups512_mask ((float *) __P, (__v16sf) __A, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_loadu_epi64 (__m512i __W, __mmask8 __U, void const *__P) +{ + return (__m512i) __builtin_ia32_loaddqudi512_mask ((const long long *) __P, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_loadu_epi64 (__mmask8 __U, void const *__P) +{ + return (__m512i) __builtin_ia32_loaddqudi512_mask ((const long long *) __P, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_storeu_epi64 (void *__P, __mmask8 __U, __m512i __A) +{ + __builtin_ia32_storedqudi512_mask ((long long *) __P, (__v8di) __A, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_loadu_si512 (void const *__P) +{ + return *(__m512i_u *)__P; +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_loadu_epi32 (__m512i __W, __mmask16 __U, void const *__P) +{ + return (__m512i) __builtin_ia32_loaddqusi512_mask ((const int *) __P, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_loadu_epi32 (__mmask16 __U, void const *__P) +{ + return (__m512i) __builtin_ia32_loaddqusi512_mask ((const int *) __P, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_storeu_si512 (void *__P, __m512i __A) +{ + *(__m512i_u *)__P = __A; +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_storeu_epi32 (void *__P, __mmask16 __U, __m512i __A) +{ + __builtin_ia32_storedqusi512_mask ((int *) __P, (__v16si) __A, + (__mmask16) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permutevar_pd (__m512d __A, __m512i __C) +{ + return (__m512d) __builtin_ia32_vpermilvarpd512_mask ((__v8df) __A, + (__v8di) __C, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permutevar_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512i __C) +{ + return (__m512d) __builtin_ia32_vpermilvarpd512_mask ((__v8df) __A, + (__v8di) __C, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permutevar_pd (__mmask8 __U, __m512d __A, __m512i __C) +{ + return (__m512d) __builtin_ia32_vpermilvarpd512_mask ((__v8df) __A, + (__v8di) __C, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permutevar_ps (__m512 __A, __m512i __C) +{ + return (__m512) __builtin_ia32_vpermilvarps512_mask ((__v16sf) __A, + (__v16si) __C, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permutevar_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512i __C) +{ + return (__m512) __builtin_ia32_vpermilvarps512_mask ((__v16sf) __A, + (__v16si) __C, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permutevar_ps (__mmask16 __U, __m512 __A, __m512i __C) +{ + return (__m512) __builtin_ia32_vpermilvarps512_mask ((__v16sf) __A, + (__v16si) __C, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permutex2var_epi64 (__m512i __A, __m512i __I, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpermt2varq512_mask ((__v8di) __I + /* idx */ , + (__v8di) __A, + (__v8di) __B, + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permutex2var_epi64 (__m512i __A, __mmask8 __U, __m512i __I, + __m512i __B) +{ + return (__m512i) __builtin_ia32_vpermt2varq512_mask ((__v8di) __I + /* idx */ , + (__v8di) __A, + (__v8di) __B, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask2_permutex2var_epi64 (__m512i __A, __m512i __I, + __mmask8 __U, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpermi2varq512_mask ((__v8di) __A, + (__v8di) __I + /* idx */ , + (__v8di) __B, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permutex2var_epi64 (__mmask8 __U, __m512i __A, + __m512i __I, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpermt2varq512_maskz ((__v8di) __I + /* idx */ , + (__v8di) __A, + (__v8di) __B, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permutex2var_epi32 (__m512i __A, __m512i __I, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpermt2vard512_mask ((__v16si) __I + /* idx */ , + (__v16si) __A, + (__v16si) __B, + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permutex2var_epi32 (__m512i __A, __mmask16 __U, + __m512i __I, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpermt2vard512_mask ((__v16si) __I + /* idx */ , + (__v16si) __A, + (__v16si) __B, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask2_permutex2var_epi32 (__m512i __A, __m512i __I, + __mmask16 __U, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpermi2vard512_mask ((__v16si) __A, + (__v16si) __I + /* idx */ , + (__v16si) __B, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permutex2var_epi32 (__mmask16 __U, __m512i __A, + __m512i __I, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpermt2vard512_maskz ((__v16si) __I + /* idx */ , + (__v16si) __A, + (__v16si) __B, + (__mmask16) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permutex2var_pd (__m512d __A, __m512i __I, __m512d __B) +{ + return (__m512d) __builtin_ia32_vpermt2varpd512_mask ((__v8di) __I + /* idx */ , + (__v8df) __A, + (__v8df) __B, + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permutex2var_pd (__m512d __A, __mmask8 __U, __m512i __I, + __m512d __B) +{ + return (__m512d) __builtin_ia32_vpermt2varpd512_mask ((__v8di) __I + /* idx */ , + (__v8df) __A, + (__v8df) __B, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask2_permutex2var_pd (__m512d __A, __m512i __I, __mmask8 __U, + __m512d __B) +{ + return (__m512d) __builtin_ia32_vpermi2varpd512_mask ((__v8df) __A, + (__v8di) __I + /* idx */ , + (__v8df) __B, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permutex2var_pd (__mmask8 __U, __m512d __A, __m512i __I, + __m512d __B) +{ + return (__m512d) __builtin_ia32_vpermt2varpd512_maskz ((__v8di) __I + /* idx */ , + (__v8df) __A, + (__v8df) __B, + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permutex2var_ps (__m512 __A, __m512i __I, __m512 __B) +{ + return (__m512) __builtin_ia32_vpermt2varps512_mask ((__v16si) __I + /* idx */ , + (__v16sf) __A, + (__v16sf) __B, + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permutex2var_ps (__m512 __A, __mmask16 __U, __m512i __I, __m512 __B) +{ + return (__m512) __builtin_ia32_vpermt2varps512_mask ((__v16si) __I + /* idx */ , + (__v16sf) __A, + (__v16sf) __B, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask2_permutex2var_ps (__m512 __A, __m512i __I, __mmask16 __U, + __m512 __B) +{ + return (__m512) __builtin_ia32_vpermi2varps512_mask ((__v16sf) __A, + (__v16si) __I + /* idx */ , + (__v16sf) __B, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permutex2var_ps (__mmask16 __U, __m512 __A, __m512i __I, + __m512 __B) +{ + return (__m512) __builtin_ia32_vpermt2varps512_maskz ((__v16si) __I + /* idx */ , + (__v16sf) __A, + (__v16sf) __B, + (__mmask16) __U); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permute_pd (__m512d __X, const int __C) +{ + return (__m512d) __builtin_ia32_vpermilpd512_mask ((__v8df) __X, __C, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permute_pd (__m512d __W, __mmask8 __U, __m512d __X, const int __C) +{ + return (__m512d) __builtin_ia32_vpermilpd512_mask ((__v8df) __X, __C, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permute_pd (__mmask8 __U, __m512d __X, const int __C) +{ + return (__m512d) __builtin_ia32_vpermilpd512_mask ((__v8df) __X, __C, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permute_ps (__m512 __X, const int __C) +{ + return (__m512) __builtin_ia32_vpermilps512_mask ((__v16sf) __X, __C, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permute_ps (__m512 __W, __mmask16 __U, __m512 __X, const int __C) +{ + return (__m512) __builtin_ia32_vpermilps512_mask ((__v16sf) __X, __C, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permute_ps (__mmask16 __U, __m512 __X, const int __C) +{ + return (__m512) __builtin_ia32_vpermilps512_mask ((__v16sf) __X, __C, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} +#else +#define _mm512_permute_pd(X, C) \ + ((__m512d) __builtin_ia32_vpermilpd512_mask ((__v8df)(__m512d)(X), (int)(C), \ + (__v8df)(__m512d)_mm512_undefined_pd(),\ + (__mmask8)(-1))) + +#define _mm512_mask_permute_pd(W, U, X, C) \ + ((__m512d) __builtin_ia32_vpermilpd512_mask ((__v8df)(__m512d)(X), (int)(C), \ + (__v8df)(__m512d)(W), \ + (__mmask8)(U))) + +#define _mm512_maskz_permute_pd(U, X, C) \ + ((__m512d) __builtin_ia32_vpermilpd512_mask ((__v8df)(__m512d)(X), (int)(C), \ + (__v8df)(__m512d)_mm512_setzero_pd(), \ + (__mmask8)(U))) + +#define _mm512_permute_ps(X, C) \ + ((__m512) __builtin_ia32_vpermilps512_mask ((__v16sf)(__m512)(X), (int)(C), \ + (__v16sf)(__m512)_mm512_undefined_ps(),\ + (__mmask16)(-1))) + +#define _mm512_mask_permute_ps(W, U, X, C) \ + ((__m512) __builtin_ia32_vpermilps512_mask ((__v16sf)(__m512)(X), (int)(C), \ + (__v16sf)(__m512)(W), \ + (__mmask16)(U))) + +#define _mm512_maskz_permute_ps(U, X, C) \ + ((__m512) __builtin_ia32_vpermilps512_mask ((__v16sf)(__m512)(X), (int)(C), \ + (__v16sf)(__m512)_mm512_setzero_ps(), \ + (__mmask16)(U))) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permutex_epi64 (__m512i __X, const int __I) +{ + return (__m512i) __builtin_ia32_permdi512_mask ((__v8di) __X, __I, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) (-1)); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permutex_epi64 (__m512i __W, __mmask8 __M, + __m512i __X, const int __I) +{ + return (__m512i) __builtin_ia32_permdi512_mask ((__v8di) __X, __I, + (__v8di) __W, + (__mmask8) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permutex_epi64 (__mmask8 __M, __m512i __X, const int __I) +{ + return (__m512i) __builtin_ia32_permdi512_mask ((__v8di) __X, __I, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __M); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permutex_pd (__m512d __X, const int __M) +{ + return (__m512d) __builtin_ia32_permdf512_mask ((__v8df) __X, __M, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permutex_pd (__m512d __W, __mmask8 __U, __m512d __X, const int __M) +{ + return (__m512d) __builtin_ia32_permdf512_mask ((__v8df) __X, __M, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permutex_pd (__mmask8 __U, __m512d __X, const int __M) +{ + return (__m512d) __builtin_ia32_permdf512_mask ((__v8df) __X, __M, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} +#else +#define _mm512_permutex_pd(X, M) \ + ((__m512d) __builtin_ia32_permdf512_mask ((__v8df)(__m512d)(X), (int)(M), \ + (__v8df)(__m512d)_mm512_undefined_pd(),\ + (__mmask8)-1)) + +#define _mm512_mask_permutex_pd(W, U, X, M) \ + ((__m512d) __builtin_ia32_permdf512_mask ((__v8df)(__m512d)(X), (int)(M), \ + (__v8df)(__m512d)(W), (__mmask8)(U))) + +#define _mm512_maskz_permutex_pd(U, X, M) \ + ((__m512d) __builtin_ia32_permdf512_mask ((__v8df)(__m512d)(X), (int)(M), \ + (__v8df)(__m512d)_mm512_setzero_pd(),\ + (__mmask8)(U))) + +#define _mm512_permutex_epi64(X, I) \ + ((__m512i) __builtin_ia32_permdi512_mask ((__v8di)(__m512i)(X), \ + (int)(I), \ + (__v8di)(__m512i) \ + (_mm512_undefined_epi32 ()),\ + (__mmask8)(-1))) + +#define _mm512_maskz_permutex_epi64(M, X, I) \ + ((__m512i) __builtin_ia32_permdi512_mask ((__v8di)(__m512i)(X), \ + (int)(I), \ + (__v8di)(__m512i) \ + (_mm512_setzero_si512 ()),\ + (__mmask8)(M))) + +#define _mm512_mask_permutex_epi64(W, M, X, I) \ + ((__m512i) __builtin_ia32_permdi512_mask ((__v8di)(__m512i)(X), \ + (int)(I), \ + (__v8di)(__m512i)(W), \ + (__mmask8)(M))) +#endif + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permutexvar_epi64 (__mmask8 __M, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_permvardi512_mask ((__v8di) __Y, + (__v8di) __X, + (__v8di) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permutexvar_epi64 (__m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_permvardi512_mask ((__v8di) __Y, + (__v8di) __X, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permutexvar_epi64 (__m512i __W, __mmask8 __M, __m512i __X, + __m512i __Y) +{ + return (__m512i) __builtin_ia32_permvardi512_mask ((__v8di) __Y, + (__v8di) __X, + (__v8di) __W, + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permutexvar_epi32 (__mmask16 __M, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_permvarsi512_mask ((__v16si) __Y, + (__v16si) __X, + (__v16si) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permutexvar_epi32 (__m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_permvarsi512_mask ((__v16si) __Y, + (__v16si) __X, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permutexvar_epi32 (__m512i __W, __mmask16 __M, __m512i __X, + __m512i __Y) +{ + return (__m512i) __builtin_ia32_permvarsi512_mask ((__v16si) __Y, + (__v16si) __X, + (__v16si) __W, + __M); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permutexvar_pd (__m512i __X, __m512d __Y) +{ + return (__m512d) __builtin_ia32_permvardf512_mask ((__v8df) __Y, + (__v8di) __X, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permutexvar_pd (__m512d __W, __mmask8 __U, __m512i __X, __m512d __Y) +{ + return (__m512d) __builtin_ia32_permvardf512_mask ((__v8df) __Y, + (__v8di) __X, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permutexvar_pd (__mmask8 __U, __m512i __X, __m512d __Y) +{ + return (__m512d) __builtin_ia32_permvardf512_mask ((__v8df) __Y, + (__v8di) __X, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permutexvar_ps (__m512i __X, __m512 __Y) +{ + return (__m512) __builtin_ia32_permvarsf512_mask ((__v16sf) __Y, + (__v16si) __X, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permutexvar_ps (__m512 __W, __mmask16 __U, __m512i __X, __m512 __Y) +{ + return (__m512) __builtin_ia32_permvarsf512_mask ((__v16sf) __Y, + (__v16si) __X, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permutexvar_ps (__mmask16 __U, __m512i __X, __m512 __Y) +{ + return (__m512) __builtin_ia32_permvarsf512_mask ((__v16sf) __Y, + (__v16si) __X, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_shuffle_ps (__m512 __M, __m512 __V, const int __imm) +{ + return (__m512) __builtin_ia32_shufps512_mask ((__v16sf) __M, + (__v16sf) __V, __imm, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_shuffle_ps (__m512 __W, __mmask16 __U, __m512 __M, + __m512 __V, const int __imm) +{ + return (__m512) __builtin_ia32_shufps512_mask ((__v16sf) __M, + (__v16sf) __V, __imm, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_shuffle_ps (__mmask16 __U, __m512 __M, __m512 __V, const int __imm) +{ + return (__m512) __builtin_ia32_shufps512_mask ((__v16sf) __M, + (__v16sf) __V, __imm, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_shuffle_pd (__m512d __M, __m512d __V, const int __imm) +{ + return (__m512d) __builtin_ia32_shufpd512_mask ((__v8df) __M, + (__v8df) __V, __imm, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_shuffle_pd (__m512d __W, __mmask8 __U, __m512d __M, + __m512d __V, const int __imm) +{ + return (__m512d) __builtin_ia32_shufpd512_mask ((__v8df) __M, + (__v8df) __V, __imm, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_shuffle_pd (__mmask8 __U, __m512d __M, __m512d __V, + const int __imm) +{ + return (__m512d) __builtin_ia32_shufpd512_mask ((__v8df) __M, + (__v8df) __V, __imm, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fixupimm_round_pd (__m512d __A, __m512d __B, __m512i __C, + const int __imm, const int __R) +{ + return (__m512d) __builtin_ia32_fixupimmpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8di) __C, + __imm, + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fixupimm_round_pd (__m512d __A, __mmask8 __U, __m512d __B, + __m512i __C, const int __imm, const int __R) +{ + return (__m512d) __builtin_ia32_fixupimmpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8di) __C, + __imm, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fixupimm_round_pd (__mmask8 __U, __m512d __A, __m512d __B, + __m512i __C, const int __imm, const int __R) +{ + return (__m512d) __builtin_ia32_fixupimmpd512_maskz ((__v8df) __A, + (__v8df) __B, + (__v8di) __C, + __imm, + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fixupimm_round_ps (__m512 __A, __m512 __B, __m512i __C, + const int __imm, const int __R) +{ + return (__m512) __builtin_ia32_fixupimmps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16si) __C, + __imm, + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fixupimm_round_ps (__m512 __A, __mmask16 __U, __m512 __B, + __m512i __C, const int __imm, const int __R) +{ + return (__m512) __builtin_ia32_fixupimmps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16si) __C, + __imm, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fixupimm_round_ps (__mmask16 __U, __m512 __A, __m512 __B, + __m512i __C, const int __imm, const int __R) +{ + return (__m512) __builtin_ia32_fixupimmps512_maskz ((__v16sf) __A, + (__v16sf) __B, + (__v16si) __C, + __imm, + (__mmask16) __U, __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fixupimm_round_sd (__m128d __A, __m128d __B, __m128i __C, + const int __imm, const int __R) +{ + return (__m128d) __builtin_ia32_fixupimmsd_mask ((__v2df) __A, + (__v2df) __B, + (__v2di) __C, __imm, + (__mmask8) -1, __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fixupimm_round_sd (__m128d __A, __mmask8 __U, __m128d __B, + __m128i __C, const int __imm, const int __R) +{ + return (__m128d) __builtin_ia32_fixupimmsd_mask ((__v2df) __A, + (__v2df) __B, + (__v2di) __C, __imm, + (__mmask8) __U, __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fixupimm_round_sd (__mmask8 __U, __m128d __A, __m128d __B, + __m128i __C, const int __imm, const int __R) +{ + return (__m128d) __builtin_ia32_fixupimmsd_maskz ((__v2df) __A, + (__v2df) __B, + (__v2di) __C, + __imm, + (__mmask8) __U, __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fixupimm_round_ss (__m128 __A, __m128 __B, __m128i __C, + const int __imm, const int __R) +{ + return (__m128) __builtin_ia32_fixupimmss_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4si) __C, __imm, + (__mmask8) -1, __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fixupimm_round_ss (__m128 __A, __mmask8 __U, __m128 __B, + __m128i __C, const int __imm, const int __R) +{ + return (__m128) __builtin_ia32_fixupimmss_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4si) __C, __imm, + (__mmask8) __U, __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fixupimm_round_ss (__mmask8 __U, __m128 __A, __m128 __B, + __m128i __C, const int __imm, const int __R) +{ + return (__m128) __builtin_ia32_fixupimmss_maskz ((__v4sf) __A, + (__v4sf) __B, + (__v4si) __C, __imm, + (__mmask8) __U, __R); +} + +#else +#define _mm512_shuffle_pd(X, Y, C) \ + ((__m512d)__builtin_ia32_shufpd512_mask ((__v8df)(__m512d)(X), \ + (__v8df)(__m512d)(Y), (int)(C),\ + (__v8df)(__m512d)_mm512_undefined_pd(),\ + (__mmask8)-1)) + +#define _mm512_mask_shuffle_pd(W, U, X, Y, C) \ + ((__m512d)__builtin_ia32_shufpd512_mask ((__v8df)(__m512d)(X), \ + (__v8df)(__m512d)(Y), (int)(C),\ + (__v8df)(__m512d)(W),\ + (__mmask8)(U))) + +#define _mm512_maskz_shuffle_pd(U, X, Y, C) \ + ((__m512d)__builtin_ia32_shufpd512_mask ((__v8df)(__m512d)(X), \ + (__v8df)(__m512d)(Y), (int)(C),\ + (__v8df)(__m512d)_mm512_setzero_pd(),\ + (__mmask8)(U))) + +#define _mm512_shuffle_ps(X, Y, C) \ + ((__m512)__builtin_ia32_shufps512_mask ((__v16sf)(__m512)(X), \ + (__v16sf)(__m512)(Y), (int)(C),\ + (__v16sf)(__m512)_mm512_undefined_ps(),\ + (__mmask16)-1)) + +#define _mm512_mask_shuffle_ps(W, U, X, Y, C) \ + ((__m512)__builtin_ia32_shufps512_mask ((__v16sf)(__m512)(X), \ + (__v16sf)(__m512)(Y), (int)(C),\ + (__v16sf)(__m512)(W),\ + (__mmask16)(U))) + +#define _mm512_maskz_shuffle_ps(U, X, Y, C) \ + ((__m512)__builtin_ia32_shufps512_mask ((__v16sf)(__m512)(X), \ + (__v16sf)(__m512)(Y), (int)(C),\ + (__v16sf)(__m512)_mm512_setzero_ps(),\ + (__mmask16)(U))) + +#define _mm512_fixupimm_round_pd(X, Y, Z, C, R) \ + ((__m512d)__builtin_ia32_fixupimmpd512_mask ((__v8df)(__m512d)(X), \ + (__v8df)(__m512d)(Y), (__v8di)(__m512i)(Z), (int)(C), \ + (__mmask8)(-1), (R))) + +#define _mm512_mask_fixupimm_round_pd(X, U, Y, Z, C, R) \ + ((__m512d)__builtin_ia32_fixupimmpd512_mask ((__v8df)(__m512d)(X), \ + (__v8df)(__m512d)(Y), (__v8di)(__m512i)(Z), (int)(C), \ + (__mmask8)(U), (R))) + +#define _mm512_maskz_fixupimm_round_pd(U, X, Y, Z, C, R) \ + ((__m512d)__builtin_ia32_fixupimmpd512_maskz ((__v8df)(__m512d)(X), \ + (__v8df)(__m512d)(Y), (__v8di)(__m512i)(Z), (int)(C), \ + (__mmask8)(U), (R))) + +#define _mm512_fixupimm_round_ps(X, Y, Z, C, R) \ + ((__m512)__builtin_ia32_fixupimmps512_mask ((__v16sf)(__m512)(X), \ + (__v16sf)(__m512)(Y), (__v16si)(__m512i)(Z), (int)(C), \ + (__mmask16)(-1), (R))) + +#define _mm512_mask_fixupimm_round_ps(X, U, Y, Z, C, R) \ + ((__m512)__builtin_ia32_fixupimmps512_mask ((__v16sf)(__m512)(X), \ + (__v16sf)(__m512)(Y), (__v16si)(__m512i)(Z), (int)(C), \ + (__mmask16)(U), (R))) + +#define _mm512_maskz_fixupimm_round_ps(U, X, Y, Z, C, R) \ + ((__m512)__builtin_ia32_fixupimmps512_maskz ((__v16sf)(__m512)(X), \ + (__v16sf)(__m512)(Y), (__v16si)(__m512i)(Z), (int)(C), \ + (__mmask16)(U), (R))) + +#define _mm_fixupimm_round_sd(X, Y, Z, C, R) \ + ((__m128d)__builtin_ia32_fixupimmsd_mask ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (__v2di)(__m128i)(Z), (int)(C), \ + (__mmask8)(-1), (R))) + +#define _mm_mask_fixupimm_round_sd(X, U, Y, Z, C, R) \ + ((__m128d)__builtin_ia32_fixupimmsd_mask ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (__v2di)(__m128i)(Z), (int)(C), \ + (__mmask8)(U), (R))) + +#define _mm_maskz_fixupimm_round_sd(U, X, Y, Z, C, R) \ + ((__m128d)__builtin_ia32_fixupimmsd_maskz ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (__v2di)(__m128i)(Z), (int)(C), \ + (__mmask8)(U), (R))) + +#define _mm_fixupimm_round_ss(X, Y, Z, C, R) \ + ((__m128)__builtin_ia32_fixupimmss_mask ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (__v4si)(__m128i)(Z), (int)(C), \ + (__mmask8)(-1), (R))) + +#define _mm_mask_fixupimm_round_ss(X, U, Y, Z, C, R) \ + ((__m128)__builtin_ia32_fixupimmss_mask ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (__v4si)(__m128i)(Z), (int)(C), \ + (__mmask8)(U), (R))) + +#define _mm_maskz_fixupimm_round_ss(U, X, Y, Z, C, R) \ + ((__m128)__builtin_ia32_fixupimmss_maskz ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (__v4si)(__m128i)(Z), (int)(C), \ + (__mmask8)(U), (R))) +#endif + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_movehdup_ps (__m512 __A) +{ + return (__m512) __builtin_ia32_movshdup512_mask ((__v16sf) __A, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_movehdup_ps (__m512 __W, __mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_movshdup512_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_movehdup_ps (__mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_movshdup512_mask ((__v16sf) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_moveldup_ps (__m512 __A) +{ + return (__m512) __builtin_ia32_movsldup512_mask ((__v16sf) __A, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_moveldup_ps (__m512 __W, __mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_movsldup512_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_moveldup_ps (__mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_movsldup512_mask ((__v16sf) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_or_si512 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v16su) __A | (__v16su) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_or_epi32 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v16su) __A | (__v16su) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_or_epi32 (__m512i __W, __mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pord512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_or_epi32 (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pord512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_or_epi64 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v8du) __A | (__v8du) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_or_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_porq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_or_epi64 (__mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_porq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_xor_si512 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v16su) __A ^ (__v16su) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_xor_epi32 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v16su) __A ^ (__v16su) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_xor_epi32 (__m512i __W, __mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pxord512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_xor_epi32 (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pxord512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_xor_epi64 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v8du) __A ^ (__v8du) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_xor_epi64 (__m512i __W, __mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pxorq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_xor_epi64 (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pxorq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_rol_epi32 (__m512i __A, const int __B) +{ + return (__m512i) __builtin_ia32_prold512_mask ((__v16si) __A, __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_rol_epi32 (__m512i __W, __mmask16 __U, __m512i __A, const int __B) +{ + return (__m512i) __builtin_ia32_prold512_mask ((__v16si) __A, __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_rol_epi32 (__mmask16 __U, __m512i __A, const int __B) +{ + return (__m512i) __builtin_ia32_prold512_mask ((__v16si) __A, __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_ror_epi32 (__m512i __A, int __B) +{ + return (__m512i) __builtin_ia32_prord512_mask ((__v16si) __A, __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_ror_epi32 (__m512i __W, __mmask16 __U, __m512i __A, int __B) +{ + return (__m512i) __builtin_ia32_prord512_mask ((__v16si) __A, __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_ror_epi32 (__mmask16 __U, __m512i __A, int __B) +{ + return (__m512i) __builtin_ia32_prord512_mask ((__v16si) __A, __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_rol_epi64 (__m512i __A, const int __B) +{ + return (__m512i) __builtin_ia32_prolq512_mask ((__v8di) __A, __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_rol_epi64 (__m512i __W, __mmask8 __U, __m512i __A, const int __B) +{ + return (__m512i) __builtin_ia32_prolq512_mask ((__v8di) __A, __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_rol_epi64 (__mmask8 __U, __m512i __A, const int __B) +{ + return (__m512i) __builtin_ia32_prolq512_mask ((__v8di) __A, __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_ror_epi64 (__m512i __A, int __B) +{ + return (__m512i) __builtin_ia32_prorq512_mask ((__v8di) __A, __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_ror_epi64 (__m512i __W, __mmask8 __U, __m512i __A, int __B) +{ + return (__m512i) __builtin_ia32_prorq512_mask ((__v8di) __A, __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_ror_epi64 (__mmask8 __U, __m512i __A, int __B) +{ + return (__m512i) __builtin_ia32_prorq512_mask ((__v8di) __A, __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +#else +#define _mm512_rol_epi32(A, B) \ + ((__m512i)__builtin_ia32_prold512_mask ((__v16si)(__m512i)(A), \ + (int)(B), \ + (__v16si)_mm512_undefined_epi32 (), \ + (__mmask16)(-1))) +#define _mm512_mask_rol_epi32(W, U, A, B) \ + ((__m512i)__builtin_ia32_prold512_mask ((__v16si)(__m512i)(A), \ + (int)(B), \ + (__v16si)(__m512i)(W), \ + (__mmask16)(U))) +#define _mm512_maskz_rol_epi32(U, A, B) \ + ((__m512i)__builtin_ia32_prold512_mask ((__v16si)(__m512i)(A), \ + (int)(B), \ + (__v16si)_mm512_setzero_si512 (), \ + (__mmask16)(U))) +#define _mm512_ror_epi32(A, B) \ + ((__m512i)__builtin_ia32_prord512_mask ((__v16si)(__m512i)(A), \ + (int)(B), \ + (__v16si)_mm512_undefined_epi32 (), \ + (__mmask16)(-1))) +#define _mm512_mask_ror_epi32(W, U, A, B) \ + ((__m512i)__builtin_ia32_prord512_mask ((__v16si)(__m512i)(A), \ + (int)(B), \ + (__v16si)(__m512i)(W), \ + (__mmask16)(U))) +#define _mm512_maskz_ror_epi32(U, A, B) \ + ((__m512i)__builtin_ia32_prord512_mask ((__v16si)(__m512i)(A), \ + (int)(B), \ + (__v16si)_mm512_setzero_si512 (), \ + (__mmask16)(U))) +#define _mm512_rol_epi64(A, B) \ + ((__m512i)__builtin_ia32_prolq512_mask ((__v8di)(__m512i)(A), \ + (int)(B), \ + (__v8di)_mm512_undefined_epi32 (), \ + (__mmask8)(-1))) +#define _mm512_mask_rol_epi64(W, U, A, B) \ + ((__m512i)__builtin_ia32_prolq512_mask ((__v8di)(__m512i)(A), \ + (int)(B), \ + (__v8di)(__m512i)(W), \ + (__mmask8)(U))) +#define _mm512_maskz_rol_epi64(U, A, B) \ + ((__m512i)__builtin_ia32_prolq512_mask ((__v8di)(__m512i)(A), \ + (int)(B), \ + (__v8di)_mm512_setzero_si512 (), \ + (__mmask8)(U))) + +#define _mm512_ror_epi64(A, B) \ + ((__m512i)__builtin_ia32_prorq512_mask ((__v8di)(__m512i)(A), \ + (int)(B), \ + (__v8di)_mm512_undefined_epi32 (), \ + (__mmask8)(-1))) +#define _mm512_mask_ror_epi64(W, U, A, B) \ + ((__m512i)__builtin_ia32_prorq512_mask ((__v8di)(__m512i)(A), \ + (int)(B), \ + (__v8di)(__m512i)(W), \ + (__mmask8)(U))) +#define _mm512_maskz_ror_epi64(U, A, B) \ + ((__m512i)__builtin_ia32_prorq512_mask ((__v8di)(__m512i)(A), \ + (int)(B), \ + (__v8di)_mm512_setzero_si512 (), \ + (__mmask8)(U))) +#endif + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_and_si512 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v16su) __A & (__v16su) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_and_epi32 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v16su) __A & (__v16su) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_and_epi32 (__m512i __W, __mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pandd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_and_epi32 (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pandd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_and_epi64 (__m512i __A, __m512i __B) +{ + return (__m512i) ((__v8du) __A & (__v8du) __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_and_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pandq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __W, __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_and_epi64 (__mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pandq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_setzero_pd (), + __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_andnot_si512 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pandnd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_andnot_epi32 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pandnd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_andnot_epi32 (__m512i __W, __mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pandnd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_andnot_epi32 (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pandnd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_andnot_epi64 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pandnq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_andnot_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pandnq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __W, __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_andnot_epi64 (__mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pandnq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_setzero_pd (), + __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_test_epi32_mask (__m512i __A, __m512i __B) +{ + return (__mmask16) __builtin_ia32_ptestmd512 ((__v16si) __A, + (__v16si) __B, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_test_epi32_mask (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__mmask16) __builtin_ia32_ptestmd512 ((__v16si) __A, + (__v16si) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_test_epi64_mask (__m512i __A, __m512i __B) +{ + return (__mmask8) __builtin_ia32_ptestmq512 ((__v8di) __A, + (__v8di) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_test_epi64_mask (__mmask8 __U, __m512i __A, __m512i __B) +{ + return (__mmask8) __builtin_ia32_ptestmq512 ((__v8di) __A, (__v8di) __B, __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_testn_epi32_mask (__m512i __A, __m512i __B) +{ + return (__mmask16) __builtin_ia32_ptestnmd512 ((__v16si) __A, + (__v16si) __B, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_testn_epi32_mask (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__mmask16) __builtin_ia32_ptestnmd512 ((__v16si) __A, + (__v16si) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_testn_epi64_mask (__m512i __A, __m512i __B) +{ + return (__mmask8) __builtin_ia32_ptestnmq512 ((__v8di) __A, + (__v8di) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_testn_epi64_mask (__mmask8 __U, __m512i __A, __m512i __B) +{ + return (__mmask8) __builtin_ia32_ptestnmq512 ((__v8di) __A, + (__v8di) __B, __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_abs_ps (__m512 __A) +{ + return (__m512) _mm512_and_epi32 ((__m512i) __A, + _mm512_set1_epi32 (0x7fffffff)); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_abs_ps (__m512 __W, __mmask16 __U, __m512 __A) +{ + return (__m512) _mm512_mask_and_epi32 ((__m512i) __W, __U, (__m512i) __A, + _mm512_set1_epi32 (0x7fffffff)); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_abs_pd (__m512 __A) +{ + return (__m512d) _mm512_and_epi64 ((__m512i) __A, + _mm512_set1_epi64 (0x7fffffffffffffffLL)); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_abs_pd (__m512d __W, __mmask8 __U, __m512 __A) +{ + return (__m512d) + _mm512_mask_and_epi64 ((__m512i) __W, __U, (__m512i) __A, + _mm512_set1_epi64 (0x7fffffffffffffffLL)); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_unpackhi_epi32 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpckhdq512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_unpackhi_epi32 (__m512i __W, __mmask16 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_punpckhdq512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_unpackhi_epi32 (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpckhdq512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_unpackhi_epi64 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpckhqdq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_unpackhi_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpckhqdq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_unpackhi_epi64 (__mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpckhqdq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_unpacklo_epi32 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpckldq512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_unpacklo_epi32 (__m512i __W, __mmask16 __U, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_punpckldq512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_unpacklo_epi32 (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpckldq512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_unpacklo_epi64 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpcklqdq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_unpacklo_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpcklqdq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_unpacklo_epi64 (__mmask8 __U, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_punpcklqdq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +#ifdef __x86_64__ +#ifdef __OPTIMIZE__ +extern __inline unsigned long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundss_u64 (__m128 __A, const int __R) +{ + return (unsigned long long) __builtin_ia32_vcvtss2usi64 ((__v4sf) __A, __R); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundss_si64 (__m128 __A, const int __R) +{ + return (long long) __builtin_ia32_vcvtss2si64 ((__v4sf) __A, __R); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundss_i64 (__m128 __A, const int __R) +{ + return (long long) __builtin_ia32_vcvtss2si64 ((__v4sf) __A, __R); +} + +extern __inline unsigned long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtt_roundss_u64 (__m128 __A, const int __R) +{ + return (unsigned long long) __builtin_ia32_vcvttss2usi64 ((__v4sf) __A, __R); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtt_roundss_i64 (__m128 __A, const int __R) +{ + return (long long) __builtin_ia32_vcvttss2si64 ((__v4sf) __A, __R); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtt_roundss_si64 (__m128 __A, const int __R) +{ + return (long long) __builtin_ia32_vcvttss2si64 ((__v4sf) __A, __R); +} +#else +#define _mm_cvt_roundss_u64(A, B) \ + ((unsigned long long)__builtin_ia32_vcvtss2usi64(A, B)) + +#define _mm_cvt_roundss_si64(A, B) \ + ((long long)__builtin_ia32_vcvtss2si64(A, B)) + +#define _mm_cvt_roundss_i64(A, B) \ + ((long long)__builtin_ia32_vcvtss2si64(A, B)) + +#define _mm_cvtt_roundss_u64(A, B) \ + ((unsigned long long)__builtin_ia32_vcvttss2usi64(A, B)) + +#define _mm_cvtt_roundss_i64(A, B) \ + ((long long)__builtin_ia32_vcvttss2si64(A, B)) + +#define _mm_cvtt_roundss_si64(A, B) \ + ((long long)__builtin_ia32_vcvttss2si64(A, B)) +#endif +#endif + +#ifdef __OPTIMIZE__ +extern __inline unsigned +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundss_u32 (__m128 __A, const int __R) +{ + return (unsigned) __builtin_ia32_vcvtss2usi32 ((__v4sf) __A, __R); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundss_si32 (__m128 __A, const int __R) +{ + return (int) __builtin_ia32_vcvtss2si32 ((__v4sf) __A, __R); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundss_i32 (__m128 __A, const int __R) +{ + return (int) __builtin_ia32_vcvtss2si32 ((__v4sf) __A, __R); +} + +extern __inline unsigned +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtt_roundss_u32 (__m128 __A, const int __R) +{ + return (unsigned) __builtin_ia32_vcvttss2usi32 ((__v4sf) __A, __R); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtt_roundss_i32 (__m128 __A, const int __R) +{ + return (int) __builtin_ia32_vcvttss2si32 ((__v4sf) __A, __R); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtt_roundss_si32 (__m128 __A, const int __R) +{ + return (int) __builtin_ia32_vcvttss2si32 ((__v4sf) __A, __R); +} +#else +#define _mm_cvt_roundss_u32(A, B) \ + ((unsigned)__builtin_ia32_vcvtss2usi32(A, B)) + +#define _mm_cvt_roundss_si32(A, B) \ + ((int)__builtin_ia32_vcvtss2si32(A, B)) + +#define _mm_cvt_roundss_i32(A, B) \ + ((int)__builtin_ia32_vcvtss2si32(A, B)) + +#define _mm_cvtt_roundss_u32(A, B) \ + ((unsigned)__builtin_ia32_vcvttss2usi32(A, B)) + +#define _mm_cvtt_roundss_si32(A, B) \ + ((int)__builtin_ia32_vcvttss2si32(A, B)) + +#define _mm_cvtt_roundss_i32(A, B) \ + ((int)__builtin_ia32_vcvttss2si32(A, B)) +#endif + +#ifdef __x86_64__ +#ifdef __OPTIMIZE__ +extern __inline unsigned long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundsd_u64 (__m128d __A, const int __R) +{ + return (unsigned long long) __builtin_ia32_vcvtsd2usi64 ((__v2df) __A, __R); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundsd_si64 (__m128d __A, const int __R) +{ + return (long long) __builtin_ia32_vcvtsd2si64 ((__v2df) __A, __R); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundsd_i64 (__m128d __A, const int __R) +{ + return (long long) __builtin_ia32_vcvtsd2si64 ((__v2df) __A, __R); +} + +extern __inline unsigned long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtt_roundsd_u64 (__m128d __A, const int __R) +{ + return (unsigned long long) __builtin_ia32_vcvttsd2usi64 ((__v2df) __A, __R); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtt_roundsd_si64 (__m128d __A, const int __R) +{ + return (long long) __builtin_ia32_vcvttsd2si64 ((__v2df) __A, __R); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtt_roundsd_i64 (__m128d __A, const int __R) +{ + return (long long) __builtin_ia32_vcvttsd2si64 ((__v2df) __A, __R); +} +#else +#define _mm_cvt_roundsd_u64(A, B) \ + ((unsigned long long)__builtin_ia32_vcvtsd2usi64(A, B)) + +#define _mm_cvt_roundsd_si64(A, B) \ + ((long long)__builtin_ia32_vcvtsd2si64(A, B)) + +#define _mm_cvt_roundsd_i64(A, B) \ + ((long long)__builtin_ia32_vcvtsd2si64(A, B)) + +#define _mm_cvtt_roundsd_u64(A, B) \ + ((unsigned long long)__builtin_ia32_vcvttsd2usi64(A, B)) + +#define _mm_cvtt_roundsd_si64(A, B) \ + ((long long)__builtin_ia32_vcvttsd2si64(A, B)) + +#define _mm_cvtt_roundsd_i64(A, B) \ + ((long long)__builtin_ia32_vcvttsd2si64(A, B)) +#endif +#endif + +#ifdef __OPTIMIZE__ +extern __inline unsigned +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundsd_u32 (__m128d __A, const int __R) +{ + return (unsigned) __builtin_ia32_vcvtsd2usi32 ((__v2df) __A, __R); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundsd_si32 (__m128d __A, const int __R) +{ + return (int) __builtin_ia32_vcvtsd2si32 ((__v2df) __A, __R); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundsd_i32 (__m128d __A, const int __R) +{ + return (int) __builtin_ia32_vcvtsd2si32 ((__v2df) __A, __R); +} + +extern __inline unsigned +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtt_roundsd_u32 (__m128d __A, const int __R) +{ + return (unsigned) __builtin_ia32_vcvttsd2usi32 ((__v2df) __A, __R); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtt_roundsd_i32 (__m128d __A, const int __R) +{ + return (int) __builtin_ia32_vcvttsd2si32 ((__v2df) __A, __R); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtt_roundsd_si32 (__m128d __A, const int __R) +{ + return (int) __builtin_ia32_vcvttsd2si32 ((__v2df) __A, __R); +} +#else +#define _mm_cvt_roundsd_u32(A, B) \ + ((unsigned)__builtin_ia32_vcvtsd2usi32(A, B)) + +#define _mm_cvt_roundsd_si32(A, B) \ + ((int)__builtin_ia32_vcvtsd2si32(A, B)) + +#define _mm_cvt_roundsd_i32(A, B) \ + ((int)__builtin_ia32_vcvtsd2si32(A, B)) + +#define _mm_cvtt_roundsd_u32(A, B) \ + ((unsigned)__builtin_ia32_vcvttsd2usi32(A, B)) + +#define _mm_cvtt_roundsd_si32(A, B) \ + ((int)__builtin_ia32_vcvttsd2si32(A, B)) + +#define _mm_cvtt_roundsd_i32(A, B) \ + ((int)__builtin_ia32_vcvttsd2si32(A, B)) +#endif + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_movedup_pd (__m512d __A) +{ + return (__m512d) __builtin_ia32_movddup512_mask ((__v8df) __A, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_movedup_pd (__m512d __W, __mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_movddup512_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_movedup_pd (__mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_movddup512_mask ((__v8df) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_unpacklo_pd (__m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_unpcklpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_unpacklo_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_unpcklpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_unpacklo_pd (__mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_unpcklpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_unpackhi_pd (__m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_unpckhpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_unpackhi_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_unpckhpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_unpackhi_pd (__mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_unpckhpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_unpackhi_ps (__m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_unpckhps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_unpackhi_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_unpckhps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_unpackhi_ps (__mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_unpckhps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundps_pd (__m256 __A, const int __R) +{ + return (__m512d) __builtin_ia32_cvtps2pd512_mask ((__v8sf) __A, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundps_pd (__m512d __W, __mmask8 __U, __m256 __A, + const int __R) +{ + return (__m512d) __builtin_ia32_cvtps2pd512_mask ((__v8sf) __A, + (__v8df) __W, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundps_pd (__mmask8 __U, __m256 __A, const int __R) +{ + return (__m512d) __builtin_ia32_cvtps2pd512_mask ((__v8sf) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundph_ps (__m256i __A, const int __R) +{ + return (__m512) __builtin_ia32_vcvtph2ps512_mask ((__v16hi) __A, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundph_ps (__m512 __W, __mmask16 __U, __m256i __A, + const int __R) +{ + return (__m512) __builtin_ia32_vcvtph2ps512_mask ((__v16hi) __A, + (__v16sf) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundph_ps (__mmask16 __U, __m256i __A, const int __R) +{ + return (__m512) __builtin_ia32_vcvtph2ps512_mask ((__v16hi) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, __R); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundps_ph (__m512 __A, const int __I) +{ + return (__m256i) __builtin_ia32_vcvtps2ph512_mask ((__v16sf) __A, + __I, + (__v16hi) + _mm256_undefined_si256 (), + -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtps_ph (__m512 __A, const int __I) +{ + return (__m256i) __builtin_ia32_vcvtps2ph512_mask ((__v16sf) __A, + __I, + (__v16hi) + _mm256_undefined_si256 (), + -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundps_ph (__m256i __U, __mmask16 __W, __m512 __A, + const int __I) +{ + return (__m256i) __builtin_ia32_vcvtps2ph512_mask ((__v16sf) __A, + __I, + (__v16hi) __U, + (__mmask16) __W); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtps_ph (__m256i __U, __mmask16 __W, __m512 __A, const int __I) +{ + return (__m256i) __builtin_ia32_vcvtps2ph512_mask ((__v16sf) __A, + __I, + (__v16hi) __U, + (__mmask16) __W); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundps_ph (__mmask16 __W, __m512 __A, const int __I) +{ + return (__m256i) __builtin_ia32_vcvtps2ph512_mask ((__v16sf) __A, + __I, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __W); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtps_ph (__mmask16 __W, __m512 __A, const int __I) +{ + return (__m256i) __builtin_ia32_vcvtps2ph512_mask ((__v16sf) __A, + __I, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __W); +} +#else +#define _mm512_cvt_roundps_pd(A, B) \ + (__m512d)__builtin_ia32_cvtps2pd512_mask(A, (__v8df)_mm512_undefined_pd(), -1, B) + +#define _mm512_mask_cvt_roundps_pd(W, U, A, B) \ + (__m512d)__builtin_ia32_cvtps2pd512_mask(A, (__v8df)(W), U, B) + +#define _mm512_maskz_cvt_roundps_pd(U, A, B) \ + (__m512d)__builtin_ia32_cvtps2pd512_mask(A, (__v8df)_mm512_setzero_pd(), U, B) + +#define _mm512_cvt_roundph_ps(A, B) \ + (__m512)__builtin_ia32_vcvtph2ps512_mask((__v16hi)(A), (__v16sf)_mm512_undefined_ps(), -1, B) + +#define _mm512_mask_cvt_roundph_ps(W, U, A, B) \ + (__m512)__builtin_ia32_vcvtph2ps512_mask((__v16hi)(A), (__v16sf)(W), U, B) + +#define _mm512_maskz_cvt_roundph_ps(U, A, B) \ + (__m512)__builtin_ia32_vcvtph2ps512_mask((__v16hi)(A), (__v16sf)_mm512_setzero_ps(), U, B) + +#define _mm512_cvt_roundps_ph(A, I) \ + ((__m256i) __builtin_ia32_vcvtps2ph512_mask ((__v16sf)(__m512) A, (int) (I),\ + (__v16hi)_mm256_undefined_si256 (), -1)) +#define _mm512_cvtps_ph(A, I) \ + ((__m256i) __builtin_ia32_vcvtps2ph512_mask ((__v16sf)(__m512) A, (int) (I),\ + (__v16hi)_mm256_undefined_si256 (), -1)) +#define _mm512_mask_cvt_roundps_ph(U, W, A, I) \ + ((__m256i) __builtin_ia32_vcvtps2ph512_mask ((__v16sf)(__m512) A, (int) (I),\ + (__v16hi)(__m256i)(U), (__mmask16) (W))) +#define _mm512_mask_cvtps_ph(U, W, A, I) \ + ((__m256i) __builtin_ia32_vcvtps2ph512_mask ((__v16sf)(__m512) A, (int) (I),\ + (__v16hi)(__m256i)(U), (__mmask16) (W))) +#define _mm512_maskz_cvt_roundps_ph(W, A, I) \ + ((__m256i) __builtin_ia32_vcvtps2ph512_mask ((__v16sf)(__m512) A, (int) (I),\ + (__v16hi)_mm256_setzero_si256 (), (__mmask16) (W))) +#define _mm512_maskz_cvtps_ph(W, A, I) \ + ((__m256i) __builtin_ia32_vcvtps2ph512_mask ((__v16sf)(__m512) A, (int) (I),\ + (__v16hi)_mm256_setzero_si256 (), (__mmask16) (W))) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvt_roundpd_ps (__m512d __A, const int __R) +{ + return (__m256) __builtin_ia32_cvtpd2ps512_mask ((__v8df) __A, + (__v8sf) + _mm256_undefined_ps (), + (__mmask8) -1, __R); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvt_roundpd_ps (__m256 __W, __mmask8 __U, __m512d __A, + const int __R) +{ + return (__m256) __builtin_ia32_cvtpd2ps512_mask ((__v8df) __A, + (__v8sf) __W, + (__mmask8) __U, __R); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvt_roundpd_ps (__mmask8 __U, __m512d __A, const int __R) +{ + return (__m256) __builtin_ia32_cvtpd2ps512_mask ((__v8df) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U, __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundsd_ss (__m128 __A, __m128d __B, const int __R) +{ + return (__m128) __builtin_ia32_cvtsd2ss_round ((__v4sf) __A, + (__v2df) __B, + __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_roundss_sd (__m128d __A, __m128 __B, const int __R) +{ + return (__m128d) __builtin_ia32_cvtss2sd_round ((__v2df) __A, + (__v4sf) __B, + __R); +} +#else +#define _mm512_cvt_roundpd_ps(A, B) \ + (__m256)__builtin_ia32_cvtpd2ps512_mask(A, (__v8sf)_mm256_undefined_ps(), -1, B) + +#define _mm512_mask_cvt_roundpd_ps(W, U, A, B) \ + (__m256)__builtin_ia32_cvtpd2ps512_mask(A, (__v8sf)(W), U, B) + +#define _mm512_maskz_cvt_roundpd_ps(U, A, B) \ + (__m256)__builtin_ia32_cvtpd2ps512_mask(A, (__v8sf)_mm256_setzero_ps(), U, B) + +#define _mm_cvt_roundsd_ss(A, B, C) \ + (__m128)__builtin_ia32_cvtsd2ss_round(A, B, C) + +#define _mm_cvt_roundss_sd(A, B, C) \ + (__m128d)__builtin_ia32_cvtss2sd_round(A, B, C) +#endif + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_stream_si512 (__m512i * __P, __m512i __A) +{ + __builtin_ia32_movntdq512 ((__v8di *) __P, (__v8di) __A); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_stream_ps (float *__P, __m512 __A) +{ + __builtin_ia32_movntps512 (__P, (__v16sf) __A); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_stream_pd (double *__P, __m512d __A) +{ + __builtin_ia32_movntpd512 (__P, (__v8df) __A); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_stream_load_si512 (void *__P) +{ + return __builtin_ia32_movntdqa512 ((__v8di *)__P); +} + +/* Constants for mantissa extraction */ +typedef enum +{ + _MM_MANT_NORM_1_2, /* interval [1, 2) */ + _MM_MANT_NORM_p5_2, /* interval [0.5, 2) */ + _MM_MANT_NORM_p5_1, /* interval [0.5, 1) */ + _MM_MANT_NORM_p75_1p5 /* interval [0.75, 1.5) */ +} _MM_MANTISSA_NORM_ENUM; + +typedef enum +{ + _MM_MANT_SIGN_src, /* sign = sign(SRC) */ + _MM_MANT_SIGN_zero, /* sign = 0 */ + _MM_MANT_SIGN_nan /* DEST = NaN if sign(SRC) = 1 */ +} _MM_MANTISSA_SIGN_ENUM; + +#ifdef __OPTIMIZE__ +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_getexp_round_ss (__m128 __A, __m128 __B, const int __R) +{ + return (__m128) __builtin_ia32_getexpss128_round ((__v4sf) __A, + (__v4sf) __B, + __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_getexp_round_sd (__m128d __A, __m128d __B, const int __R) +{ + return (__m128d) __builtin_ia32_getexpsd128_round ((__v2df) __A, + (__v2df) __B, + __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_getexp_round_ps (__m512 __A, const int __R) +{ + return (__m512) __builtin_ia32_getexpps512_mask ((__v16sf) __A, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_getexp_round_ps (__m512 __W, __mmask16 __U, __m512 __A, + const int __R) +{ + return (__m512) __builtin_ia32_getexpps512_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_getexp_round_ps (__mmask16 __U, __m512 __A, const int __R) +{ + return (__m512) __builtin_ia32_getexpps512_mask ((__v16sf) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_getexp_round_pd (__m512d __A, const int __R) +{ + return (__m512d) __builtin_ia32_getexppd512_mask ((__v8df) __A, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_getexp_round_pd (__m512d __W, __mmask8 __U, __m512d __A, + const int __R) +{ + return (__m512d) __builtin_ia32_getexppd512_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_getexp_round_pd (__mmask8 __U, __m512d __A, const int __R) +{ + return (__m512d) __builtin_ia32_getexppd512_mask ((__v8df) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_getmant_round_pd (__m512d __A, _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C, const int __R) +{ + return (__m512d) __builtin_ia32_getmantpd512_mask ((__v8df) __A, + (__C << 2) | __B, + _mm512_undefined_pd (), + (__mmask8) -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_getmant_round_pd (__m512d __W, __mmask8 __U, __m512d __A, + _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C, const int __R) +{ + return (__m512d) __builtin_ia32_getmantpd512_mask ((__v8df) __A, + (__C << 2) | __B, + (__v8df) __W, __U, + __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_getmant_round_pd (__mmask8 __U, __m512d __A, + _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C, const int __R) +{ + return (__m512d) __builtin_ia32_getmantpd512_mask ((__v8df) __A, + (__C << 2) | __B, + (__v8df) + _mm512_setzero_pd (), + __U, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_getmant_round_ps (__m512 __A, _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C, const int __R) +{ + return (__m512) __builtin_ia32_getmantps512_mask ((__v16sf) __A, + (__C << 2) | __B, + _mm512_undefined_ps (), + (__mmask16) -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_getmant_round_ps (__m512 __W, __mmask16 __U, __m512 __A, + _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C, const int __R) +{ + return (__m512) __builtin_ia32_getmantps512_mask ((__v16sf) __A, + (__C << 2) | __B, + (__v16sf) __W, __U, + __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_getmant_round_ps (__mmask16 __U, __m512 __A, + _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C, const int __R) +{ + return (__m512) __builtin_ia32_getmantps512_mask ((__v16sf) __A, + (__C << 2) | __B, + (__v16sf) + _mm512_setzero_ps (), + __U, __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_getmant_round_sd (__m128d __A, __m128d __B, + _MM_MANTISSA_NORM_ENUM __C, + _MM_MANTISSA_SIGN_ENUM __D, const int __R) +{ + return (__m128d) __builtin_ia32_getmantsd_round ((__v2df) __A, + (__v2df) __B, + (__D << 2) | __C, + __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_getmant_round_ss (__m128 __A, __m128 __B, + _MM_MANTISSA_NORM_ENUM __C, + _MM_MANTISSA_SIGN_ENUM __D, const int __R) +{ + return (__m128) __builtin_ia32_getmantss_round ((__v4sf) __A, + (__v4sf) __B, + (__D << 2) | __C, + __R); +} + +#else +#define _mm512_getmant_round_pd(X, B, C, R) \ + ((__m512d)__builtin_ia32_getmantpd512_mask ((__v8df)(__m512d)(X), \ + (int)(((C)<<2) | (B)), \ + (__v8df)(__m512d)_mm512_undefined_pd(), \ + (__mmask8)-1,\ + (R))) + +#define _mm512_mask_getmant_round_pd(W, U, X, B, C, R) \ + ((__m512d)__builtin_ia32_getmantpd512_mask ((__v8df)(__m512d)(X), \ + (int)(((C)<<2) | (B)), \ + (__v8df)(__m512d)(W), \ + (__mmask8)(U),\ + (R))) + +#define _mm512_maskz_getmant_round_pd(U, X, B, C, R) \ + ((__m512d)__builtin_ia32_getmantpd512_mask ((__v8df)(__m512d)(X), \ + (int)(((C)<<2) | (B)), \ + (__v8df)(__m512d)_mm512_setzero_pd(), \ + (__mmask8)(U),\ + (R))) +#define _mm512_getmant_round_ps(X, B, C, R) \ + ((__m512)__builtin_ia32_getmantps512_mask ((__v16sf)(__m512)(X), \ + (int)(((C)<<2) | (B)), \ + (__v16sf)(__m512)_mm512_undefined_ps(), \ + (__mmask16)-1,\ + (R))) + +#define _mm512_mask_getmant_round_ps(W, U, X, B, C, R) \ + ((__m512)__builtin_ia32_getmantps512_mask ((__v16sf)(__m512)(X), \ + (int)(((C)<<2) | (B)), \ + (__v16sf)(__m512)(W), \ + (__mmask16)(U),\ + (R))) + +#define _mm512_maskz_getmant_round_ps(U, X, B, C, R) \ + ((__m512)__builtin_ia32_getmantps512_mask ((__v16sf)(__m512)(X), \ + (int)(((C)<<2) | (B)), \ + (__v16sf)(__m512)_mm512_setzero_ps(), \ + (__mmask16)(U),\ + (R))) +#define _mm_getmant_round_sd(X, Y, C, D, R) \ + ((__m128d)__builtin_ia32_getmantsd_round ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), \ + (int)(((D)<<2) | (C)), \ + (R))) + +#define _mm_getmant_round_ss(X, Y, C, D, R) \ + ((__m128)__builtin_ia32_getmantss_round ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), \ + (int)(((D)<<2) | (C)), \ + (R))) + +#define _mm_getexp_round_ss(A, B, R) \ + ((__m128)__builtin_ia32_getexpss128_round((__v4sf)(__m128)(A), (__v4sf)(__m128)(B), R)) + +#define _mm_getexp_round_sd(A, B, R) \ + ((__m128d)__builtin_ia32_getexpsd128_round((__v2df)(__m128d)(A), (__v2df)(__m128d)(B), R)) + +#define _mm512_getexp_round_ps(A, R) \ + ((__m512)__builtin_ia32_getexpps512_mask((__v16sf)(__m512)(A), \ + (__v16sf)_mm512_undefined_ps(), (__mmask16)-1, R)) + +#define _mm512_mask_getexp_round_ps(W, U, A, R) \ + ((__m512)__builtin_ia32_getexpps512_mask((__v16sf)(__m512)(A), \ + (__v16sf)(__m512)(W), (__mmask16)(U), R)) + +#define _mm512_maskz_getexp_round_ps(U, A, R) \ + ((__m512)__builtin_ia32_getexpps512_mask((__v16sf)(__m512)(A), \ + (__v16sf)_mm512_setzero_ps(), (__mmask16)(U), R)) + +#define _mm512_getexp_round_pd(A, R) \ + ((__m512d)__builtin_ia32_getexppd512_mask((__v8df)(__m512d)(A), \ + (__v8df)_mm512_undefined_pd(), (__mmask8)-1, R)) + +#define _mm512_mask_getexp_round_pd(W, U, A, R) \ + ((__m512d)__builtin_ia32_getexppd512_mask((__v8df)(__m512d)(A), \ + (__v8df)(__m512d)(W), (__mmask8)(U), R)) + +#define _mm512_maskz_getexp_round_pd(U, A, R) \ + ((__m512d)__builtin_ia32_getexppd512_mask((__v8df)(__m512d)(A), \ + (__v8df)_mm512_setzero_pd(), (__mmask8)(U), R)) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_roundscale_round_ps (__m512 __A, const int __imm, const int __R) +{ + return (__m512) __builtin_ia32_rndscaleps_mask ((__v16sf) __A, __imm, + (__v16sf) + _mm512_undefined_ps (), + -1, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_roundscale_round_ps (__m512 __A, __mmask16 __B, __m512 __C, + const int __imm, const int __R) +{ + return (__m512) __builtin_ia32_rndscaleps_mask ((__v16sf) __C, __imm, + (__v16sf) __A, + (__mmask16) __B, __R); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_roundscale_round_ps (__mmask16 __A, __m512 __B, + const int __imm, const int __R) +{ + return (__m512) __builtin_ia32_rndscaleps_mask ((__v16sf) __B, + __imm, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __A, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_roundscale_round_pd (__m512d __A, const int __imm, const int __R) +{ + return (__m512d) __builtin_ia32_rndscalepd_mask ((__v8df) __A, __imm, + (__v8df) + _mm512_undefined_pd (), + -1, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_roundscale_round_pd (__m512d __A, __mmask8 __B, + __m512d __C, const int __imm, const int __R) +{ + return (__m512d) __builtin_ia32_rndscalepd_mask ((__v8df) __C, __imm, + (__v8df) __A, + (__mmask8) __B, __R); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_roundscale_round_pd (__mmask8 __A, __m512d __B, + const int __imm, const int __R) +{ + return (__m512d) __builtin_ia32_rndscalepd_mask ((__v8df) __B, + __imm, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __A, __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_roundscale_round_ss (__m128 __A, __m128 __B, const int __imm, const int __R) +{ + return (__m128) __builtin_ia32_rndscaless_round ((__v4sf) __A, + (__v4sf) __B, __imm, __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_roundscale_round_sd (__m128d __A, __m128d __B, const int __imm, + const int __R) +{ + return (__m128d) __builtin_ia32_rndscalesd_round ((__v2df) __A, + (__v2df) __B, __imm, __R); +} + +#else +#define _mm512_roundscale_round_ps(A, B, R) \ + ((__m512) __builtin_ia32_rndscaleps_mask ((__v16sf)(__m512)(A), (int)(B),\ + (__v16sf)_mm512_undefined_ps(), (__mmask16)(-1), R)) +#define _mm512_mask_roundscale_round_ps(A, B, C, D, R) \ + ((__m512) __builtin_ia32_rndscaleps_mask ((__v16sf)(__m512)(C), \ + (int)(D), \ + (__v16sf)(__m512)(A), \ + (__mmask16)(B), R)) +#define _mm512_maskz_roundscale_round_ps(A, B, C, R) \ + ((__m512) __builtin_ia32_rndscaleps_mask ((__v16sf)(__m512)(B), \ + (int)(C), \ + (__v16sf)_mm512_setzero_ps(),\ + (__mmask16)(A), R)) +#define _mm512_roundscale_round_pd(A, B, R) \ + ((__m512d) __builtin_ia32_rndscalepd_mask ((__v8df)(__m512d)(A), (int)(B),\ + (__v8df)_mm512_undefined_pd(), (__mmask8)(-1), R)) +#define _mm512_mask_roundscale_round_pd(A, B, C, D, R) \ + ((__m512d) __builtin_ia32_rndscalepd_mask ((__v8df)(__m512d)(C), \ + (int)(D), \ + (__v8df)(__m512d)(A), \ + (__mmask8)(B), R)) +#define _mm512_maskz_roundscale_round_pd(A, B, C, R) \ + ((__m512d) __builtin_ia32_rndscalepd_mask ((__v8df)(__m512d)(B), \ + (int)(C), \ + (__v8df)_mm512_setzero_pd(),\ + (__mmask8)(A), R)) +#define _mm_roundscale_round_ss(A, B, C, R) \ + ((__m128) __builtin_ia32_rndscaless_round ((__v4sf)(__m128)(A), \ + (__v4sf)(__m128)(B), (int)(C), R)) +#define _mm_roundscale_round_sd(A, B, C, R) \ + ((__m128d) __builtin_ia32_rndscalesd_round ((__v2df)(__m128d)(A), \ + (__v2df)(__m128d)(B), (int)(C), R)) +#endif + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_floor_ps (__m512 __A) +{ + return (__m512) __builtin_ia32_rndscaleps_mask ((__v16sf) __A, + _MM_FROUND_FLOOR, + (__v16sf) __A, -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_floor_pd (__m512d __A) +{ + return (__m512d) __builtin_ia32_rndscalepd_mask ((__v8df) __A, + _MM_FROUND_FLOOR, + (__v8df) __A, -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_ceil_ps (__m512 __A) +{ + return (__m512) __builtin_ia32_rndscaleps_mask ((__v16sf) __A, + _MM_FROUND_CEIL, + (__v16sf) __A, -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_ceil_pd (__m512d __A) +{ + return (__m512d) __builtin_ia32_rndscalepd_mask ((__v8df) __A, + _MM_FROUND_CEIL, + (__v8df) __A, -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_floor_ps (__m512 __W, __mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_rndscaleps_mask ((__v16sf) __A, + _MM_FROUND_FLOOR, + (__v16sf) __W, __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_floor_pd (__m512d __W, __mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_rndscalepd_mask ((__v8df) __A, + _MM_FROUND_FLOOR, + (__v8df) __W, __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_ceil_ps (__m512 __W, __mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_rndscaleps_mask ((__v16sf) __A, + _MM_FROUND_CEIL, + (__v16sf) __W, __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_ceil_pd (__m512d __W, __mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_rndscalepd_mask ((__v8df) __A, + _MM_FROUND_CEIL, + (__v8df) __W, __U, + _MM_FROUND_CUR_DIRECTION); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_alignr_epi32 (__m512i __A, __m512i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_alignd512_mask ((__v16si) __A, + (__v16si) __B, __imm, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_alignr_epi32 (__m512i __W, __mmask16 __U, __m512i __A, + __m512i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_alignd512_mask ((__v16si) __A, + (__v16si) __B, __imm, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_alignr_epi32 (__mmask16 __U, __m512i __A, __m512i __B, + const int __imm) +{ + return (__m512i) __builtin_ia32_alignd512_mask ((__v16si) __A, + (__v16si) __B, __imm, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_alignr_epi64 (__m512i __A, __m512i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_alignq512_mask ((__v8di) __A, + (__v8di) __B, __imm, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_alignr_epi64 (__m512i __W, __mmask8 __U, __m512i __A, + __m512i __B, const int __imm) +{ + return (__m512i) __builtin_ia32_alignq512_mask ((__v8di) __A, + (__v8di) __B, __imm, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_alignr_epi64 (__mmask8 __U, __m512i __A, __m512i __B, + const int __imm) +{ + return (__m512i) __builtin_ia32_alignq512_mask ((__v8di) __A, + (__v8di) __B, __imm, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} +#else +#define _mm512_alignr_epi32(X, Y, C) \ + ((__m512i)__builtin_ia32_alignd512_mask ((__v16si)(__m512i)(X), \ + (__v16si)(__m512i)(Y), (int)(C), (__v16si)_mm512_undefined_epi32 (),\ + (__mmask16)-1)) + +#define _mm512_mask_alignr_epi32(W, U, X, Y, C) \ + ((__m512i)__builtin_ia32_alignd512_mask ((__v16si)(__m512i)(X), \ + (__v16si)(__m512i)(Y), (int)(C), (__v16si)(__m512i)(W), \ + (__mmask16)(U))) + +#define _mm512_maskz_alignr_epi32(U, X, Y, C) \ + ((__m512i)__builtin_ia32_alignd512_mask ((__v16si)(__m512i)(X), \ + (__v16si)(__m512i)(Y), (int)(C), (__v16si)_mm512_setzero_si512 (),\ + (__mmask16)(U))) + +#define _mm512_alignr_epi64(X, Y, C) \ + ((__m512i)__builtin_ia32_alignq512_mask ((__v8di)(__m512i)(X), \ + (__v8di)(__m512i)(Y), (int)(C), (__v8di)_mm512_undefined_epi32 (), \ + (__mmask8)-1)) + +#define _mm512_mask_alignr_epi64(W, U, X, Y, C) \ + ((__m512i)__builtin_ia32_alignq512_mask ((__v8di)(__m512i)(X), \ + (__v8di)(__m512i)(Y), (int)(C), (__v8di)(__m512i)(W), (__mmask8)(U))) + +#define _mm512_maskz_alignr_epi64(U, X, Y, C) \ + ((__m512i)__builtin_ia32_alignq512_mask ((__v8di)(__m512i)(X), \ + (__v8di)(__m512i)(Y), (int)(C), (__v8di)_mm512_setzero_si512 (),\ + (__mmask8)(U))) +#endif + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpeq_epi32_mask (__m512i __A, __m512i __B) +{ + return (__mmask16) __builtin_ia32_pcmpeqd512_mask ((__v16si) __A, + (__v16si) __B, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpeq_epi32_mask (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__mmask16) __builtin_ia32_pcmpeqd512_mask ((__v16si) __A, + (__v16si) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpeq_epi64_mask (__mmask8 __U, __m512i __A, __m512i __B) +{ + return (__mmask8) __builtin_ia32_pcmpeqq512_mask ((__v8di) __A, + (__v8di) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpeq_epi64_mask (__m512i __A, __m512i __B) +{ + return (__mmask8) __builtin_ia32_pcmpeqq512_mask ((__v8di) __A, + (__v8di) __B, + (__mmask8) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpgt_epi32_mask (__m512i __A, __m512i __B) +{ + return (__mmask16) __builtin_ia32_pcmpgtd512_mask ((__v16si) __A, + (__v16si) __B, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpgt_epi32_mask (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__mmask16) __builtin_ia32_pcmpgtd512_mask ((__v16si) __A, + (__v16si) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpgt_epi64_mask (__mmask8 __U, __m512i __A, __m512i __B) +{ + return (__mmask8) __builtin_ia32_pcmpgtq512_mask ((__v8di) __A, + (__v8di) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpgt_epi64_mask (__m512i __A, __m512i __B) +{ + return (__mmask8) __builtin_ia32_pcmpgtq512_mask ((__v8di) __A, + (__v8di) __B, + (__mmask8) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpge_epi32_mask (__m512i __X, __m512i __Y) +{ + return (__mmask16) __builtin_ia32_cmpd512_mask ((__v16si) __X, + (__v16si) __Y, 5, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpge_epi32_mask (__mmask16 __M, __m512i __X, __m512i __Y) +{ + return (__mmask16) __builtin_ia32_cmpd512_mask ((__v16si) __X, + (__v16si) __Y, 5, + (__mmask16) __M); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpge_epu32_mask (__mmask16 __M, __m512i __X, __m512i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpd512_mask ((__v16si) __X, + (__v16si) __Y, 5, + (__mmask16) __M); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpge_epu32_mask (__m512i __X, __m512i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpd512_mask ((__v16si) __X, + (__v16si) __Y, 5, + (__mmask16) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpge_epi64_mask (__mmask8 __M, __m512i __X, __m512i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq512_mask ((__v8di) __X, + (__v8di) __Y, 5, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpge_epi64_mask (__m512i __X, __m512i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq512_mask ((__v8di) __X, + (__v8di) __Y, 5, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpge_epu64_mask (__mmask8 __M, __m512i __X, __m512i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq512_mask ((__v8di) __X, + (__v8di) __Y, 5, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpge_epu64_mask (__m512i __X, __m512i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq512_mask ((__v8di) __X, + (__v8di) __Y, 5, + (__mmask8) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmple_epi32_mask (__mmask16 __M, __m512i __X, __m512i __Y) +{ + return (__mmask16) __builtin_ia32_cmpd512_mask ((__v16si) __X, + (__v16si) __Y, 2, + (__mmask16) __M); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmple_epi32_mask (__m512i __X, __m512i __Y) +{ + return (__mmask16) __builtin_ia32_cmpd512_mask ((__v16si) __X, + (__v16si) __Y, 2, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmple_epu32_mask (__mmask16 __M, __m512i __X, __m512i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpd512_mask ((__v16si) __X, + (__v16si) __Y, 2, + (__mmask16) __M); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmple_epu32_mask (__m512i __X, __m512i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpd512_mask ((__v16si) __X, + (__v16si) __Y, 2, + (__mmask16) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmple_epi64_mask (__mmask8 __M, __m512i __X, __m512i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq512_mask ((__v8di) __X, + (__v8di) __Y, 2, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmple_epi64_mask (__m512i __X, __m512i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq512_mask ((__v8di) __X, + (__v8di) __Y, 2, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmple_epu64_mask (__mmask8 __M, __m512i __X, __m512i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq512_mask ((__v8di) __X, + (__v8di) __Y, 2, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmple_epu64_mask (__m512i __X, __m512i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq512_mask ((__v8di) __X, + (__v8di) __Y, 2, + (__mmask8) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmplt_epi32_mask (__mmask16 __M, __m512i __X, __m512i __Y) +{ + return (__mmask16) __builtin_ia32_cmpd512_mask ((__v16si) __X, + (__v16si) __Y, 1, + (__mmask16) __M); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmplt_epi32_mask (__m512i __X, __m512i __Y) +{ + return (__mmask16) __builtin_ia32_cmpd512_mask ((__v16si) __X, + (__v16si) __Y, 1, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmplt_epu32_mask (__mmask16 __M, __m512i __X, __m512i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpd512_mask ((__v16si) __X, + (__v16si) __Y, 1, + (__mmask16) __M); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmplt_epu32_mask (__m512i __X, __m512i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpd512_mask ((__v16si) __X, + (__v16si) __Y, 1, + (__mmask16) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmplt_epi64_mask (__mmask8 __M, __m512i __X, __m512i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq512_mask ((__v8di) __X, + (__v8di) __Y, 1, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmplt_epi64_mask (__m512i __X, __m512i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq512_mask ((__v8di) __X, + (__v8di) __Y, 1, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmplt_epu64_mask (__mmask8 __M, __m512i __X, __m512i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq512_mask ((__v8di) __X, + (__v8di) __Y, 1, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmplt_epu64_mask (__m512i __X, __m512i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq512_mask ((__v8di) __X, + (__v8di) __Y, 1, + (__mmask8) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpneq_epi32_mask (__m512i __X, __m512i __Y) +{ + return (__mmask16) __builtin_ia32_cmpd512_mask ((__v16si) __X, + (__v16si) __Y, 4, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpneq_epi32_mask (__mmask16 __M, __m512i __X, __m512i __Y) +{ + return (__mmask16) __builtin_ia32_cmpd512_mask ((__v16si) __X, + (__v16si) __Y, 4, + (__mmask16) __M); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpneq_epu32_mask (__mmask16 __M, __m512i __X, __m512i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpd512_mask ((__v16si) __X, + (__v16si) __Y, 4, + (__mmask16) __M); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpneq_epu32_mask (__m512i __X, __m512i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpd512_mask ((__v16si) __X, + (__v16si) __Y, 4, + (__mmask16) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpneq_epi64_mask (__mmask16 __M, __m512i __X, __m512i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq512_mask ((__v8di) __X, + (__v8di) __Y, 4, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpneq_epi64_mask (__m512i __X, __m512i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq512_mask ((__v8di) __X, + (__v8di) __Y, 4, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpneq_epu64_mask (__mmask8 __M, __m512i __X, __m512i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq512_mask ((__v8di) __X, + (__v8di) __Y, 4, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpneq_epu64_mask (__m512i __X, __m512i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq512_mask ((__v8di) __X, + (__v8di) __Y, 4, + (__mmask8) -1); +} + +#define _MM_CMPINT_EQ 0x0 +#define _MM_CMPINT_LT 0x1 +#define _MM_CMPINT_LE 0x2 +#define _MM_CMPINT_UNUSED 0x3 +#define _MM_CMPINT_NE 0x4 +#define _MM_CMPINT_NLT 0x5 +#define _MM_CMPINT_GE 0x5 +#define _MM_CMPINT_NLE 0x6 +#define _MM_CMPINT_GT 0x6 + +#ifdef __OPTIMIZE__ +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kshiftli_mask16 (__mmask16 __A, unsigned int __B) +{ + return (__mmask16) __builtin_ia32_kshiftlihi ((__mmask16) __A, + (__mmask8) __B); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kshiftri_mask16 (__mmask16 __A, unsigned int __B) +{ + return (__mmask16) __builtin_ia32_kshiftrihi ((__mmask16) __A, + (__mmask8) __B); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmp_epi64_mask (__m512i __X, __m512i __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_cmpq512_mask ((__v8di) __X, + (__v8di) __Y, __P, + (__mmask8) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmp_epi32_mask (__m512i __X, __m512i __Y, const int __P) +{ + return (__mmask16) __builtin_ia32_cmpd512_mask ((__v16si) __X, + (__v16si) __Y, __P, + (__mmask16) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmp_epu64_mask (__m512i __X, __m512i __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_ucmpq512_mask ((__v8di) __X, + (__v8di) __Y, __P, + (__mmask8) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmp_epu32_mask (__m512i __X, __m512i __Y, const int __P) +{ + return (__mmask16) __builtin_ia32_ucmpd512_mask ((__v16si) __X, + (__v16si) __Y, __P, + (__mmask16) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmp_round_pd_mask (__m512d __X, __m512d __Y, const int __P, + const int __R) +{ + return (__mmask8) __builtin_ia32_cmppd512_mask ((__v8df) __X, + (__v8df) __Y, __P, + (__mmask8) -1, __R); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmp_round_ps_mask (__m512 __X, __m512 __Y, const int __P, const int __R) +{ + return (__mmask16) __builtin_ia32_cmpps512_mask ((__v16sf) __X, + (__v16sf) __Y, __P, + (__mmask16) -1, __R); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmp_epi64_mask (__mmask8 __U, __m512i __X, __m512i __Y, + const int __P) +{ + return (__mmask8) __builtin_ia32_cmpq512_mask ((__v8di) __X, + (__v8di) __Y, __P, + (__mmask8) __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmp_epi32_mask (__mmask16 __U, __m512i __X, __m512i __Y, + const int __P) +{ + return (__mmask16) __builtin_ia32_cmpd512_mask ((__v16si) __X, + (__v16si) __Y, __P, + (__mmask16) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmp_epu64_mask (__mmask8 __U, __m512i __X, __m512i __Y, + const int __P) +{ + return (__mmask8) __builtin_ia32_ucmpq512_mask ((__v8di) __X, + (__v8di) __Y, __P, + (__mmask8) __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmp_epu32_mask (__mmask16 __U, __m512i __X, __m512i __Y, + const int __P) +{ + return (__mmask16) __builtin_ia32_ucmpd512_mask ((__v16si) __X, + (__v16si) __Y, __P, + (__mmask16) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmp_round_pd_mask (__mmask8 __U, __m512d __X, __m512d __Y, + const int __P, const int __R) +{ + return (__mmask8) __builtin_ia32_cmppd512_mask ((__v8df) __X, + (__v8df) __Y, __P, + (__mmask8) __U, __R); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmp_round_ps_mask (__mmask16 __U, __m512 __X, __m512 __Y, + const int __P, const int __R) +{ + return (__mmask16) __builtin_ia32_cmpps512_mask ((__v16sf) __X, + (__v16sf) __Y, __P, + (__mmask16) __U, __R); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_round_sd_mask (__m128d __X, __m128d __Y, const int __P, const int __R) +{ + return (__mmask8) __builtin_ia32_cmpsd_mask ((__v2df) __X, + (__v2df) __Y, __P, + (__mmask8) -1, __R); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmp_round_sd_mask (__mmask8 __M, __m128d __X, __m128d __Y, + const int __P, const int __R) +{ + return (__mmask8) __builtin_ia32_cmpsd_mask ((__v2df) __X, + (__v2df) __Y, __P, + (__mmask8) __M, __R); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_round_ss_mask (__m128 __X, __m128 __Y, const int __P, const int __R) +{ + return (__mmask8) __builtin_ia32_cmpss_mask ((__v4sf) __X, + (__v4sf) __Y, __P, + (__mmask8) -1, __R); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmp_round_ss_mask (__mmask8 __M, __m128 __X, __m128 __Y, + const int __P, const int __R) +{ + return (__mmask8) __builtin_ia32_cmpss_mask ((__v4sf) __X, + (__v4sf) __Y, __P, + (__mmask8) __M, __R); +} + +#else +#define _kshiftli_mask16(X, Y) \ + ((__mmask16) __builtin_ia32_kshiftlihi ((__mmask16)(X), (__mmask8)(Y))) + +#define _kshiftri_mask16(X, Y) \ + ((__mmask16) __builtin_ia32_kshiftrihi ((__mmask16)(X), (__mmask8)(Y))) + +#define _mm512_cmp_epi64_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpq512_mask ((__v8di)(__m512i)(X), \ + (__v8di)(__m512i)(Y), (int)(P),\ + (__mmask8)-1)) + +#define _mm512_cmp_epi32_mask(X, Y, P) \ + ((__mmask16) __builtin_ia32_cmpd512_mask ((__v16si)(__m512i)(X), \ + (__v16si)(__m512i)(Y), (int)(P), \ + (__mmask16)-1)) + +#define _mm512_cmp_epu64_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_ucmpq512_mask ((__v8di)(__m512i)(X), \ + (__v8di)(__m512i)(Y), (int)(P),\ + (__mmask8)-1)) + +#define _mm512_cmp_epu32_mask(X, Y, P) \ + ((__mmask16) __builtin_ia32_ucmpd512_mask ((__v16si)(__m512i)(X), \ + (__v16si)(__m512i)(Y), (int)(P), \ + (__mmask16)-1)) + +#define _mm512_cmp_round_pd_mask(X, Y, P, R) \ + ((__mmask8) __builtin_ia32_cmppd512_mask ((__v8df)(__m512d)(X), \ + (__v8df)(__m512d)(Y), (int)(P),\ + (__mmask8)-1, R)) + +#define _mm512_cmp_round_ps_mask(X, Y, P, R) \ + ((__mmask16) __builtin_ia32_cmpps512_mask ((__v16sf)(__m512)(X), \ + (__v16sf)(__m512)(Y), (int)(P),\ + (__mmask16)-1, R)) + +#define _mm512_mask_cmp_epi64_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpq512_mask ((__v8di)(__m512i)(X), \ + (__v8di)(__m512i)(Y), (int)(P),\ + (__mmask8)M)) + +#define _mm512_mask_cmp_epi32_mask(M, X, Y, P) \ + ((__mmask16) __builtin_ia32_cmpd512_mask ((__v16si)(__m512i)(X), \ + (__v16si)(__m512i)(Y), (int)(P), \ + (__mmask16)M)) + +#define _mm512_mask_cmp_epu64_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_ucmpq512_mask ((__v8di)(__m512i)(X), \ + (__v8di)(__m512i)(Y), (int)(P),\ + (__mmask8)M)) + +#define _mm512_mask_cmp_epu32_mask(M, X, Y, P) \ + ((__mmask16) __builtin_ia32_ucmpd512_mask ((__v16si)(__m512i)(X), \ + (__v16si)(__m512i)(Y), (int)(P), \ + (__mmask16)M)) + +#define _mm512_mask_cmp_round_pd_mask(M, X, Y, P, R) \ + ((__mmask8) __builtin_ia32_cmppd512_mask ((__v8df)(__m512d)(X), \ + (__v8df)(__m512d)(Y), (int)(P),\ + (__mmask8)M, R)) + +#define _mm512_mask_cmp_round_ps_mask(M, X, Y, P, R) \ + ((__mmask16) __builtin_ia32_cmpps512_mask ((__v16sf)(__m512)(X), \ + (__v16sf)(__m512)(Y), (int)(P),\ + (__mmask16)M, R)) + +#define _mm_cmp_round_sd_mask(X, Y, P, R) \ + ((__mmask8) __builtin_ia32_cmpsd_mask ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (int)(P),\ + (__mmask8)-1, R)) + +#define _mm_mask_cmp_round_sd_mask(M, X, Y, P, R) \ + ((__mmask8) __builtin_ia32_cmpsd_mask ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (int)(P),\ + (M), R)) + +#define _mm_cmp_round_ss_mask(X, Y, P, R) \ + ((__mmask8) __builtin_ia32_cmpss_mask ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (int)(P), \ + (__mmask8)-1, R)) + +#define _mm_mask_cmp_round_ss_mask(M, X, Y, P, R) \ + ((__mmask8) __builtin_ia32_cmpss_mask ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (int)(P), \ + (M), R)) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_i32gather_ps (__m512i __index, void const *__addr, int __scale) +{ + __m512 __v1_old = _mm512_undefined_ps (); + __mmask16 __mask = 0xFFFF; + + return (__m512) __builtin_ia32_gathersiv16sf ((__v16sf) __v1_old, + __addr, + (__v16si) __index, + __mask, __scale); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_i32gather_ps (__m512 __v1_old, __mmask16 __mask, + __m512i __index, void const *__addr, int __scale) +{ + return (__m512) __builtin_ia32_gathersiv16sf ((__v16sf) __v1_old, + __addr, + (__v16si) __index, + __mask, __scale); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_i32gather_pd (__m256i __index, void const *__addr, int __scale) +{ + __m512d __v1_old = _mm512_undefined_pd (); + __mmask8 __mask = 0xFF; + + return (__m512d) __builtin_ia32_gathersiv8df ((__v8df) __v1_old, + __addr, + (__v8si) __index, __mask, + __scale); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_i32gather_pd (__m512d __v1_old, __mmask8 __mask, + __m256i __index, void const *__addr, int __scale) +{ + return (__m512d) __builtin_ia32_gathersiv8df ((__v8df) __v1_old, + __addr, + (__v8si) __index, + __mask, __scale); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_i64gather_ps (__m512i __index, void const *__addr, int __scale) +{ + __m256 __v1_old = _mm256_undefined_ps (); + __mmask8 __mask = 0xFF; + + return (__m256) __builtin_ia32_gatherdiv16sf ((__v8sf) __v1_old, + __addr, + (__v8di) __index, __mask, + __scale); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_i64gather_ps (__m256 __v1_old, __mmask8 __mask, + __m512i __index, void const *__addr, int __scale) +{ + return (__m256) __builtin_ia32_gatherdiv16sf ((__v8sf) __v1_old, + __addr, + (__v8di) __index, + __mask, __scale); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_i64gather_pd (__m512i __index, void const *__addr, int __scale) +{ + __m512d __v1_old = _mm512_undefined_pd (); + __mmask8 __mask = 0xFF; + + return (__m512d) __builtin_ia32_gatherdiv8df ((__v8df) __v1_old, + __addr, + (__v8di) __index, __mask, + __scale); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_i64gather_pd (__m512d __v1_old, __mmask8 __mask, + __m512i __index, void const *__addr, int __scale) +{ + return (__m512d) __builtin_ia32_gatherdiv8df ((__v8df) __v1_old, + __addr, + (__v8di) __index, + __mask, __scale); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_i32gather_epi32 (__m512i __index, void const *__addr, int __scale) +{ + __m512i __v1_old = _mm512_undefined_epi32 (); + __mmask16 __mask = 0xFFFF; + + return (__m512i) __builtin_ia32_gathersiv16si ((__v16si) __v1_old, + __addr, + (__v16si) __index, + __mask, __scale); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_i32gather_epi32 (__m512i __v1_old, __mmask16 __mask, + __m512i __index, void const *__addr, int __scale) +{ + return (__m512i) __builtin_ia32_gathersiv16si ((__v16si) __v1_old, + __addr, + (__v16si) __index, + __mask, __scale); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_i32gather_epi64 (__m256i __index, void const *__addr, int __scale) +{ + __m512i __v1_old = _mm512_undefined_epi32 (); + __mmask8 __mask = 0xFF; + + return (__m512i) __builtin_ia32_gathersiv8di ((__v8di) __v1_old, + __addr, + (__v8si) __index, __mask, + __scale); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_i32gather_epi64 (__m512i __v1_old, __mmask8 __mask, + __m256i __index, void const *__addr, + int __scale) +{ + return (__m512i) __builtin_ia32_gathersiv8di ((__v8di) __v1_old, + __addr, + (__v8si) __index, + __mask, __scale); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_i64gather_epi32 (__m512i __index, void const *__addr, int __scale) +{ + __m256i __v1_old = _mm256_undefined_si256 (); + __mmask8 __mask = 0xFF; + + return (__m256i) __builtin_ia32_gatherdiv16si ((__v8si) __v1_old, + __addr, + (__v8di) __index, + __mask, __scale); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_i64gather_epi32 (__m256i __v1_old, __mmask8 __mask, + __m512i __index, void const *__addr, int __scale) +{ + return (__m256i) __builtin_ia32_gatherdiv16si ((__v8si) __v1_old, + __addr, + (__v8di) __index, + __mask, __scale); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_i64gather_epi64 (__m512i __index, void const *__addr, int __scale) +{ + __m512i __v1_old = _mm512_undefined_epi32 (); + __mmask8 __mask = 0xFF; + + return (__m512i) __builtin_ia32_gatherdiv8di ((__v8di) __v1_old, + __addr, + (__v8di) __index, __mask, + __scale); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_i64gather_epi64 (__m512i __v1_old, __mmask8 __mask, + __m512i __index, void const *__addr, + int __scale) +{ + return (__m512i) __builtin_ia32_gatherdiv8di ((__v8di) __v1_old, + __addr, + (__v8di) __index, + __mask, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_i32scatter_ps (void *__addr, __m512i __index, __m512 __v1, int __scale) +{ + __builtin_ia32_scattersiv16sf (__addr, (__mmask16) 0xFFFF, + (__v16si) __index, (__v16sf) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_i32scatter_ps (void *__addr, __mmask16 __mask, + __m512i __index, __m512 __v1, int __scale) +{ + __builtin_ia32_scattersiv16sf (__addr, __mask, (__v16si) __index, + (__v16sf) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_i32scatter_pd (void *__addr, __m256i __index, __m512d __v1, + int __scale) +{ + __builtin_ia32_scattersiv8df (__addr, (__mmask8) 0xFF, + (__v8si) __index, (__v8df) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_i32scatter_pd (void *__addr, __mmask8 __mask, + __m256i __index, __m512d __v1, int __scale) +{ + __builtin_ia32_scattersiv8df (__addr, __mask, (__v8si) __index, + (__v8df) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_i64scatter_ps (void *__addr, __m512i __index, __m256 __v1, int __scale) +{ + __builtin_ia32_scatterdiv16sf (__addr, (__mmask8) 0xFF, + (__v8di) __index, (__v8sf) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_i64scatter_ps (void *__addr, __mmask8 __mask, + __m512i __index, __m256 __v1, int __scale) +{ + __builtin_ia32_scatterdiv16sf (__addr, __mask, (__v8di) __index, + (__v8sf) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_i64scatter_pd (void *__addr, __m512i __index, __m512d __v1, + int __scale) +{ + __builtin_ia32_scatterdiv8df (__addr, (__mmask8) 0xFF, + (__v8di) __index, (__v8df) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_i64scatter_pd (void *__addr, __mmask8 __mask, + __m512i __index, __m512d __v1, int __scale) +{ + __builtin_ia32_scatterdiv8df (__addr, __mask, (__v8di) __index, + (__v8df) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_i32scatter_epi32 (void *__addr, __m512i __index, + __m512i __v1, int __scale) +{ + __builtin_ia32_scattersiv16si (__addr, (__mmask16) 0xFFFF, + (__v16si) __index, (__v16si) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_i32scatter_epi32 (void *__addr, __mmask16 __mask, + __m512i __index, __m512i __v1, int __scale) +{ + __builtin_ia32_scattersiv16si (__addr, __mask, (__v16si) __index, + (__v16si) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_i32scatter_epi64 (void *__addr, __m256i __index, + __m512i __v1, int __scale) +{ + __builtin_ia32_scattersiv8di (__addr, (__mmask8) 0xFF, + (__v8si) __index, (__v8di) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_i32scatter_epi64 (void *__addr, __mmask8 __mask, + __m256i __index, __m512i __v1, int __scale) +{ + __builtin_ia32_scattersiv8di (__addr, __mask, (__v8si) __index, + (__v8di) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_i64scatter_epi32 (void *__addr, __m512i __index, + __m256i __v1, int __scale) +{ + __builtin_ia32_scatterdiv16si (__addr, (__mmask8) 0xFF, + (__v8di) __index, (__v8si) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_i64scatter_epi32 (void *__addr, __mmask8 __mask, + __m512i __index, __m256i __v1, int __scale) +{ + __builtin_ia32_scatterdiv16si (__addr, __mask, (__v8di) __index, + (__v8si) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_i64scatter_epi64 (void *__addr, __m512i __index, + __m512i __v1, int __scale) +{ + __builtin_ia32_scatterdiv8di (__addr, (__mmask8) 0xFF, + (__v8di) __index, (__v8di) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_i64scatter_epi64 (void *__addr, __mmask8 __mask, + __m512i __index, __m512i __v1, int __scale) +{ + __builtin_ia32_scatterdiv8di (__addr, __mask, (__v8di) __index, + (__v8di) __v1, __scale); +} +#else +#define _mm512_i32gather_ps(INDEX, ADDR, SCALE) \ + (__m512) __builtin_ia32_gathersiv16sf ((__v16sf)_mm512_undefined_ps(),\ + (void const *)ADDR, \ + (__v16si)(__m512i)INDEX, \ + (__mmask16)0xFFFF, (int)SCALE) + +#define _mm512_mask_i32gather_ps(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m512) __builtin_ia32_gathersiv16sf ((__v16sf)(__m512)V1OLD, \ + (void const *)ADDR, \ + (__v16si)(__m512i)INDEX, \ + (__mmask16)MASK, (int)SCALE) + +#define _mm512_i32gather_pd(INDEX, ADDR, SCALE) \ + (__m512d) __builtin_ia32_gathersiv8df ((__v8df)_mm512_undefined_pd(), \ + (void const *)ADDR, \ + (__v8si)(__m256i)INDEX, \ + (__mmask8)0xFF, (int)SCALE) + +#define _mm512_mask_i32gather_pd(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m512d) __builtin_ia32_gathersiv8df ((__v8df)(__m512d)V1OLD, \ + (void const *)ADDR, \ + (__v8si)(__m256i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm512_i64gather_ps(INDEX, ADDR, SCALE) \ + (__m256) __builtin_ia32_gatherdiv16sf ((__v8sf)_mm256_undefined_ps(), \ + (void const *)ADDR, \ + (__v8di)(__m512i)INDEX, \ + (__mmask8)0xFF, (int)SCALE) + +#define _mm512_mask_i64gather_ps(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m256) __builtin_ia32_gatherdiv16sf ((__v8sf)(__m256)V1OLD, \ + (void const *)ADDR, \ + (__v8di)(__m512i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm512_i64gather_pd(INDEX, ADDR, SCALE) \ + (__m512d) __builtin_ia32_gatherdiv8df ((__v8df)_mm512_undefined_pd(), \ + (void const *)ADDR, \ + (__v8di)(__m512i)INDEX, \ + (__mmask8)0xFF, (int)SCALE) + +#define _mm512_mask_i64gather_pd(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m512d) __builtin_ia32_gatherdiv8df ((__v8df)(__m512d)V1OLD, \ + (void const *)ADDR, \ + (__v8di)(__m512i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm512_i32gather_epi32(INDEX, ADDR, SCALE) \ + (__m512i) __builtin_ia32_gathersiv16si ((__v16si)_mm512_undefined_epi32 (), \ + (void const *)ADDR, \ + (__v16si)(__m512i)INDEX, \ + (__mmask16)0xFFFF, (int)SCALE) + +#define _mm512_mask_i32gather_epi32(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m512i) __builtin_ia32_gathersiv16si ((__v16si)(__m512i)V1OLD, \ + (void const *)ADDR, \ + (__v16si)(__m512i)INDEX, \ + (__mmask16)MASK, (int)SCALE) + +#define _mm512_i32gather_epi64(INDEX, ADDR, SCALE) \ + (__m512i) __builtin_ia32_gathersiv8di ((__v8di)_mm512_undefined_epi32 (), \ + (void const *)ADDR, \ + (__v8si)(__m256i)INDEX, \ + (__mmask8)0xFF, (int)SCALE) + +#define _mm512_mask_i32gather_epi64(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m512i) __builtin_ia32_gathersiv8di ((__v8di)(__m512i)V1OLD, \ + (void const *)ADDR, \ + (__v8si)(__m256i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm512_i64gather_epi32(INDEX, ADDR, SCALE) \ + (__m256i) __builtin_ia32_gatherdiv16si ((__v8si)_mm256_undefined_si256(), \ + (void const *)ADDR, \ + (__v8di)(__m512i)INDEX, \ + (__mmask8)0xFF, (int)SCALE) + +#define _mm512_mask_i64gather_epi32(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m256i) __builtin_ia32_gatherdiv16si ((__v8si)(__m256i)V1OLD, \ + (void const *)ADDR, \ + (__v8di)(__m512i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm512_i64gather_epi64(INDEX, ADDR, SCALE) \ + (__m512i) __builtin_ia32_gatherdiv8di ((__v8di)_mm512_undefined_epi32 (), \ + (void const *)ADDR, \ + (__v8di)(__m512i)INDEX, \ + (__mmask8)0xFF, (int)SCALE) + +#define _mm512_mask_i64gather_epi64(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m512i) __builtin_ia32_gatherdiv8di ((__v8di)(__m512i)V1OLD, \ + (void const *)ADDR, \ + (__v8di)(__m512i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm512_i32scatter_ps(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv16sf ((void *)ADDR, (__mmask16)0xFFFF, \ + (__v16si)(__m512i)INDEX, \ + (__v16sf)(__m512)V1, (int)SCALE) + +#define _mm512_mask_i32scatter_ps(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv16sf ((void *)ADDR, (__mmask16)MASK, \ + (__v16si)(__m512i)INDEX, \ + (__v16sf)(__m512)V1, (int)SCALE) + +#define _mm512_i32scatter_pd(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv8df ((void *)ADDR, (__mmask8)0xFF, \ + (__v8si)(__m256i)INDEX, \ + (__v8df)(__m512d)V1, (int)SCALE) + +#define _mm512_mask_i32scatter_pd(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv8df ((void *)ADDR, (__mmask8)MASK, \ + (__v8si)(__m256i)INDEX, \ + (__v8df)(__m512d)V1, (int)SCALE) + +#define _mm512_i64scatter_ps(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv16sf ((void *)ADDR, (__mmask8)0xFF, \ + (__v8di)(__m512i)INDEX, \ + (__v8sf)(__m256)V1, (int)SCALE) + +#define _mm512_mask_i64scatter_ps(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv16sf ((void *)ADDR, (__mmask16)MASK, \ + (__v8di)(__m512i)INDEX, \ + (__v8sf)(__m256)V1, (int)SCALE) + +#define _mm512_i64scatter_pd(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv8df ((void *)ADDR, (__mmask8)0xFF, \ + (__v8di)(__m512i)INDEX, \ + (__v8df)(__m512d)V1, (int)SCALE) + +#define _mm512_mask_i64scatter_pd(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv8df ((void *)ADDR, (__mmask8)MASK, \ + (__v8di)(__m512i)INDEX, \ + (__v8df)(__m512d)V1, (int)SCALE) + +#define _mm512_i32scatter_epi32(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv16si ((void *)ADDR, (__mmask16)0xFFFF, \ + (__v16si)(__m512i)INDEX, \ + (__v16si)(__m512i)V1, (int)SCALE) + +#define _mm512_mask_i32scatter_epi32(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv16si ((void *)ADDR, (__mmask16)MASK, \ + (__v16si)(__m512i)INDEX, \ + (__v16si)(__m512i)V1, (int)SCALE) + +#define _mm512_i32scatter_epi64(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv8di ((void *)ADDR, (__mmask8)0xFF, \ + (__v8si)(__m256i)INDEX, \ + (__v8di)(__m512i)V1, (int)SCALE) + +#define _mm512_mask_i32scatter_epi64(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv8di ((void *)ADDR, (__mmask8)MASK, \ + (__v8si)(__m256i)INDEX, \ + (__v8di)(__m512i)V1, (int)SCALE) + +#define _mm512_i64scatter_epi32(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv16si ((void *)ADDR, (__mmask8)0xFF, \ + (__v8di)(__m512i)INDEX, \ + (__v8si)(__m256i)V1, (int)SCALE) + +#define _mm512_mask_i64scatter_epi32(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv16si ((void *)ADDR, (__mmask8)MASK, \ + (__v8di)(__m512i)INDEX, \ + (__v8si)(__m256i)V1, (int)SCALE) + +#define _mm512_i64scatter_epi64(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv8di ((void *)ADDR, (__mmask8)0xFF, \ + (__v8di)(__m512i)INDEX, \ + (__v8di)(__m512i)V1, (int)SCALE) + +#define _mm512_mask_i64scatter_epi64(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv8di ((void *)ADDR, (__mmask8)MASK, \ + (__v8di)(__m512i)INDEX, \ + (__v8di)(__m512i)V1, (int)SCALE) +#endif + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_compress_pd (__m512d __W, __mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_compressdf512_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_compress_pd (__mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_compressdf512_mask ((__v8df) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_compressstoreu_pd (void *__P, __mmask8 __U, __m512d __A) +{ + __builtin_ia32_compressstoredf512_mask ((__v8df *) __P, (__v8df) __A, + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_compress_ps (__m512 __W, __mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_compresssf512_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_compress_ps (__mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_compresssf512_mask ((__v16sf) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_compressstoreu_ps (void *__P, __mmask16 __U, __m512 __A) +{ + __builtin_ia32_compressstoresf512_mask ((__v16sf *) __P, (__v16sf) __A, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_compress_epi64 (__m512i __W, __mmask8 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_compressdi512_mask ((__v8di) __A, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_compress_epi64 (__mmask8 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_compressdi512_mask ((__v8di) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_compressstoreu_epi64 (void *__P, __mmask8 __U, __m512i __A) +{ + __builtin_ia32_compressstoredi512_mask ((__v8di *) __P, (__v8di) __A, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_compress_epi32 (__m512i __W, __mmask16 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_compresssi512_mask ((__v16si) __A, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_compress_epi32 (__mmask16 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_compresssi512_mask ((__v16si) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_compressstoreu_epi32 (void *__P, __mmask16 __U, __m512i __A) +{ + __builtin_ia32_compressstoresi512_mask ((__v16si *) __P, (__v16si) __A, + (__mmask16) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_expand_pd (__m512d __W, __mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_expanddf512_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_expand_pd (__mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_expanddf512_maskz ((__v8df) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_expandloadu_pd (__m512d __W, __mmask8 __U, void const *__P) +{ + return (__m512d) __builtin_ia32_expandloaddf512_mask ((const __v8df *) __P, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_expandloadu_pd (__mmask8 __U, void const *__P) +{ + return (__m512d) __builtin_ia32_expandloaddf512_maskz ((const __v8df *) __P, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_expand_ps (__m512 __W, __mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_expandsf512_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_expand_ps (__mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_expandsf512_maskz ((__v16sf) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_expandloadu_ps (__m512 __W, __mmask16 __U, void const *__P) +{ + return (__m512) __builtin_ia32_expandloadsf512_mask ((const __v16sf *) __P, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_expandloadu_ps (__mmask16 __U, void const *__P) +{ + return (__m512) __builtin_ia32_expandloadsf512_maskz ((const __v16sf *) __P, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_expand_epi64 (__m512i __W, __mmask8 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_expanddi512_mask ((__v8di) __A, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_expand_epi64 (__mmask8 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_expanddi512_maskz ((__v8di) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_expandloadu_epi64 (__m512i __W, __mmask8 __U, void const *__P) +{ + return (__m512i) __builtin_ia32_expandloaddi512_mask ((const __v8di *) __P, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_expandloadu_epi64 (__mmask8 __U, void const *__P) +{ + return (__m512i) + __builtin_ia32_expandloaddi512_maskz ((const __v8di *) __P, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_expand_epi32 (__m512i __W, __mmask16 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_expandsi512_mask ((__v16si) __A, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_expand_epi32 (__mmask16 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_expandsi512_maskz ((__v16si) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_expandloadu_epi32 (__m512i __W, __mmask16 __U, void const *__P) +{ + return (__m512i) __builtin_ia32_expandloadsi512_mask ((const __v16si *) __P, + (__v16si) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_expandloadu_epi32 (__mmask16 __U, void const *__P) +{ + return (__m512i) __builtin_ia32_expandloadsi512_maskz ((const __v16si *) __P, + (__v16si) + _mm512_setzero_si512 + (), (__mmask16) __U); +} + +/* Mask arithmetic operations */ +#define _kand_mask16 _mm512_kand +#define _kandn_mask16 _mm512_kandn +#define _knot_mask16 _mm512_knot +#define _kor_mask16 _mm512_kor +#define _kxnor_mask16 _mm512_kxnor +#define _kxor_mask16 _mm512_kxor + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kortest_mask16_u8 (__mmask16 __A, __mmask16 __B, unsigned char *__CF) +{ + *__CF = (unsigned char) __builtin_ia32_kortestchi (__A, __B); + return (unsigned char) __builtin_ia32_kortestzhi (__A, __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kortestz_mask16_u8 (__mmask16 __A, __mmask16 __B) +{ + return (unsigned char) __builtin_ia32_kortestzhi ((__mmask16) __A, + (__mmask16) __B); +} + +extern __inline unsigned char +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kortestc_mask16_u8 (__mmask16 __A, __mmask16 __B) +{ + return (unsigned char) __builtin_ia32_kortestchi ((__mmask16) __A, + (__mmask16) __B); +} + +extern __inline unsigned int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_cvtmask16_u32 (__mmask16 __A) +{ + return (unsigned int) __builtin_ia32_kmovw ((__mmask16 ) __A); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_cvtu32_mask16 (unsigned int __A) +{ + return (__mmask16) __builtin_ia32_kmovw ((__mmask16 ) __A); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_load_mask16 (__mmask16 *__A) +{ + return (__mmask16) __builtin_ia32_kmovw (*(__mmask16 *) __A); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_store_mask16 (__mmask16 *__A, __mmask16 __B) +{ + *(__mmask16 *) __A = __builtin_ia32_kmovw (__B); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_kand (__mmask16 __A, __mmask16 __B) +{ + return (__mmask16) __builtin_ia32_kandhi ((__mmask16) __A, (__mmask16) __B); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_kandn (__mmask16 __A, __mmask16 __B) +{ + return (__mmask16) __builtin_ia32_kandnhi ((__mmask16) __A, + (__mmask16) __B); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_kor (__mmask16 __A, __mmask16 __B) +{ + return (__mmask16) __builtin_ia32_korhi ((__mmask16) __A, (__mmask16) __B); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_kortestz (__mmask16 __A, __mmask16 __B) +{ + return (__mmask16) __builtin_ia32_kortestzhi ((__mmask16) __A, + (__mmask16) __B); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_kortestc (__mmask16 __A, __mmask16 __B) +{ + return (__mmask16) __builtin_ia32_kortestchi ((__mmask16) __A, + (__mmask16) __B); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_kxnor (__mmask16 __A, __mmask16 __B) +{ + return (__mmask16) __builtin_ia32_kxnorhi ((__mmask16) __A, (__mmask16) __B); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_kxor (__mmask16 __A, __mmask16 __B) +{ + return (__mmask16) __builtin_ia32_kxorhi ((__mmask16) __A, (__mmask16) __B); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_knot (__mmask16 __A) +{ + return (__mmask16) __builtin_ia32_knothi ((__mmask16) __A); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_kunpackb (__mmask16 __A, __mmask16 __B) +{ + return (__mmask16) __builtin_ia32_kunpckhi ((__mmask16) __A, (__mmask16) __B); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_kunpackb_mask16 (__mmask8 __A, __mmask8 __B) +{ + return (__mmask16) __builtin_ia32_kunpckhi ((__mmask16) __A, (__mmask16) __B); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_inserti32x4 (__mmask16 __B, __m512i __C, __m128i __D, + const int __imm) +{ + return (__m512i) __builtin_ia32_inserti32x4_mask ((__v16si) __C, + (__v4si) __D, + __imm, + (__v16si) + _mm512_setzero_si512 (), + __B); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_insertf32x4 (__mmask16 __B, __m512 __C, __m128 __D, + const int __imm) +{ + return (__m512) __builtin_ia32_insertf32x4_mask ((__v16sf) __C, + (__v4sf) __D, + __imm, + (__v16sf) + _mm512_setzero_ps (), __B); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_inserti32x4 (__m512i __A, __mmask16 __B, __m512i __C, + __m128i __D, const int __imm) +{ + return (__m512i) __builtin_ia32_inserti32x4_mask ((__v16si) __C, + (__v4si) __D, + __imm, + (__v16si) __A, + __B); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_insertf32x4 (__m512 __A, __mmask16 __B, __m512 __C, + __m128 __D, const int __imm) +{ + return (__m512) __builtin_ia32_insertf32x4_mask ((__v16sf) __C, + (__v4sf) __D, + __imm, + (__v16sf) __A, __B); +} +#else +#define _mm512_maskz_insertf32x4(A, X, Y, C) \ + ((__m512) __builtin_ia32_insertf32x4_mask ((__v16sf)(__m512) (X), \ + (__v4sf)(__m128) (Y), (int) (C), (__v16sf)_mm512_setzero_ps(), \ + (__mmask8)(A))) + +#define _mm512_maskz_inserti32x4(A, X, Y, C) \ + ((__m512i) __builtin_ia32_inserti32x4_mask ((__v16si)(__m512i) (X), \ + (__v4si)(__m128i) (Y), (int) (C), (__v16si)_mm512_setzero_si512 (), \ + (__mmask8)(A))) + +#define _mm512_mask_insertf32x4(A, B, X, Y, C) \ + ((__m512) __builtin_ia32_insertf32x4_mask ((__v16sf)(__m512) (X), \ + (__v4sf)(__m128) (Y), (int) (C), (__v16sf)(__m512) (A), \ + (__mmask8)(B))) + +#define _mm512_mask_inserti32x4(A, B, X, Y, C) \ + ((__m512i) __builtin_ia32_inserti32x4_mask ((__v16si)(__m512i) (X), \ + (__v4si)(__m128i) (Y), (int) (C), (__v16si)(__m512i) (A), \ + (__mmask8)(B))) +#endif + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_max_epi64 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxsq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_max_epi64 (__mmask8 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxsq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_max_epi64 (__m512i __W, __mmask8 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxsq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __W, __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_min_epi64 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminsq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_min_epi64 (__m512i __W, __mmask8 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminsq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __W, __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_min_epi64 (__mmask8 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminsq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_max_epu64 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxuq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_max_epu64 (__mmask8 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxuq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_max_epu64 (__m512i __W, __mmask8 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxuq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __W, __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_min_epu64 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminuq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_undefined_epi32 (), + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_min_epu64 (__m512i __W, __mmask8 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminuq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) __W, __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_min_epu64 (__mmask8 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminuq512_mask ((__v8di) __A, + (__v8di) __B, + (__v8di) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_max_epi32 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxsd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_max_epi32 (__mmask16 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxsd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_max_epi32 (__m512i __W, __mmask16 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxsd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __W, __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_min_epi32 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminsd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_min_epi32 (__mmask16 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminsd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_min_epi32 (__m512i __W, __mmask16 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminsd512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __W, __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_max_epu32 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxud512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_max_epu32 (__mmask16 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxud512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_max_epu32 (__m512i __W, __mmask16 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pmaxud512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __W, __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_min_epu32 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminud512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_min_epu32 (__mmask16 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminud512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) + _mm512_setzero_si512 (), + __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_min_epu32 (__m512i __W, __mmask16 __M, __m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_pminud512_mask ((__v16si) __A, + (__v16si) __B, + (__v16si) __W, __M); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_unpacklo_ps (__m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_unpcklps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_unpacklo_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_unpcklps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_unpacklo_ps (__mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_unpcklps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U); +} + +#ifdef __OPTIMIZE__ +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_max_round_sd (__m128d __A, __m128d __B, const int __R) +{ + return (__m128d) __builtin_ia32_maxsd_round ((__v2df) __A, + (__v2df) __B, + __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_max_round_ss (__m128 __A, __m128 __B, const int __R) +{ + return (__m128) __builtin_ia32_maxss_round ((__v4sf) __A, + (__v4sf) __B, + __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_min_round_sd (__m128d __A, __m128d __B, const int __R) +{ + return (__m128d) __builtin_ia32_minsd_round ((__v2df) __A, + (__v2df) __B, + __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_min_round_ss (__m128 __A, __m128 __B, const int __R) +{ + return (__m128) __builtin_ia32_minss_round ((__v4sf) __A, + (__v4sf) __B, + __R); +} + +#else +#define _mm_max_round_sd(A, B, C) \ + (__m128d)__builtin_ia32_addsd_round(A, B, C) + +#define _mm_max_round_ss(A, B, C) \ + (__m128)__builtin_ia32_addss_round(A, B, C) + +#define _mm_min_round_sd(A, B, C) \ + (__m128d)__builtin_ia32_subsd_round(A, B, C) + +#define _mm_min_round_ss(A, B, C) \ + (__m128)__builtin_ia32_subss_round(A, B, C) +#endif + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_blend_pd (__mmask8 __U, __m512d __A, __m512d __W) +{ + return (__m512d) __builtin_ia32_blendmpd_512_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) __U); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_blend_ps (__mmask16 __U, __m512 __A, __m512 __W) +{ + return (__m512) __builtin_ia32_blendmps_512_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_blend_epi64 (__mmask8 __U, __m512i __A, __m512i __W) +{ + return (__m512i) __builtin_ia32_blendmq_512_mask ((__v8di) __A, + (__v8di) __W, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_blend_epi32 (__mmask16 __U, __m512i __A, __m512i __W) +{ + return (__m512i) __builtin_ia32_blendmd_512_mask ((__v16si) __A, + (__v16si) __W, + (__mmask16) __U); +} + +#ifdef __OPTIMIZE__ +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fmadd_round_sd (__m128d __W, __m128d __A, __m128d __B, const int __R) +{ + return (__m128d) __builtin_ia32_vfmaddsd3_round ((__v2df) __W, + (__v2df) __A, + (__v2df) __B, + __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fmadd_round_ss (__m128 __W, __m128 __A, __m128 __B, const int __R) +{ + return (__m128) __builtin_ia32_vfmaddss3_round ((__v4sf) __W, + (__v4sf) __A, + (__v4sf) __B, + __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fmsub_round_sd (__m128d __W, __m128d __A, __m128d __B, const int __R) +{ + return (__m128d) __builtin_ia32_vfmaddsd3_round ((__v2df) __W, + (__v2df) __A, + -(__v2df) __B, + __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fmsub_round_ss (__m128 __W, __m128 __A, __m128 __B, const int __R) +{ + return (__m128) __builtin_ia32_vfmaddss3_round ((__v4sf) __W, + (__v4sf) __A, + -(__v4sf) __B, + __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fnmadd_round_sd (__m128d __W, __m128d __A, __m128d __B, const int __R) +{ + return (__m128d) __builtin_ia32_vfmaddsd3_round ((__v2df) __W, + -(__v2df) __A, + (__v2df) __B, + __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fnmadd_round_ss (__m128 __W, __m128 __A, __m128 __B, const int __R) +{ + return (__m128) __builtin_ia32_vfmaddss3_round ((__v4sf) __W, + -(__v4sf) __A, + (__v4sf) __B, + __R); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fnmsub_round_sd (__m128d __W, __m128d __A, __m128d __B, const int __R) +{ + return (__m128d) __builtin_ia32_vfmaddsd3_round ((__v2df) __W, + -(__v2df) __A, + -(__v2df) __B, + __R); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fnmsub_round_ss (__m128 __W, __m128 __A, __m128 __B, const int __R) +{ + return (__m128) __builtin_ia32_vfmaddss3_round ((__v4sf) __W, + -(__v4sf) __A, + -(__v4sf) __B, + __R); +} +#else +#define _mm_fmadd_round_sd(A, B, C, R) \ + (__m128d)__builtin_ia32_vfmaddsd3_round(A, B, C, R) + +#define _mm_fmadd_round_ss(A, B, C, R) \ + (__m128)__builtin_ia32_vfmaddss3_round(A, B, C, R) + +#define _mm_fmsub_round_sd(A, B, C, R) \ + (__m128d)__builtin_ia32_vfmaddsd3_round(A, B, -(C), R) + +#define _mm_fmsub_round_ss(A, B, C, R) \ + (__m128)__builtin_ia32_vfmaddss3_round(A, B, -(C), R) + +#define _mm_fnmadd_round_sd(A, B, C, R) \ + (__m128d)__builtin_ia32_vfmaddsd3_round(A, -(B), C, R) + +#define _mm_fnmadd_round_ss(A, B, C, R) \ + (__m128)__builtin_ia32_vfmaddss3_round(A, -(B), C, R) + +#define _mm_fnmsub_round_sd(A, B, C, R) \ + (__m128d)__builtin_ia32_vfmaddsd3_round(A, -(B), -(C), R) + +#define _mm_fnmsub_round_ss(A, B, C, R) \ + (__m128)__builtin_ia32_vfmaddss3_round(A, -(B), -(C), R) +#endif + +#ifdef __OPTIMIZE__ +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comi_round_ss (__m128 __A, __m128 __B, const int __P, const int __R) +{ + return __builtin_ia32_vcomiss ((__v4sf) __A, (__v4sf) __B, __P, __R); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comi_round_sd (__m128d __A, __m128d __B, const int __P, const int __R) +{ + return __builtin_ia32_vcomisd ((__v2df) __A, (__v2df) __B, __P, __R); +} +#else +#define _mm_comi_round_ss(A, B, C, D)\ +__builtin_ia32_vcomiss(A, B, C, D) +#define _mm_comi_round_sd(A, B, C, D)\ +__builtin_ia32_vcomisd(A, B, C, D) +#endif + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sqrt_pd (__m512d __A) +{ + return (__m512d) __builtin_ia32_sqrtpd512_mask ((__v8df) __A, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sqrt_pd (__m512d __W, __mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_sqrtpd512_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sqrt_pd (__mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_sqrtpd512_mask ((__v8df) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sqrt_ps (__m512 __A) +{ + return (__m512) __builtin_ia32_sqrtps512_mask ((__v16sf) __A, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sqrt_ps (__m512 __W, __mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_sqrtps512_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sqrt_ps (__mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_sqrtps512_mask ((__v16sf) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_add_pd (__m512d __A, __m512d __B) +{ + return (__m512d) ((__v8df)__A + (__v8df)__B); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_add_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_addpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_add_pd (__mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_addpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_add_ps (__m512 __A, __m512 __B) +{ + return (__m512) ((__v16sf)__A + (__v16sf)__B); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_add_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_addps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_add_ps (__mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_addps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sub_pd (__m512d __A, __m512d __B) +{ + return (__m512d) ((__v8df)__A - (__v8df)__B); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sub_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_subpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sub_pd (__mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_subpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_sub_ps (__m512 __A, __m512 __B) +{ + return (__m512) ((__v16sf)__A - (__v16sf)__B); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_sub_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_subps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_sub_ps (__mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_subps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mul_pd (__m512d __A, __m512d __B) +{ + return (__m512d) ((__v8df)__A * (__v8df)__B); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mul_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_mulpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mul_pd (__mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_mulpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mul_ps (__m512 __A, __m512 __B) +{ + return (__m512) ((__v16sf)__A * (__v16sf)__B); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_mul_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_mulps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_mul_ps (__mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_mulps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_div_pd (__m512d __M, __m512d __V) +{ + return (__m512d) ((__v8df)__M / (__v8df)__V); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_div_pd (__m512d __W, __mmask8 __U, __m512d __M, __m512d __V) +{ + return (__m512d) __builtin_ia32_divpd512_mask ((__v8df) __M, + (__v8df) __V, + (__v8df) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_div_pd (__mmask8 __U, __m512d __M, __m512d __V) +{ + return (__m512d) __builtin_ia32_divpd512_mask ((__v8df) __M, + (__v8df) __V, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_div_ps (__m512 __A, __m512 __B) +{ + return (__m512) ((__v16sf)__A / (__v16sf)__B); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_div_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_divps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_div_ps (__mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_divps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_max_pd (__m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_maxpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_max_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_maxpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_max_pd (__mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_maxpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_max_ps (__m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_maxps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_max_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_maxps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_max_ps (__mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_maxps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_min_pd (__m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_minpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_min_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_minpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_min_pd (__mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_minpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_min_ps (__m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_minps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_min_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_minps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_min_ps (__mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_minps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_scalef_pd (__m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_scalefpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_scalef_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_scalefpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_scalef_pd (__mmask8 __U, __m512d __A, __m512d __B) +{ + return (__m512d) __builtin_ia32_scalefpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_scalef_ps (__m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_scalefps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_scalef_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_scalefps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_scalef_ps (__mmask16 __U, __m512 __A, __m512 __B) +{ + return (__m512) __builtin_ia32_scalefps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_scalef_sd (__m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_scalefsd_round ((__v2df) __A, + (__v2df) __B, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_scalef_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_scalefss_round ((__v4sf) __A, + (__v4sf) __B, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fmadd_pd (__m512d __A, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fmadd_pd (__m512d __A, __mmask8 __U, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fmadd_pd (__m512d __A, __m512d __B, __m512d __C, __mmask8 __U) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_mask3 ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fmadd_pd (__mmask8 __U, __m512d __A, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_maskz ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fmadd_ps (__m512 __A, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fmadd_ps (__m512 __A, __mmask16 __U, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fmadd_ps (__m512 __A, __m512 __B, __m512 __C, __mmask16 __U) +{ + return (__m512) __builtin_ia32_vfmaddps512_mask3 ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fmadd_ps (__mmask16 __U, __m512 __A, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfmaddps512_maskz ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fmsub_pd (__m512d __A, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A, + (__v8df) __B, + -(__v8df) __C, + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fmsub_pd (__m512d __A, __mmask8 __U, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_mask ((__v8df) __A, + (__v8df) __B, + -(__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fmsub_pd (__m512d __A, __m512d __B, __m512d __C, __mmask8 __U) +{ + return (__m512d) __builtin_ia32_vfmsubpd512_mask3 ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fmsub_pd (__mmask8 __U, __m512d __A, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_maskz ((__v8df) __A, + (__v8df) __B, + -(__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fmsub_ps (__m512 __A, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A, + (__v16sf) __B, + -(__v16sf) __C, + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fmsub_ps (__m512 __A, __mmask16 __U, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfmaddps512_mask ((__v16sf) __A, + (__v16sf) __B, + -(__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fmsub_ps (__m512 __A, __m512 __B, __m512 __C, __mmask16 __U) +{ + return (__m512) __builtin_ia32_vfmsubps512_mask3 ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fmsub_ps (__mmask16 __U, __m512 __A, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfmaddps512_maskz ((__v16sf) __A, + (__v16sf) __B, + -(__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fmaddsub_pd (__m512d __A, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfmaddsubpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fmaddsub_pd (__m512d __A, __mmask8 __U, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfmaddsubpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fmaddsub_pd (__m512d __A, __m512d __B, __m512d __C, __mmask8 __U) +{ + return (__m512d) __builtin_ia32_vfmaddsubpd512_mask3 ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fmaddsub_pd (__mmask8 __U, __m512d __A, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfmaddsubpd512_maskz ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fmaddsub_ps (__m512 __A, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfmaddsubps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fmaddsub_ps (__m512 __A, __mmask16 __U, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfmaddsubps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fmaddsub_ps (__m512 __A, __m512 __B, __m512 __C, __mmask16 __U) +{ + return (__m512) __builtin_ia32_vfmaddsubps512_mask3 ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fmaddsub_ps (__mmask16 __U, __m512 __A, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfmaddsubps512_maskz ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fmsubadd_pd (__m512d __A, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfmaddsubpd512_mask ((__v8df) __A, + (__v8df) __B, + -(__v8df) __C, + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fmsubadd_pd (__m512d __A, __mmask8 __U, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfmaddsubpd512_mask ((__v8df) __A, + (__v8df) __B, + -(__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fmsubadd_pd (__m512d __A, __m512d __B, __m512d __C, __mmask8 __U) +{ + return (__m512d) __builtin_ia32_vfmsubaddpd512_mask3 ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fmsubadd_pd (__mmask8 __U, __m512d __A, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfmaddsubpd512_maskz ((__v8df) __A, + (__v8df) __B, + -(__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fmsubadd_ps (__m512 __A, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfmaddsubps512_mask ((__v16sf) __A, + (__v16sf) __B, + -(__v16sf) __C, + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fmsubadd_ps (__m512 __A, __mmask16 __U, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfmaddsubps512_mask ((__v16sf) __A, + (__v16sf) __B, + -(__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fmsubadd_ps (__m512 __A, __m512 __B, __m512 __C, __mmask16 __U) +{ + return (__m512) __builtin_ia32_vfmsubaddps512_mask3 ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fmsubadd_ps (__mmask16 __U, __m512 __A, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfmaddsubps512_maskz ((__v16sf) __A, + (__v16sf) __B, + -(__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fnmadd_pd (__m512d __A, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_mask (-(__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fnmadd_pd (__m512d __A, __mmask8 __U, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfnmaddpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fnmadd_pd (__m512d __A, __m512d __B, __m512d __C, __mmask8 __U) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_mask3 (-(__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fnmadd_pd (__mmask8 __U, __m512d __A, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_maskz (-(__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fnmadd_ps (__m512 __A, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfmaddps512_mask (-(__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fnmadd_ps (__m512 __A, __mmask16 __U, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfnmaddps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fnmadd_ps (__m512 __A, __m512 __B, __m512 __C, __mmask16 __U) +{ + return (__m512) __builtin_ia32_vfmaddps512_mask3 (-(__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fnmadd_ps (__mmask16 __U, __m512 __A, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfmaddps512_maskz (-(__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fnmsub_pd (__m512d __A, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_mask (-(__v8df) __A, + (__v8df) __B, + -(__v8df) __C, + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fnmsub_pd (__m512d __A, __mmask8 __U, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfnmsubpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fnmsub_pd (__m512d __A, __m512d __B, __m512d __C, __mmask8 __U) +{ + return (__m512d) __builtin_ia32_vfnmsubpd512_mask3 ((__v8df) __A, + (__v8df) __B, + (__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fnmsub_pd (__mmask8 __U, __m512d __A, __m512d __B, __m512d __C) +{ + return (__m512d) __builtin_ia32_vfmaddpd512_maskz (-(__v8df) __A, + (__v8df) __B, + -(__v8df) __C, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fnmsub_ps (__m512 __A, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfmaddps512_mask (-(__v16sf) __A, + (__v16sf) __B, + -(__v16sf) __C, + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fnmsub_ps (__m512 __A, __mmask16 __U, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfnmsubps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask3_fnmsub_ps (__m512 __A, __m512 __B, __m512 __C, __mmask16 __U) +{ + return (__m512) __builtin_ia32_vfnmsubps512_mask3 ((__v16sf) __A, + (__v16sf) __B, + (__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fnmsub_ps (__mmask16 __U, __m512 __A, __m512 __B, __m512 __C) +{ + return (__m512) __builtin_ia32_vfmaddps512_maskz (-(__v16sf) __A, + (__v16sf) __B, + -(__v16sf) __C, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvttpd_epi32 (__m512d __A) +{ + return (__m256i) __builtin_ia32_cvttpd2dq512_mask ((__v8df) __A, + (__v8si) + _mm256_undefined_si256 (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvttpd_epi32 (__m256i __W, __mmask8 __U, __m512d __A) +{ + return (__m256i) __builtin_ia32_cvttpd2dq512_mask ((__v8df) __A, + (__v8si) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvttpd_epi32 (__mmask8 __U, __m512d __A) +{ + return (__m256i) __builtin_ia32_cvttpd2dq512_mask ((__v8df) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvttpd_epu32 (__m512d __A) +{ + return (__m256i) __builtin_ia32_cvttpd2udq512_mask ((__v8df) __A, + (__v8si) + _mm256_undefined_si256 (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvttpd_epu32 (__m256i __W, __mmask8 __U, __m512d __A) +{ + return (__m256i) __builtin_ia32_cvttpd2udq512_mask ((__v8df) __A, + (__v8si) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvttpd_epu32 (__mmask8 __U, __m512d __A) +{ + return (__m256i) __builtin_ia32_cvttpd2udq512_mask ((__v8df) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtpd_epi32 (__m512d __A) +{ + return (__m256i) __builtin_ia32_cvtpd2dq512_mask ((__v8df) __A, + (__v8si) + _mm256_undefined_si256 (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtpd_epi32 (__m256i __W, __mmask8 __U, __m512d __A) +{ + return (__m256i) __builtin_ia32_cvtpd2dq512_mask ((__v8df) __A, + (__v8si) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtpd_epi32 (__mmask8 __U, __m512d __A) +{ + return (__m256i) __builtin_ia32_cvtpd2dq512_mask ((__v8df) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtpd_epu32 (__m512d __A) +{ + return (__m256i) __builtin_ia32_cvtpd2udq512_mask ((__v8df) __A, + (__v8si) + _mm256_undefined_si256 (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtpd_epu32 (__m256i __W, __mmask8 __U, __m512d __A) +{ + return (__m256i) __builtin_ia32_cvtpd2udq512_mask ((__v8df) __A, + (__v8si) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtpd_epu32 (__mmask8 __U, __m512d __A) +{ + return (__m256i) __builtin_ia32_cvtpd2udq512_mask ((__v8df) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvttps_epi32 (__m512 __A) +{ + return (__m512i) __builtin_ia32_cvttps2dq512_mask ((__v16sf) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvttps_epi32 (__m512i __W, __mmask16 __U, __m512 __A) +{ + return (__m512i) __builtin_ia32_cvttps2dq512_mask ((__v16sf) __A, + (__v16si) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvttps_epi32 (__mmask16 __U, __m512 __A) +{ + return (__m512i) __builtin_ia32_cvttps2dq512_mask ((__v16sf) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvttps_epu32 (__m512 __A) +{ + return (__m512i) __builtin_ia32_cvttps2udq512_mask ((__v16sf) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvttps_epu32 (__m512i __W, __mmask16 __U, __m512 __A) +{ + return (__m512i) __builtin_ia32_cvttps2udq512_mask ((__v16sf) __A, + (__v16si) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvttps_epu32 (__mmask16 __U, __m512 __A) +{ + return (__m512i) __builtin_ia32_cvttps2udq512_mask ((__v16sf) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtps_epi32 (__m512 __A) +{ + return (__m512i) __builtin_ia32_cvtps2dq512_mask ((__v16sf) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtps_epi32 (__m512i __W, __mmask16 __U, __m512 __A) +{ + return (__m512i) __builtin_ia32_cvtps2dq512_mask ((__v16sf) __A, + (__v16si) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtps_epi32 (__mmask16 __U, __m512 __A) +{ + return (__m512i) __builtin_ia32_cvtps2dq512_mask ((__v16sf) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtps_epu32 (__m512 __A) +{ + return (__m512i) __builtin_ia32_cvtps2udq512_mask ((__v16sf) __A, + (__v16si) + _mm512_undefined_epi32 (), + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtps_epu32 (__m512i __W, __mmask16 __U, __m512 __A) +{ + return (__m512i) __builtin_ia32_cvtps2udq512_mask ((__v16sf) __A, + (__v16si) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtps_epu32 (__mmask16 __U, __m512 __A) +{ + return (__m512i) __builtin_ia32_cvtps2udq512_mask ((__v16sf) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline double +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtsd_f64 (__m512d __A) +{ + return __A[0]; +} + +extern __inline float +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtss_f32 (__m512 __A) +{ + return __A[0]; +} + +#ifdef __x86_64__ +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtu64_ss (__m128 __A, unsigned long long __B) +{ + return (__m128) __builtin_ia32_cvtusi2ss64 ((__v4sf) __A, __B, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtu64_sd (__m128d __A, unsigned long long __B) +{ + return (__m128d) __builtin_ia32_cvtusi2sd64 ((__v2df) __A, __B, + _MM_FROUND_CUR_DIRECTION); +} +#endif + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtu32_ss (__m128 __A, unsigned __B) +{ + return (__m128) __builtin_ia32_cvtusi2ss32 ((__v4sf) __A, __B, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepi32_ps (__m512i __A) +{ + return (__m512) __builtin_ia32_cvtdq2ps512_mask ((__v16si) __A, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepi32_ps (__m512 __W, __mmask16 __U, __m512i __A) +{ + return (__m512) __builtin_ia32_cvtdq2ps512_mask ((__v16si) __A, + (__v16sf) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepi32_ps (__mmask16 __U, __m512i __A) +{ + return (__m512) __builtin_ia32_cvtdq2ps512_mask ((__v16si) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtepu32_ps (__m512i __A) +{ + return (__m512) __builtin_ia32_cvtudq2ps512_mask ((__v16si) __A, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtepu32_ps (__m512 __W, __mmask16 __U, __m512i __A) +{ + return (__m512) __builtin_ia32_cvtudq2ps512_mask ((__v16si) __A, + (__v16sf) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtepu32_ps (__mmask16 __U, __m512i __A) +{ + return (__m512) __builtin_ia32_cvtudq2ps512_mask ((__v16si) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fixupimm_pd (__m512d __A, __m512d __B, __m512i __C, const int __imm) +{ + return (__m512d) __builtin_ia32_fixupimmpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8di) __C, + __imm, + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fixupimm_pd (__m512d __A, __mmask8 __U, __m512d __B, + __m512i __C, const int __imm) +{ + return (__m512d) __builtin_ia32_fixupimmpd512_mask ((__v8df) __A, + (__v8df) __B, + (__v8di) __C, + __imm, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fixupimm_pd (__mmask8 __U, __m512d __A, __m512d __B, + __m512i __C, const int __imm) +{ + return (__m512d) __builtin_ia32_fixupimmpd512_maskz ((__v8df) __A, + (__v8df) __B, + (__v8di) __C, + __imm, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_fixupimm_ps (__m512 __A, __m512 __B, __m512i __C, const int __imm) +{ + return (__m512) __builtin_ia32_fixupimmps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16si) __C, + __imm, + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_fixupimm_ps (__m512 __A, __mmask16 __U, __m512 __B, + __m512i __C, const int __imm) +{ + return (__m512) __builtin_ia32_fixupimmps512_mask ((__v16sf) __A, + (__v16sf) __B, + (__v16si) __C, + __imm, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_fixupimm_ps (__mmask16 __U, __m512 __A, __m512 __B, + __m512i __C, const int __imm) +{ + return (__m512) __builtin_ia32_fixupimmps512_maskz ((__v16sf) __A, + (__v16sf) __B, + (__v16si) __C, + __imm, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fixupimm_sd (__m128d __A, __m128d __B, __m128i __C, const int __imm) +{ + return (__m128d) __builtin_ia32_fixupimmsd_mask ((__v2df) __A, + (__v2df) __B, + (__v2di) __C, __imm, + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fixupimm_sd (__m128d __A, __mmask8 __U, __m128d __B, + __m128i __C, const int __imm) +{ + return (__m128d) __builtin_ia32_fixupimmsd_mask ((__v2df) __A, + (__v2df) __B, + (__v2di) __C, __imm, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fixupimm_sd (__mmask8 __U, __m128d __A, __m128d __B, + __m128i __C, const int __imm) +{ + return (__m128d) __builtin_ia32_fixupimmsd_maskz ((__v2df) __A, + (__v2df) __B, + (__v2di) __C, + __imm, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fixupimm_ss (__m128 __A, __m128 __B, __m128i __C, const int __imm) +{ + return (__m128) __builtin_ia32_fixupimmss_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4si) __C, __imm, + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fixupimm_ss (__m128 __A, __mmask8 __U, __m128 __B, + __m128i __C, const int __imm) +{ + return (__m128) __builtin_ia32_fixupimmss_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4si) __C, __imm, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fixupimm_ss (__mmask8 __U, __m128 __A, __m128 __B, + __m128i __C, const int __imm) +{ + return (__m128) __builtin_ia32_fixupimmss_maskz ((__v4sf) __A, + (__v4sf) __B, + (__v4si) __C, __imm, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} +#else +#define _mm512_fixupimm_pd(X, Y, Z, C) \ + ((__m512d)__builtin_ia32_fixupimmpd512_mask ((__v8df)(__m512d)(X), \ + (__v8df)(__m512d)(Y), (__v8di)(__m512i)(Z), (int)(C), \ + (__mmask8)(-1), _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_mask_fixupimm_pd(X, U, Y, Z, C) \ + ((__m512d)__builtin_ia32_fixupimmpd512_mask ((__v8df)(__m512d)(X), \ + (__v8df)(__m512d)(Y), (__v8di)(__m512i)(Z), (int)(C), \ + (__mmask8)(U), _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_maskz_fixupimm_pd(U, X, Y, Z, C) \ + ((__m512d)__builtin_ia32_fixupimmpd512_maskz ((__v8df)(__m512d)(X), \ + (__v8df)(__m512d)(Y), (__v8di)(__m512i)(Z), (int)(C), \ + (__mmask8)(U), _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_fixupimm_ps(X, Y, Z, C) \ + ((__m512)__builtin_ia32_fixupimmps512_mask ((__v16sf)(__m512)(X), \ + (__v16sf)(__m512)(Y), (__v16si)(__m512i)(Z), (int)(C), \ + (__mmask16)(-1), _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_mask_fixupimm_ps(X, U, Y, Z, C) \ + ((__m512)__builtin_ia32_fixupimmps512_mask ((__v16sf)(__m512)(X), \ + (__v16sf)(__m512)(Y), (__v16si)(__m512i)(Z), (int)(C), \ + (__mmask16)(U), _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_maskz_fixupimm_ps(U, X, Y, Z, C) \ + ((__m512)__builtin_ia32_fixupimmps512_maskz ((__v16sf)(__m512)(X), \ + (__v16sf)(__m512)(Y), (__v16si)(__m512i)(Z), (int)(C), \ + (__mmask16)(U), _MM_FROUND_CUR_DIRECTION)) + +#define _mm_fixupimm_sd(X, Y, Z, C) \ + ((__m128d)__builtin_ia32_fixupimmsd_mask ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (__v2di)(__m128i)(Z), (int)(C), \ + (__mmask8)(-1), _MM_FROUND_CUR_DIRECTION)) + +#define _mm_mask_fixupimm_sd(X, U, Y, Z, C) \ + ((__m128d)__builtin_ia32_fixupimmsd_mask ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (__v2di)(__m128i)(Z), (int)(C), \ + (__mmask8)(U), _MM_FROUND_CUR_DIRECTION)) + +#define _mm_maskz_fixupimm_sd(U, X, Y, Z, C) \ + ((__m128d)__builtin_ia32_fixupimmsd_maskz ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (__v2di)(__m128i)(Z), (int)(C), \ + (__mmask8)(U), _MM_FROUND_CUR_DIRECTION)) + +#define _mm_fixupimm_ss(X, Y, Z, C) \ + ((__m128)__builtin_ia32_fixupimmss_mask ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (__v4si)(__m128i)(Z), (int)(C), \ + (__mmask8)(-1), _MM_FROUND_CUR_DIRECTION)) + +#define _mm_mask_fixupimm_ss(X, U, Y, Z, C) \ + ((__m128)__builtin_ia32_fixupimmss_mask ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (__v4si)(__m128i)(Z), (int)(C), \ + (__mmask8)(U), _MM_FROUND_CUR_DIRECTION)) + +#define _mm_maskz_fixupimm_ss(U, X, Y, Z, C) \ + ((__m128)__builtin_ia32_fixupimmss_maskz ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (__v4si)(__m128i)(Z), (int)(C), \ + (__mmask8)(U), _MM_FROUND_CUR_DIRECTION)) +#endif + +#ifdef __x86_64__ +extern __inline unsigned long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtss_u64 (__m128 __A) +{ + return (unsigned long long) __builtin_ia32_vcvtss2usi64 ((__v4sf) + __A, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline unsigned long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttss_u64 (__m128 __A) +{ + return (unsigned long long) __builtin_ia32_vcvttss2usi64 ((__v4sf) + __A, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttss_i64 (__m128 __A) +{ + return (long long) __builtin_ia32_vcvttss2si64 ((__v4sf) __A, + _MM_FROUND_CUR_DIRECTION); +} +#endif /* __x86_64__ */ + +extern __inline unsigned +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtss_u32 (__m128 __A) +{ + return (unsigned) __builtin_ia32_vcvtss2usi32 ((__v4sf) __A, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline unsigned +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttss_u32 (__m128 __A) +{ + return (unsigned) __builtin_ia32_vcvttss2usi32 ((__v4sf) __A, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttss_i32 (__m128 __A) +{ + return (int) __builtin_ia32_vcvttss2si32 ((__v4sf) __A, + _MM_FROUND_CUR_DIRECTION); +} + +#ifdef __x86_64__ +extern __inline unsigned long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsd_u64 (__m128d __A) +{ + return (unsigned long long) __builtin_ia32_vcvtsd2usi64 ((__v2df) + __A, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline unsigned long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttsd_u64 (__m128d __A) +{ + return (unsigned long long) __builtin_ia32_vcvttsd2usi64 ((__v2df) + __A, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttsd_i64 (__m128d __A) +{ + return (long long) __builtin_ia32_vcvttsd2si64 ((__v2df) __A, + _MM_FROUND_CUR_DIRECTION); +} +#endif /* __x86_64__ */ + +extern __inline unsigned +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsd_u32 (__m128d __A) +{ + return (unsigned) __builtin_ia32_vcvtsd2usi32 ((__v2df) __A, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline unsigned +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttsd_u32 (__m128d __A) +{ + return (unsigned) __builtin_ia32_vcvttsd2usi32 ((__v2df) __A, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttsd_i32 (__m128d __A) +{ + return (int) __builtin_ia32_vcvttsd2si32 ((__v2df) __A, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtps_pd (__m256 __A) +{ + return (__m512d) __builtin_ia32_cvtps2pd512_mask ((__v8sf) __A, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtps_pd (__m512d __W, __mmask8 __U, __m256 __A) +{ + return (__m512d) __builtin_ia32_cvtps2pd512_mask ((__v8sf) __A, + (__v8df) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtps_pd (__mmask8 __U, __m256 __A) +{ + return (__m512d) __builtin_ia32_cvtps2pd512_mask ((__v8sf) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtph_ps (__m256i __A) +{ + return (__m512) __builtin_ia32_vcvtph2ps512_mask ((__v16hi) __A, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtph_ps (__m512 __W, __mmask16 __U, __m256i __A) +{ + return (__m512) __builtin_ia32_vcvtph2ps512_mask ((__v16hi) __A, + (__v16sf) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtph_ps (__mmask16 __U, __m256i __A) +{ + return (__m512) __builtin_ia32_vcvtph2ps512_mask ((__v16hi) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cvtpd_ps (__m512d __A) +{ + return (__m256) __builtin_ia32_cvtpd2ps512_mask ((__v8df) __A, + (__v8sf) + _mm256_undefined_ps (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cvtpd_ps (__m256 __W, __mmask8 __U, __m512d __A) +{ + return (__m256) __builtin_ia32_cvtpd2ps512_mask ((__v8df) __A, + (__v8sf) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_cvtpd_ps (__mmask8 __U, __m512d __A) +{ + return (__m256) __builtin_ia32_cvtpd2ps512_mask ((__v8df) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +#ifdef __OPTIMIZE__ +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_getexp_ps (__m512 __A) +{ + return (__m512) __builtin_ia32_getexpps512_mask ((__v16sf) __A, + (__v16sf) + _mm512_undefined_ps (), + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_getexp_ps (__m512 __W, __mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_getexpps512_mask ((__v16sf) __A, + (__v16sf) __W, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_getexp_ps (__mmask16 __U, __m512 __A) +{ + return (__m512) __builtin_ia32_getexpps512_mask ((__v16sf) __A, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_getexp_pd (__m512d __A) +{ + return (__m512d) __builtin_ia32_getexppd512_mask ((__v8df) __A, + (__v8df) + _mm512_undefined_pd (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_getexp_pd (__m512d __W, __mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_getexppd512_mask ((__v8df) __A, + (__v8df) __W, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_getexp_pd (__mmask8 __U, __m512d __A) +{ + return (__m512d) __builtin_ia32_getexppd512_mask ((__v8df) __A, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_getexp_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_getexpss128_round ((__v4sf) __A, + (__v4sf) __B, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_getexp_sd (__m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_getexpsd128_round ((__v2df) __A, + (__v2df) __B, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_getmant_pd (__m512d __A, _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m512d) __builtin_ia32_getmantpd512_mask ((__v8df) __A, + (__C << 2) | __B, + _mm512_undefined_pd (), + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_getmant_pd (__m512d __W, __mmask8 __U, __m512d __A, + _MM_MANTISSA_NORM_ENUM __B, _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m512d) __builtin_ia32_getmantpd512_mask ((__v8df) __A, + (__C << 2) | __B, + (__v8df) __W, __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_getmant_pd (__mmask8 __U, __m512d __A, + _MM_MANTISSA_NORM_ENUM __B, _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m512d) __builtin_ia32_getmantpd512_mask ((__v8df) __A, + (__C << 2) | __B, + (__v8df) + _mm512_setzero_pd (), + __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_getmant_ps (__m512 __A, _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m512) __builtin_ia32_getmantps512_mask ((__v16sf) __A, + (__C << 2) | __B, + _mm512_undefined_ps (), + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_getmant_ps (__m512 __W, __mmask16 __U, __m512 __A, + _MM_MANTISSA_NORM_ENUM __B, _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m512) __builtin_ia32_getmantps512_mask ((__v16sf) __A, + (__C << 2) | __B, + (__v16sf) __W, __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_getmant_ps (__mmask16 __U, __m512 __A, + _MM_MANTISSA_NORM_ENUM __B, _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m512) __builtin_ia32_getmantps512_mask ((__v16sf) __A, + (__C << 2) | __B, + (__v16sf) + _mm512_setzero_ps (), + __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_getmant_sd (__m128d __A, __m128d __B, _MM_MANTISSA_NORM_ENUM __C, + _MM_MANTISSA_SIGN_ENUM __D) +{ + return (__m128d) __builtin_ia32_getmantsd_round ((__v2df) __A, + (__v2df) __B, + (__D << 2) | __C, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_getmant_ss (__m128 __A, __m128 __B, _MM_MANTISSA_NORM_ENUM __C, + _MM_MANTISSA_SIGN_ENUM __D) +{ + return (__m128) __builtin_ia32_getmantss_round ((__v4sf) __A, + (__v4sf) __B, + (__D << 2) | __C, + _MM_FROUND_CUR_DIRECTION); +} + +#else +#define _mm512_getmant_pd(X, B, C) \ + ((__m512d)__builtin_ia32_getmantpd512_mask ((__v8df)(__m512d)(X), \ + (int)(((C)<<2) | (B)), \ + (__v8df)_mm512_undefined_pd(), \ + (__mmask8)-1,\ + _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_mask_getmant_pd(W, U, X, B, C) \ + ((__m512d)__builtin_ia32_getmantpd512_mask ((__v8df)(__m512d)(X), \ + (int)(((C)<<2) | (B)), \ + (__v8df)(__m512d)(W), \ + (__mmask8)(U),\ + _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_maskz_getmant_pd(U, X, B, C) \ + ((__m512d)__builtin_ia32_getmantpd512_mask ((__v8df)(__m512d)(X), \ + (int)(((C)<<2) | (B)), \ + (__v8df)_mm512_setzero_pd(), \ + (__mmask8)(U),\ + _MM_FROUND_CUR_DIRECTION)) +#define _mm512_getmant_ps(X, B, C) \ + ((__m512)__builtin_ia32_getmantps512_mask ((__v16sf)(__m512)(X), \ + (int)(((C)<<2) | (B)), \ + (__v16sf)_mm512_undefined_ps(), \ + (__mmask16)-1,\ + _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_mask_getmant_ps(W, U, X, B, C) \ + ((__m512)__builtin_ia32_getmantps512_mask ((__v16sf)(__m512)(X), \ + (int)(((C)<<2) | (B)), \ + (__v16sf)(__m512)(W), \ + (__mmask16)(U),\ + _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_maskz_getmant_ps(U, X, B, C) \ + ((__m512)__builtin_ia32_getmantps512_mask ((__v16sf)(__m512)(X), \ + (int)(((C)<<2) | (B)), \ + (__v16sf)_mm512_setzero_ps(), \ + (__mmask16)(U),\ + _MM_FROUND_CUR_DIRECTION)) +#define _mm_getmant_sd(X, Y, C, D) \ + ((__m128d)__builtin_ia32_getmantsd_round ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), \ + (int)(((D)<<2) | (C)), \ + _MM_FROUND_CUR_DIRECTION)) + +#define _mm_getmant_ss(X, Y, C, D) \ + ((__m128)__builtin_ia32_getmantss_round ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), \ + (int)(((D)<<2) | (C)), \ + _MM_FROUND_CUR_DIRECTION)) + +#define _mm_getexp_ss(A, B) \ + ((__m128)__builtin_ia32_getexpss128_mask((__v4sf)(__m128)(A), (__v4sf)(__m128)(B), \ + _MM_FROUND_CUR_DIRECTION)) + +#define _mm_getexp_sd(A, B) \ + ((__m128d)__builtin_ia32_getexpsd128_mask((__v2df)(__m128d)(A), (__v2df)(__m128d)(B),\ + _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_getexp_ps(A) \ + ((__m512)__builtin_ia32_getexpps512_mask((__v16sf)(__m512)(A), \ + (__v16sf)_mm512_undefined_ps(), (__mmask16)-1, _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_mask_getexp_ps(W, U, A) \ + ((__m512)__builtin_ia32_getexpps512_mask((__v16sf)(__m512)(A), \ + (__v16sf)(__m512)(W), (__mmask16)(U), _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_maskz_getexp_ps(U, A) \ + ((__m512)__builtin_ia32_getexpps512_mask((__v16sf)(__m512)(A), \ + (__v16sf)_mm512_setzero_ps(), (__mmask16)(U), _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_getexp_pd(A) \ + ((__m512d)__builtin_ia32_getexppd512_mask((__v8df)(__m512d)(A), \ + (__v8df)_mm512_undefined_pd(), (__mmask8)-1, _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_mask_getexp_pd(W, U, A) \ + ((__m512d)__builtin_ia32_getexppd512_mask((__v8df)(__m512d)(A), \ + (__v8df)(__m512d)(W), (__mmask8)(U), _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_maskz_getexp_pd(U, A) \ + ((__m512d)__builtin_ia32_getexppd512_mask((__v8df)(__m512d)(A), \ + (__v8df)_mm512_setzero_pd(), (__mmask8)(U), _MM_FROUND_CUR_DIRECTION)) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_roundscale_ps (__m512 __A, const int __imm) +{ + return (__m512) __builtin_ia32_rndscaleps_mask ((__v16sf) __A, __imm, + (__v16sf) + _mm512_undefined_ps (), + -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_roundscale_ps (__m512 __A, __mmask16 __B, __m512 __C, + const int __imm) +{ + return (__m512) __builtin_ia32_rndscaleps_mask ((__v16sf) __C, __imm, + (__v16sf) __A, + (__mmask16) __B, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_roundscale_ps (__mmask16 __A, __m512 __B, const int __imm) +{ + return (__m512) __builtin_ia32_rndscaleps_mask ((__v16sf) __B, + __imm, + (__v16sf) + _mm512_setzero_ps (), + (__mmask16) __A, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_roundscale_pd (__m512d __A, const int __imm) +{ + return (__m512d) __builtin_ia32_rndscalepd_mask ((__v8df) __A, __imm, + (__v8df) + _mm512_undefined_pd (), + -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_roundscale_pd (__m512d __A, __mmask8 __B, __m512d __C, + const int __imm) +{ + return (__m512d) __builtin_ia32_rndscalepd_mask ((__v8df) __C, __imm, + (__v8df) __A, + (__mmask8) __B, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_roundscale_pd (__mmask8 __A, __m512d __B, const int __imm) +{ + return (__m512d) __builtin_ia32_rndscalepd_mask ((__v8df) __B, + __imm, + (__v8df) + _mm512_setzero_pd (), + (__mmask8) __A, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_roundscale_ss (__m128 __A, __m128 __B, const int __imm) +{ + return (__m128) __builtin_ia32_rndscaless_round ((__v4sf) __A, + (__v4sf) __B, __imm, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_roundscale_sd (__m128d __A, __m128d __B, const int __imm) +{ + return (__m128d) __builtin_ia32_rndscalesd_round ((__v2df) __A, + (__v2df) __B, __imm, + _MM_FROUND_CUR_DIRECTION); +} + +#else +#define _mm512_roundscale_ps(A, B) \ + ((__m512) __builtin_ia32_rndscaleps_mask ((__v16sf)(__m512)(A), (int)(B),\ + (__v16sf)_mm512_undefined_ps(), (__mmask16)(-1), _MM_FROUND_CUR_DIRECTION)) +#define _mm512_mask_roundscale_ps(A, B, C, D) \ + ((__m512) __builtin_ia32_rndscaleps_mask ((__v16sf)(__m512)(C), \ + (int)(D), \ + (__v16sf)(__m512)(A), \ + (__mmask16)(B), _MM_FROUND_CUR_DIRECTION)) +#define _mm512_maskz_roundscale_ps(A, B, C) \ + ((__m512) __builtin_ia32_rndscaleps_mask ((__v16sf)(__m512)(B), \ + (int)(C), \ + (__v16sf)_mm512_setzero_ps(),\ + (__mmask16)(A), _MM_FROUND_CUR_DIRECTION)) +#define _mm512_roundscale_pd(A, B) \ + ((__m512d) __builtin_ia32_rndscalepd_mask ((__v8df)(__m512d)(A), (int)(B),\ + (__v8df)_mm512_undefined_pd(), (__mmask8)(-1), _MM_FROUND_CUR_DIRECTION)) +#define _mm512_mask_roundscale_pd(A, B, C, D) \ + ((__m512d) __builtin_ia32_rndscalepd_mask ((__v8df)(__m512d)(C), \ + (int)(D), \ + (__v8df)(__m512d)(A), \ + (__mmask8)(B), _MM_FROUND_CUR_DIRECTION)) +#define _mm512_maskz_roundscale_pd(A, B, C) \ + ((__m512d) __builtin_ia32_rndscalepd_mask ((__v8df)(__m512d)(B), \ + (int)(C), \ + (__v8df)_mm512_setzero_pd(),\ + (__mmask8)(A), _MM_FROUND_CUR_DIRECTION)) +#define _mm_roundscale_ss(A, B, C) \ + ((__m128) __builtin_ia32_rndscaless_round ((__v4sf)(__m128)(A), \ + (__v4sf)(__m128)(B), (int)(C), _MM_FROUND_CUR_DIRECTION)) +#define _mm_roundscale_sd(A, B, C) \ + ((__m128d) __builtin_ia32_rndscalesd_round ((__v2df)(__m128d)(A), \ + (__v2df)(__m128d)(B), (int)(C), _MM_FROUND_CUR_DIRECTION)) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmp_pd_mask (__m512d __X, __m512d __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_cmppd512_mask ((__v8df) __X, + (__v8df) __Y, __P, + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmp_ps_mask (__m512 __X, __m512 __Y, const int __P) +{ + return (__mmask16) __builtin_ia32_cmpps512_mask ((__v16sf) __X, + (__v16sf) __Y, __P, + (__mmask16) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmp_ps_mask (__mmask16 __U, __m512 __X, __m512 __Y, const int __P) +{ + return (__mmask16) __builtin_ia32_cmpps512_mask ((__v16sf) __X, + (__v16sf) __Y, __P, + (__mmask16) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmp_pd_mask (__mmask8 __U, __m512d __X, __m512d __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_cmppd512_mask ((__v8df) __X, + (__v8df) __Y, __P, + (__mmask8) __U, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_sd_mask (__m128d __X, __m128d __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_cmpsd_mask ((__v2df) __X, + (__v2df) __Y, __P, + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmp_sd_mask (__mmask8 __M, __m128d __X, __m128d __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_cmpsd_mask ((__v2df) __X, + (__v2df) __Y, __P, + (__mmask8) __M, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_ss_mask (__m128 __X, __m128 __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_cmpss_mask ((__v4sf) __X, + (__v4sf) __Y, __P, + (__mmask8) -1, + _MM_FROUND_CUR_DIRECTION); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmp_ss_mask (__mmask8 __M, __m128 __X, __m128 __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_cmpss_mask ((__v4sf) __X, + (__v4sf) __Y, __P, + (__mmask8) __M, + _MM_FROUND_CUR_DIRECTION); +} + +#else +#define _mm512_cmp_pd_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_cmppd512_mask ((__v8df)(__m512d)(X), \ + (__v8df)(__m512d)(Y), (int)(P),\ + (__mmask8)-1,_MM_FROUND_CUR_DIRECTION)) + +#define _mm512_cmp_ps_mask(X, Y, P) \ + ((__mmask16) __builtin_ia32_cmpps512_mask ((__v16sf)(__m512)(X), \ + (__v16sf)(__m512)(Y), (int)(P),\ + (__mmask16)-1,_MM_FROUND_CUR_DIRECTION)) + +#define _mm512_mask_cmp_pd_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_cmppd512_mask ((__v8df)(__m512d)(X), \ + (__v8df)(__m512d)(Y), (int)(P),\ + (__mmask8)M, _MM_FROUND_CUR_DIRECTION)) + +#define _mm512_mask_cmp_ps_mask(M, X, Y, P) \ + ((__mmask16) __builtin_ia32_cmpps512_mask ((__v16sf)(__m512)(X), \ + (__v16sf)(__m512)(Y), (int)(P),\ + (__mmask16)M,_MM_FROUND_CUR_DIRECTION)) + +#define _mm_cmp_sd_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpsd_mask ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (int)(P),\ + (__mmask8)-1,_MM_FROUND_CUR_DIRECTION)) + +#define _mm_mask_cmp_sd_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpsd_mask ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (int)(P),\ + M,_MM_FROUND_CUR_DIRECTION)) + +#define _mm_cmp_ss_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpss_mask ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (int)(P), \ + (__mmask8)-1,_MM_FROUND_CUR_DIRECTION)) + +#define _mm_mask_cmp_ss_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpss_mask ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (int)(P), \ + M,_MM_FROUND_CUR_DIRECTION)) +#endif + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_kmov (__mmask16 __A) +{ + return __builtin_ia32_kmovw (__A); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castpd_ps (__m512d __A) +{ + return (__m512) (__A); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castpd_si512 (__m512d __A) +{ + return (__m512i) (__A); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castps_pd (__m512 __A) +{ + return (__m512d) (__A); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castps_si512 (__m512 __A) +{ + return (__m512i) (__A); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castsi512_ps (__m512i __A) +{ + return (__m512) (__A); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castsi512_pd (__m512i __A) +{ + return (__m512d) (__A); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castpd512_pd128 (__m512d __A) +{ + return (__m128d)_mm512_extractf32x4_ps((__m512)__A, 0); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castps512_ps128 (__m512 __A) +{ + return _mm512_extractf32x4_ps(__A, 0); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castsi512_si128 (__m512i __A) +{ + return (__m128i)_mm512_extracti32x4_epi32((__m512i)__A, 0); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castpd512_pd256 (__m512d __A) +{ + return _mm512_extractf64x4_pd(__A, 0); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castps512_ps256 (__m512 __A) +{ + return (__m256)_mm512_extractf64x4_pd((__m512d)__A, 0); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castsi512_si256 (__m512i __A) +{ + return (__m256i)_mm512_extractf64x4_pd((__m512d)__A, 0); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castpd128_pd512 (__m128d __A) +{ + return (__m512d) __builtin_ia32_pd512_pd((__m128d)__A); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castps128_ps512 (__m128 __A) +{ + return (__m512) __builtin_ia32_ps512_ps((__m128)__A); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castsi128_si512 (__m128i __A) +{ + return (__m512i) __builtin_ia32_si512_si((__v4si)__A); +} + +extern __inline __m512d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castpd256_pd512 (__m256d __A) +{ + return __builtin_ia32_pd512_256pd (__A); +} + +extern __inline __m512 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castps256_ps512 (__m256 __A) +{ + return __builtin_ia32_ps512_256ps (__A); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_castsi256_si512 (__m256i __A) +{ + return (__m512i)__builtin_ia32_si512_256si ((__v8si)__A); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpeq_epu32_mask (__m512i __A, __m512i __B) +{ + return (__mmask16) __builtin_ia32_ucmpd512_mask ((__v16si) __A, + (__v16si) __B, 0, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpeq_epu32_mask (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__mmask16) __builtin_ia32_ucmpd512_mask ((__v16si) __A, + (__v16si) __B, 0, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpeq_epu64_mask (__mmask8 __U, __m512i __A, __m512i __B) +{ + return (__mmask8) __builtin_ia32_ucmpq512_mask ((__v8di) __A, + (__v8di) __B, 0, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpeq_epu64_mask (__m512i __A, __m512i __B) +{ + return (__mmask8) __builtin_ia32_ucmpq512_mask ((__v8di) __A, + (__v8di) __B, 0, + (__mmask8) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpgt_epu32_mask (__m512i __A, __m512i __B) +{ + return (__mmask16) __builtin_ia32_ucmpd512_mask ((__v16si) __A, + (__v16si) __B, 6, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpgt_epu32_mask (__mmask16 __U, __m512i __A, __m512i __B) +{ + return (__mmask16) __builtin_ia32_ucmpd512_mask ((__v16si) __A, + (__v16si) __B, 6, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_cmpgt_epu64_mask (__mmask8 __U, __m512i __A, __m512i __B) +{ + return (__mmask8) __builtin_ia32_ucmpq512_mask ((__v8di) __A, + (__v8di) __B, 6, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_cmpgt_epu64_mask (__m512i __A, __m512i __B) +{ + return (__mmask8) __builtin_ia32_ucmpq512_mask ((__v8di) __A, + (__v8di) __B, 6, + (__mmask8) -1); +} + +#undef __MM512_REDUCE_OP +#define __MM512_REDUCE_OP(op) \ + __v8si __T1 = (__v8si) _mm512_extracti64x4_epi64 (__A, 1); \ + __v8si __T2 = (__v8si) _mm512_extracti64x4_epi64 (__A, 0); \ + __m256i __T3 = (__m256i) (__T1 op __T2); \ + __v4si __T4 = (__v4si) _mm256_extracti128_si256 (__T3, 1); \ + __v4si __T5 = (__v4si) _mm256_extracti128_si256 (__T3, 0); \ + __v4si __T6 = __T4 op __T5; \ + __v4si __T7 = __builtin_shuffle (__T6, (__v4si) { 2, 3, 0, 1 }); \ + __v4si __T8 = __T6 op __T7; \ + return __T8[0] op __T8[1] + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_add_epi32 (__m512i __A) +{ + __MM512_REDUCE_OP (+); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_mul_epi32 (__m512i __A) +{ + __MM512_REDUCE_OP (*); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_and_epi32 (__m512i __A) +{ + __MM512_REDUCE_OP (&); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_or_epi32 (__m512i __A) +{ + __MM512_REDUCE_OP (|); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_add_epi32 (__mmask16 __U, __m512i __A) +{ + __A = _mm512_maskz_mov_epi32 (__U, __A); + __MM512_REDUCE_OP (+); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_mul_epi32 (__mmask16 __U, __m512i __A) +{ + __A = _mm512_mask_mov_epi32 (_mm512_set1_epi32 (1), __U, __A); + __MM512_REDUCE_OP (*); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_and_epi32 (__mmask16 __U, __m512i __A) +{ + __A = _mm512_mask_mov_epi32 (_mm512_set1_epi32 (~0), __U, __A); + __MM512_REDUCE_OP (&); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_or_epi32 (__mmask16 __U, __m512i __A) +{ + __A = _mm512_maskz_mov_epi32 (__U, __A); + __MM512_REDUCE_OP (|); +} + +#undef __MM512_REDUCE_OP +#define __MM512_REDUCE_OP(op) \ + __m256i __T1 = (__m256i) _mm512_extracti64x4_epi64 (__A, 1); \ + __m256i __T2 = (__m256i) _mm512_extracti64x4_epi64 (__A, 0); \ + __m256i __T3 = _mm256_##op (__T1, __T2); \ + __m128i __T4 = (__m128i) _mm256_extracti128_si256 (__T3, 1); \ + __m128i __T5 = (__m128i) _mm256_extracti128_si256 (__T3, 0); \ + __m128i __T6 = _mm_##op (__T4, __T5); \ + __m128i __T7 = (__m128i) __builtin_shuffle ((__v4si) __T6, \ + (__v4si) { 2, 3, 0, 1 }); \ + __m128i __T8 = _mm_##op (__T6, __T7); \ + __m128i __T9 = (__m128i) __builtin_shuffle ((__v4si) __T8, \ + (__v4si) { 1, 0, 1, 0 }); \ + __v4si __T10 = (__v4si) _mm_##op (__T8, __T9); \ + return __T10[0] + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_min_epi32 (__m512i __A) +{ + __MM512_REDUCE_OP (min_epi32); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_max_epi32 (__m512i __A) +{ + __MM512_REDUCE_OP (max_epi32); +} + +extern __inline unsigned int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_min_epu32 (__m512i __A) +{ + __MM512_REDUCE_OP (min_epu32); +} + +extern __inline unsigned int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_max_epu32 (__m512i __A) +{ + __MM512_REDUCE_OP (max_epu32); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_min_epi32 (__mmask16 __U, __m512i __A) +{ + __A = _mm512_mask_mov_epi32 (_mm512_set1_epi32 (__INT_MAX__), __U, __A); + __MM512_REDUCE_OP (min_epi32); +} + +extern __inline int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_max_epi32 (__mmask16 __U, __m512i __A) +{ + __A = _mm512_mask_mov_epi32 (_mm512_set1_epi32 (-__INT_MAX__ - 1), __U, __A); + __MM512_REDUCE_OP (max_epi32); +} + +extern __inline unsigned int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_min_epu32 (__mmask16 __U, __m512i __A) +{ + __A = _mm512_mask_mov_epi32 (_mm512_set1_epi32 (~0), __U, __A); + __MM512_REDUCE_OP (min_epu32); +} + +extern __inline unsigned int +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_max_epu32 (__mmask16 __U, __m512i __A) +{ + __A = _mm512_maskz_mov_epi32 (__U, __A); + __MM512_REDUCE_OP (max_epu32); +} + +#undef __MM512_REDUCE_OP +#define __MM512_REDUCE_OP(op) \ + __m256 __T1 = (__m256) _mm512_extractf64x4_pd ((__m512d) __A, 1); \ + __m256 __T2 = (__m256) _mm512_extractf64x4_pd ((__m512d) __A, 0); \ + __m256 __T3 = __T1 op __T2; \ + __m128 __T4 = _mm256_extractf128_ps (__T3, 1); \ + __m128 __T5 = _mm256_extractf128_ps (__T3, 0); \ + __m128 __T6 = __T4 op __T5; \ + __m128 __T7 = __builtin_shuffle (__T6, (__v4si) { 2, 3, 0, 1 }); \ + __m128 __T8 = __T6 op __T7; \ + return __T8[0] op __T8[1] + +extern __inline float +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_add_ps (__m512 __A) +{ + __MM512_REDUCE_OP (+); +} + +extern __inline float +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_mul_ps (__m512 __A) +{ + __MM512_REDUCE_OP (*); +} + +extern __inline float +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_add_ps (__mmask16 __U, __m512 __A) +{ + __A = _mm512_maskz_mov_ps (__U, __A); + __MM512_REDUCE_OP (+); +} + +extern __inline float +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_mul_ps (__mmask16 __U, __m512 __A) +{ + __A = _mm512_mask_mov_ps (_mm512_set1_ps (1.0f), __U, __A); + __MM512_REDUCE_OP (*); +} + +#undef __MM512_REDUCE_OP +#define __MM512_REDUCE_OP(op) \ + __m256 __T1 = (__m256) _mm512_extractf64x4_pd ((__m512d) __A, 1); \ + __m256 __T2 = (__m256) _mm512_extractf64x4_pd ((__m512d) __A, 0); \ + __m256 __T3 = _mm256_##op (__T1, __T2); \ + __m128 __T4 = _mm256_extractf128_ps (__T3, 1); \ + __m128 __T5 = _mm256_extractf128_ps (__T3, 0); \ + __m128 __T6 = _mm_##op (__T4, __T5); \ + __m128 __T7 = __builtin_shuffle (__T6, (__v4si) { 2, 3, 0, 1 }); \ + __m128 __T8 = _mm_##op (__T6, __T7); \ + __m128 __T9 = __builtin_shuffle (__T8, (__v4si) { 1, 0, 1, 0 }); \ + __m128 __T10 = _mm_##op (__T8, __T9); \ + return __T10[0] + +extern __inline float +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_min_ps (__m512 __A) +{ + __MM512_REDUCE_OP (min_ps); +} + +extern __inline float +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_max_ps (__m512 __A) +{ + __MM512_REDUCE_OP (max_ps); +} + +extern __inline float +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_min_ps (__mmask16 __U, __m512 __A) +{ + __A = _mm512_mask_mov_ps (_mm512_set1_ps (__builtin_inff ()), __U, __A); + __MM512_REDUCE_OP (min_ps); +} + +extern __inline float +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_max_ps (__mmask16 __U, __m512 __A) +{ + __A = _mm512_mask_mov_ps (_mm512_set1_ps (-__builtin_inff ()), __U, __A); + __MM512_REDUCE_OP (max_ps); +} + +#undef __MM512_REDUCE_OP +#define __MM512_REDUCE_OP(op) \ + __v4di __T1 = (__v4di) _mm512_extracti64x4_epi64 (__A, 1); \ + __v4di __T2 = (__v4di) _mm512_extracti64x4_epi64 (__A, 0); \ + __m256i __T3 = (__m256i) (__T1 op __T2); \ + __v2di __T4 = (__v2di) _mm256_extracti128_si256 (__T3, 1); \ + __v2di __T5 = (__v2di) _mm256_extracti128_si256 (__T3, 0); \ + __v2di __T6 = __T4 op __T5; \ + return __T6[0] op __T6[1] + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_add_epi64 (__m512i __A) +{ + __MM512_REDUCE_OP (+); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_mul_epi64 (__m512i __A) +{ + __MM512_REDUCE_OP (*); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_and_epi64 (__m512i __A) +{ + __MM512_REDUCE_OP (&); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_or_epi64 (__m512i __A) +{ + __MM512_REDUCE_OP (|); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_add_epi64 (__mmask8 __U, __m512i __A) +{ + __A = _mm512_maskz_mov_epi64 (__U, __A); + __MM512_REDUCE_OP (+); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_mul_epi64 (__mmask8 __U, __m512i __A) +{ + __A = _mm512_mask_mov_epi64 (_mm512_set1_epi64 (1LL), __U, __A); + __MM512_REDUCE_OP (*); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_and_epi64 (__mmask8 __U, __m512i __A) +{ + __A = _mm512_mask_mov_epi64 (_mm512_set1_epi64 (~0LL), __U, __A); + __MM512_REDUCE_OP (&); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_or_epi64 (__mmask8 __U, __m512i __A) +{ + __A = _mm512_maskz_mov_epi64 (__U, __A); + __MM512_REDUCE_OP (|); +} + +#undef __MM512_REDUCE_OP +#define __MM512_REDUCE_OP(op) \ + __m512i __T1 = _mm512_shuffle_i64x2 (__A, __A, 0x4e); \ + __m512i __T2 = _mm512_##op (__A, __T1); \ + __m512i __T3 \ + = (__m512i) __builtin_shuffle ((__v8di) __T2, \ + (__v8di) { 2, 3, 0, 1, 6, 7, 4, 5 });\ + __m512i __T4 = _mm512_##op (__T2, __T3); \ + __m512i __T5 \ + = (__m512i) __builtin_shuffle ((__v8di) __T4, \ + (__v8di) { 1, 0, 3, 2, 5, 4, 7, 6 });\ + __v8di __T6 = (__v8di) _mm512_##op (__T4, __T5); \ + return __T6[0] + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_min_epi64 (__m512i __A) +{ + __MM512_REDUCE_OP (min_epi64); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_max_epi64 (__m512i __A) +{ + __MM512_REDUCE_OP (max_epi64); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_min_epi64 (__mmask8 __U, __m512i __A) +{ + __A = _mm512_mask_mov_epi64 (_mm512_set1_epi64 (__LONG_LONG_MAX__), + __U, __A); + __MM512_REDUCE_OP (min_epi64); +} + +extern __inline long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_max_epi64 (__mmask8 __U, __m512i __A) +{ + __A = _mm512_mask_mov_epi64 (_mm512_set1_epi64 (-__LONG_LONG_MAX__ - 1), + __U, __A); + __MM512_REDUCE_OP (max_epi64); +} + +extern __inline unsigned long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_min_epu64 (__m512i __A) +{ + __MM512_REDUCE_OP (min_epu64); +} + +extern __inline unsigned long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_max_epu64 (__m512i __A) +{ + __MM512_REDUCE_OP (max_epu64); +} + +extern __inline unsigned long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_min_epu64 (__mmask8 __U, __m512i __A) +{ + __A = _mm512_mask_mov_epi64 (_mm512_set1_epi64 (~0LL), __U, __A); + __MM512_REDUCE_OP (min_epu64); +} + +extern __inline unsigned long long +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_max_epu64 (__mmask8 __U, __m512i __A) +{ + __A = _mm512_maskz_mov_epi64 (__U, __A); + __MM512_REDUCE_OP (max_epu64); +} + +#undef __MM512_REDUCE_OP +#define __MM512_REDUCE_OP(op) \ + __m256d __T1 = (__m256d) _mm512_extractf64x4_pd (__A, 1); \ + __m256d __T2 = (__m256d) _mm512_extractf64x4_pd (__A, 0); \ + __m256d __T3 = __T1 op __T2; \ + __m128d __T4 = _mm256_extractf128_pd (__T3, 1); \ + __m128d __T5 = _mm256_extractf128_pd (__T3, 0); \ + __m128d __T6 = __T4 op __T5; \ + return __T6[0] op __T6[1] + +extern __inline double +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_add_pd (__m512d __A) +{ + __MM512_REDUCE_OP (+); +} + +extern __inline double +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_mul_pd (__m512d __A) +{ + __MM512_REDUCE_OP (*); +} + +extern __inline double +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_add_pd (__mmask8 __U, __m512d __A) +{ + __A = _mm512_maskz_mov_pd (__U, __A); + __MM512_REDUCE_OP (+); +} + +extern __inline double +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_mul_pd (__mmask8 __U, __m512d __A) +{ + __A = _mm512_mask_mov_pd (_mm512_set1_pd (1.0), __U, __A); + __MM512_REDUCE_OP (*); +} + +#undef __MM512_REDUCE_OP +#define __MM512_REDUCE_OP(op) \ + __m256d __T1 = (__m256d) _mm512_extractf64x4_pd (__A, 1); \ + __m256d __T2 = (__m256d) _mm512_extractf64x4_pd (__A, 0); \ + __m256d __T3 = _mm256_##op (__T1, __T2); \ + __m128d __T4 = _mm256_extractf128_pd (__T3, 1); \ + __m128d __T5 = _mm256_extractf128_pd (__T3, 0); \ + __m128d __T6 = _mm_##op (__T4, __T5); \ + __m128d __T7 = (__m128d) __builtin_shuffle (__T6, (__v2di) { 1, 0 }); \ + __m128d __T8 = _mm_##op (__T6, __T7); \ + return __T8[0] + +extern __inline double +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_min_pd (__m512d __A) +{ + __MM512_REDUCE_OP (min_pd); +} + +extern __inline double +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_reduce_max_pd (__m512d __A) +{ + __MM512_REDUCE_OP (max_pd); +} + +extern __inline double +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_min_pd (__mmask8 __U, __m512d __A) +{ + __A = _mm512_mask_mov_pd (_mm512_set1_pd (__builtin_inf ()), __U, __A); + __MM512_REDUCE_OP (min_pd); +} + +extern __inline double +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_reduce_max_pd (__mmask8 __U, __m512d __A) +{ + __A = _mm512_mask_mov_pd (_mm512_set1_pd (-__builtin_inf ()), __U, __A); + __MM512_REDUCE_OP (max_pd); +} + +#undef __MM512_REDUCE_OP + +#ifdef __DISABLE_AVX512F__ +#undef __DISABLE_AVX512F__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX512F__ */ + +#endif /* _AVX512FINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512ifmaintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512ifmaintrin.h new file mode 100644 index 0000000000..56502d29ba --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512ifmaintrin.h @@ -0,0 +1,104 @@ +/* Copyright (C) 2013-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +#error "Never use directly; include instead." +#endif + +#ifndef _AVX512IFMAINTRIN_H_INCLUDED +#define _AVX512IFMAINTRIN_H_INCLUDED + +#ifndef __AVX512IFMA__ +#pragma GCC push_options +//#pragma GCC target("avx512ifma") +#define __DISABLE_AVX512IFMA__ +#endif /* __AVX512IFMA__ */ + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_madd52lo_epu64 (__m512i __X, __m512i __Y, __m512i __Z) +{ + return (__m512i) __builtin_ia32_vpmadd52luq512_mask ((__v8di) __X, + (__v8di) __Y, + (__v8di) __Z, + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_madd52hi_epu64 (__m512i __X, __m512i __Y, __m512i __Z) +{ + return (__m512i) __builtin_ia32_vpmadd52huq512_mask ((__v8di) __X, + (__v8di) __Y, + (__v8di) __Z, + (__mmask8) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_madd52lo_epu64 (__m512i __W, __mmask8 __M, __m512i __X, + __m512i __Y) +{ + return (__m512i) __builtin_ia32_vpmadd52luq512_mask ((__v8di) __W, + (__v8di) __X, + (__v8di) __Y, + (__mmask8) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_madd52hi_epu64 (__m512i __W, __mmask8 __M, __m512i __X, + __m512i __Y) +{ + return (__m512i) __builtin_ia32_vpmadd52huq512_mask ((__v8di) __W, + (__v8di) __X, + (__v8di) __Y, + (__mmask8) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_madd52lo_epu64 (__mmask8 __M, __m512i __X, __m512i __Y, __m512i __Z) +{ + return (__m512i) __builtin_ia32_vpmadd52luq512_maskz ((__v8di) __X, + (__v8di) __Y, + (__v8di) __Z, + (__mmask8) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_madd52hi_epu64 (__mmask8 __M, __m512i __X, __m512i __Y, __m512i __Z) +{ + return (__m512i) __builtin_ia32_vpmadd52huq512_maskz ((__v8di) __X, + (__v8di) __Y, + (__v8di) __Z, + (__mmask8) __M); +} + +#ifdef __DISABLE_AVX512IFMA__ +#undef __DISABLE_AVX512IFMA__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX512IFMA__ */ + +#endif /* _AVX512IFMAINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512ifmavlintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512ifmavlintrin.h new file mode 100644 index 0000000000..2522dd054f --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512ifmavlintrin.h @@ -0,0 +1,164 @@ +/* Copyright (C) 2013-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +#error "Never use directly; include instead." +#endif + +#ifndef _AVX512IFMAVLINTRIN_H_INCLUDED +#define _AVX512IFMAVLINTRIN_H_INCLUDED + +#if !defined(__AVX512VL__) || !defined(__AVX512IFMA__) +#pragma GCC push_options +//#pragma GCC target("avx512ifma,avx512vl") +#define __DISABLE_AVX512IFMAVL__ +#endif /* __AVX512IFMAVL__ */ + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_madd52lo_epu64 (__m128i __X, __m128i __Y, __m128i __Z) +{ + return (__m128i) __builtin_ia32_vpmadd52luq128_mask ((__v2di) __X, + (__v2di) __Y, + (__v2di) __Z, + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_madd52hi_epu64 (__m128i __X, __m128i __Y, __m128i __Z) +{ + return (__m128i) __builtin_ia32_vpmadd52huq128_mask ((__v2di) __X, + (__v2di) __Y, + (__v2di) __Z, + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_madd52lo_epu64 (__m256i __X, __m256i __Y, __m256i __Z) +{ + return (__m256i) __builtin_ia32_vpmadd52luq256_mask ((__v4di) __X, + (__v4di) __Y, + (__v4di) __Z, + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_madd52hi_epu64 (__m256i __X, __m256i __Y, __m256i __Z) +{ + return (__m256i) __builtin_ia32_vpmadd52huq256_mask ((__v4di) __X, + (__v4di) __Y, + (__v4di) __Z, + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_madd52lo_epu64 (__m128i __W, __mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_vpmadd52luq128_mask ((__v2di) __W, + (__v2di) __X, + (__v2di) __Y, + (__mmask8) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_madd52hi_epu64 (__m128i __W, __mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_vpmadd52huq128_mask ((__v2di) __W, + (__v2di) __X, + (__v2di) __Y, + (__mmask8) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_madd52lo_epu64 (__m256i __W, __mmask8 __M, __m256i __X, + __m256i __Y) +{ + return (__m256i) __builtin_ia32_vpmadd52luq256_mask ((__v4di) __W, + (__v4di) __X, + (__v4di) __Y, + (__mmask8) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_madd52hi_epu64 (__m256i __W, __mmask8 __M, __m256i __X, + __m256i __Y) +{ + return (__m256i) __builtin_ia32_vpmadd52huq256_mask ((__v4di) __W, + (__v4di) __X, + (__v4di) __Y, + (__mmask8) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_madd52lo_epu64 (__mmask8 __M, __m128i __X, __m128i __Y, __m128i __Z) +{ + return (__m128i) __builtin_ia32_vpmadd52luq128_maskz ((__v2di) __X, + (__v2di) __Y, + (__v2di) __Z, + (__mmask8) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_madd52hi_epu64 (__mmask8 __M, __m128i __X, __m128i __Y, __m128i __Z) +{ + return (__m128i) __builtin_ia32_vpmadd52huq128_maskz ((__v2di) __X, + (__v2di) __Y, + (__v2di) __Z, + (__mmask8) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_madd52lo_epu64 (__mmask8 __M, __m256i __X, __m256i __Y, __m256i __Z) +{ + return (__m256i) __builtin_ia32_vpmadd52luq256_maskz ((__v4di) __X, + (__v4di) __Y, + (__v4di) __Z, + (__mmask8) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_madd52hi_epu64 (__mmask8 __M, __m256i __X, __m256i __Y, __m256i __Z) +{ + return (__m256i) __builtin_ia32_vpmadd52huq256_maskz ((__v4di) __X, + (__v4di) __Y, + (__v4di) __Z, + (__mmask8) __M); +} + +#ifdef __DISABLE_AVX512IFMAVL__ +#undef __DISABLE_AVX512IFMAVL__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX512IFMAVL__ */ + +#endif /* _AVX512IFMAVLINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512pfintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512pfintrin.h new file mode 100644 index 0000000000..6e988954de --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512pfintrin.h @@ -0,0 +1,264 @@ +/* Copyright (C) 2013-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +#error "Never use directly; include instead." +#endif + +#ifndef _AVX512PFINTRIN_H_INCLUDED +#define _AVX512PFINTRIN_H_INCLUDED + +#ifndef __AVX512PF__ +#pragma GCC push_options +//#pragma GCC target("avx512pf") +#define __DISABLE_AVX512PF__ +#endif /* __AVX512PF__ */ + +/* Internal data types for implementing the intrinsics. */ +typedef long long __v8di __attribute__ ((__vector_size__ (64))); +typedef int __v16si __attribute__ ((__vector_size__ (64))); + +/* The Intel API is flexible enough that we must allow aliasing with other + vector types, and their scalar components. */ +typedef long long __m512i __attribute__ ((__vector_size__ (64), __may_alias__)); + +typedef unsigned char __mmask8; +typedef unsigned short __mmask16; + +#ifdef __OPTIMIZE__ +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_prefetch_i32gather_pd (__m256i __index, void const *__addr, + int __scale, int __hint) +{ + __builtin_ia32_gatherpfdpd ((__mmask8) 0xFF, (__v8si) __index, __addr, + __scale, __hint); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_prefetch_i32gather_ps (__m512i __index, void const *__addr, + int __scale, int __hint) +{ + __builtin_ia32_gatherpfdps ((__mmask16) 0xFFFF, (__v16si) __index, __addr, + __scale, __hint); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_prefetch_i32gather_pd (__m256i __index, __mmask8 __mask, + void const *__addr, int __scale, int __hint) +{ + __builtin_ia32_gatherpfdpd (__mask, (__v8si) __index, __addr, __scale, + __hint); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_prefetch_i32gather_ps (__m512i __index, __mmask16 __mask, + void const *__addr, int __scale, int __hint) +{ + __builtin_ia32_gatherpfdps (__mask, (__v16si) __index, __addr, __scale, + __hint); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_prefetch_i64gather_pd (__m512i __index, void const *__addr, + int __scale, int __hint) +{ + __builtin_ia32_gatherpfqpd ((__mmask8) 0xFF, (__v8di) __index, __addr, + __scale, __hint); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_prefetch_i64gather_ps (__m512i __index, void const *__addr, + int __scale, int __hint) +{ + __builtin_ia32_gatherpfqps ((__mmask8) 0xFF, (__v8di) __index, __addr, + __scale, __hint); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_prefetch_i64gather_pd (__m512i __index, __mmask8 __mask, + void const *__addr, int __scale, int __hint) +{ + __builtin_ia32_gatherpfqpd (__mask, (__v8di) __index, __addr, __scale, + __hint); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_prefetch_i64gather_ps (__m512i __index, __mmask8 __mask, + void const *__addr, int __scale, int __hint) +{ + __builtin_ia32_gatherpfqps (__mask, (__v8di) __index, __addr, __scale, + __hint); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_prefetch_i32scatter_pd (void *__addr, __m256i __index, int __scale, + int __hint) +{ + __builtin_ia32_scatterpfdpd ((__mmask8) 0xFF, (__v8si) __index, __addr, + __scale, __hint); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_prefetch_i32scatter_ps (void *__addr, __m512i __index, int __scale, + int __hint) +{ + __builtin_ia32_scatterpfdps ((__mmask16) 0xFFFF, (__v16si) __index, __addr, + __scale, __hint); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_prefetch_i32scatter_pd (void *__addr, __mmask8 __mask, + __m256i __index, int __scale, int __hint) +{ + __builtin_ia32_scatterpfdpd (__mask, (__v8si) __index, __addr, __scale, + __hint); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_prefetch_i32scatter_ps (void *__addr, __mmask16 __mask, + __m512i __index, int __scale, int __hint) +{ + __builtin_ia32_scatterpfdps (__mask, (__v16si) __index, __addr, __scale, + __hint); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_prefetch_i64scatter_pd (void *__addr, __m512i __index, int __scale, + int __hint) +{ + __builtin_ia32_scatterpfqpd ((__mmask8) 0xFF, (__v8di) __index,__addr, + __scale, __hint); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_prefetch_i64scatter_ps (void *__addr, __m512i __index, int __scale, + int __hint) +{ + __builtin_ia32_scatterpfqps ((__mmask8) 0xFF, (__v8di) __index, __addr, + __scale, __hint); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_prefetch_i64scatter_pd (void *__addr, __mmask16 __mask, + __m512i __index, int __scale, int __hint) +{ + __builtin_ia32_scatterpfqpd (__mask, (__v8di) __index, __addr, __scale, + __hint); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_prefetch_i64scatter_ps (void *__addr, __mmask16 __mask, + __m512i __index, int __scale, int __hint) +{ + __builtin_ia32_scatterpfqps (__mask, (__v8di) __index, __addr, __scale, + __hint); +} + +#else +#define _mm512_prefetch_i32gather_pd(INDEX, ADDR, SCALE, HINT) \ + __builtin_ia32_gatherpfdpd ((__mmask8)0xFF, (__v8si)(__m256i)INDEX, \ + (void const *)ADDR, (int)SCALE, (int)HINT) + +#define _mm512_prefetch_i32gather_ps(INDEX, ADDR, SCALE, HINT) \ + __builtin_ia32_gatherpfdps ((__mmask16)0xFFFF, (__v16si)(__m512i)INDEX, \ + (void const *)ADDR, (int)SCALE, (int)HINT) + +#define _mm512_mask_prefetch_i32gather_pd(INDEX, MASK, ADDR, SCALE, HINT) \ + __builtin_ia32_gatherpfdpd ((__mmask8)MASK, (__v8si)(__m256i)INDEX, \ + (void const *)ADDR, (int)SCALE, (int)HINT) + +#define _mm512_mask_prefetch_i32gather_ps(INDEX, MASK, ADDR, SCALE, HINT) \ + __builtin_ia32_gatherpfdps ((__mmask16)MASK, (__v16si)(__m512i)INDEX, \ + (void const *)ADDR, (int)SCALE, (int)HINT) + +#define _mm512_prefetch_i64gather_pd(INDEX, ADDR, SCALE, HINT) \ + __builtin_ia32_gatherpfqpd ((__mmask8)0xFF, (__v8di)(__m512i)INDEX, \ + (void *)ADDR, (int)SCALE, (int)HINT) + +#define _mm512_prefetch_i64gather_ps(INDEX, ADDR, SCALE, HINT) \ + __builtin_ia32_gatherpfqps ((__mmask8)0xFF, (__v8di)(__m512i)INDEX, \ + (void *)ADDR, (int)SCALE, (int)HINT) + +#define _mm512_mask_prefetch_i64gather_pd(INDEX, MASK, ADDR, SCALE, HINT) \ + __builtin_ia32_gatherpfqpd ((__mmask8)MASK, (__v8di)(__m512i)INDEX, \ + (void *)ADDR, (int)SCALE, (int)HINT) + +#define _mm512_mask_prefetch_i64gather_ps(INDEX, MASK, ADDR, SCALE, HINT) \ + __builtin_ia32_gatherpfqps ((__mmask8)MASK, (__v8di)(__m512i)INDEX, \ + (void *)ADDR, (int)SCALE, (int)HINT) + +#define _mm512_prefetch_i32scatter_pd(ADDR, INDEX, SCALE, HINT) \ + __builtin_ia32_scatterpfdpd ((__mmask8)0xFF, (__v8si)(__m256i)INDEX, \ + (void *)ADDR, (int)SCALE, (int)HINT) + +#define _mm512_prefetch_i32scatter_ps(ADDR, INDEX, SCALE, HINT) \ + __builtin_ia32_scatterpfdps ((__mmask16)0xFFFF, (__v16si)(__m512i)INDEX, \ + (void *)ADDR, (int)SCALE, (int)HINT) + +#define _mm512_mask_prefetch_i32scatter_pd(ADDR, MASK, INDEX, SCALE, HINT) \ + __builtin_ia32_scatterpfdpd ((__mmask8)MASK, (__v8si)(__m256i)INDEX, \ + (void *)ADDR, (int)SCALE, (int)HINT) + +#define _mm512_mask_prefetch_i32scatter_ps(ADDR, MASK, INDEX, SCALE, HINT) \ + __builtin_ia32_scatterpfdps ((__mmask16)MASK, (__v16si)(__m512i)INDEX, \ + (void *)ADDR, (int)SCALE, (int)HINT) + +#define _mm512_prefetch_i64scatter_pd(ADDR, INDEX, SCALE, HINT) \ + __builtin_ia32_scatterpfqpd ((__mmask8)0xFF, (__v8di)(__m512i)INDEX, \ + (void *)ADDR, (int)SCALE, (int)HINT) + +#define _mm512_prefetch_i64scatter_ps(ADDR, INDEX, SCALE, HINT) \ + __builtin_ia32_scatterpfqps ((__mmask8)0xFF, (__v8di)(__m512i)INDEX, \ + (void *)ADDR, (int)SCALE, (int)HINT) + +#define _mm512_mask_prefetch_i64scatter_pd(ADDR, MASK, INDEX, SCALE, HINT) \ + __builtin_ia32_scatterpfqpd ((__mmask8)MASK, (__v8di)(__m512i)INDEX, \ + (void *)ADDR, (int)SCALE, (int)HINT) + +#define _mm512_mask_prefetch_i64scatter_ps(ADDR, MASK, INDEX, SCALE, HINT) \ + __builtin_ia32_scatterpfqps ((__mmask8)MASK, (__v8di)(__m512i)INDEX, \ + (void *)ADDR, (int)SCALE, (int)HINT) +#endif + +#ifdef __DISABLE_AVX512PF__ +#undef __DISABLE_AVX512PF__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX512PF__ */ + +#endif /* _AVX512PFINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512vbmiintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512vbmiintrin.h new file mode 100644 index 0000000000..bb6d383a7c --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512vbmiintrin.h @@ -0,0 +1,158 @@ +/* Copyright (C) 2013-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +#error "Never use directly; include instead." +#endif + +#ifndef _AVX512VBMIINTRIN_H_INCLUDED +#define _AVX512VBMIINTRIN_H_INCLUDED + +#ifndef __AVX512VBMI__ +#pragma GCC push_options +//#pragma GCC target("avx512vbmi") +#define __DISABLE_AVX512VBMI__ +#endif /* __AVX512VBMI__ */ + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_multishift_epi64_epi8 (__m512i __W, __mmask64 __M, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_vpmultishiftqb512_mask ((__v64qi) __X, + (__v64qi) __Y, + (__v64qi) __W, + (__mmask64) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_multishift_epi64_epi8 (__mmask64 __M, __m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_vpmultishiftqb512_mask ((__v64qi) __X, + (__v64qi) __Y, + (__v64qi) + _mm512_setzero_si512 (), + (__mmask64) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_multishift_epi64_epi8 (__m512i __X, __m512i __Y) +{ + return (__m512i) __builtin_ia32_vpmultishiftqb512_mask ((__v64qi) __X, + (__v64qi) __Y, + (__v64qi) + _mm512_undefined_epi32 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permutexvar_epi8 (__m512i __A, __m512i __B) +{ + return (__m512i) __builtin_ia32_permvarqi512_mask ((__v64qi) __B, + (__v64qi) __A, + (__v64qi) + _mm512_undefined_epi32 (), + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permutexvar_epi8 (__mmask64 __M, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_permvarqi512_mask ((__v64qi) __B, + (__v64qi) __A, + (__v64qi) + _mm512_setzero_si512(), + (__mmask64) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permutexvar_epi8 (__m512i __W, __mmask64 __M, __m512i __A, + __m512i __B) +{ + return (__m512i) __builtin_ia32_permvarqi512_mask ((__v64qi) __B, + (__v64qi) __A, + (__v64qi) __W, + (__mmask64) __M); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_permutex2var_epi8 (__m512i __A, __m512i __I, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpermt2varqi512_mask ((__v64qi) __I + /* idx */ , + (__v64qi) __A, + (__v64qi) __B, + (__mmask64) -1); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_permutex2var_epi8 (__m512i __A, __mmask64 __U, + __m512i __I, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpermt2varqi512_mask ((__v64qi) __I + /* idx */ , + (__v64qi) __A, + (__v64qi) __B, + (__mmask64) + __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask2_permutex2var_epi8 (__m512i __A, __m512i __I, + __mmask64 __U, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpermi2varqi512_mask ((__v64qi) __A, + (__v64qi) __I + /* idx */ , + (__v64qi) __B, + (__mmask64) + __U); +} + +extern __inline __m512i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_permutex2var_epi8 (__mmask64 __U, __m512i __A, + __m512i __I, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpermt2varqi512_maskz ((__v64qi) __I + /* idx */ , + (__v64qi) __A, + (__v64qi) __B, + (__mmask64) + __U); +} + +#ifdef __DISABLE_AVX512VBMI__ +#undef __DISABLE_AVX512VBMI__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX512VBMI__ */ + +#endif /* _AVX512VBMIINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512vbmivlintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512vbmivlintrin.h new file mode 100644 index 0000000000..58f0cb26f2 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512vbmivlintrin.h @@ -0,0 +1,273 @@ +/* Copyright (C) 2013-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +#error "Never use directly; include instead." +#endif + +#ifndef _AVX512VBMIVLINTRIN_H_INCLUDED +#define _AVX512VBMIVLINTRIN_H_INCLUDED + +#if !defined(__AVX512VL__) || !defined(__AVX512VBMI__) +#pragma GCC push_options +//#pragma GCC target("avx512vbmi,avx512vl") +#define __DISABLE_AVX512VBMIVL__ +#endif /* __AVX512VBMIVL__ */ + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_multishift_epi64_epi8 (__m256i __W, __mmask32 __M, __m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_vpmultishiftqb256_mask ((__v32qi) __X, + (__v32qi) __Y, + (__v32qi) __W, + (__mmask32) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_multishift_epi64_epi8 (__mmask32 __M, __m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_vpmultishiftqb256_mask ((__v32qi) __X, + (__v32qi) __Y, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_multishift_epi64_epi8 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_vpmultishiftqb256_mask ((__v32qi) __X, + (__v32qi) __Y, + (__v32qi) + _mm256_undefined_si256 (), + (__mmask32) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_multishift_epi64_epi8 (__m128i __W, __mmask16 __M, __m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_vpmultishiftqb128_mask ((__v16qi) __X, + (__v16qi) __Y, + (__v16qi) __W, + (__mmask16) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_multishift_epi64_epi8 (__mmask16 __M, __m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_vpmultishiftqb128_mask ((__v16qi) __X, + (__v16qi) __Y, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_multishift_epi64_epi8 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_vpmultishiftqb128_mask ((__v16qi) __X, + (__v16qi) __Y, + (__v16qi) + _mm_undefined_si128 (), + (__mmask16) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permutexvar_epi8 (__m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_permvarqi256_mask ((__v32qi) __B, + (__v32qi) __A, + (__v32qi) + _mm256_undefined_si256 (), + (__mmask32) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permutexvar_epi8 (__mmask32 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_permvarqi256_mask ((__v32qi) __B, + (__v32qi) __A, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permutexvar_epi8 (__m256i __W, __mmask32 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_permvarqi256_mask ((__v32qi) __B, + (__v32qi) __A, + (__v32qi) __W, + (__mmask32) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permutexvar_epi8 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_permvarqi128_mask ((__v16qi) __B, + (__v16qi) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask16) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_permutexvar_epi8 (__mmask16 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_permvarqi128_mask ((__v16qi) __B, + (__v16qi) __A, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_permutexvar_epi8 (__m128i __W, __mmask16 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_permvarqi128_mask ((__v16qi) __B, + (__v16qi) __A, + (__v16qi) __W, + (__mmask16) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permutex2var_epi8 (__m256i __A, __m256i __I, __m256i __B) +{ + return (__m256i) __builtin_ia32_vpermt2varqi256_mask ((__v32qi) __I + /* idx */ , + (__v32qi) __A, + (__v32qi) __B, + (__mmask32) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permutex2var_epi8 (__m256i __A, __mmask32 __U, + __m256i __I, __m256i __B) +{ + return (__m256i) __builtin_ia32_vpermt2varqi256_mask ((__v32qi) __I + /* idx */ , + (__v32qi) __A, + (__v32qi) __B, + (__mmask32) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask2_permutex2var_epi8 (__m256i __A, __m256i __I, + __mmask32 __U, __m256i __B) +{ + return (__m256i) __builtin_ia32_vpermi2varqi256_mask ((__v32qi) __A, + (__v32qi) __I + /* idx */ , + (__v32qi) __B, + (__mmask32) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permutex2var_epi8 (__mmask32 __U, __m256i __A, + __m256i __I, __m256i __B) +{ + return (__m256i) __builtin_ia32_vpermt2varqi256_maskz ((__v32qi) __I + /* idx */ , + (__v32qi) __A, + (__v32qi) __B, + (__mmask32) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permutex2var_epi8 (__m128i __A, __m128i __I, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpermt2varqi128_mask ((__v16qi) __I + /* idx */ , + (__v16qi) __A, + (__v16qi) __B, + (__mmask16) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_permutex2var_epi8 (__m128i __A, __mmask16 __U, __m128i __I, + __m128i __B) +{ + return (__m128i) __builtin_ia32_vpermt2varqi128_mask ((__v16qi) __I + /* idx */ , + (__v16qi) __A, + (__v16qi) __B, + (__mmask16) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask2_permutex2var_epi8 (__m128i __A, __m128i __I, __mmask16 __U, + __m128i __B) +{ + return (__m128i) __builtin_ia32_vpermi2varqi128_mask ((__v16qi) __A, + (__v16qi) __I + /* idx */ , + (__v16qi) __B, + (__mmask16) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_permutex2var_epi8 (__mmask16 __U, __m128i __A, __m128i __I, + __m128i __B) +{ + return (__m128i) __builtin_ia32_vpermt2varqi128_maskz ((__v16qi) __I + /* idx */ , + (__v16qi) __A, + (__v16qi) __B, + (__mmask16) + __U); +} + +#ifdef __DISABLE_AVX512VBMIVL__ +#undef __DISABLE_AVX512VBMIVL__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX512VBMIVL__ */ + +#endif /* _AVX512VBMIVLINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512vlbwintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512vlbwintrin.h new file mode 100644 index 0000000000..6836814ff3 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512vlbwintrin.h @@ -0,0 +1,4651 @@ +/* Copyright (C) 2014-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +#error "Never use directly; include instead." +#endif + +#ifndef _AVX512VLBWINTRIN_H_INCLUDED +#define _AVX512VLBWINTRIN_H_INCLUDED + +#if !defined(__AVX512VL__) || !defined(__AVX512BW__) +#pragma GCC push_options +//#pragma GCC target("avx512vl,avx512bw") +#define __DISABLE_AVX512VLBW__ +#endif /* __AVX512VLBW__ */ + + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_mov_epi8 (__m256i __W, __mmask32 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_movdquqi256_mask ((__v32qi) __A, + (__v32qi) __W, + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_mov_epi8 (__mmask32 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_movdquqi256_mask ((__v32qi) __A, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_mov_epi8 (__m128i __W, __mmask16 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_movdquqi128_mask ((__v16qi) __A, + (__v16qi) __W, + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_mov_epi8 (__mmask16 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_movdquqi128_mask ((__v16qi) __A, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_storeu_epi8 (void *__P, __mmask32 __U, __m256i __A) +{ + __builtin_ia32_storedquqi256_mask ((char *) __P, + (__v32qi) __A, + (__mmask32) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_storeu_epi8 (void *__P, __mmask16 __U, __m128i __A) +{ + __builtin_ia32_storedquqi128_mask ((char *) __P, + (__v16qi) __A, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_loadu_epi16 (__m256i __W, __mmask16 __U, void const *__P) +{ + return (__m256i) __builtin_ia32_loaddquhi256_mask ((const short *) __P, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_loadu_epi16 (__mmask16 __U, void const *__P) +{ + return (__m256i) __builtin_ia32_loaddquhi256_mask ((const short *) __P, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_loadu_epi16 (__m128i __W, __mmask8 __U, void const *__P) +{ + return (__m128i) __builtin_ia32_loaddquhi128_mask ((const short *) __P, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_loadu_epi16 (__mmask8 __U, void const *__P) +{ + return (__m128i) __builtin_ia32_loaddquhi128_mask ((const short *) __P, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_mov_epi16 (__m256i __W, __mmask16 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_movdquhi256_mask ((__v16hi) __A, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_mov_epi16 (__mmask16 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_movdquhi256_mask ((__v16hi) __A, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_mov_epi16 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_movdquhi128_mask ((__v8hi) __A, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_mov_epi16 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_movdquhi128_mask ((__v8hi) __A, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_loadu_epi8 (__m256i __W, __mmask32 __U, void const *__P) +{ + return (__m256i) __builtin_ia32_loaddquqi256_mask ((const char *) __P, + (__v32qi) __W, + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_loadu_epi8 (__mmask32 __U, void const *__P) +{ + return (__m256i) __builtin_ia32_loaddquqi256_mask ((const char *) __P, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_loadu_epi8 (__m128i __W, __mmask16 __U, void const *__P) +{ + return (__m128i) __builtin_ia32_loaddquqi128_mask ((const char *) __P, + (__v16qi) __W, + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_loadu_epi8 (__mmask16 __U, void const *__P) +{ + return (__m128i) __builtin_ia32_loaddquqi128_mask ((const char *) __P, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi16_epi8 (__m256i __A) +{ + + return (__m128i) __builtin_ia32_pmovwb256_mask ((__v16hi) __A, + (__v16qi)_mm_undefined_si128(), + (__mmask16) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi16_epi8 (__m128i __O, __mmask16 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovwb256_mask ((__v16hi) __A, + (__v16qi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepi16_epi8 (__mmask16 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovwb256_mask ((__v16hi) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsepi16_epi8 (__m128i __A) +{ + + return (__m128i) __builtin_ia32_pmovswb128_mask ((__v8hi) __A, + (__v16qi)_mm_undefined_si128(), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtsepi16_epi8 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovswb128_mask ((__v8hi) __A, + (__v16qi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtsepi16_epi8 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovswb128_mask ((__v8hi) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtsepi16_epi8 (__m256i __A) +{ + + return (__m128i) __builtin_ia32_pmovswb256_mask ((__v16hi) __A, + (__v16qi)_mm_undefined_si128(), + (__mmask16) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtsepi16_epi8 (__m128i __O, __mmask16 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovswb256_mask ((__v16hi) __A, + (__v16qi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtsepi16_epi8 (__mmask16 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovswb256_mask ((__v16hi) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtusepi16_epi8 (__m128i __A) +{ + + return (__m128i) __builtin_ia32_pmovuswb128_mask ((__v8hi) __A, + (__v16qi)_mm_undefined_si128(), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtusepi16_epi8 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovuswb128_mask ((__v8hi) __A, + (__v16qi) __O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtusepi16_epi8 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovuswb128_mask ((__v8hi) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtusepi16_epi8 (__m256i __A) +{ + + return (__m128i) __builtin_ia32_pmovuswb256_mask ((__v16hi) __A, + (__v16qi)_mm_undefined_si128(), + (__mmask16) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtusepi16_epi8 (__m128i __O, __mmask16 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovuswb256_mask ((__v16hi) __A, + (__v16qi) __O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtusepi16_epi8 (__mmask16 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovuswb256_mask ((__v16hi) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_broadcastb_epi8 (__m256i __O, __mmask32 __M, __m128i __A) +{ + return (__m256i) __builtin_ia32_pbroadcastb256_mask ((__v16qi) __A, + (__v32qi) __O, + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_broadcastb_epi8 (__mmask32 __M, __m128i __A) +{ + return (__m256i) __builtin_ia32_pbroadcastb256_mask ((__v16qi) __A, + (__v32qi) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_set1_epi8 (__m256i __O, __mmask32 __M, char __A) +{ + return (__m256i) __builtin_ia32_pbroadcastb256_gpr_mask (__A, + (__v32qi) __O, + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_set1_epi8 (__mmask32 __M, char __A) +{ + return (__m256i) __builtin_ia32_pbroadcastb256_gpr_mask (__A, + (__v32qi) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_broadcastb_epi8 (__m128i __O, __mmask16 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pbroadcastb128_mask ((__v16qi) __A, + (__v16qi) __O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_broadcastb_epi8 (__mmask16 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pbroadcastb128_mask ((__v16qi) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_set1_epi8 (__m128i __O, __mmask16 __M, char __A) +{ + return (__m128i) __builtin_ia32_pbroadcastb128_gpr_mask (__A, + (__v16qi) __O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_set1_epi8 (__mmask16 __M, char __A) +{ + return (__m128i) __builtin_ia32_pbroadcastb128_gpr_mask (__A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_broadcastw_epi16 (__m256i __O, __mmask16 __M, __m128i __A) +{ + return (__m256i) __builtin_ia32_pbroadcastw256_mask ((__v8hi) __A, + (__v16hi) __O, + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_broadcastw_epi16 (__mmask16 __M, __m128i __A) +{ + return (__m256i) __builtin_ia32_pbroadcastw256_mask ((__v8hi) __A, + (__v16hi) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_set1_epi16 (__m256i __O, __mmask16 __M, short __A) +{ + return (__m256i) __builtin_ia32_pbroadcastw256_gpr_mask (__A, + (__v16hi) __O, + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_set1_epi16 (__mmask16 __M, short __A) +{ + return (__m256i) __builtin_ia32_pbroadcastw256_gpr_mask (__A, + (__v16hi) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_broadcastw_epi16 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pbroadcastw128_mask ((__v8hi) __A, + (__v8hi) __O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_broadcastw_epi16 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pbroadcastw128_mask ((__v8hi) __A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_set1_epi16 (__m128i __O, __mmask8 __M, short __A) +{ + return (__m128i) __builtin_ia32_pbroadcastw128_gpr_mask (__A, + (__v8hi) __O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_set1_epi16 (__mmask8 __M, short __A) +{ + return (__m128i) __builtin_ia32_pbroadcastw128_gpr_mask (__A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permutexvar_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_permvarhi256_mask ((__v16hi) __B, + (__v16hi) __A, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permutexvar_epi16 (__mmask16 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_permvarhi256_mask ((__v16hi) __B, + (__v16hi) __A, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permutexvar_epi16 (__m256i __W, __mmask16 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_permvarhi256_mask ((__v16hi) __B, + (__v16hi) __A, + (__v16hi) __W, + (__mmask16) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permutexvar_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_permvarhi128_mask ((__v8hi) __B, + (__v8hi) __A, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_permutexvar_epi16 (__mmask8 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_permvarhi128_mask ((__v8hi) __B, + (__v8hi) __A, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_permutexvar_epi16 (__m128i __W, __mmask8 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_permvarhi128_mask ((__v8hi) __B, + (__v8hi) __A, + (__v8hi) __W, + (__mmask8) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permutex2var_epi16 (__m256i __A, __m256i __I, __m256i __B) +{ + return (__m256i) __builtin_ia32_vpermt2varhi256_mask ((__v16hi) __I + /* idx */ , + (__v16hi) __A, + (__v16hi) __B, + (__mmask16) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permutex2var_epi16 (__m256i __A, __mmask16 __U, + __m256i __I, __m256i __B) +{ + return (__m256i) __builtin_ia32_vpermt2varhi256_mask ((__v16hi) __I + /* idx */ , + (__v16hi) __A, + (__v16hi) __B, + (__mmask16) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask2_permutex2var_epi16 (__m256i __A, __m256i __I, + __mmask16 __U, __m256i __B) +{ + return (__m256i) __builtin_ia32_vpermi2varhi256_mask ((__v16hi) __A, + (__v16hi) __I + /* idx */ , + (__v16hi) __B, + (__mmask16) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permutex2var_epi16 (__mmask16 __U, __m256i __A, + __m256i __I, __m256i __B) +{ + return (__m256i) __builtin_ia32_vpermt2varhi256_maskz ((__v16hi) __I + /* idx */ , + (__v16hi) __A, + (__v16hi) __B, + (__mmask16) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permutex2var_epi16 (__m128i __A, __m128i __I, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpermt2varhi128_mask ((__v8hi) __I + /* idx */ , + (__v8hi) __A, + (__v8hi) __B, + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_permutex2var_epi16 (__m128i __A, __mmask8 __U, __m128i __I, + __m128i __B) +{ + return (__m128i) __builtin_ia32_vpermt2varhi128_mask ((__v8hi) __I + /* idx */ , + (__v8hi) __A, + (__v8hi) __B, + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask2_permutex2var_epi16 (__m128i __A, __m128i __I, __mmask8 __U, + __m128i __B) +{ + return (__m128i) __builtin_ia32_vpermi2varhi128_mask ((__v8hi) __A, + (__v8hi) __I + /* idx */ , + (__v8hi) __B, + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_permutex2var_epi16 (__mmask8 __U, __m128i __A, __m128i __I, + __m128i __B) +{ + return (__m128i) __builtin_ia32_vpermt2varhi128_maskz ((__v8hi) __I + /* idx */ , + (__v8hi) __A, + (__v8hi) __B, + (__mmask8) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_maddubs_epi16 (__m256i __W, __mmask16 __U, __m256i __X, + __m256i __Y) +{ + return (__m256i) __builtin_ia32_pmaddubsw256_mask ((__v32qi) __X, + (__v32qi) __Y, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_maddubs_epi16 (__mmask16 __U, __m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_pmaddubsw256_mask ((__v32qi) __X, + (__v32qi) __Y, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_maddubs_epi16 (__m128i __W, __mmask8 __U, __m128i __X, + __m128i __Y) +{ + return (__m128i) __builtin_ia32_pmaddubsw128_mask ((__v16qi) __X, + (__v16qi) __Y, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_maddubs_epi16 (__mmask8 __U, __m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_pmaddubsw128_mask ((__v16qi) __X, + (__v16qi) __Y, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_madd_epi16 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaddwd256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_madd_epi16 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaddwd256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_madd_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaddwd128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_madd_epi16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaddwd128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movepi8_mask (__m128i __A) +{ + return (__mmask16) __builtin_ia32_cvtb2mask128 ((__v16qi) __A); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_movepi8_mask (__m256i __A) +{ + return (__mmask32) __builtin_ia32_cvtb2mask256 ((__v32qi) __A); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movepi16_mask (__m128i __A) +{ + return (__mmask8) __builtin_ia32_cvtw2mask128 ((__v8hi) __A); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_movepi16_mask (__m256i __A) +{ + return (__mmask16) __builtin_ia32_cvtw2mask256 ((__v16hi) __A); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movm_epi8 (__mmask16 __A) +{ + return (__m128i) __builtin_ia32_cvtmask2b128 (__A); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_movm_epi8 (__mmask32 __A) +{ + return (__m256i) __builtin_ia32_cvtmask2b256 (__A); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movm_epi16 (__mmask8 __A) +{ + return (__m128i) __builtin_ia32_cvtmask2w128 (__A); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_movm_epi16 (__mmask16 __A) +{ + return (__m256i) __builtin_ia32_cvtmask2w256 (__A); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_test_epi8_mask (__m128i __A, __m128i __B) +{ + return (__mmask16) __builtin_ia32_ptestmb128 ((__v16qi) __A, + (__v16qi) __B, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_test_epi8_mask (__mmask16 __U, __m128i __A, __m128i __B) +{ + return (__mmask16) __builtin_ia32_ptestmb128 ((__v16qi) __A, + (__v16qi) __B, __U); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_test_epi8_mask (__m256i __A, __m256i __B) +{ + return (__mmask32) __builtin_ia32_ptestmb256 ((__v32qi) __A, + (__v32qi) __B, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_test_epi8_mask (__mmask32 __U, __m256i __A, __m256i __B) +{ + return (__mmask32) __builtin_ia32_ptestmb256 ((__v32qi) __A, + (__v32qi) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_test_epi16_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ptestmw128 ((__v8hi) __A, + (__v8hi) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_test_epi16_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ptestmw128 ((__v8hi) __A, + (__v8hi) __B, __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_test_epi16_mask (__m256i __A, __m256i __B) +{ + return (__mmask16) __builtin_ia32_ptestmw256 ((__v16hi) __A, + (__v16hi) __B, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_test_epi16_mask (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__mmask16) __builtin_ia32_ptestmw256 ((__v16hi) __A, + (__v16hi) __B, __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_min_epu16 (__mmask16 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pminuw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_min_epu16 (__m256i __W, __mmask16 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pminuw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_min_epu16 (__mmask8 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pminuw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_min_epu16 (__m128i __W, __mmask8 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pminuw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_min_epi16 (__mmask16 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pminsw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_min_epi16 (__m256i __W, __mmask16 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pminsw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_max_epu8 (__mmask32 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxub256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_max_epu8 (__m256i __W, __mmask32 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxub256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) __W, + (__mmask32) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_max_epu8 (__mmask16 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxub128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_max_epu8 (__m128i __W, __mmask16 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxub128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) __W, + (__mmask16) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_max_epi8 (__mmask32 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxsb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_max_epi8 (__m256i __W, __mmask32 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxsb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) __W, + (__mmask32) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_max_epi8 (__mmask16 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxsb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_max_epi8 (__m128i __W, __mmask16 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxsb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) __W, + (__mmask16) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_min_epu8 (__mmask32 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pminub256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_min_epu8 (__m256i __W, __mmask32 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pminub256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) __W, + (__mmask32) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_min_epu8 (__mmask16 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pminub128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_min_epu8 (__m128i __W, __mmask16 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pminub128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) __W, + (__mmask16) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_min_epi8 (__mmask32 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pminsb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_min_epi8 (__m256i __W, __mmask32 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pminsb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) __W, + (__mmask32) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_min_epi8 (__mmask16 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pminsb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_min_epi8 (__m128i __W, __mmask16 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pminsb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) __W, + (__mmask16) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_max_epi16 (__mmask16 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxsw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_max_epi16 (__m256i __W, __mmask16 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxsw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_max_epi16 (__mmask8 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxsw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_max_epi16 (__m128i __W, __mmask8 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxsw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_max_epu16 (__mmask16 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxuw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_max_epu16 (__m256i __W, __mmask16 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxuw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_max_epu16 (__mmask8 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxuw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_max_epu16 (__m128i __W, __mmask8 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxuw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_min_epi16 (__mmask8 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pminsw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_min_epi16 (__m128i __W, __mmask8 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pminsw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __M); +} + +#ifdef __OPTIMIZE__ +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_alignr_epi8 (__m256i __W, __mmask32 __U, __m256i __A, + __m256i __B, const int __N) +{ + return (__m256i) __builtin_ia32_palignr256_mask ((__v4di) __A, + (__v4di) __B, + __N * 8, + (__v4di) __W, + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_alignr_epi8 (__mmask32 __U, __m256i __A, __m256i __B, + const int __N) +{ + return (__m256i) __builtin_ia32_palignr256_mask ((__v4di) __A, + (__v4di) __B, + __N * 8, + (__v4di) + _mm256_setzero_si256 (), + (__mmask32) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_alignr_epi8 (__m128i __W, __mmask16 __U, __m128i __A, + __m128i __B, const int __N) +{ + return (__m128i) __builtin_ia32_palignr128_mask ((__v2di) __A, + (__v2di) __B, + __N * 8, + (__v2di) __W, + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_alignr_epi8 (__mmask16 __U, __m128i __A, __m128i __B, + const int __N) +{ + return (__m128i) __builtin_ia32_palignr128_mask ((__v2di) __A, + (__v2di) __B, + __N * 8, + (__v2di) + _mm_setzero_si128 (), + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_dbsad_epu8 (__m256i __A, __m256i __B, const int __imm) +{ + return (__m256i) __builtin_ia32_dbpsadbw256_mask ((__v32qi) __A, + (__v32qi) __B, + __imm, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_dbsad_epu8 (__m256i __W, __mmask16 __U, __m256i __A, + __m256i __B, const int __imm) +{ + return (__m256i) __builtin_ia32_dbpsadbw256_mask ((__v32qi) __A, + (__v32qi) __B, + __imm, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_dbsad_epu8 (__mmask16 __U, __m256i __A, __m256i __B, + const int __imm) +{ + return (__m256i) __builtin_ia32_dbpsadbw256_mask ((__v32qi) __A, + (__v32qi) __B, + __imm, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_dbsad_epu8 (__m128i __A, __m128i __B, const int __imm) +{ + return (__m128i) __builtin_ia32_dbpsadbw128_mask ((__v16qi) __A, + (__v16qi) __B, + __imm, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_dbsad_epu8 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B, const int __imm) +{ + return (__m128i) __builtin_ia32_dbpsadbw128_mask ((__v16qi) __A, + (__v16qi) __B, + __imm, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_dbsad_epu8 (__mmask8 __U, __m128i __A, __m128i __B, + const int __imm) +{ + return (__m128i) __builtin_ia32_dbpsadbw128_mask ((__v16qi) __A, + (__v16qi) __B, + __imm, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_blend_epi16 (__mmask8 __U, __m128i __A, __m128i __W) +{ + return (__m128i) __builtin_ia32_blendmw_128_mask ((__v8hi) __A, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_blend_epi8 (__mmask16 __U, __m128i __A, __m128i __W) +{ + return (__m128i) __builtin_ia32_blendmb_128_mask ((__v16qi) __A, + (__v16qi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_blend_epi16 (__mmask16 __U, __m256i __A, __m256i __W) +{ + return (__m256i) __builtin_ia32_blendmw_256_mask ((__v16hi) __A, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_blend_epi8 (__mmask32 __U, __m256i __A, __m256i __W) +{ + return (__m256i) __builtin_ia32_blendmb_256_mask ((__v32qi) __A, + (__v32qi) __W, + (__mmask32) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmp_epi16_mask (__mmask8 __U, __m128i __X, __m128i __Y, + const int __P) +{ + return (__mmask8) __builtin_ia32_cmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, __P, + (__mmask8) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_epi16_mask (__m128i __X, __m128i __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_cmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, __P, + (__mmask8) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmp_epi16_mask (__mmask16 __U, __m256i __X, __m256i __Y, + const int __P) +{ + return (__mmask16) __builtin_ia32_cmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, __P, + (__mmask16) __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmp_epi16_mask (__m256i __X, __m256i __Y, const int __P) +{ + return (__mmask16) __builtin_ia32_cmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, __P, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmp_epi8_mask (__mmask8 __U, __m128i __X, __m128i __Y, + const int __P) +{ + return (__mmask16) __builtin_ia32_cmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, __P, + (__mmask16) __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_epi8_mask (__m128i __X, __m128i __Y, const int __P) +{ + return (__mmask16) __builtin_ia32_cmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, __P, + (__mmask16) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmp_epi8_mask (__mmask16 __U, __m256i __X, __m256i __Y, + const int __P) +{ + return (__mmask32) __builtin_ia32_cmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, __P, + (__mmask32) __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmp_epi8_mask (__m256i __X, __m256i __Y, const int __P) +{ + return (__mmask32) __builtin_ia32_cmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, __P, + (__mmask32) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmp_epu16_mask (__mmask8 __U, __m128i __X, __m128i __Y, + const int __P) +{ + return (__mmask8) __builtin_ia32_ucmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, __P, + (__mmask8) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_epu16_mask (__m128i __X, __m128i __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_ucmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, __P, + (__mmask8) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmp_epu16_mask (__mmask16 __U, __m256i __X, __m256i __Y, + const int __P) +{ + return (__mmask16) __builtin_ia32_ucmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, __P, + (__mmask16) __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmp_epu16_mask (__m256i __X, __m256i __Y, const int __P) +{ + return (__mmask16) __builtin_ia32_ucmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, __P, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmp_epu8_mask (__mmask8 __U, __m128i __X, __m128i __Y, + const int __P) +{ + return (__mmask16) __builtin_ia32_ucmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, __P, + (__mmask16) __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_epu8_mask (__m128i __X, __m128i __Y, const int __P) +{ + return (__mmask16) __builtin_ia32_ucmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, __P, + (__mmask16) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmp_epu8_mask (__mmask16 __U, __m256i __X, __m256i __Y, + const int __P) +{ + return (__mmask32) __builtin_ia32_ucmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, __P, + (__mmask32) __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmp_epu8_mask (__m256i __X, __m256i __Y, const int __P) +{ + return (__mmask32) __builtin_ia32_ucmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, __P, + (__mmask32) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_srli_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + const int __imm) +{ + return (__m256i) __builtin_ia32_psrlwi256_mask ((__v16hi) __A, __imm, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_srli_epi16 (__mmask16 __U, __m256i __A, const int __imm) +{ + return (__m256i) __builtin_ia32_psrlwi256_mask ((__v16hi) __A, __imm, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_srli_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + const int __imm) +{ + return (__m128i) __builtin_ia32_psrlwi128_mask ((__v8hi) __A, __imm, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_srli_epi16 (__mmask8 __U, __m128i __A, const int __imm) +{ + return (__m128i) __builtin_ia32_psrlwi128_mask ((__v8hi) __A, __imm, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_shufflehi_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + const int __imm) +{ + return (__m256i) __builtin_ia32_pshufhw256_mask ((__v16hi) __A, + __imm, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_shufflehi_epi16 (__mmask16 __U, __m256i __A, + const int __imm) +{ + return (__m256i) __builtin_ia32_pshufhw256_mask ((__v16hi) __A, + __imm, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_shufflehi_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + const int __imm) +{ + return (__m128i) __builtin_ia32_pshufhw128_mask ((__v8hi) __A, __imm, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_shufflehi_epi16 (__mmask8 __U, __m128i __A, const int __imm) +{ + return (__m128i) __builtin_ia32_pshufhw128_mask ((__v8hi) __A, __imm, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_shufflelo_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + const int __imm) +{ + return (__m256i) __builtin_ia32_pshuflw256_mask ((__v16hi) __A, + __imm, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_shufflelo_epi16 (__mmask16 __U, __m256i __A, + const int __imm) +{ + return (__m256i) __builtin_ia32_pshuflw256_mask ((__v16hi) __A, + __imm, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_shufflelo_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + const int __imm) +{ + return (__m128i) __builtin_ia32_pshuflw128_mask ((__v8hi) __A, __imm, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_shufflelo_epi16 (__mmask8 __U, __m128i __A, const int __imm) +{ + return (__m128i) __builtin_ia32_pshuflw128_mask ((__v8hi) __A, __imm, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_srai_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + const int __imm) +{ + return (__m256i) __builtin_ia32_psrawi256_mask ((__v16hi) __A, __imm, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_srai_epi16 (__mmask16 __U, __m256i __A, const int __imm) +{ + return (__m256i) __builtin_ia32_psrawi256_mask ((__v16hi) __A, __imm, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_srai_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + const int __imm) +{ + return (__m128i) __builtin_ia32_psrawi128_mask ((__v8hi) __A, __imm, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_srai_epi16 (__mmask8 __U, __m128i __A, const int __imm) +{ + return (__m128i) __builtin_ia32_psrawi128_mask ((__v8hi) __A, __imm, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_slli_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + int __B) +{ + return (__m256i) __builtin_ia32_psllwi256_mask ((__v16hi) __A, __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_slli_epi16 (__mmask16 __U, __m256i __A, int __B) +{ + return (__m256i) __builtin_ia32_psllwi256_mask ((__v16hi) __A, __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_slli_epi16 (__m128i __W, __mmask8 __U, __m128i __A, int __B) +{ + return (__m128i) __builtin_ia32_psllwi128_mask ((__v8hi) __A, __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_slli_epi16 (__mmask8 __U, __m128i __A, int __B) +{ + return (__m128i) __builtin_ia32_psllwi128_mask ((__v8hi) __A, __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +#else +#define _mm256_mask_alignr_epi8(W, U, X, Y, N) \ + ((__m256i) __builtin_ia32_palignr256_mask ((__v4di)(__m256i)(X), \ + (__v4di)(__m256i)(Y), (int)(N * 8), \ + (__v4di)(__m256i)(X), (__mmask32)(U))) + +#define _mm256_mask_srli_epi16(W, U, A, B) \ + ((__m256i) __builtin_ia32_psrlwi256_mask ((__v16hi)(__m256i)(A), \ + (int)(B), (__v16hi)(__m256i)(W), (__mmask16)(U))) + +#define _mm256_maskz_srli_epi16(U, A, B) \ + ((__m256i) __builtin_ia32_psrlwi256_mask ((__v16hi)(__m256i)(A), \ + (int)(B), (__v16hi)_mm256_setzero_si256 (), (__mmask16)(U))) + +#define _mm_mask_srli_epi16(W, U, A, B) \ + ((__m128i) __builtin_ia32_psrlwi128_mask ((__v8hi)(__m128i)(A), \ + (int)(B), (__v8hi)(__m128i)(W), (__mmask8)(U))) + +#define _mm_maskz_srli_epi16(U, A, B) \ + ((__m128i) __builtin_ia32_psrlwi128_mask ((__v8hi)(__m128i)(A), \ + (int)(B), (__v8hi)_mm_setzero_si128(), (__mmask8)(U))) + +#define _mm256_mask_srai_epi16(W, U, A, B) \ + ((__m256i) __builtin_ia32_psrawi256_mask ((__v16hi)(__m256i)(A), \ + (int)(B), (__v16hi)(__m256i)(W), (__mmask16)(U))) + +#define _mm256_maskz_srai_epi16(U, A, B) \ + ((__m256i) __builtin_ia32_psrawi256_mask ((__v16hi)(__m256i)(A), \ + (int)(B), (__v16hi)_mm256_setzero_si256 (), (__mmask16)(U))) + +#define _mm_mask_srai_epi16(W, U, A, B) \ + ((__m128i) __builtin_ia32_psrawi128_mask ((__v8hi)(__m128i)(A), \ + (int)(B), (__v8hi)(__m128i)(W), (__mmask8)(U))) + +#define _mm_maskz_srai_epi16(U, A, B) \ + ((__m128i) __builtin_ia32_psrawi128_mask ((__v8hi)(__m128i)(A), \ + (int)(B), (__v8hi)_mm_setzero_si128(), (__mmask8)(U))) + +#define _mm256_mask_shufflehi_epi16(W, U, A, B) \ + ((__m256i) __builtin_ia32_pshufhw256_mask ((__v16hi)(__m256i)(A), (int)(B), \ + (__v16hi)(__m256i)(W), \ + (__mmask16)(U))) + +#define _mm256_maskz_shufflehi_epi16(U, A, B) \ + ((__m256i) __builtin_ia32_pshufhw256_mask ((__v16hi)(__m256i)(A), (int)(B), \ + (__v16hi)(__m256i)_mm256_setzero_si256 (), \ + (__mmask16)(U))) + +#define _mm_mask_shufflehi_epi16(W, U, A, B) \ + ((__m128i) __builtin_ia32_pshufhw128_mask ((__v8hi)(__m128i)(A), (int)(B), \ + (__v8hi)(__m128i)(W), \ + (__mmask8)(U))) + +#define _mm_maskz_shufflehi_epi16(U, A, B) \ + ((__m128i) __builtin_ia32_pshufhw128_mask ((__v8hi)(__m128i)(A), (int)(B), \ + (__v8hi)(__m128i)_mm_setzero_si128 (), \ + (__mmask8)(U))) + +#define _mm256_mask_shufflelo_epi16(W, U, A, B) \ + ((__m256i) __builtin_ia32_pshuflw256_mask ((__v16hi)(__m256i)(A), (int)(B), \ + (__v16hi)(__m256i)(W), \ + (__mmask16)(U))) + +#define _mm256_maskz_shufflelo_epi16(U, A, B) \ + ((__m256i) __builtin_ia32_pshuflw256_mask ((__v16hi)(__m256i)(A), (int)(B), \ + (__v16hi)(__m256i)_mm256_setzero_si256 (), \ + (__mmask16)(U))) + +#define _mm_mask_shufflelo_epi16(W, U, A, B) \ + ((__m128i) __builtin_ia32_pshuflw128_mask ((__v8hi)(__m128i)(A), (int)(B), \ + (__v8hi)(__m128i)(W), \ + (__mmask8)(U))) + +#define _mm_maskz_shufflelo_epi16(U, A, B) \ + ((__m128i) __builtin_ia32_pshuflw128_mask ((__v8hi)(__m128i)(A), (int)(B), \ + (__v8hi)(__m128i)_mm_setzero_si128 (), \ + (__mmask8)(U))) + +#define _mm256_maskz_alignr_epi8(U, X, Y, N) \ + ((__m256i) __builtin_ia32_palignr256_mask ((__v4di)(__m256i)(X), \ + (__v4di)(__m256i)(Y), (int)(N * 8), \ + (__v4di)(__m256i)_mm256_setzero_si256 (), \ + (__mmask32)(U))) + +#define _mm_mask_alignr_epi8(W, U, X, Y, N) \ + ((__m128i) __builtin_ia32_palignr128_mask ((__v2di)(__m128i)(X), \ + (__v2di)(__m128i)(Y), (int)(N * 8), \ + (__v2di)(__m128i)(X), (__mmask16)(U))) + +#define _mm_maskz_alignr_epi8(U, X, Y, N) \ + ((__m128i) __builtin_ia32_palignr128_mask ((__v2di)(__m128i)(X), \ + (__v2di)(__m128i)(Y), (int)(N * 8), \ + (__v2di)(__m128i)_mm_setzero_si128 (), \ + (__mmask16)(U))) + +#define _mm_mask_slli_epi16(W, U, X, C) \ + ((__m128i)__builtin_ia32_psllwi128_mask ((__v8hi)(__m128i)(X), (int)(C),\ + (__v8hi)(__m128i)(W),\ + (__mmask8)(U))) + +#define _mm_maskz_slli_epi16(U, X, C) \ + ((__m128i)__builtin_ia32_psllwi128_mask ((__v8hi)(__m128i)(X), (int)(C),\ + (__v8hi)(__m128i)_mm_setzero_si128 (),\ + (__mmask8)(U))) + +#define _mm256_dbsad_epu8(X, Y, C) \ + ((__m256i) __builtin_ia32_dbpsadbw256_mask ((__v32qi)(__m256i) (X), \ + (__v32qi)(__m256i) (Y), (int) (C), \ + (__v16hi)(__m256i)_mm256_setzero_si256(),\ + (__mmask16)-1)) + +#define _mm256_mask_slli_epi16(W, U, X, C) \ + ((__m256i)__builtin_ia32_psllwi256_mask ((__v16hi)(__m256i)(X), (int)(C),\ + (__v16hi)(__m256i)(W),\ + (__mmask16)(U))) + +#define _mm256_maskz_slli_epi16(U, X, C) \ + ((__m256i)__builtin_ia32_psllwi256_mask ((__v16hi)(__m256i)(X), (int)(C),\ + (__v16hi)(__m256i)_mm256_setzero_si256 (),\ + (__mmask16)(U))) + +#define _mm256_mask_dbsad_epu8(W, U, X, Y, C) \ + ((__m256i) __builtin_ia32_dbpsadbw256_mask ((__v32qi)(__m256i) (X), \ + (__v32qi)(__m256i) (Y), (int) (C), \ + (__v16hi)(__m256i)(W), \ + (__mmask16)(U))) + +#define _mm256_maskz_dbsad_epu8(U, X, Y, C) \ + ((__m256i) __builtin_ia32_dbpsadbw256_mask ((__v32qi)(__m256i) (X), \ + (__v32qi)(__m256i) (Y), (int) (C), \ + (__v16hi)(__m256i)_mm256_setzero_si256(),\ + (__mmask16)(U))) + +#define _mm_dbsad_epu8(X, Y, C) \ + ((__m128i) __builtin_ia32_dbpsadbw128_mask ((__v16qi)(__m128i) (X), \ + (__v16qi)(__m128i) (Y), (int) (C), \ + (__v8hi)(__m128i)_mm_setzero_si128(), \ + (__mmask8)-1)) + +#define _mm_mask_dbsad_epu8(W, U, X, Y, C) \ + ((__m128i) __builtin_ia32_dbpsadbw128_mask ((__v16qi)(__m128i) (X), \ + (__v16qi)(__m128i) (Y), (int) (C), \ + (__v8hi)(__m128i)(W), \ + (__mmask8)(U))) + +#define _mm_maskz_dbsad_epu8(U, X, Y, C) \ + ((__m128i) __builtin_ia32_dbpsadbw128_mask ((__v16qi)(__m128i) (X), \ + (__v16qi)(__m128i) (Y), (int) (C), \ + (__v8hi)(__m128i)_mm_setzero_si128(), \ + (__mmask8)(U))) + +#define _mm_mask_blend_epi16(__U, __A, __W) \ + ((__m128i) __builtin_ia32_blendmw_128_mask ((__v8hi) (__A), \ + (__v8hi) (__W), \ + (__mmask8) (__U))) + +#define _mm_mask_blend_epi8(__U, __A, __W) \ + ((__m128i) __builtin_ia32_blendmb_128_mask ((__v16qi) (__A), \ + (__v16qi) (__W), \ + (__mmask16) (__U))) + +#define _mm256_mask_blend_epi16(__U, __A, __W) \ + ((__m256i) __builtin_ia32_blendmw_256_mask ((__v16hi) (__A), \ + (__v16hi) (__W), \ + (__mmask16) (__U))) + +#define _mm256_mask_blend_epi8(__U, __A, __W) \ + ((__m256i) __builtin_ia32_blendmb_256_mask ((__v32qi) (__A), \ + (__v32qi) (__W), \ + (__mmask32) (__U))) + +#define _mm_cmp_epi16_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpw128_mask ((__v8hi)(__m128i)(X), \ + (__v8hi)(__m128i)(Y), (int)(P),\ + (__mmask8)(-1))) + +#define _mm_cmp_epi8_mask(X, Y, P) \ + ((__mmask16) __builtin_ia32_cmpb128_mask ((__v16qi)(__m128i)(X), \ + (__v16qi)(__m128i)(Y), (int)(P),\ + (__mmask16)(-1))) + +#define _mm256_cmp_epi16_mask(X, Y, P) \ + ((__mmask16) __builtin_ia32_cmpw256_mask ((__v16hi)(__m256i)(X), \ + (__v16hi)(__m256i)(Y), (int)(P),\ + (__mmask16)(-1))) + +#define _mm256_cmp_epi8_mask(X, Y, P) \ + ((__mmask32) __builtin_ia32_cmpb256_mask ((__v32qi)(__m256i)(X), \ + (__v32qi)(__m256i)(Y), (int)(P),\ + (__mmask32)(-1))) + +#define _mm_cmp_epu16_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_ucmpw128_mask ((__v8hi)(__m128i)(X), \ + (__v8hi)(__m128i)(Y), (int)(P),\ + (__mmask8)(-1))) + +#define _mm_cmp_epu8_mask(X, Y, P) \ + ((__mmask16) __builtin_ia32_ucmpb128_mask ((__v16qi)(__m128i)(X), \ + (__v16qi)(__m128i)(Y), (int)(P),\ + (__mmask16)(-1))) + +#define _mm256_cmp_epu16_mask(X, Y, P) \ + ((__mmask16) __builtin_ia32_ucmpw256_mask ((__v16hi)(__m256i)(X), \ + (__v16hi)(__m256i)(Y), (int)(P),\ + (__mmask16)(-1))) + +#define _mm256_cmp_epu8_mask(X, Y, P) \ + ((__mmask32) __builtin_ia32_ucmpb256_mask ((__v32qi)(__m256i)(X), \ + (__v32qi)(__m256i)(Y), (int)(P),\ + (__mmask32)-1)) + +#define _mm_mask_cmp_epi16_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpw128_mask ((__v8hi)(__m128i)(X), \ + (__v8hi)(__m128i)(Y), (int)(P),\ + (__mmask16)(M))) + +#define _mm_mask_cmp_epi8_mask(M, X, Y, P) \ + ((__mmask16) __builtin_ia32_cmpb128_mask ((__v16qi)(__m128i)(X), \ + (__v16qi)(__m128i)(Y), (int)(P),\ + (__mmask16)(M))) + +#define _mm256_mask_cmp_epi16_mask(M, X, Y, P) \ + ((__mmask16) __builtin_ia32_cmpw256_mask ((__v16hi)(__m256i)(X), \ + (__v16hi)(__m256i)(Y), (int)(P),\ + (__mmask16)(M))) + +#define _mm256_mask_cmp_epi8_mask(M, X, Y, P) \ + ((__mmask32) __builtin_ia32_cmpb256_mask ((__v32qi)(__m256i)(X), \ + (__v32qi)(__m256i)(Y), (int)(P),\ + (__mmask32)(M))) + +#define _mm_mask_cmp_epu16_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_ucmpw128_mask ((__v8hi)(__m128i)(X), \ + (__v8hi)(__m128i)(Y), (int)(P),\ + (__mmask8)(M))) + +#define _mm_mask_cmp_epu8_mask(M, X, Y, P) \ + ((__mmask16) __builtin_ia32_ucmpb128_mask ((__v16qi)(__m128i)(X), \ + (__v16qi)(__m128i)(Y), (int)(P),\ + (__mmask16)(M))) + +#define _mm256_mask_cmp_epu16_mask(M, X, Y, P) \ + ((__mmask16) __builtin_ia32_ucmpw256_mask ((__v16hi)(__m256i)(X), \ + (__v16hi)(__m256i)(Y), (int)(P),\ + (__mmask16)(M))) + +#define _mm256_mask_cmp_epu8_mask(M, X, Y, P) \ + ((__mmask32) __builtin_ia32_ucmpb256_mask ((__v32qi)(__m256i)(X), \ + (__v32qi)(__m256i)(Y), (int)(P),\ + (__mmask32)M)) +#endif + +extern __inline __mmask32 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpneq_epi8_mask (__m256i __X, __m256i __Y) +{ + return (__mmask32) __builtin_ia32_cmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, 4, + (__mmask32) -1); +} + +extern __inline __mmask32 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmplt_epi8_mask (__m256i __X, __m256i __Y) +{ + return (__mmask32) __builtin_ia32_cmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, 1, + (__mmask32) -1); +} + +extern __inline __mmask32 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpge_epi8_mask (__m256i __X, __m256i __Y) +{ + return (__mmask32) __builtin_ia32_cmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, 5, + (__mmask32) -1); +} + +extern __inline __mmask32 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmple_epi8_mask (__m256i __X, __m256i __Y) +{ + return (__mmask32) __builtin_ia32_cmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, 2, + (__mmask32) -1); +} + +extern __inline __mmask16 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpneq_epi16_mask (__m256i __X, __m256i __Y) +{ + return (__mmask16) __builtin_ia32_cmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, 4, + (__mmask16) -1); +} + +extern __inline __mmask16 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmplt_epi16_mask (__m256i __X, __m256i __Y) +{ + return (__mmask16) __builtin_ia32_cmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, 1, + (__mmask16) -1); +} + +extern __inline __mmask16 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpge_epi16_mask (__m256i __X, __m256i __Y) +{ + return (__mmask16) __builtin_ia32_cmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, 5, + (__mmask16) -1); +} + +extern __inline __mmask16 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmple_epi16_mask (__m256i __X, __m256i __Y) +{ + return (__mmask16) __builtin_ia32_cmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, 2, + (__mmask16) -1); +} + +extern __inline __mmask16 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpneq_epu8_mask (__m128i __X, __m128i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, 4, + (__mmask16) -1); +} + +extern __inline __mmask16 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmplt_epu8_mask (__m128i __X, __m128i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, 1, + (__mmask16) -1); +} + +extern __inline __mmask16 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpge_epu8_mask (__m128i __X, __m128i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, 5, + (__mmask16) -1); +} + +extern __inline __mmask16 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmple_epu8_mask (__m128i __X, __m128i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, 2, + (__mmask16) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpneq_epu16_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, 4, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmplt_epu16_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, 1, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpge_epu16_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, 5, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmple_epu16_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, 2, + (__mmask8) -1); +} + +extern __inline __mmask16 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpneq_epi8_mask (__m128i __X, __m128i __Y) +{ + return (__mmask16) __builtin_ia32_cmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, 4, + (__mmask16) -1); +} + +extern __inline __mmask16 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmplt_epi8_mask (__m128i __X, __m128i __Y) +{ + return (__mmask16) __builtin_ia32_cmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, 1, + (__mmask16) -1); +} + +extern __inline __mmask16 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpge_epi8_mask (__m128i __X, __m128i __Y) +{ + return (__mmask16) __builtin_ia32_cmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, 5, + (__mmask16) -1); +} + +extern __inline __mmask16 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmple_epi8_mask (__m128i __X, __m128i __Y) +{ + return (__mmask16) __builtin_ia32_cmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, 2, + (__mmask16) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpneq_epi16_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, 4, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmplt_epi16_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, 1, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpge_epi16_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, 5, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmple_epi16_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, 2, + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_mulhrs_epi16 (__m256i __W, __mmask16 __U, __m256i __X, + __m256i __Y) +{ + return (__m256i) __builtin_ia32_pmulhrsw256_mask ((__v16hi) __X, + (__v16hi) __Y, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_mulhrs_epi16 (__mmask16 __U, __m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_pmulhrsw256_mask ((__v16hi) __X, + (__v16hi) __Y, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_mulhi_epu16 (__m256i __W, __mmask16 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pmulhuw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_mulhi_epu16 (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pmulhuw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_mulhi_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pmulhw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_mulhi_epi16 (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pmulhw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_mulhi_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pmulhw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_mulhi_epi16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pmulhw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_mulhi_epu16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pmulhuw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_mulhi_epu16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pmulhuw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_mulhrs_epi16 (__m128i __W, __mmask8 __U, __m128i __X, + __m128i __Y) +{ + return (__m128i) __builtin_ia32_pmulhrsw128_mask ((__v8hi) __X, + (__v8hi) __Y, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_mulhrs_epi16 (__mmask8 __U, __m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_pmulhrsw128_mask ((__v8hi) __X, + (__v8hi) __Y, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_mullo_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pmullw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_mullo_epi16 (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pmullw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_mullo_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pmullw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_mullo_epi16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pmullw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi8_epi16 (__m256i __W, __mmask32 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovsxbw256_mask ((__v16qi) __A, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepi8_epi16 (__mmask16 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovsxbw256_mask ((__v16qi) __A, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi8_epi16 (__m128i __W, __mmask32 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsxbw128_mask ((__v16qi) __A, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepi8_epi16 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsxbw128_mask ((__v16qi) __A, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepu8_epi16 (__m256i __W, __mmask32 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovzxbw256_mask ((__v16qi) __A, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepu8_epi16 (__mmask16 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovzxbw256_mask ((__v16qi) __A, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepu8_epi16 (__m128i __W, __mmask32 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovzxbw128_mask ((__v16qi) __A, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepu8_epi16 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovzxbw128_mask ((__v16qi) __A, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_avg_epu8 (__m256i __W, __mmask32 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pavgb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) __W, + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_avg_epu8 (__mmask32 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pavgb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_avg_epu8 (__m128i __W, __mmask16 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pavgb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) __W, + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_avg_epu8 (__mmask16 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pavgb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_avg_epu16 (__m256i __W, __mmask16 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pavgw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_avg_epu16 (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pavgw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_avg_epu16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pavgw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_avg_epu16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pavgw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_add_epi8 (__m256i __W, __mmask32 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_paddb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) __W, + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_add_epi8 (__mmask32 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_paddb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_add_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_paddw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_add_epi16 (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_paddw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_adds_epi8 (__m256i __W, __mmask32 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_paddsb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) __W, + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_adds_epi8 (__mmask32 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_paddsb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_adds_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_paddsw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_adds_epi16 (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_paddsw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_adds_epu8 (__m256i __W, __mmask32 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_paddusb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) __W, + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_adds_epu8 (__mmask32 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_paddusb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_adds_epu16 (__m256i __W, __mmask16 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_paddusw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_adds_epu16 (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_paddusw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sub_epi8 (__m256i __W, __mmask32 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_psubb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) __W, + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sub_epi8 (__mmask32 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_psubb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sub_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_psubw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sub_epi16 (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_psubw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_subs_epi8 (__m256i __W, __mmask32 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_psubsb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) __W, + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_subs_epi8 (__mmask32 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_psubsb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_subs_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_psubsw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_subs_epi16 (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_psubsw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_subs_epu8 (__m256i __W, __mmask32 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_psubusb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) __W, + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_subs_epu8 (__mmask32 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_psubusb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_subs_epu16 (__m256i __W, __mmask16 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_psubusw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_subs_epu16 (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_psubusw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_add_epi8 (__m128i __W, __mmask16 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_paddb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) __W, + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_add_epi8 (__mmask16 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_paddb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_add_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_paddw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_add_epi16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_paddw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_unpackhi_epi8 (__m256i __W, __mmask32 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_punpckhbw256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) __W, + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_unpackhi_epi8 (__mmask32 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_punpckhbw256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_unpackhi_epi8 (__m128i __W, __mmask16 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_punpckhbw128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) __W, + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_unpackhi_epi8 (__mmask16 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_punpckhbw128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_unpackhi_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_punpckhwd256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_unpackhi_epi16 (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_punpckhwd256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_unpackhi_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_punpckhwd128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_unpackhi_epi16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_punpckhwd128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_unpacklo_epi8 (__m256i __W, __mmask32 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_punpcklbw256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) __W, + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_unpacklo_epi8 (__mmask32 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_punpcklbw256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_unpacklo_epi8 (__m128i __W, __mmask16 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_punpcklbw128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) __W, + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_unpacklo_epi8 (__mmask16 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_punpcklbw128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_unpacklo_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_punpcklwd256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_unpacklo_epi16 (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_punpcklwd256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_unpacklo_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_punpcklwd128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_unpacklo_epi16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_punpcklwd128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_epi8_mask (__m128i __A, __m128i __B) +{ + return (__mmask16) __builtin_ia32_pcmpeqb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_epu8_mask (__m128i __A, __m128i __B) +{ + return (__mmask16) __builtin_ia32_ucmpb128_mask ((__v16qi) __A, + (__v16qi) __B, 0, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpeq_epu8_mask (__mmask16 __U, __m128i __A, __m128i __B) +{ + return (__mmask16) __builtin_ia32_ucmpb128_mask ((__v16qi) __A, + (__v16qi) __B, 0, + __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpeq_epi8_mask (__mmask16 __U, __m128i __A, __m128i __B) +{ + return (__mmask16) __builtin_ia32_pcmpeqb128_mask ((__v16qi) __A, + (__v16qi) __B, + __U); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpeq_epu8_mask (__m256i __A, __m256i __B) +{ + return (__mmask32) __builtin_ia32_ucmpb256_mask ((__v32qi) __A, + (__v32qi) __B, 0, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpeq_epi8_mask (__m256i __A, __m256i __B) +{ + return (__mmask32) __builtin_ia32_pcmpeqb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpeq_epu8_mask (__mmask32 __U, __m256i __A, __m256i __B) +{ + return (__mmask32) __builtin_ia32_ucmpb256_mask ((__v32qi) __A, + (__v32qi) __B, 0, + __U); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpeq_epi8_mask (__mmask32 __U, __m256i __A, __m256i __B) +{ + return (__mmask32) __builtin_ia32_pcmpeqb256_mask ((__v32qi) __A, + (__v32qi) __B, + __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_epu16_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ucmpw128_mask ((__v8hi) __A, + (__v8hi) __B, 0, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_epi16_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_pcmpeqw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpeq_epu16_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ucmpw128_mask ((__v8hi) __A, + (__v8hi) __B, 0, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpeq_epi16_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_pcmpeqw128_mask ((__v8hi) __A, + (__v8hi) __B, __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpeq_epu16_mask (__m256i __A, __m256i __B) +{ + return (__mmask16) __builtin_ia32_ucmpw256_mask ((__v16hi) __A, + (__v16hi) __B, 0, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpeq_epi16_mask (__m256i __A, __m256i __B) +{ + return (__mmask16) __builtin_ia32_pcmpeqw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpeq_epu16_mask (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__mmask16) __builtin_ia32_ucmpw256_mask ((__v16hi) __A, + (__v16hi) __B, 0, + __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpeq_epi16_mask (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__mmask16) __builtin_ia32_pcmpeqw256_mask ((__v16hi) __A, + (__v16hi) __B, + __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_epu8_mask (__m128i __A, __m128i __B) +{ + return (__mmask16) __builtin_ia32_ucmpb128_mask ((__v16qi) __A, + (__v16qi) __B, 6, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_epi8_mask (__m128i __A, __m128i __B) +{ + return (__mmask16) __builtin_ia32_pcmpgtb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpgt_epu8_mask (__mmask16 __U, __m128i __A, __m128i __B) +{ + return (__mmask16) __builtin_ia32_ucmpb128_mask ((__v16qi) __A, + (__v16qi) __B, 6, + __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpgt_epi8_mask (__mmask16 __U, __m128i __A, __m128i __B) +{ + return (__mmask16) __builtin_ia32_pcmpgtb128_mask ((__v16qi) __A, + (__v16qi) __B, + __U); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpgt_epu8_mask (__m256i __A, __m256i __B) +{ + return (__mmask32) __builtin_ia32_ucmpb256_mask ((__v32qi) __A, + (__v32qi) __B, 6, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpgt_epi8_mask (__m256i __A, __m256i __B) +{ + return (__mmask32) __builtin_ia32_pcmpgtb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpgt_epu8_mask (__mmask32 __U, __m256i __A, __m256i __B) +{ + return (__mmask32) __builtin_ia32_ucmpb256_mask ((__v32qi) __A, + (__v32qi) __B, 6, + __U); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpgt_epi8_mask (__mmask32 __U, __m256i __A, __m256i __B) +{ + return (__mmask32) __builtin_ia32_pcmpgtb256_mask ((__v32qi) __A, + (__v32qi) __B, + __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_epu16_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ucmpw128_mask ((__v8hi) __A, + (__v8hi) __B, 6, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_epi16_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_pcmpgtw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpgt_epu16_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ucmpw128_mask ((__v8hi) __A, + (__v8hi) __B, 6, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpgt_epi16_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_pcmpgtw128_mask ((__v8hi) __A, + (__v8hi) __B, __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpgt_epu16_mask (__m256i __A, __m256i __B) +{ + return (__mmask16) __builtin_ia32_ucmpw256_mask ((__v16hi) __A, + (__v16hi) __B, 6, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpgt_epi16_mask (__m256i __A, __m256i __B) +{ + return (__mmask16) __builtin_ia32_pcmpgtw256_mask ((__v16hi) __A, + (__v16hi) __B, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpgt_epu16_mask (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__mmask16) __builtin_ia32_ucmpw256_mask ((__v16hi) __A, + (__v16hi) __B, 6, + __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpgt_epi16_mask (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__mmask16) __builtin_ia32_pcmpgtw256_mask ((__v16hi) __A, + (__v16hi) __B, + __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_testn_epi8_mask (__m128i __A, __m128i __B) +{ + return (__mmask16) __builtin_ia32_ptestnmb128 ((__v16qi) __A, + (__v16qi) __B, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_testn_epi8_mask (__mmask16 __U, __m128i __A, __m128i __B) +{ + return (__mmask16) __builtin_ia32_ptestnmb128 ((__v16qi) __A, + (__v16qi) __B, __U); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testn_epi8_mask (__m256i __A, __m256i __B) +{ + return (__mmask32) __builtin_ia32_ptestnmb256 ((__v32qi) __A, + (__v32qi) __B, + (__mmask32) -1); +} + +extern __inline __mmask32 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_testn_epi8_mask (__mmask32 __U, __m256i __A, __m256i __B) +{ + return (__mmask32) __builtin_ia32_ptestnmb256 ((__v32qi) __A, + (__v32qi) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_testn_epi16_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ptestnmw128 ((__v8hi) __A, + (__v8hi) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_testn_epi16_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ptestnmw128 ((__v8hi) __A, + (__v8hi) __B, __U); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testn_epi16_mask (__m256i __A, __m256i __B) +{ + return (__mmask16) __builtin_ia32_ptestnmw256 ((__v16hi) __A, + (__v16hi) __B, + (__mmask16) -1); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_testn_epi16_mask (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__mmask16) __builtin_ia32_ptestnmw256 ((__v16hi) __A, + (__v16hi) __B, __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_shuffle_epi8 (__m256i __W, __mmask32 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pshufb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) __W, + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_shuffle_epi8 (__mmask32 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pshufb256_mask ((__v32qi) __A, + (__v32qi) __B, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_shuffle_epi8 (__m128i __W, __mmask16 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pshufb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) __W, + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_shuffle_epi8 (__mmask16 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pshufb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_packs_epi16 (__mmask32 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_packsswb256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v32qi) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_packs_epi16 (__m256i __W, __mmask32 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_packsswb256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v32qi) __W, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_packs_epi16 (__mmask16 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_packsswb128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_packs_epi16 (__m128i __W, __mmask16 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_packsswb128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v16qi) __W, + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_packus_epi16 (__mmask32 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_packuswb256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v32qi) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_packus_epi16 (__m256i __W, __mmask32 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_packuswb256_mask ((__v16hi) __A, + (__v16hi) __B, + (__v32qi) __W, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_packus_epi16 (__mmask16 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_packuswb128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_packus_epi16 (__m128i __W, __mmask16 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_packuswb128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v16qi) __W, + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_abs_epi8 (__m256i __W, __mmask32 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_pabsb256_mask ((__v32qi) __A, + (__v32qi) __W, + (__mmask32) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_abs_epi8 (__mmask32 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_pabsb256_mask ((__v32qi) __A, + (__v32qi) + _mm256_setzero_si256 (), + (__mmask32) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_abs_epi8 (__m128i __W, __mmask16 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pabsb128_mask ((__v16qi) __A, + (__v16qi) __W, + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_abs_epi8 (__mmask16 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pabsb128_mask ((__v16qi) __A, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_abs_epi16 (__m256i __W, __mmask16 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_pabsw256_mask ((__v16hi) __A, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_abs_epi16 (__mmask16 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_pabsw256_mask ((__v16hi) __A, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_abs_epi16 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pabsw128_mask ((__v8hi) __A, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_abs_epi16 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pabsw128_mask ((__v8hi) __A, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __mmask32 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpneq_epu8_mask (__m256i __X, __m256i __Y) +{ + return (__mmask32) __builtin_ia32_ucmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, 4, + (__mmask32) -1); +} + +extern __inline __mmask32 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmplt_epu8_mask (__m256i __X, __m256i __Y) +{ + return (__mmask32) __builtin_ia32_ucmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, 1, + (__mmask32) -1); +} + +extern __inline __mmask32 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpge_epu8_mask (__m256i __X, __m256i __Y) +{ + return (__mmask32) __builtin_ia32_ucmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, 5, + (__mmask32) -1); +} + +extern __inline __mmask32 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmple_epu8_mask (__m256i __X, __m256i __Y) +{ + return (__mmask32) __builtin_ia32_ucmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, 2, + (__mmask32) -1); +} + +extern __inline __mmask16 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpneq_epu16_mask (__m256i __X, __m256i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, 4, + (__mmask16) -1); +} + +extern __inline __mmask16 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmplt_epu16_mask (__m256i __X, __m256i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, 1, + (__mmask16) -1); +} + +extern __inline __mmask16 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpge_epu16_mask (__m256i __X, __m256i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, 5, + (__mmask16) -1); +} + +extern __inline __mmask16 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmple_epu16_mask (__m256i __X, __m256i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, 2, + (__mmask16) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_storeu_epi16 (void *__P, __mmask16 __U, __m256i __A) +{ + __builtin_ia32_storedquhi256_mask ((short *) __P, + (__v16hi) __A, + (__mmask16) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_storeu_epi16 (void *__P, __mmask8 __U, __m128i __A) +{ + __builtin_ia32_storedquhi128_mask ((short *) __P, + (__v8hi) __A, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_adds_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_paddsw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_subs_epi8 (__m128i __W, __mmask16 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psubsb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) __W, + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_subs_epi8 (__mmask16 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psubsb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_subs_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psubsw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_subs_epi16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psubsw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_subs_epu8 (__m128i __W, __mmask16 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psubusb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) __W, + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_subs_epu8 (__mmask16 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psubusb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_subs_epu16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psubusw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_subs_epu16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psubusw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_srl_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + __m128i __B) +{ + return (__m256i) __builtin_ia32_psrlw256_mask ((__v16hi) __A, + (__v8hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_srl_epi16 (__mmask16 __U, __m256i __A, __m128i __B) +{ + return (__m256i) __builtin_ia32_psrlw256_mask ((__v16hi) __A, + (__v8hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_srl_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psrlw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_srl_epi16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psrlw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sra_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + __m128i __B) +{ + return (__m256i) __builtin_ia32_psraw256_mask ((__v16hi) __A, + (__v8hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sra_epi16 (__mmask16 __U, __m256i __A, __m128i __B) +{ + return (__m256i) __builtin_ia32_psraw256_mask ((__v16hi) __A, + (__v8hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sra_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psraw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sra_epi16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psraw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_adds_epi16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_paddsw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_adds_epu8 (__m128i __W, __mmask16 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_paddusb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) __W, + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_adds_epu8 (__mmask16 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_paddusb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_adds_epu16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_paddusw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_adds_epu16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_paddusw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sub_epi8 (__m128i __W, __mmask16 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psubb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) __W, + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sub_epi8 (__mmask16 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psubb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sub_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psubw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sub_epi16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psubw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_adds_epi8 (__m128i __W, __mmask16 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_paddsb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) __W, + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_adds_epi8 (__mmask16 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_paddsb128_mask ((__v16qi) __A, + (__v16qi) __B, + (__v16qi) + _mm_setzero_si128 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepi16_epi8 (__m128i __A) +{ + + return (__m128i) __builtin_ia32_pmovwb128_mask ((__v8hi) __A, + (__v16qi)_mm_undefined_si128(), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi16_epi8 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovwb128_mask ((__v8hi) __A, + (__v16qi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepi16_epi8 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovwb128_mask ((__v8hi) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_srav_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_psrav16hi_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_srav_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_psrav16hi_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_srav_epi16 (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_psrav16hi_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srav_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psrav8hi_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_srav_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psrav8hi_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_srav_epi16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psrav8hi_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_srlv_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_psrlv16hi_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_srlv_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_psrlv16hi_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_srlv_epi16 (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_psrlv16hi_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srlv_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psrlv8hi_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_srlv_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psrlv8hi_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_srlv_epi16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psrlv8hi_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sllv_epi16 (__m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_psllv16hi_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sllv_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_psllv16hi_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sllv_epi16 (__mmask16 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_psllv16hi_mask ((__v16hi) __A, + (__v16hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sllv_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psllv8hi_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sllv_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psllv8hi_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sllv_epi16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psllv8hi_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sll_epi16 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psllw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sll_epi16 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psllw128_mask ((__v8hi) __A, + (__v8hi) __B, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sll_epi16 (__m256i __W, __mmask16 __U, __m256i __A, + __m128i __B) +{ + return (__m256i) __builtin_ia32_psllw256_mask ((__v16hi) __A, + (__v8hi) __B, + (__v16hi) __W, + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sll_epi16 (__mmask16 __U, __m256i __A, __m128i __B) +{ + return (__m256i) __builtin_ia32_psllw256_mask ((__v16hi) __A, + (__v8hi) __B, + (__v16hi) + _mm256_setzero_si256 (), + (__mmask16) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_packus_epi32 (__mmask16 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_packusdw256_mask ((__v8si) __A, + (__v8si) __B, + (__v16hi) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_packus_epi32 (__m256i __W, __mmask16 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_packusdw256_mask ((__v8si) __A, + (__v8si) __B, + (__v16hi) __W, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_packus_epi32 (__mmask8 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_packusdw128_mask ((__v4si) __A, + (__v4si) __B, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_packus_epi32 (__m128i __W, __mmask16 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_packusdw128_mask ((__v4si) __A, + (__v4si) __B, + (__v8hi) __W, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_packs_epi32 (__mmask16 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_packssdw256_mask ((__v8si) __A, + (__v8si) __B, + (__v16hi) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_packs_epi32 (__m256i __W, __mmask16 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_packssdw256_mask ((__v8si) __A, + (__v8si) __B, + (__v16hi) __W, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_packs_epi32 (__mmask8 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_packssdw128_mask ((__v4si) __A, + (__v4si) __B, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_packs_epi32 (__m128i __W, __mmask16 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_packssdw128_mask ((__v4si) __A, + (__v4si) __B, + (__v8hi) __W, __M); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpneq_epu8_mask (__mmask16 __M, __m128i __X, __m128i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, 4, + (__mmask16) __M); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmplt_epu8_mask (__mmask16 __M, __m128i __X, __m128i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, 1, + (__mmask16) __M); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpge_epu8_mask (__mmask16 __M, __m128i __X, __m128i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, 5, + (__mmask16) __M); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmple_epu8_mask (__mmask16 __M, __m128i __X, __m128i __Y) +{ + return (__mmask16) __builtin_ia32_ucmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, 2, + (__mmask16) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpneq_epu16_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, 4, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmplt_epu16_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, 1, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpge_epu16_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, 5, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmple_epu16_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, 2, + (__mmask8) __M); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpneq_epi8_mask (__mmask16 __M, __m128i __X, __m128i __Y) +{ + return (__mmask16) __builtin_ia32_cmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, 4, + (__mmask16) __M); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmplt_epi8_mask (__mmask16 __M, __m128i __X, __m128i __Y) +{ + return (__mmask16) __builtin_ia32_cmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, 1, + (__mmask16) __M); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpge_epi8_mask (__mmask16 __M, __m128i __X, __m128i __Y) +{ + return (__mmask16) __builtin_ia32_cmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, 5, + (__mmask16) __M); +} + +extern __inline __mmask16 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmple_epi8_mask (__mmask16 __M, __m128i __X, __m128i __Y) +{ + return (__mmask16) __builtin_ia32_cmpb128_mask ((__v16qi) __X, + (__v16qi) __Y, 2, + (__mmask16) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpneq_epi16_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, 4, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmplt_epi16_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, 1, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpge_epi16_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, 5, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmple_epi16_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpw128_mask ((__v8hi) __X, + (__v8hi) __Y, 2, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpneq_epu8_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, 4, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmplt_epu8_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, 1, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpge_epu8_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, 5, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmple_epu8_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, 2, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpneq_epu16_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, 4, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmplt_epu16_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, 1, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpge_epu16_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, 5, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmple_epu16_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, 2, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpneq_epi8_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, 4, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmplt_epi8_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, 1, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpge_epi8_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, 5, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmple_epi8_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpb256_mask ((__v32qi) __X, + (__v32qi) __Y, 2, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpneq_epi16_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, 4, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmplt_epi16_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, 1, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpge_epi16_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, 5, + (__mmask8) __M); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmple_epi16_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpw256_mask ((__v16hi) __X, + (__v16hi) __Y, 2, + (__mmask8) __M); +} + +#ifdef __DISABLE_AVX512VLBW__ +#undef __DISABLE_AVX512VLBW__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX512VLBW__ */ + +#endif /* _AVX512VLBWINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512vldqintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512vldqintrin.h new file mode 100644 index 0000000000..dee74e4f4a --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512vldqintrin.h @@ -0,0 +1,2016 @@ +/* Copyright (C) 2014-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +#error "Never use directly; include instead." +#endif + +#ifndef _AVX512VLDQINTRIN_H_INCLUDED +#define _AVX512VLDQINTRIN_H_INCLUDED + +#if !defined(__AVX512VL__) || !defined(__AVX512DQ__) +#pragma GCC push_options +//#pragma GCC target("avx512vl,avx512dq") +#define __DISABLE_AVX512VLDQ__ +#endif /* __AVX512VLDQ__ */ + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvttpd_epi64 (__m256d __A) +{ + return (__m256i) __builtin_ia32_cvttpd2qq256_mask ((__v4df) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvttpd_epi64 (__m256i __W, __mmask8 __U, __m256d __A) +{ + return (__m256i) __builtin_ia32_cvttpd2qq256_mask ((__v4df) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvttpd_epi64 (__mmask8 __U, __m256d __A) +{ + return (__m256i) __builtin_ia32_cvttpd2qq256_mask ((__v4df) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttpd_epi64 (__m128d __A) +{ + return (__m128i) __builtin_ia32_cvttpd2qq128_mask ((__v2df) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvttpd_epi64 (__m128i __W, __mmask8 __U, __m128d __A) +{ + return (__m128i) __builtin_ia32_cvttpd2qq128_mask ((__v2df) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvttpd_epi64 (__mmask8 __U, __m128d __A) +{ + return (__m128i) __builtin_ia32_cvttpd2qq128_mask ((__v2df) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvttpd_epu64 (__m256d __A) +{ + return (__m256i) __builtin_ia32_cvttpd2uqq256_mask ((__v4df) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvttpd_epu64 (__m256i __W, __mmask8 __U, __m256d __A) +{ + return (__m256i) __builtin_ia32_cvttpd2uqq256_mask ((__v4df) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvttpd_epu64 (__mmask8 __U, __m256d __A) +{ + return (__m256i) __builtin_ia32_cvttpd2uqq256_mask ((__v4df) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttpd_epu64 (__m128d __A) +{ + return (__m128i) __builtin_ia32_cvttpd2uqq128_mask ((__v2df) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvttpd_epu64 (__m128i __W, __mmask8 __U, __m128d __A) +{ + return (__m128i) __builtin_ia32_cvttpd2uqq128_mask ((__v2df) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvttpd_epu64 (__mmask8 __U, __m128d __A) +{ + return (__m128i) __builtin_ia32_cvttpd2uqq128_mask ((__v2df) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtpd_epi64 (__m256d __A) +{ + return (__m256i) __builtin_ia32_cvtpd2qq256_mask ((__v4df) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtpd_epi64 (__m256i __W, __mmask8 __U, __m256d __A) +{ + return (__m256i) __builtin_ia32_cvtpd2qq256_mask ((__v4df) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtpd_epi64 (__mmask8 __U, __m256d __A) +{ + return (__m256i) __builtin_ia32_cvtpd2qq256_mask ((__v4df) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtpd_epi64 (__m128d __A) +{ + return (__m128i) __builtin_ia32_cvtpd2qq128_mask ((__v2df) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtpd_epi64 (__m128i __W, __mmask8 __U, __m128d __A) +{ + return (__m128i) __builtin_ia32_cvtpd2qq128_mask ((__v2df) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtpd_epi64 (__mmask8 __U, __m128d __A) +{ + return (__m128i) __builtin_ia32_cvtpd2qq128_mask ((__v2df) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtpd_epu64 (__m256d __A) +{ + return (__m256i) __builtin_ia32_cvtpd2uqq256_mask ((__v4df) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtpd_epu64 (__m256i __W, __mmask8 __U, __m256d __A) +{ + return (__m256i) __builtin_ia32_cvtpd2uqq256_mask ((__v4df) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtpd_epu64 (__mmask8 __U, __m256d __A) +{ + return (__m256i) __builtin_ia32_cvtpd2uqq256_mask ((__v4df) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtpd_epu64 (__m128d __A) +{ + return (__m128i) __builtin_ia32_cvtpd2uqq128_mask ((__v2df) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtpd_epu64 (__m128i __W, __mmask8 __U, __m128d __A) +{ + return (__m128i) __builtin_ia32_cvtpd2uqq128_mask ((__v2df) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtpd_epu64 (__mmask8 __U, __m128d __A) +{ + return (__m128i) __builtin_ia32_cvtpd2uqq128_mask ((__v2df) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvttps_epi64 (__m128 __A) +{ + return (__m256i) __builtin_ia32_cvttps2qq256_mask ((__v4sf) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvttps_epi64 (__m256i __W, __mmask8 __U, __m128 __A) +{ + return (__m256i) __builtin_ia32_cvttps2qq256_mask ((__v4sf) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvttps_epi64 (__mmask8 __U, __m128 __A) +{ + return (__m256i) __builtin_ia32_cvttps2qq256_mask ((__v4sf) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttps_epi64 (__m128 __A) +{ + return (__m128i) __builtin_ia32_cvttps2qq128_mask ((__v4sf) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvttps_epi64 (__m128i __W, __mmask8 __U, __m128 __A) +{ + return (__m128i) __builtin_ia32_cvttps2qq128_mask ((__v4sf) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvttps_epi64 (__mmask8 __U, __m128 __A) +{ + return (__m128i) __builtin_ia32_cvttps2qq128_mask ((__v4sf) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvttps_epu64 (__m128 __A) +{ + return (__m256i) __builtin_ia32_cvttps2uqq256_mask ((__v4sf) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvttps_epu64 (__m256i __W, __mmask8 __U, __m128 __A) +{ + return (__m256i) __builtin_ia32_cvttps2uqq256_mask ((__v4sf) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvttps_epu64 (__mmask8 __U, __m128 __A) +{ + return (__m256i) __builtin_ia32_cvttps2uqq256_mask ((__v4sf) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttps_epu64 (__m128 __A) +{ + return (__m128i) __builtin_ia32_cvttps2uqq128_mask ((__v4sf) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvttps_epu64 (__m128i __W, __mmask8 __U, __m128 __A) +{ + return (__m128i) __builtin_ia32_cvttps2uqq128_mask ((__v4sf) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvttps_epu64 (__mmask8 __U, __m128 __A) +{ + return (__m128i) __builtin_ia32_cvttps2uqq128_mask ((__v4sf) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcast_f64x2 (__m128d __A) +{ + return (__m256d) __builtin_ia32_broadcastf64x2_256_mask ((__v2df) + __A, + (__v4df)_mm256_undefined_pd(), + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_broadcast_f64x2 (__m256d __O, __mmask8 __M, __m128d __A) +{ + return (__m256d) __builtin_ia32_broadcastf64x2_256_mask ((__v2df) + __A, + (__v4df) + __O, __M); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_broadcast_f64x2 (__mmask8 __M, __m128d __A) +{ + return (__m256d) __builtin_ia32_broadcastf64x2_256_mask ((__v2df) + __A, + (__v4df) + _mm256_setzero_ps (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcast_i64x2 (__m128i __A) +{ + return (__m256i) __builtin_ia32_broadcasti64x2_256_mask ((__v2di) + __A, + (__v4di)_mm256_undefined_si256(), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_broadcast_i64x2 (__m256i __O, __mmask8 __M, __m128i __A) +{ + return (__m256i) __builtin_ia32_broadcasti64x2_256_mask ((__v2di) + __A, + (__v4di) + __O, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_broadcast_i64x2 (__mmask8 __M, __m128i __A) +{ + return (__m256i) __builtin_ia32_broadcasti64x2_256_mask ((__v2di) + __A, + (__v4di) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcast_f32x2 (__m128 __A) +{ + return (__m256) __builtin_ia32_broadcastf32x2_256_mask ((__v4sf) __A, + (__v8sf)_mm256_undefined_ps(), + (__mmask8) -1); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_broadcast_f32x2 (__m256 __O, __mmask8 __M, __m128 __A) +{ + return (__m256) __builtin_ia32_broadcastf32x2_256_mask ((__v4sf) __A, + (__v8sf) __O, + __M); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_broadcast_f32x2 (__mmask8 __M, __m128 __A) +{ + return (__m256) __builtin_ia32_broadcastf32x2_256_mask ((__v4sf) __A, + (__v8sf) + _mm256_setzero_ps (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcast_i32x2 (__m128i __A) +{ + return (__m256i) __builtin_ia32_broadcasti32x2_256_mask ((__v4si) + __A, + (__v8si)_mm256_undefined_si256(), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_broadcast_i32x2 (__m256i __O, __mmask8 __M, __m128i __A) +{ + return (__m256i) __builtin_ia32_broadcasti32x2_256_mask ((__v4si) + __A, + (__v8si) + __O, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_broadcast_i32x2 (__mmask8 __M, __m128i __A) +{ + return (__m256i) __builtin_ia32_broadcasti32x2_256_mask ((__v4si) + __A, + (__v8si) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_broadcast_i32x2 (__m128i __A) +{ + return (__m128i) __builtin_ia32_broadcasti32x2_128_mask ((__v4si) + __A, + (__v4si)_mm_undefined_si128(), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_broadcast_i32x2 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_broadcasti32x2_128_mask ((__v4si) + __A, + (__v4si) + __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_broadcast_i32x2 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_broadcasti32x2_128_mask ((__v4si) + __A, + (__v4si) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mullo_epi64 (__m256i __A, __m256i __B) +{ + return (__m256i) ((__v4du) __A * (__v4du) __B); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_mullo_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pmullq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_mullo_epi64 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pmullq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mullo_epi64 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v2du) __A * (__v2du) __B); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_mullo_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pmullq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_mullo_epi64 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pmullq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_andnot_pd (__m256d __W, __mmask8 __U, __m256d __A, + __m256d __B) +{ + return (__m256d) __builtin_ia32_andnpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_andnot_pd (__mmask8 __U, __m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_andnpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_andnot_pd (__m128d __W, __mmask8 __U, __m128d __A, + __m128d __B) +{ + return (__m128d) __builtin_ia32_andnpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_andnot_pd (__mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_andnpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_andnot_ps (__m256 __W, __mmask8 __U, __m256 __A, + __m256 __B) +{ + return (__m256) __builtin_ia32_andnps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_andnot_ps (__mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_andnps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_andnot_ps (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_andnps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_andnot_ps (__mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_andnps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtps_epi64 (__m128 __A) +{ + return (__m256i) __builtin_ia32_cvtps2qq256_mask ((__v4sf) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtps_epi64 (__m256i __W, __mmask8 __U, __m128 __A) +{ + return (__m256i) __builtin_ia32_cvtps2qq256_mask ((__v4sf) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtps_epi64 (__mmask8 __U, __m128 __A) +{ + return (__m256i) __builtin_ia32_cvtps2qq256_mask ((__v4sf) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtps_epi64 (__m128 __A) +{ + return (__m128i) __builtin_ia32_cvtps2qq128_mask ((__v4sf) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtps_epi64 (__m128i __W, __mmask8 __U, __m128 __A) +{ + return (__m128i) __builtin_ia32_cvtps2qq128_mask ((__v4sf) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtps_epi64 (__mmask8 __U, __m128 __A) +{ + return (__m128i) __builtin_ia32_cvtps2qq128_mask ((__v4sf) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtps_epu64 (__m128 __A) +{ + return (__m256i) __builtin_ia32_cvtps2uqq256_mask ((__v4sf) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtps_epu64 (__m256i __W, __mmask8 __U, __m128 __A) +{ + return (__m256i) __builtin_ia32_cvtps2uqq256_mask ((__v4sf) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtps_epu64 (__mmask8 __U, __m128 __A) +{ + return (__m256i) __builtin_ia32_cvtps2uqq256_mask ((__v4sf) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtps_epu64 (__m128 __A) +{ + return (__m128i) __builtin_ia32_cvtps2uqq128_mask ((__v4sf) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtps_epu64 (__m128i __W, __mmask8 __U, __m128 __A) +{ + return (__m128i) __builtin_ia32_cvtps2uqq128_mask ((__v4sf) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtps_epu64 (__mmask8 __U, __m128 __A) +{ + return (__m128i) __builtin_ia32_cvtps2uqq128_mask ((__v4sf) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi64_ps (__m256i __A) +{ + return (__m128) __builtin_ia32_cvtqq2ps256_mask ((__v4di) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi64_ps (__m128 __W, __mmask8 __U, __m256i __A) +{ + return (__m128) __builtin_ia32_cvtqq2ps256_mask ((__v4di) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepi64_ps (__mmask8 __U, __m256i __A) +{ + return (__m128) __builtin_ia32_cvtqq2ps256_mask ((__v4di) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepi64_ps (__m128i __A) +{ + return (__m128) __builtin_ia32_cvtqq2ps128_mask ((__v2di) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi64_ps (__m128 __W, __mmask8 __U, __m128i __A) +{ + return (__m128) __builtin_ia32_cvtqq2ps128_mask ((__v2di) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepi64_ps (__mmask8 __U, __m128i __A) +{ + return (__m128) __builtin_ia32_cvtqq2ps128_mask ((__v2di) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepu64_ps (__m256i __A) +{ + return (__m128) __builtin_ia32_cvtuqq2ps256_mask ((__v4di) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepu64_ps (__m128 __W, __mmask8 __U, __m256i __A) +{ + return (__m128) __builtin_ia32_cvtuqq2ps256_mask ((__v4di) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepu64_ps (__mmask8 __U, __m256i __A) +{ + return (__m128) __builtin_ia32_cvtuqq2ps256_mask ((__v4di) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepu64_ps (__m128i __A) +{ + return (__m128) __builtin_ia32_cvtuqq2ps128_mask ((__v2di) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepu64_ps (__m128 __W, __mmask8 __U, __m128i __A) +{ + return (__m128) __builtin_ia32_cvtuqq2ps128_mask ((__v2di) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepu64_ps (__mmask8 __U, __m128i __A) +{ + return (__m128) __builtin_ia32_cvtuqq2ps128_mask ((__v2di) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi64_pd (__m256i __A) +{ + return (__m256d) __builtin_ia32_cvtqq2pd256_mask ((__v4di) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi64_pd (__m256d __W, __mmask8 __U, __m256i __A) +{ + return (__m256d) __builtin_ia32_cvtqq2pd256_mask ((__v4di) __A, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepi64_pd (__mmask8 __U, __m256i __A) +{ + return (__m256d) __builtin_ia32_cvtqq2pd256_mask ((__v4di) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepi64_pd (__m128i __A) +{ + return (__m128d) __builtin_ia32_cvtqq2pd128_mask ((__v2di) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi64_pd (__m128d __W, __mmask8 __U, __m128i __A) +{ + return (__m128d) __builtin_ia32_cvtqq2pd128_mask ((__v2di) __A, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepi64_pd (__mmask8 __U, __m128i __A) +{ + return (__m128d) __builtin_ia32_cvtqq2pd128_mask ((__v2di) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepu64_pd (__m256i __A) +{ + return (__m256d) __builtin_ia32_cvtuqq2pd256_mask ((__v4di) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepu64_pd (__m256d __W, __mmask8 __U, __m256i __A) +{ + return (__m256d) __builtin_ia32_cvtuqq2pd256_mask ((__v4di) __A, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepu64_pd (__mmask8 __U, __m256i __A) +{ + return (__m256d) __builtin_ia32_cvtuqq2pd256_mask ((__v4di) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_and_pd (__m256d __W, __mmask8 __U, __m256d __A, + __m256d __B) +{ + return (__m256d) __builtin_ia32_andpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_and_pd (__mmask8 __U, __m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_andpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_and_pd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_andpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_and_pd (__mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_andpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_and_ps (__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_andps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_and_ps (__mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_andps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_and_ps (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_andps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_and_ps (__mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_andps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepu64_pd (__m128i __A) +{ + return (__m128d) __builtin_ia32_cvtuqq2pd128_mask ((__v2di) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepu64_pd (__m128d __W, __mmask8 __U, __m128i __A) +{ + return (__m128d) __builtin_ia32_cvtuqq2pd128_mask ((__v2di) __A, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepu64_pd (__mmask8 __U, __m128i __A) +{ + return (__m128d) __builtin_ia32_cvtuqq2pd128_mask ((__v2di) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_xor_pd (__m256d __W, __mmask8 __U, __m256d __A, + __m256d __B) +{ + return (__m256d) __builtin_ia32_xorpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_xor_pd (__mmask8 __U, __m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_xorpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_xor_pd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_xorpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_xor_pd (__mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_xorpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_xor_ps (__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_xorps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_xor_ps (__mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_xorps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_xor_ps (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_xorps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_xor_ps (__mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_xorps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_or_pd (__m256d __W, __mmask8 __U, __m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_orpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_or_pd (__mmask8 __U, __m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_orpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_or_pd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_orpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_or_pd (__mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_orpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_or_ps (__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_orps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_or_ps (__mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_orps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_or_ps (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_orps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_or_ps (__mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_orps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movm_epi32 (__mmask8 __A) +{ + return (__m128i) __builtin_ia32_cvtmask2d128 (__A); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_movm_epi32 (__mmask8 __A) +{ + return (__m256i) __builtin_ia32_cvtmask2d256 (__A); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movm_epi64 (__mmask8 __A) +{ + return (__m128i) __builtin_ia32_cvtmask2q128 (__A); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_movm_epi64 (__mmask8 __A) +{ + return (__m256i) __builtin_ia32_cvtmask2q256 (__A); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movepi32_mask (__m128i __A) +{ + return (__mmask8) __builtin_ia32_cvtd2mask128 ((__v4si) __A); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_movepi32_mask (__m256i __A) +{ + return (__mmask8) __builtin_ia32_cvtd2mask256 ((__v8si) __A); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movepi64_mask (__m128i __A) +{ + return (__mmask8) __builtin_ia32_cvtq2mask128 ((__v2di) __A); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_movepi64_mask (__m256i __A) +{ + return (__mmask8) __builtin_ia32_cvtq2mask256 ((__v4di) __A); +} + +#ifdef __OPTIMIZE__ +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_extractf64x2_pd (__m256d __A, const int __imm) +{ + return (__m128d) __builtin_ia32_extractf64x2_256_mask ((__v4df) __A, + __imm, + (__v2df) + _mm_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_extractf64x2_pd (__m128d __W, __mmask8 __U, __m256d __A, + const int __imm) +{ + return (__m128d) __builtin_ia32_extractf64x2_256_mask ((__v4df) __A, + __imm, + (__v2df) __W, + (__mmask8) + __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_extractf64x2_pd (__mmask8 __U, __m256d __A, + const int __imm) +{ + return (__m128d) __builtin_ia32_extractf64x2_256_mask ((__v4df) __A, + __imm, + (__v2df) + _mm_setzero_pd (), + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_extracti64x2_epi64 (__m256i __A, const int __imm) +{ + return (__m128i) __builtin_ia32_extracti64x2_256_mask ((__v4di) __A, + __imm, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_extracti64x2_epi64 (__m128i __W, __mmask8 __U, __m256i __A, + const int __imm) +{ + return (__m128i) __builtin_ia32_extracti64x2_256_mask ((__v4di) __A, + __imm, + (__v2di) __W, + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_extracti64x2_epi64 (__mmask8 __U, __m256i __A, + const int __imm) +{ + return (__m128i) __builtin_ia32_extracti64x2_256_mask ((__v4di) __A, + __imm, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) + __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_reduce_pd (__m256d __A, int __B) +{ + return (__m256d) __builtin_ia32_reducepd256_mask ((__v4df) __A, __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_reduce_pd (__m256d __W, __mmask8 __U, __m256d __A, int __B) +{ + return (__m256d) __builtin_ia32_reducepd256_mask ((__v4df) __A, __B, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_reduce_pd (__mmask8 __U, __m256d __A, int __B) +{ + return (__m256d) __builtin_ia32_reducepd256_mask ((__v4df) __A, __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_reduce_pd (__m128d __A, int __B) +{ + return (__m128d) __builtin_ia32_reducepd128_mask ((__v2df) __A, __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_reduce_pd (__m128d __W, __mmask8 __U, __m128d __A, int __B) +{ + return (__m128d) __builtin_ia32_reducepd128_mask ((__v2df) __A, __B, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_reduce_pd (__mmask8 __U, __m128d __A, int __B) +{ + return (__m128d) __builtin_ia32_reducepd128_mask ((__v2df) __A, __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_reduce_ps (__m256 __A, int __B) +{ + return (__m256) __builtin_ia32_reduceps256_mask ((__v8sf) __A, __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_reduce_ps (__m256 __W, __mmask8 __U, __m256 __A, int __B) +{ + return (__m256) __builtin_ia32_reduceps256_mask ((__v8sf) __A, __B, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_reduce_ps (__mmask8 __U, __m256 __A, int __B) +{ + return (__m256) __builtin_ia32_reduceps256_mask ((__v8sf) __A, __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_reduce_ps (__m128 __A, int __B) +{ + return (__m128) __builtin_ia32_reduceps128_mask ((__v4sf) __A, __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_reduce_ps (__m128 __W, __mmask8 __U, __m128 __A, int __B) +{ + return (__m128) __builtin_ia32_reduceps128_mask ((__v4sf) __A, __B, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_reduce_ps (__mmask8 __U, __m128 __A, int __B) +{ + return (__m128) __builtin_ia32_reduceps128_mask ((__v4sf) __A, __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_range_pd (__m256d __A, __m256d __B, int __C) +{ + return (__m256d) __builtin_ia32_rangepd256_mask ((__v4df) __A, + (__v4df) __B, __C, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_range_pd (__m256d __W, __mmask8 __U, + __m256d __A, __m256d __B, int __C) +{ + return (__m256d) __builtin_ia32_rangepd256_mask ((__v4df) __A, + (__v4df) __B, __C, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_range_pd (__mmask8 __U, __m256d __A, __m256d __B, int __C) +{ + return (__m256d) __builtin_ia32_rangepd256_mask ((__v4df) __A, + (__v4df) __B, __C, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_range_pd (__m128d __A, __m128d __B, int __C) +{ + return (__m128d) __builtin_ia32_rangepd128_mask ((__v2df) __A, + (__v2df) __B, __C, + (__v2df) + _mm_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_range_pd (__m128d __W, __mmask8 __U, + __m128d __A, __m128d __B, int __C) +{ + return (__m128d) __builtin_ia32_rangepd128_mask ((__v2df) __A, + (__v2df) __B, __C, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_range_pd (__mmask8 __U, __m128d __A, __m128d __B, int __C) +{ + return (__m128d) __builtin_ia32_rangepd128_mask ((__v2df) __A, + (__v2df) __B, __C, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_range_ps (__m256 __A, __m256 __B, int __C) +{ + return (__m256) __builtin_ia32_rangeps256_mask ((__v8sf) __A, + (__v8sf) __B, __C, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_range_ps (__m256 __W, __mmask8 __U, __m256 __A, __m256 __B, + int __C) +{ + return (__m256) __builtin_ia32_rangeps256_mask ((__v8sf) __A, + (__v8sf) __B, __C, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_range_ps (__mmask8 __U, __m256 __A, __m256 __B, int __C) +{ + return (__m256) __builtin_ia32_rangeps256_mask ((__v8sf) __A, + (__v8sf) __B, __C, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_range_ps (__m128 __A, __m128 __B, int __C) +{ + return (__m128) __builtin_ia32_rangeps128_mask ((__v4sf) __A, + (__v4sf) __B, __C, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_range_ps (__m128 __W, __mmask8 __U, + __m128 __A, __m128 __B, int __C) +{ + return (__m128) __builtin_ia32_rangeps128_mask ((__v4sf) __A, + (__v4sf) __B, __C, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_range_ps (__mmask8 __U, __m128 __A, __m128 __B, int __C) +{ + return (__m128) __builtin_ia32_rangeps128_mask ((__v4sf) __A, + (__v4sf) __B, __C, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_fpclass_pd_mask (__mmask8 __U, __m256d __A, + const int __imm) +{ + return (__mmask8) __builtin_ia32_fpclasspd256_mask ((__v4df) __A, + __imm, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_fpclass_pd_mask (__m256d __A, const int __imm) +{ + return (__mmask8) __builtin_ia32_fpclasspd256_mask ((__v4df) __A, + __imm, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_fpclass_ps_mask (__mmask8 __U, __m256 __A, const int __imm) +{ + return (__mmask8) __builtin_ia32_fpclassps256_mask ((__v8sf) __A, + __imm, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_fpclass_ps_mask (__m256 __A, const int __imm) +{ + return (__mmask8) __builtin_ia32_fpclassps256_mask ((__v8sf) __A, + __imm, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fpclass_pd_mask (__mmask8 __U, __m128d __A, const int __imm) +{ + return (__mmask8) __builtin_ia32_fpclasspd128_mask ((__v2df) __A, + __imm, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fpclass_pd_mask (__m128d __A, const int __imm) +{ + return (__mmask8) __builtin_ia32_fpclasspd128_mask ((__v2df) __A, + __imm, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fpclass_ps_mask (__mmask8 __U, __m128 __A, const int __imm) +{ + return (__mmask8) __builtin_ia32_fpclassps128_mask ((__v4sf) __A, + __imm, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fpclass_ps_mask (__m128 __A, const int __imm) +{ + return (__mmask8) __builtin_ia32_fpclassps128_mask ((__v4sf) __A, + __imm, + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_inserti64x2 (__m256i __A, __m128i __B, const int __imm) +{ + return (__m256i) __builtin_ia32_inserti64x2_256_mask ((__v4di) __A, + (__v2di) __B, + __imm, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_inserti64x2 (__m256i __W, __mmask8 __U, __m256i __A, + __m128i __B, const int __imm) +{ + return (__m256i) __builtin_ia32_inserti64x2_256_mask ((__v4di) __A, + (__v2di) __B, + __imm, + (__v4di) __W, + (__mmask8) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_inserti64x2 (__mmask8 __U, __m256i __A, __m128i __B, + const int __imm) +{ + return (__m256i) __builtin_ia32_inserti64x2_256_mask ((__v4di) __A, + (__v2di) __B, + __imm, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) + __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_insertf64x2 (__m256d __A, __m128d __B, const int __imm) +{ + return (__m256d) __builtin_ia32_insertf64x2_256_mask ((__v4df) __A, + (__v2df) __B, + __imm, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_insertf64x2 (__m256d __W, __mmask8 __U, __m256d __A, + __m128d __B, const int __imm) +{ + return (__m256d) __builtin_ia32_insertf64x2_256_mask ((__v4df) __A, + (__v2df) __B, + __imm, + (__v4df) __W, + (__mmask8) + __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_insertf64x2 (__mmask8 __U, __m256d __A, __m128d __B, + const int __imm) +{ + return (__m256d) __builtin_ia32_insertf64x2_256_mask ((__v4df) __A, + (__v2df) __B, + __imm, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) + __U); +} + +#else +#define _mm256_insertf64x2(X, Y, C) \ + ((__m256d) __builtin_ia32_insertf64x2_256_mask ((__v4df)(__m256d) (X),\ + (__v2df)(__m128d) (Y), (int) (C), \ + (__v4df)(__m256d)_mm256_setzero_pd(), \ + (__mmask8)-1)) + +#define _mm256_mask_insertf64x2(W, U, X, Y, C) \ + ((__m256d) __builtin_ia32_insertf64x2_256_mask ((__v4df)(__m256d) (X),\ + (__v2df)(__m128d) (Y), (int) (C), \ + (__v4df)(__m256d)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_insertf64x2(U, X, Y, C) \ + ((__m256d) __builtin_ia32_insertf64x2_256_mask ((__v4df)(__m256d) (X),\ + (__v2df)(__m128d) (Y), (int) (C), \ + (__v4df)(__m256d)_mm256_setzero_pd(), \ + (__mmask8)(U))) + +#define _mm256_inserti64x2(X, Y, C) \ + ((__m256i) __builtin_ia32_inserti64x2_256_mask ((__v4di)(__m256i) (X),\ + (__v2di)(__m128i) (Y), (int) (C), \ + (__v4di)(__m256i)_mm256_setzero_si256 (), \ + (__mmask8)-1)) + +#define _mm256_mask_inserti64x2(W, U, X, Y, C) \ + ((__m256i) __builtin_ia32_inserti64x2_256_mask ((__v4di)(__m256i) (X),\ + (__v2di)(__m128i) (Y), (int) (C), \ + (__v4di)(__m256i)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_inserti64x2(U, X, Y, C) \ + ((__m256i) __builtin_ia32_inserti64x2_256_mask ((__v4di)(__m256i) (X),\ + (__v2di)(__m128i) (Y), (int) (C), \ + (__v4di)(__m256i)_mm256_setzero_si256 (), \ + (__mmask8)(U))) + +#define _mm256_extractf64x2_pd(X, C) \ + ((__m128d) __builtin_ia32_extractf64x2_256_mask ((__v4df)(__m256d) (X),\ + (int) (C), (__v2df)(__m128d) _mm_setzero_pd(), (__mmask8)-1)) + +#define _mm256_mask_extractf64x2_pd(W, U, X, C) \ + ((__m128d) __builtin_ia32_extractf64x2_256_mask ((__v4df)(__m256d) (X),\ + (int) (C), (__v2df)(__m128d) (W), (__mmask8) (U))) + +#define _mm256_maskz_extractf64x2_pd(U, X, C) \ + ((__m128d) __builtin_ia32_extractf64x2_256_mask ((__v4df)(__m256d) (X),\ + (int) (C), (__v2df)(__m128d) _mm_setzero_pd(), (__mmask8) (U))) + +#define _mm256_extracti64x2_epi64(X, C) \ + ((__m128i) __builtin_ia32_extracti64x2_256_mask ((__v4di)(__m256i) (X),\ + (int) (C), (__v2di)(__m128i) _mm_setzero_si128 (), (__mmask8)-1)) + +#define _mm256_mask_extracti64x2_epi64(W, U, X, C) \ + ((__m128i) __builtin_ia32_extracti64x2_256_mask ((__v4di)(__m256i) (X),\ + (int) (C), (__v2di)(__m128i) (W), (__mmask8) (U))) + +#define _mm256_maskz_extracti64x2_epi64(U, X, C) \ + ((__m128i) __builtin_ia32_extracti64x2_256_mask ((__v4di)(__m256i) (X),\ + (int) (C), (__v2di)(__m128i) _mm_setzero_si128 (), (__mmask8) (U))) + +#define _mm256_reduce_pd(A, B) \ + ((__m256d) __builtin_ia32_reducepd256_mask ((__v4df)(__m256d)(A), \ + (int)(B), (__v4df)_mm256_setzero_pd(), (__mmask8)-1)) + +#define _mm256_mask_reduce_pd(W, U, A, B) \ + ((__m256d) __builtin_ia32_reducepd256_mask ((__v4df)(__m256d)(A), \ + (int)(B), (__v4df)(__m256d)(W), (__mmask8)(U))) + +#define _mm256_maskz_reduce_pd(U, A, B) \ + ((__m256d) __builtin_ia32_reducepd256_mask ((__v4df)(__m256d)(A), \ + (int)(B), (__v4df)_mm256_setzero_pd(), (__mmask8)(U))) + +#define _mm_reduce_pd(A, B) \ + ((__m128d) __builtin_ia32_reducepd128_mask ((__v2df)(__m128d)(A), \ + (int)(B), (__v2df)_mm_setzero_pd(), (__mmask8)-1)) + +#define _mm_mask_reduce_pd(W, U, A, B) \ + ((__m128d) __builtin_ia32_reducepd128_mask ((__v2df)(__m128d)(A), \ + (int)(B), (__v2df)(__m128d)(W), (__mmask8)(U))) + +#define _mm_maskz_reduce_pd(U, A, B) \ + ((__m128d) __builtin_ia32_reducepd128_mask ((__v2df)(__m128d)(A), \ + (int)(B), (__v2df)_mm_setzero_pd(), (__mmask8)(U))) + +#define _mm256_reduce_ps(A, B) \ + ((__m256) __builtin_ia32_reduceps256_mask ((__v8sf)(__m256)(A), \ + (int)(B), (__v8sf)_mm256_setzero_ps(), (__mmask8)-1)) + +#define _mm256_mask_reduce_ps(W, U, A, B) \ + ((__m256) __builtin_ia32_reduceps256_mask ((__v8sf)(__m256)(A), \ + (int)(B), (__v8sf)(__m256)(W), (__mmask8)(U))) + +#define _mm256_maskz_reduce_ps(U, A, B) \ + ((__m256) __builtin_ia32_reduceps256_mask ((__v8sf)(__m256)(A), \ + (int)(B), (__v8sf)_mm256_setzero_ps(), (__mmask8)(U))) + +#define _mm_reduce_ps(A, B) \ + ((__m128) __builtin_ia32_reduceps128_mask ((__v4sf)(__m128)(A), \ + (int)(B), (__v4sf)_mm_setzero_ps(), (__mmask8)-1)) + +#define _mm_mask_reduce_ps(W, U, A, B) \ + ((__m128) __builtin_ia32_reduceps128_mask ((__v4sf)(__m128)(A), \ + (int)(B), (__v4sf)(__m128)(W), (__mmask8)(U))) + +#define _mm_maskz_reduce_ps(U, A, B) \ + ((__m128) __builtin_ia32_reduceps128_mask ((__v4sf)(__m128)(A), \ + (int)(B), (__v4sf)_mm_setzero_ps(), (__mmask8)(U))) + +#define _mm256_range_pd(A, B, C) \ + ((__m256d) __builtin_ia32_rangepd256_mask ((__v4df)(__m256d)(A), \ + (__v4df)(__m256d)(B), (int)(C), \ + (__v4df)_mm256_setzero_pd(), (__mmask8)-1)) + +#define _mm256_maskz_range_pd(U, A, B, C) \ + ((__m256d) __builtin_ia32_rangepd256_mask ((__v4df)(__m256d)(A), \ + (__v4df)(__m256d)(B), (int)(C), \ + (__v4df)_mm256_setzero_pd(), (__mmask8)(U))) + +#define _mm_range_pd(A, B, C) \ + ((__m128d) __builtin_ia32_rangepd128_mask ((__v2df)(__m128d)(A), \ + (__v2df)(__m128d)(B), (int)(C), \ + (__v2df)_mm_setzero_pd(), (__mmask8)-1)) + +#define _mm256_range_ps(A, B, C) \ + ((__m256) __builtin_ia32_rangeps256_mask ((__v8sf)(__m256)(A), \ + (__v8sf)(__m256)(B), (int)(C), \ + (__v8sf)_mm256_setzero_ps(), (__mmask8)-1)) + +#define _mm256_mask_range_ps(W, U, A, B, C) \ + ((__m256) __builtin_ia32_rangeps256_mask ((__v8sf)(__m256)(A), \ + (__v8sf)(__m256)(B), (int)(C), \ + (__v8sf)(__m256)(W), (__mmask8)(U))) + +#define _mm256_maskz_range_ps(U, A, B, C) \ + ((__m256) __builtin_ia32_rangeps256_mask ((__v8sf)(__m256)(A), \ + (__v8sf)(__m256)(B), (int)(C), \ + (__v8sf)_mm256_setzero_ps(), (__mmask8)(U))) + +#define _mm_range_ps(A, B, C) \ + ((__m128) __builtin_ia32_rangeps128_mask ((__v4sf)(__m128)(A), \ + (__v4sf)(__m128)(B), (int)(C), \ + (__v4sf)_mm_setzero_ps(), (__mmask8)-1)) + +#define _mm_mask_range_ps(W, U, A, B, C) \ + ((__m128) __builtin_ia32_rangeps128_mask ((__v4sf)(__m128)(A), \ + (__v4sf)(__m128)(B), (int)(C), \ + (__v4sf)(__m128)(W), (__mmask8)(U))) + +#define _mm_maskz_range_ps(U, A, B, C) \ + ((__m128) __builtin_ia32_rangeps128_mask ((__v4sf)(__m128)(A), \ + (__v4sf)(__m128)(B), (int)(C), \ + (__v4sf)_mm_setzero_ps(), (__mmask8)(U))) + +#define _mm256_mask_range_pd(W, U, A, B, C) \ + ((__m256d) __builtin_ia32_rangepd256_mask ((__v4df)(__m256d)(A), \ + (__v4df)(__m256d)(B), (int)(C), \ + (__v4df)(__m256d)(W), (__mmask8)(U))) + +#define _mm_mask_range_pd(W, U, A, B, C) \ + ((__m128d) __builtin_ia32_rangepd128_mask ((__v2df)(__m128d)(A), \ + (__v2df)(__m128d)(B), (int)(C), \ + (__v2df)(__m128d)(W), (__mmask8)(U))) + +#define _mm_maskz_range_pd(U, A, B, C) \ + ((__m128d) __builtin_ia32_rangepd128_mask ((__v2df)(__m128d)(A), \ + (__v2df)(__m128d)(B), (int)(C), \ + (__v2df)_mm_setzero_pd(), (__mmask8)(U))) + +#define _mm256_mask_fpclass_pd_mask(u, X, C) \ + ((__mmask8) __builtin_ia32_fpclasspd256_mask ((__v4df) (__m256d) (X), \ + (int) (C),(__mmask8)(u))) + +#define _mm256_mask_fpclass_ps_mask(u, X, C) \ + ((__mmask8) __builtin_ia32_fpclassps256_mask ((__v8sf) (__m256) (X), \ + (int) (C),(__mmask8)(u))) + +#define _mm_mask_fpclass_pd_mask(u, X, C) \ + ((__mmask8) __builtin_ia32_fpclasspd128_mask ((__v2df) (__m128d) (X), \ + (int) (C),(__mmask8)(u))) + +#define _mm_mask_fpclass_ps_mask(u, X, C) \ + ((__mmask8) __builtin_ia32_fpclassps128_mask ((__v4sf) (__m128) (X), \ + (int) (C),(__mmask8)(u))) + +#define _mm256_fpclass_pd_mask(X, C) \ + ((__mmask8) __builtin_ia32_fpclasspd256_mask ((__v4df) (__m256d) (X), \ + (int) (C),(__mmask8)-1)) + +#define _mm256_fpclass_ps_mask(X, C) \ + ((__mmask8) __builtin_ia32_fpclassps256_mask ((__v8sf) (__m256) (X), \ + (int) (C),(__mmask8)-1)) + +#define _mm_fpclass_pd_mask(X, C) \ + ((__mmask8) __builtin_ia32_fpclasspd128_mask ((__v2df) (__m128d) (X), \ + (int) (C),(__mmask8)-1)) + +#define _mm_fpclass_ps_mask(X, C) \ + ((__mmask8) __builtin_ia32_fpclassps128_mask ((__v4sf) (__m128) (X), \ + (int) (C),(__mmask8)-1)) + +#endif + +#ifdef __DISABLE_AVX512VLDQ__ +#undef __DISABLE_AVX512VLDQ__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX512VLDQ__ */ + +#endif /* _AVX512VLDQINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512vlintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512vlintrin.h new file mode 100644 index 0000000000..c9df31839a --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512vlintrin.h @@ -0,0 +1,13606 @@ +/* Copyright (C) 2014-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +#error "Never use directly; include instead." +#endif + +#ifndef _AVX512VLINTRIN_H_INCLUDED +#define _AVX512VLINTRIN_H_INCLUDED + +#ifndef __AVX512VL__ +#pragma GCC push_options +//#pragma GCC target("avx512vl") +#define __DISABLE_AVX512VL__ +#endif /* __AVX512VL__ */ + +/* Internal data types for implementing the intrinsics. */ +typedef unsigned int __mmask32; + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_mov_pd (__m256d __W, __mmask8 __U, __m256d __A) +{ + return (__m256d) __builtin_ia32_movapd256_mask ((__v4df) __A, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_mov_pd (__mmask8 __U, __m256d __A) +{ + return (__m256d) __builtin_ia32_movapd256_mask ((__v4df) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_mov_pd (__m128d __W, __mmask8 __U, __m128d __A) +{ + return (__m128d) __builtin_ia32_movapd128_mask ((__v2df) __A, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_mov_pd (__mmask8 __U, __m128d __A) +{ + return (__m128d) __builtin_ia32_movapd128_mask ((__v2df) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_load_pd (__m256d __W, __mmask8 __U, void const *__P) +{ + return (__m256d) __builtin_ia32_loadapd256_mask ((__v4df *) __P, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_load_pd (__mmask8 __U, void const *__P) +{ + return (__m256d) __builtin_ia32_loadapd256_mask ((__v4df *) __P, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_load_pd (__m128d __W, __mmask8 __U, void const *__P) +{ + return (__m128d) __builtin_ia32_loadapd128_mask ((__v2df *) __P, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_load_pd (__mmask8 __U, void const *__P) +{ + return (__m128d) __builtin_ia32_loadapd128_mask ((__v2df *) __P, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_store_pd (void *__P, __mmask8 __U, __m256d __A) +{ + __builtin_ia32_storeapd256_mask ((__v4df *) __P, + (__v4df) __A, + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_store_pd (void *__P, __mmask8 __U, __m128d __A) +{ + __builtin_ia32_storeapd128_mask ((__v2df *) __P, + (__v2df) __A, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_mov_ps (__m256 __W, __mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_movaps256_mask ((__v8sf) __A, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_mov_ps (__mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_movaps256_mask ((__v8sf) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_mov_ps (__m128 __W, __mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_movaps128_mask ((__v4sf) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_mov_ps (__mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_movaps128_mask ((__v4sf) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_load_ps (__m256 __W, __mmask8 __U, void const *__P) +{ + return (__m256) __builtin_ia32_loadaps256_mask ((__v8sf *) __P, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_load_ps (__mmask8 __U, void const *__P) +{ + return (__m256) __builtin_ia32_loadaps256_mask ((__v8sf *) __P, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_load_ps (__m128 __W, __mmask8 __U, void const *__P) +{ + return (__m128) __builtin_ia32_loadaps128_mask ((__v4sf *) __P, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_load_ps (__mmask8 __U, void const *__P) +{ + return (__m128) __builtin_ia32_loadaps128_mask ((__v4sf *) __P, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_store_ps (void *__P, __mmask8 __U, __m256 __A) +{ + __builtin_ia32_storeaps256_mask ((__v8sf *) __P, + (__v8sf) __A, + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_store_ps (void *__P, __mmask8 __U, __m128 __A) +{ + __builtin_ia32_storeaps128_mask ((__v4sf *) __P, + (__v4sf) __A, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_mov_epi64 (__m256i __W, __mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_movdqa64_256_mask ((__v4di) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_mov_epi64 (__mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_movdqa64_256_mask ((__v4di) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_mov_epi64 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_movdqa64_128_mask ((__v2di) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_mov_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_movdqa64_128_mask ((__v2di) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_load_epi64 (__m256i __W, __mmask8 __U, void const *__P) +{ + return (__m256i) __builtin_ia32_movdqa64load256_mask ((__v4di *) __P, + (__v4di) __W, + (__mmask8) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_load_epi64 (__mmask8 __U, void const *__P) +{ + return (__m256i) __builtin_ia32_movdqa64load256_mask ((__v4di *) __P, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_load_epi64 (__m128i __W, __mmask8 __U, void const *__P) +{ + return (__m128i) __builtin_ia32_movdqa64load128_mask ((__v2di *) __P, + (__v2di) __W, + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_load_epi64 (__mmask8 __U, void const *__P) +{ + return (__m128i) __builtin_ia32_movdqa64load128_mask ((__v2di *) __P, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) + __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_store_epi64 (void *__P, __mmask8 __U, __m256i __A) +{ + __builtin_ia32_movdqa64store256_mask ((__v4di *) __P, + (__v4di) __A, + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_store_epi64 (void *__P, __mmask8 __U, __m128i __A) +{ + __builtin_ia32_movdqa64store128_mask ((__v2di *) __P, + (__v2di) __A, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_mov_epi32 (__m256i __W, __mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_movdqa32_256_mask ((__v8si) __A, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_mov_epi32 (__mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_movdqa32_256_mask ((__v8si) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_mov_epi32 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_movdqa32_128_mask ((__v4si) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_mov_epi32 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_movdqa32_128_mask ((__v4si) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_load_epi32 (__m256i __W, __mmask8 __U, void const *__P) +{ + return (__m256i) __builtin_ia32_movdqa32load256_mask ((__v8si *) __P, + (__v8si) __W, + (__mmask8) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_load_epi32 (__mmask8 __U, void const *__P) +{ + return (__m256i) __builtin_ia32_movdqa32load256_mask ((__v8si *) __P, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_load_epi32 (__m128i __W, __mmask8 __U, void const *__P) +{ + return (__m128i) __builtin_ia32_movdqa32load128_mask ((__v4si *) __P, + (__v4si) __W, + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_load_epi32 (__mmask8 __U, void const *__P) +{ + return (__m128i) __builtin_ia32_movdqa32load128_mask ((__v4si *) __P, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) + __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_store_epi32 (void *__P, __mmask8 __U, __m256i __A) +{ + __builtin_ia32_movdqa32store256_mask ((__v8si *) __P, + (__v8si) __A, + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_store_epi32 (void *__P, __mmask8 __U, __m128i __A) +{ + __builtin_ia32_movdqa32store128_mask ((__v4si *) __P, + (__v4si) __A, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_add_pd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_addpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_add_pd (__mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_addpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_add_pd (__m256d __W, __mmask8 __U, __m256d __A, + __m256d __B) +{ + return (__m256d) __builtin_ia32_addpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_add_pd (__mmask8 __U, __m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_addpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_add_ps (__m128 __W, __mmask16 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_addps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_add_ps (__mmask16 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_addps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_add_ps (__m256 __W, __mmask16 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_addps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_add_ps (__mmask16 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_addps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sub_pd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_subpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sub_pd (__mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_subpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sub_pd (__m256d __W, __mmask8 __U, __m256d __A, + __m256d __B) +{ + return (__m256d) __builtin_ia32_subpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sub_pd (__mmask8 __U, __m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_subpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sub_ps (__m128 __W, __mmask16 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_subps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sub_ps (__mmask16 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_subps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sub_ps (__m256 __W, __mmask16 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_subps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sub_ps (__mmask16 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_subps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_store_epi64 (void *__P, __m256i __A) +{ + *(__m256i *) __P = __A; +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_store_epi64 (void *__P, __m128i __A) +{ + *(__m128i *) __P = __A; +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_loadu_pd (__m256d __W, __mmask8 __U, void const *__P) +{ + return (__m256d) __builtin_ia32_loadupd256_mask ((const double *) __P, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_loadu_pd (__mmask8 __U, void const *__P) +{ + return (__m256d) __builtin_ia32_loadupd256_mask ((const double *) __P, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_loadu_pd (__m128d __W, __mmask8 __U, void const *__P) +{ + return (__m128d) __builtin_ia32_loadupd128_mask ((const double *) __P, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_loadu_pd (__mmask8 __U, void const *__P) +{ + return (__m128d) __builtin_ia32_loadupd128_mask ((const double *) __P, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_storeu_pd (void *__P, __mmask8 __U, __m256d __A) +{ + __builtin_ia32_storeupd256_mask ((double *) __P, + (__v4df) __A, + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_storeu_pd (void *__P, __mmask8 __U, __m128d __A) +{ + __builtin_ia32_storeupd128_mask ((double *) __P, + (__v2df) __A, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_loadu_ps (__m256 __W, __mmask8 __U, void const *__P) +{ + return (__m256) __builtin_ia32_loadups256_mask ((const float *) __P, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_loadu_ps (__mmask8 __U, void const *__P) +{ + return (__m256) __builtin_ia32_loadups256_mask ((const float *) __P, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_loadu_ps (__m128 __W, __mmask8 __U, void const *__P) +{ + return (__m128) __builtin_ia32_loadups128_mask ((const float *) __P, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_loadu_ps (__mmask8 __U, void const *__P) +{ + return (__m128) __builtin_ia32_loadups128_mask ((const float *) __P, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_storeu_ps (void *__P, __mmask8 __U, __m256 __A) +{ + __builtin_ia32_storeups256_mask ((float *) __P, + (__v8sf) __A, + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_storeu_ps (void *__P, __mmask8 __U, __m128 __A) +{ + __builtin_ia32_storeups128_mask ((float *) __P, + (__v4sf) __A, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_loadu_epi64 (__m256i __W, __mmask8 __U, void const *__P) +{ + return (__m256i) __builtin_ia32_loaddqudi256_mask ((const long long *) __P, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_loadu_epi64 (__mmask8 __U, void const *__P) +{ + return (__m256i) __builtin_ia32_loaddqudi256_mask ((const long long *) __P, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_loadu_epi64 (__m128i __W, __mmask8 __U, void const *__P) +{ + return (__m128i) __builtin_ia32_loaddqudi128_mask ((const long long *) __P, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_loadu_epi64 (__mmask8 __U, void const *__P) +{ + return (__m128i) __builtin_ia32_loaddqudi128_mask ((const long long *) __P, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_storeu_epi64 (void *__P, __mmask8 __U, __m256i __A) +{ + __builtin_ia32_storedqudi256_mask ((long long *) __P, + (__v4di) __A, + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_storeu_epi64 (void *__P, __mmask8 __U, __m128i __A) +{ + __builtin_ia32_storedqudi128_mask ((long long *) __P, + (__v2di) __A, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_loadu_epi32 (__m256i __W, __mmask8 __U, void const *__P) +{ + return (__m256i) __builtin_ia32_loaddqusi256_mask ((const int *) __P, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_loadu_epi32 (__mmask8 __U, void const *__P) +{ + return (__m256i) __builtin_ia32_loaddqusi256_mask ((const int *) __P, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_loadu_epi32 (__m128i __W, __mmask8 __U, void const *__P) +{ + return (__m128i) __builtin_ia32_loaddqusi128_mask ((const int *) __P, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_loadu_epi32 (__mmask8 __U, void const *__P) +{ + return (__m128i) __builtin_ia32_loaddqusi128_mask ((const int *) __P, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_storeu_epi32 (void *__P, __mmask8 __U, __m256i __A) +{ + __builtin_ia32_storedqusi256_mask ((int *) __P, + (__v8si) __A, + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_storeu_epi32 (void *__P, __mmask8 __U, __m128i __A) +{ + __builtin_ia32_storedqusi128_mask ((int *) __P, + (__v4si) __A, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_abs_epi32 (__m256i __W, __mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_pabsd256_mask ((__v8si) __A, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_abs_epi32 (__mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_pabsd256_mask ((__v8si) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_abs_epi32 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pabsd128_mask ((__v4si) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_abs_epi32 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pabsd128_mask ((__v4si) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_abs_epi64 (__m256i __A) +{ + return (__m256i) __builtin_ia32_pabsq256_mask ((__v4di) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_abs_epi64 (__m256i __W, __mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_pabsq256_mask ((__v4di) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_abs_epi64 (__mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_pabsq256_mask ((__v4di) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_abs_epi64 (__m128i __A) +{ + return (__m128i) __builtin_ia32_pabsq128_mask ((__v2di) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_abs_epi64 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pabsq128_mask ((__v2di) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_abs_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pabsq128_mask ((__v2di) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtpd_epu32 (__m256d __A) +{ + return (__m128i) __builtin_ia32_cvtpd2udq256_mask ((__v4df) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtpd_epu32 (__m128i __W, __mmask8 __U, __m256d __A) +{ + return (__m128i) __builtin_ia32_cvtpd2udq256_mask ((__v4df) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtpd_epu32 (__mmask8 __U, __m256d __A) +{ + return (__m128i) __builtin_ia32_cvtpd2udq256_mask ((__v4df) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtpd_epu32 (__m128d __A) +{ + return (__m128i) __builtin_ia32_cvtpd2udq128_mask ((__v2df) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtpd_epu32 (__m128i __W, __mmask8 __U, __m128d __A) +{ + return (__m128i) __builtin_ia32_cvtpd2udq128_mask ((__v2df) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtpd_epu32 (__mmask8 __U, __m128d __A) +{ + return (__m128i) __builtin_ia32_cvtpd2udq128_mask ((__v2df) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvttps_epi32 (__m256i __W, __mmask8 __U, __m256 __A) +{ + return (__m256i) __builtin_ia32_cvttps2dq256_mask ((__v8sf) __A, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvttps_epi32 (__mmask8 __U, __m256 __A) +{ + return (__m256i) __builtin_ia32_cvttps2dq256_mask ((__v8sf) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvttps_epi32 (__m128i __W, __mmask8 __U, __m128 __A) +{ + return (__m128i) __builtin_ia32_cvttps2dq128_mask ((__v4sf) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvttps_epi32 (__mmask8 __U, __m128 __A) +{ + return (__m128i) __builtin_ia32_cvttps2dq128_mask ((__v4sf) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvttps_epu32 (__m256 __A) +{ + return (__m256i) __builtin_ia32_cvttps2udq256_mask ((__v8sf) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvttps_epu32 (__m256i __W, __mmask8 __U, __m256 __A) +{ + return (__m256i) __builtin_ia32_cvttps2udq256_mask ((__v8sf) __A, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvttps_epu32 (__mmask8 __U, __m256 __A) +{ + return (__m256i) __builtin_ia32_cvttps2udq256_mask ((__v8sf) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttps_epu32 (__m128 __A) +{ + return (__m128i) __builtin_ia32_cvttps2udq128_mask ((__v4sf) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvttps_epu32 (__m128i __W, __mmask8 __U, __m128 __A) +{ + return (__m128i) __builtin_ia32_cvttps2udq128_mask ((__v4sf) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvttps_epu32 (__mmask8 __U, __m128 __A) +{ + return (__m128i) __builtin_ia32_cvttps2udq128_mask ((__v4sf) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvttpd_epi32 (__m128i __W, __mmask8 __U, __m256d __A) +{ + return (__m128i) __builtin_ia32_cvttpd2dq256_mask ((__v4df) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvttpd_epi32 (__mmask8 __U, __m256d __A) +{ + return (__m128i) __builtin_ia32_cvttpd2dq256_mask ((__v4df) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvttpd_epi32 (__m128i __W, __mmask8 __U, __m128d __A) +{ + return (__m128i) __builtin_ia32_cvttpd2dq128_mask ((__v2df) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvttpd_epi32 (__mmask8 __U, __m128d __A) +{ + return (__m128i) __builtin_ia32_cvttpd2dq128_mask ((__v2df) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvttpd_epu32 (__m256d __A) +{ + return (__m128i) __builtin_ia32_cvttpd2udq256_mask ((__v4df) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvttpd_epu32 (__m128i __W, __mmask8 __U, __m256d __A) +{ + return (__m128i) __builtin_ia32_cvttpd2udq256_mask ((__v4df) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvttpd_epu32 (__mmask8 __U, __m256d __A) +{ + return (__m128i) __builtin_ia32_cvttpd2udq256_mask ((__v4df) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttpd_epu32 (__m128d __A) +{ + return (__m128i) __builtin_ia32_cvttpd2udq128_mask ((__v2df) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvttpd_epu32 (__m128i __W, __mmask8 __U, __m128d __A) +{ + return (__m128i) __builtin_ia32_cvttpd2udq128_mask ((__v2df) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvttpd_epu32 (__mmask8 __U, __m128d __A) +{ + return (__m128i) __builtin_ia32_cvttpd2udq128_mask ((__v2df) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtpd_epi32 (__m128i __W, __mmask8 __U, __m256d __A) +{ + return (__m128i) __builtin_ia32_cvtpd2dq256_mask ((__v4df) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtpd_epi32 (__mmask8 __U, __m256d __A) +{ + return (__m128i) __builtin_ia32_cvtpd2dq256_mask ((__v4df) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtpd_epi32 (__m128i __W, __mmask8 __U, __m128d __A) +{ + return (__m128i) __builtin_ia32_cvtpd2dq128_mask ((__v2df) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtpd_epi32 (__mmask8 __U, __m128d __A) +{ + return (__m128i) __builtin_ia32_cvtpd2dq128_mask ((__v2df) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi32_pd (__m256d __W, __mmask8 __U, __m128i __A) +{ + return (__m256d) __builtin_ia32_cvtdq2pd256_mask ((__v4si) __A, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepi32_pd (__mmask8 __U, __m128i __A) +{ + return (__m256d) __builtin_ia32_cvtdq2pd256_mask ((__v4si) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi32_pd (__m128d __W, __mmask8 __U, __m128i __A) +{ + return (__m128d) __builtin_ia32_cvtdq2pd128_mask ((__v4si) __A, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepi32_pd (__mmask8 __U, __m128i __A) +{ + return (__m128d) __builtin_ia32_cvtdq2pd128_mask ((__v4si) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepu32_pd (__m128i __A) +{ + return (__m256d) __builtin_ia32_cvtudq2pd256_mask ((__v4si) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepu32_pd (__m256d __W, __mmask8 __U, __m128i __A) +{ + return (__m256d) __builtin_ia32_cvtudq2pd256_mask ((__v4si) __A, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepu32_pd (__mmask8 __U, __m128i __A) +{ + return (__m256d) __builtin_ia32_cvtudq2pd256_mask ((__v4si) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepu32_pd (__m128i __A) +{ + return (__m128d) __builtin_ia32_cvtudq2pd128_mask ((__v4si) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepu32_pd (__m128d __W, __mmask8 __U, __m128i __A) +{ + return (__m128d) __builtin_ia32_cvtudq2pd128_mask ((__v4si) __A, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepu32_pd (__mmask8 __U, __m128i __A) +{ + return (__m128d) __builtin_ia32_cvtudq2pd128_mask ((__v4si) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi32_ps (__m256 __W, __mmask8 __U, __m256i __A) +{ + return (__m256) __builtin_ia32_cvtdq2ps256_mask ((__v8si) __A, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepi32_ps (__mmask16 __U, __m256i __A) +{ + return (__m256) __builtin_ia32_cvtdq2ps256_mask ((__v8si) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi32_ps (__m128 __W, __mmask8 __U, __m128i __A) +{ + return (__m128) __builtin_ia32_cvtdq2ps128_mask ((__v4si) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepi32_ps (__mmask16 __U, __m128i __A) +{ + return (__m128) __builtin_ia32_cvtdq2ps128_mask ((__v4si) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepu32_ps (__m256i __A) +{ + return (__m256) __builtin_ia32_cvtudq2ps256_mask ((__v8si) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepu32_ps (__m256 __W, __mmask8 __U, __m256i __A) +{ + return (__m256) __builtin_ia32_cvtudq2ps256_mask ((__v8si) __A, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepu32_ps (__mmask8 __U, __m256i __A) +{ + return (__m256) __builtin_ia32_cvtudq2ps256_mask ((__v8si) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepu32_ps (__m128i __A) +{ + return (__m128) __builtin_ia32_cvtudq2ps128_mask ((__v4si) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepu32_ps (__m128 __W, __mmask8 __U, __m128i __A) +{ + return (__m128) __builtin_ia32_cvtudq2ps128_mask ((__v4si) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepu32_ps (__mmask8 __U, __m128i __A) +{ + return (__m128) __builtin_ia32_cvtudq2ps128_mask ((__v4si) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtps_pd (__m256d __W, __mmask8 __U, __m128 __A) +{ + return (__m256d) __builtin_ia32_cvtps2pd256_mask ((__v4sf) __A, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtps_pd (__mmask8 __U, __m128 __A) +{ + return (__m256d) __builtin_ia32_cvtps2pd256_mask ((__v4sf) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtps_pd (__m128d __W, __mmask8 __U, __m128 __A) +{ + return (__m128d) __builtin_ia32_cvtps2pd128_mask ((__v4sf) __A, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtps_pd (__mmask8 __U, __m128 __A) +{ + return (__m128d) __builtin_ia32_cvtps2pd128_mask ((__v4sf) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepi32_epi8 (__m128i __A) +{ + return (__m128i) __builtin_ia32_pmovdb128_mask ((__v4si) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi32_storeu_epi8 (void * __P, __mmask8 __M, __m128i __A) +{ + __builtin_ia32_pmovdb128mem_mask ((__v16qi *) __P, (__v4si) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi32_epi8 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovdb128_mask ((__v4si) __A, + (__v16qi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepi32_epi8 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovdb128_mask ((__v4si) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi32_epi8 (__m256i __A) +{ + return (__m128i) __builtin_ia32_pmovdb256_mask ((__v8si) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi32_epi8 (__m128i __O, __mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovdb256_mask ((__v8si) __A, + (__v16qi) __O, __M); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi32_storeu_epi8 (void * __P, __mmask8 __M, __m256i __A) +{ + __builtin_ia32_pmovdb256mem_mask ((__v16qi *) __P, (__v8si) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepi32_epi8 (__mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovdb256_mask ((__v8si) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsepi32_epi8 (__m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsdb128_mask ((__v4si) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtsepi32_storeu_epi8 (void * __P, __mmask8 __M, __m128i __A) +{ + __builtin_ia32_pmovsdb128mem_mask ((__v16qi *) __P, (__v4si) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtsepi32_epi8 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsdb128_mask ((__v4si) __A, + (__v16qi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtsepi32_epi8 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsdb128_mask ((__v4si) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtsepi32_epi8 (__m256i __A) +{ + return (__m128i) __builtin_ia32_pmovsdb256_mask ((__v8si) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtsepi32_storeu_epi8 (void * __P, __mmask8 __M, __m256i __A) +{ + __builtin_ia32_pmovsdb256mem_mask ((__v16qi *) __P, (__v8si) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtsepi32_epi8 (__m128i __O, __mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovsdb256_mask ((__v8si) __A, + (__v16qi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtsepi32_epi8 (__mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovsdb256_mask ((__v8si) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtusepi32_epi8 (__m128i __A) +{ + return (__m128i) __builtin_ia32_pmovusdb128_mask ((__v4si) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtusepi32_storeu_epi8 (void * __P, __mmask8 __M, __m128i __A) +{ + __builtin_ia32_pmovusdb128mem_mask ((__v16qi *) __P, (__v4si) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtusepi32_epi8 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovusdb128_mask ((__v4si) __A, + (__v16qi) __O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtusepi32_epi8 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovusdb128_mask ((__v4si) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtusepi32_epi8 (__m256i __A) +{ + return (__m128i) __builtin_ia32_pmovusdb256_mask ((__v8si) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtusepi32_storeu_epi8 (void * __P, __mmask8 __M, __m256i __A) +{ + __builtin_ia32_pmovusdb256mem_mask ((__v16qi*) __P, (__v8si) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtusepi32_epi8 (__m128i __O, __mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovusdb256_mask ((__v8si) __A, + (__v16qi) __O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtusepi32_epi8 (__mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovusdb256_mask ((__v8si) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepi32_epi16 (__m128i __A) +{ + return (__m128i) __builtin_ia32_pmovdw128_mask ((__v4si) __A, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi32_storeu_epi16 (void * __P, __mmask8 __M, __m128i __A) +{ + __builtin_ia32_pmovdw128mem_mask ((__v8hi *) __P, (__v4si) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi32_epi16 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovdw128_mask ((__v4si) __A, + (__v8hi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepi32_epi16 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovdw128_mask ((__v4si) __A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi32_epi16 (__m256i __A) +{ + return (__m128i) __builtin_ia32_pmovdw256_mask ((__v8si) __A, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi32_storeu_epi16 (void * __P, __mmask8 __M, __m256i __A) +{ + __builtin_ia32_pmovdw256mem_mask ((__v8hi *) __P, (__v8si) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi32_epi16 (__m128i __O, __mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovdw256_mask ((__v8si) __A, + (__v8hi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepi32_epi16 (__mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovdw256_mask ((__v8si) __A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsepi32_epi16 (__m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsdw128_mask ((__v4si) __A, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtsepi32_storeu_epi16 (void * __P, __mmask8 __M, __m128i __A) +{ + __builtin_ia32_pmovsdw128mem_mask ((__v8hi *) __P, (__v4si) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtsepi32_epi16 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsdw128_mask ((__v4si) __A, + (__v8hi)__O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtsepi32_epi16 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsdw128_mask ((__v4si) __A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtsepi32_epi16 (__m256i __A) +{ + return (__m128i) __builtin_ia32_pmovsdw256_mask ((__v8si) __A, + (__v8hi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtsepi32_storeu_epi16 (void * __P, __mmask8 __M, __m256i __A) +{ + __builtin_ia32_pmovsdw256mem_mask ((__v8hi *) __P, (__v8si) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtsepi32_epi16 (__m128i __O, __mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovsdw256_mask ((__v8si) __A, + (__v8hi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtsepi32_epi16 (__mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovsdw256_mask ((__v8si) __A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtusepi32_epi16 (__m128i __A) +{ + return (__m128i) __builtin_ia32_pmovusdw128_mask ((__v4si) __A, + (__v8hi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtusepi32_storeu_epi16 (void * __P, __mmask8 __M, __m128i __A) +{ + __builtin_ia32_pmovusdw128mem_mask ((__v8hi *) __P, (__v4si) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtusepi32_epi16 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovusdw128_mask ((__v4si) __A, + (__v8hi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtusepi32_epi16 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovusdw128_mask ((__v4si) __A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtusepi32_epi16 (__m256i __A) +{ + return (__m128i) __builtin_ia32_pmovusdw256_mask ((__v8si) __A, + (__v8hi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtusepi32_storeu_epi16 (void * __P, __mmask8 __M, __m256i __A) +{ + __builtin_ia32_pmovusdw256mem_mask ((__v8hi *) __P, (__v8si) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtusepi32_epi16 (__m128i __O, __mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovusdw256_mask ((__v8si) __A, + (__v8hi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtusepi32_epi16 (__mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovusdw256_mask ((__v8si) __A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepi64_epi8 (__m128i __A) +{ + return (__m128i) __builtin_ia32_pmovqb128_mask ((__v2di) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi64_storeu_epi8 (void * __P, __mmask8 __M, __m128i __A) +{ + __builtin_ia32_pmovqb128mem_mask ((__v16qi *) __P, (__v2di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi64_epi8 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovqb128_mask ((__v2di) __A, + (__v16qi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepi64_epi8 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovqb128_mask ((__v2di) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi64_epi8 (__m256i __A) +{ + return (__m128i) __builtin_ia32_pmovqb256_mask ((__v4di) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi64_storeu_epi8 (void * __P, __mmask8 __M, __m256i __A) +{ + __builtin_ia32_pmovqb256mem_mask ((__v16qi *) __P, (__v4di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi64_epi8 (__m128i __O, __mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovqb256_mask ((__v4di) __A, + (__v16qi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepi64_epi8 (__mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovqb256_mask ((__v4di) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsepi64_epi8 (__m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsqb128_mask ((__v2di) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtsepi64_storeu_epi8 (void * __P, __mmask8 __M, __m128i __A) +{ + __builtin_ia32_pmovsqb128mem_mask ((__v16qi *) __P, (__v2di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtsepi64_epi8 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsqb128_mask ((__v2di) __A, + (__v16qi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtsepi64_epi8 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsqb128_mask ((__v2di) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtsepi64_epi8 (__m256i __A) +{ + return (__m128i) __builtin_ia32_pmovsqb256_mask ((__v4di) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtsepi64_storeu_epi8 (void * __P, __mmask8 __M, __m256i __A) +{ + __builtin_ia32_pmovsqb256mem_mask ((__v16qi *) __P, (__v4di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtsepi64_epi8 (__m128i __O, __mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovsqb256_mask ((__v4di) __A, + (__v16qi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtsepi64_epi8 (__mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovsqb256_mask ((__v4di) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtusepi64_epi8 (__m128i __A) +{ + return (__m128i) __builtin_ia32_pmovusqb128_mask ((__v2di) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtusepi64_storeu_epi8 (void * __P, __mmask8 __M, __m128i __A) +{ + __builtin_ia32_pmovusqb128mem_mask ((__v16qi *) __P, (__v2di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtusepi64_epi8 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovusqb128_mask ((__v2di) __A, + (__v16qi) __O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtusepi64_epi8 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovusqb128_mask ((__v2di) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtusepi64_epi8 (__m256i __A) +{ + return (__m128i) __builtin_ia32_pmovusqb256_mask ((__v4di) __A, + (__v16qi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtusepi64_storeu_epi8 (void * __P, __mmask8 __M, __m256i __A) +{ + __builtin_ia32_pmovusqb256mem_mask ((__v16qi *) __P, (__v4di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtusepi64_epi8 (__m128i __O, __mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovusqb256_mask ((__v4di) __A, + (__v16qi) __O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtusepi64_epi8 (__mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovusqb256_mask ((__v4di) __A, + (__v16qi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepi64_epi16 (__m128i __A) +{ + return (__m128i) __builtin_ia32_pmovqw128_mask ((__v2di) __A, + (__v8hi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi64_storeu_epi16 (void * __P, __mmask8 __M, __m128i __A) +{ + __builtin_ia32_pmovqw128mem_mask ((__v8hi *) __P, (__v2di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi64_epi16 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovqw128_mask ((__v2di) __A, + (__v8hi)__O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepi64_epi16 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovqw128_mask ((__v2di) __A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi64_epi16 (__m256i __A) +{ + return (__m128i) __builtin_ia32_pmovqw256_mask ((__v4di) __A, + (__v8hi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi64_storeu_epi16 (void * __P, __mmask8 __M, __m256i __A) +{ + __builtin_ia32_pmovqw256mem_mask ((__v8hi *) __P, (__v4di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi64_epi16 (__m128i __O, __mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovqw256_mask ((__v4di) __A, + (__v8hi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepi64_epi16 (__mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovqw256_mask ((__v4di) __A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsepi64_epi16 (__m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsqw128_mask ((__v2di) __A, + (__v8hi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtsepi64_storeu_epi16 (void * __P, __mmask8 __M, __m128i __A) +{ + __builtin_ia32_pmovsqw128mem_mask ((__v8hi *) __P, (__v2di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtsepi64_epi16 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsqw128_mask ((__v2di) __A, + (__v8hi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtsepi64_epi16 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsqw128_mask ((__v2di) __A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtsepi64_epi16 (__m256i __A) +{ + return (__m128i) __builtin_ia32_pmovsqw256_mask ((__v4di) __A, + (__v8hi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtsepi64_storeu_epi16 (void * __P, __mmask8 __M, __m256i __A) +{ + __builtin_ia32_pmovsqw256mem_mask ((__v8hi *) __P, (__v4di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtsepi64_epi16 (__m128i __O, __mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovsqw256_mask ((__v4di) __A, + (__v8hi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtsepi64_epi16 (__mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovsqw256_mask ((__v4di) __A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtusepi64_epi16 (__m128i __A) +{ + return (__m128i) __builtin_ia32_pmovusqw128_mask ((__v2di) __A, + (__v8hi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtusepi64_storeu_epi16 (void * __P, __mmask8 __M, __m128i __A) +{ + __builtin_ia32_pmovusqw128mem_mask ((__v8hi *) __P, (__v2di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtusepi64_epi16 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovusqw128_mask ((__v2di) __A, + (__v8hi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtusepi64_epi16 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovusqw128_mask ((__v2di) __A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtusepi64_epi16 (__m256i __A) +{ + return (__m128i) __builtin_ia32_pmovusqw256_mask ((__v4di) __A, + (__v8hi) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtusepi64_storeu_epi16 (void * __P, __mmask8 __M, __m256i __A) +{ + __builtin_ia32_pmovusqw256mem_mask ((__v8hi *) __P, (__v4di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtusepi64_epi16 (__m128i __O, __mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovusqw256_mask ((__v4di) __A, + (__v8hi) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtusepi64_epi16 (__mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovusqw256_mask ((__v4di) __A, + (__v8hi) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepi64_epi32 (__m128i __A) +{ + return (__m128i) __builtin_ia32_pmovqd128_mask ((__v2di) __A, + (__v4si) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi64_storeu_epi32 (void * __P, __mmask8 __M, __m128i __A) +{ + __builtin_ia32_pmovqd128mem_mask ((__v4si *) __P, (__v2di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi64_epi32 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovqd128_mask ((__v2di) __A, + (__v4si) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepi64_epi32 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovqd128_mask ((__v2di) __A, + (__v4si) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi64_epi32 (__m256i __A) +{ + return (__m128i) __builtin_ia32_pmovqd256_mask ((__v4di) __A, + (__v4si) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi64_storeu_epi32 (void * __P, __mmask8 __M, __m256i __A) +{ + __builtin_ia32_pmovqd256mem_mask ((__v4si *) __P, (__v4di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi64_epi32 (__m128i __O, __mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovqd256_mask ((__v4di) __A, + (__v4si) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepi64_epi32 (__mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovqd256_mask ((__v4di) __A, + (__v4si) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsepi64_epi32 (__m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsqd128_mask ((__v2di) __A, + (__v4si) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtsepi64_storeu_epi32 (void * __P, __mmask8 __M, __m128i __A) +{ + __builtin_ia32_pmovsqd128mem_mask ((__v4si *) __P, (__v2di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtsepi64_epi32 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsqd128_mask ((__v2di) __A, + (__v4si) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtsepi64_epi32 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsqd128_mask ((__v2di) __A, + (__v4si) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtsepi64_epi32 (__m256i __A) +{ + return (__m128i) __builtin_ia32_pmovsqd256_mask ((__v4di) __A, + (__v4si) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtsepi64_storeu_epi32 (void * __P, __mmask8 __M, __m256i __A) +{ + __builtin_ia32_pmovsqd256mem_mask ((__v4si *) __P, (__v4di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtsepi64_epi32 (__m128i __O, __mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovsqd256_mask ((__v4di) __A, + (__v4si)__O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtsepi64_epi32 (__mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovsqd256_mask ((__v4di) __A, + (__v4si) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtusepi64_epi32 (__m128i __A) +{ + return (__m128i) __builtin_ia32_pmovusqd128_mask ((__v2di) __A, + (__v4si) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtusepi64_storeu_epi32 (void * __P, __mmask8 __M, __m128i __A) +{ + __builtin_ia32_pmovusqd128mem_mask ((__v4si *) __P, (__v2di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtusepi64_epi32 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovusqd128_mask ((__v2di) __A, + (__v4si) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtusepi64_epi32 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovusqd128_mask ((__v2di) __A, + (__v4si) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtusepi64_epi32 (__m256i __A) +{ + return (__m128i) __builtin_ia32_pmovusqd256_mask ((__v4di) __A, + (__v4si) + _mm_undefined_si128 (), + (__mmask8) -1); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtusepi64_storeu_epi32 (void * __P, __mmask8 __M, __m256i __A) +{ + __builtin_ia32_pmovusqd256mem_mask ((__v4si *) __P, (__v4di) __A, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtusepi64_epi32 (__m128i __O, __mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovusqd256_mask ((__v4di) __A, + (__v4si) __O, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtusepi64_epi32 (__mmask8 __M, __m256i __A) +{ + return (__m128i) __builtin_ia32_pmovusqd256_mask ((__v4di) __A, + (__v4si) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_broadcastss_ps (__m256 __O, __mmask8 __M, __m128 __A) +{ + return (__m256) __builtin_ia32_broadcastss256_mask ((__v4sf) __A, + (__v8sf) __O, + __M); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_broadcastss_ps (__mmask8 __M, __m128 __A) +{ + return (__m256) __builtin_ia32_broadcastss256_mask ((__v4sf) __A, + (__v8sf) + _mm256_setzero_ps (), + __M); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_broadcastss_ps (__m128 __O, __mmask8 __M, __m128 __A) +{ + return (__m128) __builtin_ia32_broadcastss128_mask ((__v4sf) __A, + (__v4sf) __O, + __M); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_broadcastss_ps (__mmask8 __M, __m128 __A) +{ + return (__m128) __builtin_ia32_broadcastss128_mask ((__v4sf) __A, + (__v4sf) + _mm_setzero_ps (), + __M); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_broadcastsd_pd (__m256d __O, __mmask8 __M, __m128d __A) +{ + return (__m256d) __builtin_ia32_broadcastsd256_mask ((__v2df) __A, + (__v4df) __O, + __M); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_broadcastsd_pd (__mmask8 __M, __m128d __A) +{ + return (__m256d) __builtin_ia32_broadcastsd256_mask ((__v2df) __A, + (__v4df) + _mm256_setzero_pd (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_broadcastd_epi32 (__m256i __O, __mmask8 __M, __m128i __A) +{ + return (__m256i) __builtin_ia32_pbroadcastd256_mask ((__v4si) __A, + (__v8si) __O, + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_broadcastd_epi32 (__mmask8 __M, __m128i __A) +{ + return (__m256i) __builtin_ia32_pbroadcastd256_mask ((__v4si) __A, + (__v8si) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_set1_epi32 (__m256i __O, __mmask8 __M, int __A) +{ + return (__m256i) __builtin_ia32_pbroadcastd256_gpr_mask (__A, (__v8si) __O, + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_set1_epi32 (__mmask8 __M, int __A) +{ + return (__m256i) __builtin_ia32_pbroadcastd256_gpr_mask (__A, + (__v8si) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_broadcastd_epi32 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pbroadcastd128_mask ((__v4si) __A, + (__v4si) __O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_broadcastd_epi32 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pbroadcastd128_mask ((__v4si) __A, + (__v4si) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_set1_epi32 (__m128i __O, __mmask8 __M, int __A) +{ + return (__m128i) __builtin_ia32_pbroadcastd128_gpr_mask (__A, (__v4si) __O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_set1_epi32 (__mmask8 __M, int __A) +{ + return (__m128i) + __builtin_ia32_pbroadcastd128_gpr_mask (__A, + (__v4si) _mm_setzero_si128 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_broadcastq_epi64 (__m256i __O, __mmask8 __M, __m128i __A) +{ + return (__m256i) __builtin_ia32_pbroadcastq256_mask ((__v2di) __A, + (__v4di) __O, + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_broadcastq_epi64 (__mmask8 __M, __m128i __A) +{ + return (__m256i) __builtin_ia32_pbroadcastq256_mask ((__v2di) __A, + (__v4di) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_set1_epi64 (__m256i __O, __mmask8 __M, long long __A) +{ + return (__m256i) __builtin_ia32_pbroadcastq256_gpr_mask (__A, (__v4di) __O, + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_set1_epi64 (__mmask8 __M, long long __A) +{ + return (__m256i) __builtin_ia32_pbroadcastq256_gpr_mask (__A, + (__v4di) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_broadcastq_epi64 (__m128i __O, __mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pbroadcastq128_mask ((__v2di) __A, + (__v2di) __O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_broadcastq_epi64 (__mmask8 __M, __m128i __A) +{ + return (__m128i) __builtin_ia32_pbroadcastq128_mask ((__v2di) __A, + (__v2di) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_set1_epi64 (__m128i __O, __mmask8 __M, long long __A) +{ + return (__m128i) __builtin_ia32_pbroadcastq128_gpr_mask (__A, (__v2di) __O, + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_set1_epi64 (__mmask8 __M, long long __A) +{ + return (__m128i) + __builtin_ia32_pbroadcastq128_gpr_mask (__A, + (__v2di) _mm_setzero_si128 (), + __M); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcast_f32x4 (__m128 __A) +{ + return (__m256) __builtin_ia32_broadcastf32x4_256_mask ((__v4sf) __A, + (__v8sf)_mm256_undefined_pd (), + (__mmask8) -1); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_broadcast_f32x4 (__m256 __O, __mmask8 __M, __m128 __A) +{ + return (__m256) __builtin_ia32_broadcastf32x4_256_mask ((__v4sf) __A, + (__v8sf) __O, + __M); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_broadcast_f32x4 (__mmask8 __M, __m128 __A) +{ + return (__m256) __builtin_ia32_broadcastf32x4_256_mask ((__v4sf) __A, + (__v8sf) + _mm256_setzero_ps (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcast_i32x4 (__m128i __A) +{ + return (__m256i) __builtin_ia32_broadcasti32x4_256_mask ((__v4si) + __A, + (__v8si)_mm256_undefined_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_broadcast_i32x4 (__m256i __O, __mmask8 __M, __m128i __A) +{ + return (__m256i) __builtin_ia32_broadcasti32x4_256_mask ((__v4si) + __A, + (__v8si) + __O, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_broadcast_i32x4 (__mmask8 __M, __m128i __A) +{ + return (__m256i) __builtin_ia32_broadcasti32x4_256_mask ((__v4si) + __A, + (__v8si) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi8_epi32 (__m256i __W, __mmask8 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovsxbd256_mask ((__v16qi) __A, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepi8_epi32 (__mmask8 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovsxbd256_mask ((__v16qi) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi8_epi32 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsxbd128_mask ((__v16qi) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepi8_epi32 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsxbd128_mask ((__v16qi) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi8_epi64 (__m256i __W, __mmask8 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovsxbq256_mask ((__v16qi) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepi8_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovsxbq256_mask ((__v16qi) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi8_epi64 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsxbq128_mask ((__v16qi) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepi8_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsxbq128_mask ((__v16qi) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi16_epi32 (__m256i __W, __mmask8 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovsxwd256_mask ((__v8hi) __A, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepi16_epi32 (__mmask8 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovsxwd256_mask ((__v8hi) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi16_epi32 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsxwd128_mask ((__v8hi) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepi16_epi32 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsxwd128_mask ((__v8hi) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi16_epi64 (__m256i __W, __mmask8 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovsxwq256_mask ((__v8hi) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepi16_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovsxwq256_mask ((__v8hi) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi16_epi64 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsxwq128_mask ((__v8hi) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepi16_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovsxwq128_mask ((__v8hi) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepi32_epi64 (__m256i __W, __mmask8 __U, __m128i __X) +{ + return (__m256i) __builtin_ia32_pmovsxdq256_mask ((__v4si) __X, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepi32_epi64 (__mmask8 __U, __m128i __X) +{ + return (__m256i) __builtin_ia32_pmovsxdq256_mask ((__v4si) __X, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepi32_epi64 (__m128i __W, __mmask8 __U, __m128i __X) +{ + return (__m128i) __builtin_ia32_pmovsxdq128_mask ((__v4si) __X, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepi32_epi64 (__mmask8 __U, __m128i __X) +{ + return (__m128i) __builtin_ia32_pmovsxdq128_mask ((__v4si) __X, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepu8_epi32 (__m256i __W, __mmask8 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovzxbd256_mask ((__v16qi) __A, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepu8_epi32 (__mmask8 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovzxbd256_mask ((__v16qi) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepu8_epi32 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovzxbd128_mask ((__v16qi) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepu8_epi32 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovzxbd128_mask ((__v16qi) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepu8_epi64 (__m256i __W, __mmask8 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovzxbq256_mask ((__v16qi) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepu8_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovzxbq256_mask ((__v16qi) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepu8_epi64 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovzxbq128_mask ((__v16qi) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepu8_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovzxbq128_mask ((__v16qi) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepu16_epi32 (__m256i __W, __mmask8 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovzxwd256_mask ((__v8hi) __A, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepu16_epi32 (__mmask8 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovzxwd256_mask ((__v8hi) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepu16_epi32 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovzxwd128_mask ((__v8hi) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepu16_epi32 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovzxwd128_mask ((__v8hi) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepu16_epi64 (__m256i __W, __mmask8 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovzxwq256_mask ((__v8hi) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepu16_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m256i) __builtin_ia32_pmovzxwq256_mask ((__v8hi) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepu16_epi64 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovzxwq128_mask ((__v8hi) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepu16_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_pmovzxwq128_mask ((__v8hi) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtepu32_epi64 (__m256i __W, __mmask8 __U, __m128i __X) +{ + return (__m256i) __builtin_ia32_pmovzxdq256_mask ((__v4si) __X, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtepu32_epi64 (__mmask8 __U, __m128i __X) +{ + return (__m256i) __builtin_ia32_pmovzxdq256_mask ((__v4si) __X, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtepu32_epi64 (__m128i __W, __mmask8 __U, __m128i __X) +{ + return (__m128i) __builtin_ia32_pmovzxdq128_mask ((__v4si) __X, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtepu32_epi64 (__mmask8 __U, __m128i __X) +{ + return (__m128i) __builtin_ia32_pmovzxdq128_mask ((__v4si) __X, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_rcp14_pd (__m256d __A) +{ + return (__m256d) __builtin_ia32_rcp14pd256_mask ((__v4df) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_rcp14_pd (__m256d __W, __mmask8 __U, __m256d __A) +{ + return (__m256d) __builtin_ia32_rcp14pd256_mask ((__v4df) __A, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_rcp14_pd (__mmask8 __U, __m256d __A) +{ + return (__m256d) __builtin_ia32_rcp14pd256_mask ((__v4df) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rcp14_pd (__m128d __A) +{ + return (__m128d) __builtin_ia32_rcp14pd128_mask ((__v2df) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_rcp14_pd (__m128d __W, __mmask8 __U, __m128d __A) +{ + return (__m128d) __builtin_ia32_rcp14pd128_mask ((__v2df) __A, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_rcp14_pd (__mmask8 __U, __m128d __A) +{ + return (__m128d) __builtin_ia32_rcp14pd128_mask ((__v2df) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_rcp14_ps (__m256 __A) +{ + return (__m256) __builtin_ia32_rcp14ps256_mask ((__v8sf) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_rcp14_ps (__m256 __W, __mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_rcp14ps256_mask ((__v8sf) __A, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_rcp14_ps (__mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_rcp14ps256_mask ((__v8sf) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rcp14_ps (__m128 __A) +{ + return (__m128) __builtin_ia32_rcp14ps128_mask ((__v4sf) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_rcp14_ps (__m128 __W, __mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_rcp14ps128_mask ((__v4sf) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_rcp14_ps (__mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_rcp14ps128_mask ((__v4sf) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_rsqrt14_pd (__m256d __A) +{ + return (__m256d) __builtin_ia32_rsqrt14pd256_mask ((__v4df) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_rsqrt14_pd (__m256d __W, __mmask8 __U, __m256d __A) +{ + return (__m256d) __builtin_ia32_rsqrt14pd256_mask ((__v4df) __A, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_rsqrt14_pd (__mmask8 __U, __m256d __A) +{ + return (__m256d) __builtin_ia32_rsqrt14pd256_mask ((__v4df) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rsqrt14_pd (__m128d __A) +{ + return (__m128d) __builtin_ia32_rsqrt14pd128_mask ((__v2df) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_rsqrt14_pd (__m128d __W, __mmask8 __U, __m128d __A) +{ + return (__m128d) __builtin_ia32_rsqrt14pd128_mask ((__v2df) __A, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_rsqrt14_pd (__mmask8 __U, __m128d __A) +{ + return (__m128d) __builtin_ia32_rsqrt14pd128_mask ((__v2df) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_rsqrt14_ps (__m256 __A) +{ + return (__m256) __builtin_ia32_rsqrt14ps256_mask ((__v8sf) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_rsqrt14_ps (__m256 __W, __mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_rsqrt14ps256_mask ((__v8sf) __A, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_rsqrt14_ps (__mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_rsqrt14ps256_mask ((__v8sf) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rsqrt14_ps (__m128 __A) +{ + return (__m128) __builtin_ia32_rsqrt14ps128_mask ((__v4sf) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_rsqrt14_ps (__m128 __W, __mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_rsqrt14ps128_mask ((__v4sf) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_rsqrt14_ps (__mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_rsqrt14ps128_mask ((__v4sf) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sqrt_pd (__m256d __W, __mmask8 __U, __m256d __A) +{ + return (__m256d) __builtin_ia32_sqrtpd256_mask ((__v4df) __A, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sqrt_pd (__mmask8 __U, __m256d __A) +{ + return (__m256d) __builtin_ia32_sqrtpd256_mask ((__v4df) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sqrt_pd (__m128d __W, __mmask8 __U, __m128d __A) +{ + return (__m128d) __builtin_ia32_sqrtpd128_mask ((__v2df) __A, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sqrt_pd (__mmask8 __U, __m128d __A) +{ + return (__m128d) __builtin_ia32_sqrtpd128_mask ((__v2df) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sqrt_ps (__m256 __W, __mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_sqrtps256_mask ((__v8sf) __A, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sqrt_ps (__mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_sqrtps256_mask ((__v8sf) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sqrt_ps (__m128 __W, __mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_sqrtps128_mask ((__v4sf) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sqrt_ps (__mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_sqrtps128_mask ((__v4sf) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_add_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_paddd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_add_epi32 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_paddd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_add_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_paddq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_add_epi64 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_paddq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sub_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_psubd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sub_epi32 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_psubd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sub_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_psubq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sub_epi64 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_psubq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_add_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_paddd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_add_epi32 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_paddd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_add_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_paddq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_add_epi64 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_paddq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sub_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psubd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sub_epi32 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psubd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sub_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psubq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sub_epi64 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psubq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_getexp_ps (__m256 __A) +{ + return (__m256) __builtin_ia32_getexpps256_mask ((__v8sf) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_getexp_ps (__m256 __W, __mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_getexpps256_mask ((__v8sf) __A, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_getexp_ps (__mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_getexpps256_mask ((__v8sf) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_getexp_pd (__m256d __A) +{ + return (__m256d) __builtin_ia32_getexppd256_mask ((__v4df) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_getexp_pd (__m256d __W, __mmask8 __U, __m256d __A) +{ + return (__m256d) __builtin_ia32_getexppd256_mask ((__v4df) __A, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_getexp_pd (__mmask8 __U, __m256d __A) +{ + return (__m256d) __builtin_ia32_getexppd256_mask ((__v4df) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_getexp_ps (__m128 __A) +{ + return (__m128) __builtin_ia32_getexpps128_mask ((__v4sf) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_getexp_ps (__m128 __W, __mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_getexpps128_mask ((__v4sf) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_getexp_ps (__mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_getexpps128_mask ((__v4sf) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_getexp_pd (__m128d __A) +{ + return (__m128d) __builtin_ia32_getexppd128_mask ((__v2df) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_getexp_pd (__m128d __W, __mmask8 __U, __m128d __A) +{ + return (__m128d) __builtin_ia32_getexppd128_mask ((__v2df) __A, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_getexp_pd (__mmask8 __U, __m128d __A) +{ + return (__m128d) __builtin_ia32_getexppd128_mask ((__v2df) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_srl_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + __m128i __B) +{ + return (__m256i) __builtin_ia32_psrld256_mask ((__v8si) __A, + (__v4si) __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_srl_epi32 (__mmask8 __U, __m256i __A, __m128i __B) +{ + return (__m256i) __builtin_ia32_psrld256_mask ((__v8si) __A, + (__v4si) __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_srl_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psrld128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_srl_epi32 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psrld128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_srl_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + __m128i __B) +{ + return (__m256i) __builtin_ia32_psrlq256_mask ((__v4di) __A, + (__v2di) __B, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_srl_epi64 (__mmask8 __U, __m256i __A, __m128i __B) +{ + return (__m256i) __builtin_ia32_psrlq256_mask ((__v4di) __A, + (__v2di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_srl_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psrlq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_srl_epi64 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psrlq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_and_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pandd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_and_epi32 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pandd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_scalef_pd (__m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_scalefpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_scalef_pd (__m256d __W, __mmask8 __U, __m256d __A, + __m256d __B) +{ + return (__m256d) __builtin_ia32_scalefpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_scalef_pd (__mmask8 __U, __m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_scalefpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_scalef_ps (__m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_scalefps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_scalef_ps (__m256 __W, __mmask8 __U, __m256 __A, + __m256 __B) +{ + return (__m256) __builtin_ia32_scalefps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_scalef_ps (__mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_scalefps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_scalef_pd (__m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_scalefpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_scalef_pd (__m128d __W, __mmask8 __U, __m128d __A, + __m128d __B) +{ + return (__m128d) __builtin_ia32_scalefpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_scalef_pd (__mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_scalefpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_scalef_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_scalefps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_scalef_ps (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_scalefps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_scalef_ps (__mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_scalefps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_fmadd_pd (__m256d __A, __mmask8 __U, __m256d __B, + __m256d __C) +{ + return (__m256d) __builtin_ia32_vfmaddpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __C, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask3_fmadd_pd (__m256d __A, __m256d __B, __m256d __C, + __mmask8 __U) +{ + return (__m256d) __builtin_ia32_vfmaddpd256_mask3 ((__v4df) __A, + (__v4df) __B, + (__v4df) __C, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_fmadd_pd (__mmask8 __U, __m256d __A, __m256d __B, + __m256d __C) +{ + return (__m256d) __builtin_ia32_vfmaddpd256_maskz ((__v4df) __A, + (__v4df) __B, + (__v4df) __C, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fmadd_pd (__m128d __A, __mmask8 __U, __m128d __B, __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __C, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask3_fmadd_pd (__m128d __A, __m128d __B, __m128d __C, + __mmask8 __U) +{ + return (__m128d) __builtin_ia32_vfmaddpd128_mask3 ((__v2df) __A, + (__v2df) __B, + (__v2df) __C, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fmadd_pd (__mmask8 __U, __m128d __A, __m128d __B, + __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddpd128_maskz ((__v2df) __A, + (__v2df) __B, + (__v2df) __C, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_fmadd_ps (__m256 __A, __mmask8 __U, __m256 __B, __m256 __C) +{ + return (__m256) __builtin_ia32_vfmaddps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask3_fmadd_ps (__m256 __A, __m256 __B, __m256 __C, + __mmask8 __U) +{ + return (__m256) __builtin_ia32_vfmaddps256_mask3 ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_fmadd_ps (__mmask8 __U, __m256 __A, __m256 __B, + __m256 __C) +{ + return (__m256) __builtin_ia32_vfmaddps256_maskz ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fmadd_ps (__m128 __A, __mmask8 __U, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask3_fmadd_ps (__m128 __A, __m128 __B, __m128 __C, __mmask8 __U) +{ + return (__m128) __builtin_ia32_vfmaddps128_mask3 ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fmadd_ps (__mmask8 __U, __m128 __A, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddps128_maskz ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_fmsub_pd (__m256d __A, __mmask8 __U, __m256d __B, + __m256d __C) +{ + return (__m256d) __builtin_ia32_vfmaddpd256_mask ((__v4df) __A, + (__v4df) __B, + -(__v4df) __C, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask3_fmsub_pd (__m256d __A, __m256d __B, __m256d __C, + __mmask8 __U) +{ + return (__m256d) __builtin_ia32_vfmsubpd256_mask3 ((__v4df) __A, + (__v4df) __B, + (__v4df) __C, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_fmsub_pd (__mmask8 __U, __m256d __A, __m256d __B, + __m256d __C) +{ + return (__m256d) __builtin_ia32_vfmaddpd256_maskz ((__v4df) __A, + (__v4df) __B, + -(__v4df) __C, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fmsub_pd (__m128d __A, __mmask8 __U, __m128d __B, __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddpd128_mask ((__v2df) __A, + (__v2df) __B, + -(__v2df) __C, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask3_fmsub_pd (__m128d __A, __m128d __B, __m128d __C, + __mmask8 __U) +{ + return (__m128d) __builtin_ia32_vfmsubpd128_mask3 ((__v2df) __A, + (__v2df) __B, + (__v2df) __C, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fmsub_pd (__mmask8 __U, __m128d __A, __m128d __B, + __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddpd128_maskz ((__v2df) __A, + (__v2df) __B, + -(__v2df) __C, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_fmsub_ps (__m256 __A, __mmask8 __U, __m256 __B, __m256 __C) +{ + return (__m256) __builtin_ia32_vfmaddps256_mask ((__v8sf) __A, + (__v8sf) __B, + -(__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask3_fmsub_ps (__m256 __A, __m256 __B, __m256 __C, + __mmask8 __U) +{ + return (__m256) __builtin_ia32_vfmsubps256_mask3 ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_fmsub_ps (__mmask8 __U, __m256 __A, __m256 __B, + __m256 __C) +{ + return (__m256) __builtin_ia32_vfmaddps256_maskz ((__v8sf) __A, + (__v8sf) __B, + -(__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fmsub_ps (__m128 __A, __mmask8 __U, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddps128_mask ((__v4sf) __A, + (__v4sf) __B, + -(__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask3_fmsub_ps (__m128 __A, __m128 __B, __m128 __C, __mmask8 __U) +{ + return (__m128) __builtin_ia32_vfmsubps128_mask3 ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fmsub_ps (__mmask8 __U, __m128 __A, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddps128_maskz ((__v4sf) __A, + (__v4sf) __B, + -(__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_fmaddsub_pd (__m256d __A, __mmask8 __U, __m256d __B, + __m256d __C) +{ + return (__m256d) __builtin_ia32_vfmaddsubpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __C, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask3_fmaddsub_pd (__m256d __A, __m256d __B, __m256d __C, + __mmask8 __U) +{ + return (__m256d) __builtin_ia32_vfmaddsubpd256_mask3 ((__v4df) __A, + (__v4df) __B, + (__v4df) __C, + (__mmask8) + __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_fmaddsub_pd (__mmask8 __U, __m256d __A, __m256d __B, + __m256d __C) +{ + return (__m256d) __builtin_ia32_vfmaddsubpd256_maskz ((__v4df) __A, + (__v4df) __B, + (__v4df) __C, + (__mmask8) + __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fmaddsub_pd (__m128d __A, __mmask8 __U, __m128d __B, + __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddsubpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __C, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask3_fmaddsub_pd (__m128d __A, __m128d __B, __m128d __C, + __mmask8 __U) +{ + return (__m128d) __builtin_ia32_vfmaddsubpd128_mask3 ((__v2df) __A, + (__v2df) __B, + (__v2df) __C, + (__mmask8) + __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fmaddsub_pd (__mmask8 __U, __m128d __A, __m128d __B, + __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddsubpd128_maskz ((__v2df) __A, + (__v2df) __B, + (__v2df) __C, + (__mmask8) + __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_fmaddsub_ps (__m256 __A, __mmask8 __U, __m256 __B, + __m256 __C) +{ + return (__m256) __builtin_ia32_vfmaddsubps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask3_fmaddsub_ps (__m256 __A, __m256 __B, __m256 __C, + __mmask8 __U) +{ + return (__m256) __builtin_ia32_vfmaddsubps256_mask3 ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_fmaddsub_ps (__mmask8 __U, __m256 __A, __m256 __B, + __m256 __C) +{ + return (__m256) __builtin_ia32_vfmaddsubps256_maskz ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fmaddsub_ps (__m128 __A, __mmask8 __U, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddsubps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask3_fmaddsub_ps (__m128 __A, __m128 __B, __m128 __C, + __mmask8 __U) +{ + return (__m128) __builtin_ia32_vfmaddsubps128_mask3 ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fmaddsub_ps (__mmask8 __U, __m128 __A, __m128 __B, + __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddsubps128_maskz ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_fmsubadd_pd (__m256d __A, __mmask8 __U, __m256d __B, + __m256d __C) +{ + return (__m256d) __builtin_ia32_vfmaddsubpd256_mask ((__v4df) __A, + (__v4df) __B, + -(__v4df) __C, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask3_fmsubadd_pd (__m256d __A, __m256d __B, __m256d __C, + __mmask8 __U) +{ + return (__m256d) __builtin_ia32_vfmsubaddpd256_mask3 ((__v4df) __A, + (__v4df) __B, + (__v4df) __C, + (__mmask8) + __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_fmsubadd_pd (__mmask8 __U, __m256d __A, __m256d __B, + __m256d __C) +{ + return (__m256d) __builtin_ia32_vfmaddsubpd256_maskz ((__v4df) __A, + (__v4df) __B, + -(__v4df) __C, + (__mmask8) + __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fmsubadd_pd (__m128d __A, __mmask8 __U, __m128d __B, + __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddsubpd128_mask ((__v2df) __A, + (__v2df) __B, + -(__v2df) __C, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask3_fmsubadd_pd (__m128d __A, __m128d __B, __m128d __C, + __mmask8 __U) +{ + return (__m128d) __builtin_ia32_vfmsubaddpd128_mask3 ((__v2df) __A, + (__v2df) __B, + (__v2df) __C, + (__mmask8) + __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fmsubadd_pd (__mmask8 __U, __m128d __A, __m128d __B, + __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddsubpd128_maskz ((__v2df) __A, + (__v2df) __B, + -(__v2df) __C, + (__mmask8) + __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_fmsubadd_ps (__m256 __A, __mmask8 __U, __m256 __B, + __m256 __C) +{ + return (__m256) __builtin_ia32_vfmaddsubps256_mask ((__v8sf) __A, + (__v8sf) __B, + -(__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask3_fmsubadd_ps (__m256 __A, __m256 __B, __m256 __C, + __mmask8 __U) +{ + return (__m256) __builtin_ia32_vfmsubaddps256_mask3 ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_fmsubadd_ps (__mmask8 __U, __m256 __A, __m256 __B, + __m256 __C) +{ + return (__m256) __builtin_ia32_vfmaddsubps256_maskz ((__v8sf) __A, + (__v8sf) __B, + -(__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fmsubadd_ps (__m128 __A, __mmask8 __U, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddsubps128_mask ((__v4sf) __A, + (__v4sf) __B, + -(__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask3_fmsubadd_ps (__m128 __A, __m128 __B, __m128 __C, + __mmask8 __U) +{ + return (__m128) __builtin_ia32_vfmsubaddps128_mask3 ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fmsubadd_ps (__mmask8 __U, __m128 __A, __m128 __B, + __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddsubps128_maskz ((__v4sf) __A, + (__v4sf) __B, + -(__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_fnmadd_pd (__m256d __A, __mmask8 __U, __m256d __B, + __m256d __C) +{ + return (__m256d) __builtin_ia32_vfnmaddpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __C, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask3_fnmadd_pd (__m256d __A, __m256d __B, __m256d __C, + __mmask8 __U) +{ + return (__m256d) __builtin_ia32_vfmaddpd256_mask3 (-(__v4df) __A, + (__v4df) __B, + (__v4df) __C, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_fnmadd_pd (__mmask8 __U, __m256d __A, __m256d __B, + __m256d __C) +{ + return (__m256d) __builtin_ia32_vfmaddpd256_maskz (-(__v4df) __A, + (__v4df) __B, + (__v4df) __C, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fnmadd_pd (__m128d __A, __mmask8 __U, __m128d __B, + __m128d __C) +{ + return (__m128d) __builtin_ia32_vfnmaddpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __C, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask3_fnmadd_pd (__m128d __A, __m128d __B, __m128d __C, + __mmask8 __U) +{ + return (__m128d) __builtin_ia32_vfmaddpd128_mask3 (-(__v2df) __A, + (__v2df) __B, + (__v2df) __C, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fnmadd_pd (__mmask8 __U, __m128d __A, __m128d __B, + __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddpd128_maskz (-(__v2df) __A, + (__v2df) __B, + (__v2df) __C, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_fnmadd_ps (__m256 __A, __mmask8 __U, __m256 __B, + __m256 __C) +{ + return (__m256) __builtin_ia32_vfnmaddps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask3_fnmadd_ps (__m256 __A, __m256 __B, __m256 __C, + __mmask8 __U) +{ + return (__m256) __builtin_ia32_vfmaddps256_mask3 (-(__v8sf) __A, + (__v8sf) __B, + (__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_fnmadd_ps (__mmask8 __U, __m256 __A, __m256 __B, + __m256 __C) +{ + return (__m256) __builtin_ia32_vfmaddps256_maskz (-(__v8sf) __A, + (__v8sf) __B, + (__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fnmadd_ps (__m128 __A, __mmask8 __U, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfnmaddps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask3_fnmadd_ps (__m128 __A, __m128 __B, __m128 __C, __mmask8 __U) +{ + return (__m128) __builtin_ia32_vfmaddps128_mask3 (-(__v4sf) __A, + (__v4sf) __B, + (__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fnmadd_ps (__mmask8 __U, __m128 __A, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddps128_maskz (-(__v4sf) __A, + (__v4sf) __B, + (__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_fnmsub_pd (__m256d __A, __mmask8 __U, __m256d __B, + __m256d __C) +{ + return (__m256d) __builtin_ia32_vfnmsubpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __C, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask3_fnmsub_pd (__m256d __A, __m256d __B, __m256d __C, + __mmask8 __U) +{ + return (__m256d) __builtin_ia32_vfnmsubpd256_mask3 ((__v4df) __A, + (__v4df) __B, + (__v4df) __C, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_fnmsub_pd (__mmask8 __U, __m256d __A, __m256d __B, + __m256d __C) +{ + return (__m256d) __builtin_ia32_vfmaddpd256_maskz (-(__v4df) __A, + (__v4df) __B, + -(__v4df) __C, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fnmsub_pd (__m128d __A, __mmask8 __U, __m128d __B, + __m128d __C) +{ + return (__m128d) __builtin_ia32_vfnmsubpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __C, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask3_fnmsub_pd (__m128d __A, __m128d __B, __m128d __C, + __mmask8 __U) +{ + return (__m128d) __builtin_ia32_vfnmsubpd128_mask3 ((__v2df) __A, + (__v2df) __B, + (__v2df) __C, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fnmsub_pd (__mmask8 __U, __m128d __A, __m128d __B, + __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddpd128_maskz (-(__v2df) __A, + (__v2df) __B, + -(__v2df) __C, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_fnmsub_ps (__m256 __A, __mmask8 __U, __m256 __B, + __m256 __C) +{ + return (__m256) __builtin_ia32_vfnmsubps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask3_fnmsub_ps (__m256 __A, __m256 __B, __m256 __C, + __mmask8 __U) +{ + return (__m256) __builtin_ia32_vfnmsubps256_mask3 ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_fnmsub_ps (__mmask8 __U, __m256 __A, __m256 __B, + __m256 __C) +{ + return (__m256) __builtin_ia32_vfmaddps256_maskz (-(__v8sf) __A, + (__v8sf) __B, + -(__v8sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fnmsub_ps (__m128 __A, __mmask8 __U, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfnmsubps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask3_fnmsub_ps (__m128 __A, __m128 __B, __m128 __C, __mmask8 __U) +{ + return (__m128) __builtin_ia32_vfnmsubps128_mask3 ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fnmsub_ps (__mmask8 __U, __m128 __A, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddps128_maskz (-(__v4sf) __A, + (__v4sf) __B, + -(__v4sf) __C, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_and_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pandd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_and_epi32 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pandd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_andnot_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pandnd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_andnot_epi32 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pandnd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_andnot_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pandnd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_andnot_epi32 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pandnd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_or_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pord256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_or_epi32 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pord256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_or_epi32 (__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pord128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_or_epi32 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pord128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_xor_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pxord256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_xor_epi32 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pxord256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_xor_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pxord128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_xor_epi32 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pxord128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtpd_ps (__m128 __W, __mmask8 __U, __m128d __A) +{ + return (__m128) __builtin_ia32_cvtpd2ps_mask ((__v2df) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtpd_ps (__mmask8 __U, __m128d __A) +{ + return (__m128) __builtin_ia32_cvtpd2ps_mask ((__v2df) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtpd_ps (__m128 __W, __mmask8 __U, __m256d __A) +{ + return (__m128) __builtin_ia32_cvtpd2ps256_mask ((__v4df) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtpd_ps (__mmask8 __U, __m256d __A) +{ + return (__m128) __builtin_ia32_cvtpd2ps256_mask ((__v4df) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtps_epi32 (__m256i __W, __mmask8 __U, __m256 __A) +{ + return (__m256i) __builtin_ia32_cvtps2dq256_mask ((__v8sf) __A, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtps_epi32 (__mmask8 __U, __m256 __A) +{ + return (__m256i) __builtin_ia32_cvtps2dq256_mask ((__v8sf) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtps_epi32 (__m128i __W, __mmask8 __U, __m128 __A) +{ + return (__m128i) __builtin_ia32_cvtps2dq128_mask ((__v4sf) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtps_epi32 (__mmask8 __U, __m128 __A) +{ + return (__m128i) __builtin_ia32_cvtps2dq128_mask ((__v4sf) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtps_epu32 (__m256 __A) +{ + return (__m256i) __builtin_ia32_cvtps2udq256_mask ((__v8sf) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtps_epu32 (__m256i __W, __mmask8 __U, __m256 __A) +{ + return (__m256i) __builtin_ia32_cvtps2udq256_mask ((__v8sf) __A, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtps_epu32 (__mmask8 __U, __m256 __A) +{ + return (__m256i) __builtin_ia32_cvtps2udq256_mask ((__v8sf) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtps_epu32 (__m128 __A) +{ + return (__m128i) __builtin_ia32_cvtps2udq128_mask ((__v4sf) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtps_epu32 (__m128i __W, __mmask8 __U, __m128 __A) +{ + return (__m128i) __builtin_ia32_cvtps2udq128_mask ((__v4sf) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtps_epu32 (__mmask8 __U, __m128 __A) +{ + return (__m128i) __builtin_ia32_cvtps2udq128_mask ((__v4sf) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_movedup_pd (__m256d __W, __mmask8 __U, __m256d __A) +{ + return (__m256d) __builtin_ia32_movddup256_mask ((__v4df) __A, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_movedup_pd (__mmask8 __U, __m256d __A) +{ + return (__m256d) __builtin_ia32_movddup256_mask ((__v4df) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_movedup_pd (__m128d __W, __mmask8 __U, __m128d __A) +{ + return (__m128d) __builtin_ia32_movddup128_mask ((__v2df) __A, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_movedup_pd (__mmask8 __U, __m128d __A) +{ + return (__m128d) __builtin_ia32_movddup128_mask ((__v2df) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_movehdup_ps (__m256 __W, __mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_movshdup256_mask ((__v8sf) __A, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_movehdup_ps (__mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_movshdup256_mask ((__v8sf) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_movehdup_ps (__m128 __W, __mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_movshdup128_mask ((__v4sf) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_movehdup_ps (__mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_movshdup128_mask ((__v4sf) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_moveldup_ps (__m256 __W, __mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_movsldup256_mask ((__v8sf) __A, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_moveldup_ps (__mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_movsldup256_mask ((__v8sf) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_moveldup_ps (__m128 __W, __mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_movsldup128_mask ((__v4sf) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_moveldup_ps (__mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_movsldup128_mask ((__v4sf) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_unpackhi_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_punpckhdq128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_unpackhi_epi32 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_punpckhdq128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_unpackhi_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_punpckhdq256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_unpackhi_epi32 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_punpckhdq256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_unpackhi_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_punpckhqdq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_unpackhi_epi64 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_punpckhqdq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_unpackhi_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_punpckhqdq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_unpackhi_epi64 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_punpckhqdq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_unpacklo_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_punpckldq128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_unpacklo_epi32 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_punpckldq128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_unpacklo_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_punpckldq256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_unpacklo_epi32 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_punpckldq256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_unpacklo_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_punpcklqdq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_unpacklo_epi64 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_punpcklqdq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_unpacklo_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_punpcklqdq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_unpacklo_epi64 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_punpcklqdq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_epu32_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ucmpd128_mask ((__v4si) __A, + (__v4si) __B, 0, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_epi32_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_pcmpeqd128_mask ((__v4si) __A, + (__v4si) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpeq_epu32_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ucmpd128_mask ((__v4si) __A, + (__v4si) __B, 0, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpeq_epi32_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_pcmpeqd128_mask ((__v4si) __A, + (__v4si) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpeq_epu32_mask (__m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_ucmpd256_mask ((__v8si) __A, + (__v8si) __B, 0, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpeq_epi32_mask (__m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_pcmpeqd256_mask ((__v8si) __A, + (__v8si) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpeq_epu32_mask (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_ucmpd256_mask ((__v8si) __A, + (__v8si) __B, 0, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpeq_epi32_mask (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_pcmpeqd256_mask ((__v8si) __A, + (__v8si) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_epu64_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ucmpq128_mask ((__v2di) __A, + (__v2di) __B, 0, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_epi64_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_pcmpeqq128_mask ((__v2di) __A, + (__v2di) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpeq_epu64_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ucmpq128_mask ((__v2di) __A, + (__v2di) __B, 0, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpeq_epi64_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_pcmpeqq128_mask ((__v2di) __A, + (__v2di) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpeq_epu64_mask (__m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_ucmpq256_mask ((__v4di) __A, + (__v4di) __B, 0, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpeq_epi64_mask (__m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_pcmpeqq256_mask ((__v4di) __A, + (__v4di) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpeq_epu64_mask (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_ucmpq256_mask ((__v4di) __A, + (__v4di) __B, 0, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpeq_epi64_mask (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_pcmpeqq256_mask ((__v4di) __A, + (__v4di) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_epu32_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ucmpd128_mask ((__v4si) __A, + (__v4si) __B, 6, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_epi32_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_pcmpgtd128_mask ((__v4si) __A, + (__v4si) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpgt_epu32_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ucmpd128_mask ((__v4si) __A, + (__v4si) __B, 6, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpgt_epi32_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_pcmpgtd128_mask ((__v4si) __A, + (__v4si) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpgt_epu32_mask (__m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_ucmpd256_mask ((__v8si) __A, + (__v8si) __B, 6, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpgt_epi32_mask (__m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_pcmpgtd256_mask ((__v8si) __A, + (__v8si) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpgt_epu32_mask (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_ucmpd256_mask ((__v8si) __A, + (__v8si) __B, 6, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpgt_epi32_mask (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_pcmpgtd256_mask ((__v8si) __A, + (__v8si) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_epu64_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ucmpq128_mask ((__v2di) __A, + (__v2di) __B, 6, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_epi64_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_pcmpgtq128_mask ((__v2di) __A, + (__v2di) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpgt_epu64_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ucmpq128_mask ((__v2di) __A, + (__v2di) __B, 6, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpgt_epi64_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_pcmpgtq128_mask ((__v2di) __A, + (__v2di) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpgt_epu64_mask (__m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_ucmpq256_mask ((__v4di) __A, + (__v4di) __B, 6, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpgt_epi64_mask (__m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_pcmpgtq256_mask ((__v4di) __A, + (__v4di) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpgt_epu64_mask (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_ucmpq256_mask ((__v4di) __A, + (__v4di) __B, 6, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpgt_epi64_mask (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_pcmpgtq256_mask ((__v4di) __A, + (__v4di) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_test_epi32_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ptestmd128 ((__v4si) __A, + (__v4si) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_test_epi32_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ptestmd128 ((__v4si) __A, + (__v4si) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_test_epi32_mask (__m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_ptestmd256 ((__v8si) __A, + (__v8si) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_test_epi32_mask (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_ptestmd256 ((__v8si) __A, + (__v8si) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_test_epi64_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ptestmq128 ((__v2di) __A, + (__v2di) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_test_epi64_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ptestmq128 ((__v2di) __A, + (__v2di) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_test_epi64_mask (__m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_ptestmq256 ((__v4di) __A, + (__v4di) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_test_epi64_mask (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_ptestmq256 ((__v4di) __A, + (__v4di) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_testn_epi32_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ptestnmd128 ((__v4si) __A, + (__v4si) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_testn_epi32_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ptestnmd128 ((__v4si) __A, + (__v4si) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testn_epi32_mask (__m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_ptestnmd256 ((__v8si) __A, + (__v8si) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_testn_epi32_mask (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_ptestnmd256 ((__v8si) __A, + (__v8si) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_testn_epi64_mask (__m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ptestnmq128 ((__v2di) __A, + (__v2di) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_testn_epi64_mask (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__mmask8) __builtin_ia32_ptestnmq128 ((__v2di) __A, + (__v2di) __B, __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testn_epi64_mask (__m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_ptestnmq256 ((__v4di) __A, + (__v4di) __B, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_testn_epi64_mask (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__mmask8) __builtin_ia32_ptestnmq256 ((__v4di) __A, + (__v4di) __B, __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_compress_pd (__m256d __W, __mmask8 __U, __m256d __A) +{ + return (__m256d) __builtin_ia32_compressdf256_mask ((__v4df) __A, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_compress_pd (__mmask8 __U, __m256d __A) +{ + return (__m256d) __builtin_ia32_compressdf256_mask ((__v4df) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_compressstoreu_pd (void *__P, __mmask8 __U, __m256d __A) +{ + __builtin_ia32_compressstoredf256_mask ((__v4df *) __P, + (__v4df) __A, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_compress_pd (__m128d __W, __mmask8 __U, __m128d __A) +{ + return (__m128d) __builtin_ia32_compressdf128_mask ((__v2df) __A, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_compress_pd (__mmask8 __U, __m128d __A) +{ + return (__m128d) __builtin_ia32_compressdf128_mask ((__v2df) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_compressstoreu_pd (void *__P, __mmask8 __U, __m128d __A) +{ + __builtin_ia32_compressstoredf128_mask ((__v2df *) __P, + (__v2df) __A, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_compress_ps (__m256 __W, __mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_compresssf256_mask ((__v8sf) __A, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_compress_ps (__mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_compresssf256_mask ((__v8sf) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_compressstoreu_ps (void *__P, __mmask8 __U, __m256 __A) +{ + __builtin_ia32_compressstoresf256_mask ((__v8sf *) __P, + (__v8sf) __A, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_compress_ps (__m128 __W, __mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_compresssf128_mask ((__v4sf) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_compress_ps (__mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_compresssf128_mask ((__v4sf) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_compressstoreu_ps (void *__P, __mmask8 __U, __m128 __A) +{ + __builtin_ia32_compressstoresf128_mask ((__v4sf *) __P, + (__v4sf) __A, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_compress_epi64 (__m256i __W, __mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_compressdi256_mask ((__v4di) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_compress_epi64 (__mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_compressdi256_mask ((__v4di) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_compressstoreu_epi64 (void *__P, __mmask8 __U, __m256i __A) +{ + __builtin_ia32_compressstoredi256_mask ((__v4di *) __P, + (__v4di) __A, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_compress_epi64 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_compressdi128_mask ((__v2di) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_compress_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_compressdi128_mask ((__v2di) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_compressstoreu_epi64 (void *__P, __mmask8 __U, __m128i __A) +{ + __builtin_ia32_compressstoredi128_mask ((__v2di *) __P, + (__v2di) __A, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_compress_epi32 (__m256i __W, __mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_compresssi256_mask ((__v8si) __A, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_compress_epi32 (__mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_compresssi256_mask ((__v8si) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_compressstoreu_epi32 (void *__P, __mmask8 __U, __m256i __A) +{ + __builtin_ia32_compressstoresi256_mask ((__v8si *) __P, + (__v8si) __A, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_compress_epi32 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_compresssi128_mask ((__v4si) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_compress_epi32 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_compresssi128_mask ((__v4si) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_compressstoreu_epi32 (void *__P, __mmask8 __U, __m128i __A) +{ + __builtin_ia32_compressstoresi128_mask ((__v4si *) __P, + (__v4si) __A, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_expand_pd (__m256d __W, __mmask8 __U, __m256d __A) +{ + return (__m256d) __builtin_ia32_expanddf256_mask ((__v4df) __A, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_expand_pd (__mmask8 __U, __m256d __A) +{ + return (__m256d) __builtin_ia32_expanddf256_maskz ((__v4df) __A, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_expandloadu_pd (__m256d __W, __mmask8 __U, void const *__P) +{ + return (__m256d) __builtin_ia32_expandloaddf256_mask ((__v4df *) __P, + (__v4df) __W, + (__mmask8) + __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_expandloadu_pd (__mmask8 __U, void const *__P) +{ + return (__m256d) __builtin_ia32_expandloaddf256_maskz ((__v4df *) __P, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) + __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_expand_pd (__m128d __W, __mmask8 __U, __m128d __A) +{ + return (__m128d) __builtin_ia32_expanddf128_mask ((__v2df) __A, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_expand_pd (__mmask8 __U, __m128d __A) +{ + return (__m128d) __builtin_ia32_expanddf128_maskz ((__v2df) __A, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_expandloadu_pd (__m128d __W, __mmask8 __U, void const *__P) +{ + return (__m128d) __builtin_ia32_expandloaddf128_mask ((__v2df *) __P, + (__v2df) __W, + (__mmask8) + __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_expandloadu_pd (__mmask8 __U, void const *__P) +{ + return (__m128d) __builtin_ia32_expandloaddf128_maskz ((__v2df *) __P, + (__v2df) + _mm_setzero_pd (), + (__mmask8) + __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_expand_ps (__m256 __W, __mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_expandsf256_mask ((__v8sf) __A, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_expand_ps (__mmask8 __U, __m256 __A) +{ + return (__m256) __builtin_ia32_expandsf256_maskz ((__v8sf) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_expandloadu_ps (__m256 __W, __mmask8 __U, void const *__P) +{ + return (__m256) __builtin_ia32_expandloadsf256_mask ((__v8sf *) __P, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_expandloadu_ps (__mmask8 __U, void const *__P) +{ + return (__m256) __builtin_ia32_expandloadsf256_maskz ((__v8sf *) __P, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) + __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_expand_ps (__m128 __W, __mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_expandsf128_mask ((__v4sf) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_expand_ps (__mmask8 __U, __m128 __A) +{ + return (__m128) __builtin_ia32_expandsf128_maskz ((__v4sf) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_expandloadu_ps (__m128 __W, __mmask8 __U, void const *__P) +{ + return (__m128) __builtin_ia32_expandloadsf128_mask ((__v4sf *) __P, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_expandloadu_ps (__mmask8 __U, void const *__P) +{ + return (__m128) __builtin_ia32_expandloadsf128_maskz ((__v4sf *) __P, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_expand_epi64 (__m256i __W, __mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_expanddi256_mask ((__v4di) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_expand_epi64 (__mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_expanddi256_maskz ((__v4di) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_expandloadu_epi64 (__m256i __W, __mmask8 __U, + void const *__P) +{ + return (__m256i) __builtin_ia32_expandloaddi256_mask ((__v4di *) __P, + (__v4di) __W, + (__mmask8) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_expandloadu_epi64 (__mmask8 __U, void const *__P) +{ + return (__m256i) __builtin_ia32_expandloaddi256_maskz ((__v4di *) __P, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_expand_epi64 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_expanddi128_mask ((__v2di) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_expand_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_expanddi128_maskz ((__v2di) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_expandloadu_epi64 (__m128i __W, __mmask8 __U, void const *__P) +{ + return (__m128i) __builtin_ia32_expandloaddi128_mask ((__v2di *) __P, + (__v2di) __W, + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_expandloadu_epi64 (__mmask8 __U, void const *__P) +{ + return (__m128i) __builtin_ia32_expandloaddi128_maskz ((__v2di *) __P, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_expand_epi32 (__m256i __W, __mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_expandsi256_mask ((__v8si) __A, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_expand_epi32 (__mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_expandsi256_maskz ((__v8si) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_expandloadu_epi32 (__m256i __W, __mmask8 __U, + void const *__P) +{ + return (__m256i) __builtin_ia32_expandloadsi256_mask ((__v8si *) __P, + (__v8si) __W, + (__mmask8) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_expandloadu_epi32 (__mmask8 __U, void const *__P) +{ + return (__m256i) __builtin_ia32_expandloadsi256_maskz ((__v8si *) __P, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_expand_epi32 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_expandsi128_mask ((__v4si) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_expand_epi32 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_expandsi128_maskz ((__v4si) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_expandloadu_epi32 (__m128i __W, __mmask8 __U, void const *__P) +{ + return (__m128i) __builtin_ia32_expandloadsi128_mask ((__v4si *) __P, + (__v4si) __W, + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_expandloadu_epi32 (__mmask8 __U, void const *__P) +{ + return (__m128i) __builtin_ia32_expandloadsi128_maskz ((__v4si *) __P, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) + __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permutex2var_pd (__m256d __A, __m256i __I, __m256d __B) +{ + return (__m256d) __builtin_ia32_vpermt2varpd256_mask ((__v4di) __I + /* idx */ , + (__v4df) __A, + (__v4df) __B, + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permutex2var_pd (__m256d __A, __mmask8 __U, __m256i __I, + __m256d __B) +{ + return (__m256d) __builtin_ia32_vpermt2varpd256_mask ((__v4di) __I + /* idx */ , + (__v4df) __A, + (__v4df) __B, + (__mmask8) + __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask2_permutex2var_pd (__m256d __A, __m256i __I, __mmask8 __U, + __m256d __B) +{ + return (__m256d) __builtin_ia32_vpermi2varpd256_mask ((__v4df) __A, + (__v4di) __I + /* idx */ , + (__v4df) __B, + (__mmask8) + __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permutex2var_pd (__mmask8 __U, __m256d __A, __m256i __I, + __m256d __B) +{ + return (__m256d) __builtin_ia32_vpermt2varpd256_maskz ((__v4di) __I + /* idx */ , + (__v4df) __A, + (__v4df) __B, + (__mmask8) + __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permutex2var_ps (__m256 __A, __m256i __I, __m256 __B) +{ + return (__m256) __builtin_ia32_vpermt2varps256_mask ((__v8si) __I + /* idx */ , + (__v8sf) __A, + (__v8sf) __B, + (__mmask8) -1); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permutex2var_ps (__m256 __A, __mmask8 __U, __m256i __I, + __m256 __B) +{ + return (__m256) __builtin_ia32_vpermt2varps256_mask ((__v8si) __I + /* idx */ , + (__v8sf) __A, + (__v8sf) __B, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask2_permutex2var_ps (__m256 __A, __m256i __I, __mmask8 __U, + __m256 __B) +{ + return (__m256) __builtin_ia32_vpermi2varps256_mask ((__v8sf) __A, + (__v8si) __I + /* idx */ , + (__v8sf) __B, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permutex2var_ps (__mmask8 __U, __m256 __A, __m256i __I, + __m256 __B) +{ + return (__m256) __builtin_ia32_vpermt2varps256_maskz ((__v8si) __I + /* idx */ , + (__v8sf) __A, + (__v8sf) __B, + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permutex2var_epi64 (__m128i __A, __m128i __I, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpermt2varq128_mask ((__v2di) __I + /* idx */ , + (__v2di) __A, + (__v2di) __B, + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_permutex2var_epi64 (__m128i __A, __mmask8 __U, __m128i __I, + __m128i __B) +{ + return (__m128i) __builtin_ia32_vpermt2varq128_mask ((__v2di) __I + /* idx */ , + (__v2di) __A, + (__v2di) __B, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask2_permutex2var_epi64 (__m128i __A, __m128i __I, __mmask8 __U, + __m128i __B) +{ + return (__m128i) __builtin_ia32_vpermi2varq128_mask ((__v2di) __A, + (__v2di) __I + /* idx */ , + (__v2di) __B, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_permutex2var_epi64 (__mmask8 __U, __m128i __A, __m128i __I, + __m128i __B) +{ + return (__m128i) __builtin_ia32_vpermt2varq128_maskz ((__v2di) __I + /* idx */ , + (__v2di) __A, + (__v2di) __B, + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permutex2var_epi32 (__m128i __A, __m128i __I, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpermt2vard128_mask ((__v4si) __I + /* idx */ , + (__v4si) __A, + (__v4si) __B, + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_permutex2var_epi32 (__m128i __A, __mmask8 __U, __m128i __I, + __m128i __B) +{ + return (__m128i) __builtin_ia32_vpermt2vard128_mask ((__v4si) __I + /* idx */ , + (__v4si) __A, + (__v4si) __B, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask2_permutex2var_epi32 (__m128i __A, __m128i __I, __mmask8 __U, + __m128i __B) +{ + return (__m128i) __builtin_ia32_vpermi2vard128_mask ((__v4si) __A, + (__v4si) __I + /* idx */ , + (__v4si) __B, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_permutex2var_epi32 (__mmask8 __U, __m128i __A, __m128i __I, + __m128i __B) +{ + return (__m128i) __builtin_ia32_vpermt2vard128_maskz ((__v4si) __I + /* idx */ , + (__v4si) __A, + (__v4si) __B, + (__mmask8) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permutex2var_epi64 (__m256i __A, __m256i __I, __m256i __B) +{ + return (__m256i) __builtin_ia32_vpermt2varq256_mask ((__v4di) __I + /* idx */ , + (__v4di) __A, + (__v4di) __B, + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permutex2var_epi64 (__m256i __A, __mmask8 __U, __m256i __I, + __m256i __B) +{ + return (__m256i) __builtin_ia32_vpermt2varq256_mask ((__v4di) __I + /* idx */ , + (__v4di) __A, + (__v4di) __B, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask2_permutex2var_epi64 (__m256i __A, __m256i __I, + __mmask8 __U, __m256i __B) +{ + return (__m256i) __builtin_ia32_vpermi2varq256_mask ((__v4di) __A, + (__v4di) __I + /* idx */ , + (__v4di) __B, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permutex2var_epi64 (__mmask8 __U, __m256i __A, + __m256i __I, __m256i __B) +{ + return (__m256i) __builtin_ia32_vpermt2varq256_maskz ((__v4di) __I + /* idx */ , + (__v4di) __A, + (__v4di) __B, + (__mmask8) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permutex2var_epi32 (__m256i __A, __m256i __I, __m256i __B) +{ + return (__m256i) __builtin_ia32_vpermt2vard256_mask ((__v8si) __I + /* idx */ , + (__v8si) __A, + (__v8si) __B, + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permutex2var_epi32 (__m256i __A, __mmask8 __U, __m256i __I, + __m256i __B) +{ + return (__m256i) __builtin_ia32_vpermt2vard256_mask ((__v8si) __I + /* idx */ , + (__v8si) __A, + (__v8si) __B, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask2_permutex2var_epi32 (__m256i __A, __m256i __I, + __mmask8 __U, __m256i __B) +{ + return (__m256i) __builtin_ia32_vpermi2vard256_mask ((__v8si) __A, + (__v8si) __I + /* idx */ , + (__v8si) __B, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permutex2var_epi32 (__mmask8 __U, __m256i __A, + __m256i __I, __m256i __B) +{ + return (__m256i) __builtin_ia32_vpermt2vard256_maskz ((__v8si) __I + /* idx */ , + (__v8si) __A, + (__v8si) __B, + (__mmask8) + __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permutex2var_pd (__m128d __A, __m128i __I, __m128d __B) +{ + return (__m128d) __builtin_ia32_vpermt2varpd128_mask ((__v2di) __I + /* idx */ , + (__v2df) __A, + (__v2df) __B, + (__mmask8) -1); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_permutex2var_pd (__m128d __A, __mmask8 __U, __m128i __I, + __m128d __B) +{ + return (__m128d) __builtin_ia32_vpermt2varpd128_mask ((__v2di) __I + /* idx */ , + (__v2df) __A, + (__v2df) __B, + (__mmask8) + __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask2_permutex2var_pd (__m128d __A, __m128i __I, __mmask8 __U, + __m128d __B) +{ + return (__m128d) __builtin_ia32_vpermi2varpd128_mask ((__v2df) __A, + (__v2di) __I + /* idx */ , + (__v2df) __B, + (__mmask8) + __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_permutex2var_pd (__mmask8 __U, __m128d __A, __m128i __I, + __m128d __B) +{ + return (__m128d) __builtin_ia32_vpermt2varpd128_maskz ((__v2di) __I + /* idx */ , + (__v2df) __A, + (__v2df) __B, + (__mmask8) + __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permutex2var_ps (__m128 __A, __m128i __I, __m128 __B) +{ + return (__m128) __builtin_ia32_vpermt2varps128_mask ((__v4si) __I + /* idx */ , + (__v4sf) __A, + (__v4sf) __B, + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_permutex2var_ps (__m128 __A, __mmask8 __U, __m128i __I, + __m128 __B) +{ + return (__m128) __builtin_ia32_vpermt2varps128_mask ((__v4si) __I + /* idx */ , + (__v4sf) __A, + (__v4sf) __B, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask2_permutex2var_ps (__m128 __A, __m128i __I, __mmask8 __U, + __m128 __B) +{ + return (__m128) __builtin_ia32_vpermi2varps128_mask ((__v4sf) __A, + (__v4si) __I + /* idx */ , + (__v4sf) __B, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_permutex2var_ps (__mmask8 __U, __m128 __A, __m128i __I, + __m128 __B) +{ + return (__m128) __builtin_ia32_vpermt2varps128_maskz ((__v4si) __I + /* idx */ , + (__v4sf) __A, + (__v4sf) __B, + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srav_epi64 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_psravq128_mask ((__v2di) __X, + (__v2di) __Y, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_srav_epi64 (__m128i __W, __mmask8 __U, __m128i __X, + __m128i __Y) +{ + return (__m128i) __builtin_ia32_psravq128_mask ((__v2di) __X, + (__v2di) __Y, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_srav_epi64 (__mmask8 __U, __m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_psravq128_mask ((__v2di) __X, + (__v2di) __Y, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sllv_epi32 (__m256i __W, __mmask8 __U, __m256i __X, + __m256i __Y) +{ + return (__m256i) __builtin_ia32_psllv8si_mask ((__v8si) __X, + (__v8si) __Y, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sllv_epi32 (__mmask8 __U, __m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_psllv8si_mask ((__v8si) __X, + (__v8si) __Y, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sllv_epi32 (__m128i __W, __mmask8 __U, __m128i __X, + __m128i __Y) +{ + return (__m128i) __builtin_ia32_psllv4si_mask ((__v4si) __X, + (__v4si) __Y, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sllv_epi32 (__mmask8 __U, __m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_psllv4si_mask ((__v4si) __X, + (__v4si) __Y, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sllv_epi64 (__m256i __W, __mmask8 __U, __m256i __X, + __m256i __Y) +{ + return (__m256i) __builtin_ia32_psllv4di_mask ((__v4di) __X, + (__v4di) __Y, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sllv_epi64 (__mmask8 __U, __m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_psllv4di_mask ((__v4di) __X, + (__v4di) __Y, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sllv_epi64 (__m128i __W, __mmask8 __U, __m128i __X, + __m128i __Y) +{ + return (__m128i) __builtin_ia32_psllv2di_mask ((__v2di) __X, + (__v2di) __Y, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sllv_epi64 (__mmask8 __U, __m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_psllv2di_mask ((__v2di) __X, + (__v2di) __Y, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_srav_epi32 (__m256i __W, __mmask8 __U, __m256i __X, + __m256i __Y) +{ + return (__m256i) __builtin_ia32_psrav8si_mask ((__v8si) __X, + (__v8si) __Y, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_srav_epi32 (__mmask8 __U, __m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_psrav8si_mask ((__v8si) __X, + (__v8si) __Y, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_srav_epi32 (__m128i __W, __mmask8 __U, __m128i __X, + __m128i __Y) +{ + return (__m128i) __builtin_ia32_psrav4si_mask ((__v4si) __X, + (__v4si) __Y, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_srav_epi32 (__mmask8 __U, __m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_psrav4si_mask ((__v4si) __X, + (__v4si) __Y, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_srlv_epi32 (__m256i __W, __mmask8 __U, __m256i __X, + __m256i __Y) +{ + return (__m256i) __builtin_ia32_psrlv8si_mask ((__v8si) __X, + (__v8si) __Y, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_srlv_epi32 (__mmask8 __U, __m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_psrlv8si_mask ((__v8si) __X, + (__v8si) __Y, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_srlv_epi32 (__m128i __W, __mmask8 __U, __m128i __X, + __m128i __Y) +{ + return (__m128i) __builtin_ia32_psrlv4si_mask ((__v4si) __X, + (__v4si) __Y, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_srlv_epi32 (__mmask8 __U, __m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_psrlv4si_mask ((__v4si) __X, + (__v4si) __Y, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_srlv_epi64 (__m256i __W, __mmask8 __U, __m256i __X, + __m256i __Y) +{ + return (__m256i) __builtin_ia32_psrlv4di_mask ((__v4di) __X, + (__v4di) __Y, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_srlv_epi64 (__mmask8 __U, __m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_psrlv4di_mask ((__v4di) __X, + (__v4di) __Y, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_srlv_epi64 (__m128i __W, __mmask8 __U, __m128i __X, + __m128i __Y) +{ + return (__m128i) __builtin_ia32_psrlv2di_mask ((__v2di) __X, + (__v2di) __Y, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_srlv_epi64 (__mmask8 __U, __m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_psrlv2di_mask ((__v2di) __X, + (__v2di) __Y, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_rolv_epi32 (__m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_prolvd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_rolv_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_prolvd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_rolv_epi32 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_prolvd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rolv_epi32 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_prolvd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_rolv_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_prolvd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_rolv_epi32 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_prolvd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_rorv_epi32 (__m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_prorvd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_rorv_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_prorvd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_rorv_epi32 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_prorvd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rorv_epi32 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_prorvd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_rorv_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_prorvd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_rorv_epi32 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_prorvd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_rolv_epi64 (__m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_prolvq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_rolv_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_prolvq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_rolv_epi64 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_prolvq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rolv_epi64 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_prolvq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_rolv_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_prolvq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_rolv_epi64 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_prolvq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_rorv_epi64 (__m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_prorvq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_rorv_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_prorvq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_rorv_epi64 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_prorvq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rorv_epi64 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_prorvq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_rorv_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_prorvq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_rorv_epi64 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_prorvq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_srav_epi64 (__m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_psravq256_mask ((__v4di) __X, + (__v4di) __Y, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_srav_epi64 (__m256i __W, __mmask8 __U, __m256i __X, + __m256i __Y) +{ + return (__m256i) __builtin_ia32_psravq256_mask ((__v4di) __X, + (__v4di) __Y, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_srav_epi64 (__mmask8 __U, __m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_psravq256_mask ((__v4di) __X, + (__v4di) __Y, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_and_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pandq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __W, __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_and_epi64 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pandq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_pd (), + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_and_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pandq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_and_epi64 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pandq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_pd (), + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_andnot_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pandnq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __W, __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_andnot_epi64 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pandnq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_pd (), + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_andnot_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pandnq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_andnot_epi64 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pandnq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_pd (), + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_or_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_porq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_or_epi64 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_porq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_or_epi64 (__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_porq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_or_epi64 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_porq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_xor_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pxorq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_xor_epi64 (__mmask8 __U, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pxorq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_xor_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pxorq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_xor_epi64 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pxorq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_max_pd (__m256d __W, __mmask8 __U, __m256d __A, + __m256d __B) +{ + return (__m256d) __builtin_ia32_maxpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_max_pd (__mmask8 __U, __m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_maxpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_max_ps (__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_maxps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_max_ps (__mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_maxps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_div_ps (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_divps_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_div_ps (__mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_divps_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_div_pd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_divpd_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_div_pd (__mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_divpd_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_min_pd (__m256d __W, __mmask8 __U, __m256d __A, + __m256d __B) +{ + return (__m256d) __builtin_ia32_minpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_div_pd (__m256d __W, __mmask8 __U, __m256d __A, + __m256d __B) +{ + return (__m256d) __builtin_ia32_divpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_min_pd (__mmask8 __U, __m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_minpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_min_ps (__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_minps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_div_pd (__mmask8 __U, __m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_divpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_div_ps (__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_divps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_min_ps (__mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_minps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_div_ps (__mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_divps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_min_ps (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_minps_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_mul_ps (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_mulps_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_min_ps (__mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_minps_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_mul_ps (__mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_mulps_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_max_ps (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_maxps_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_max_ps (__mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_maxps_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_min_pd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_minpd_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_min_pd (__mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_minpd_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_max_pd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_maxpd_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_max_pd (__mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_maxpd_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_mul_pd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_mulpd_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_mul_pd (__mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_mulpd_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_mul_ps (__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_mulps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_mul_ps (__mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_mulps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_mul_pd (__m256d __W, __mmask8 __U, __m256d __A, + __m256d __B) +{ + return (__m256d) __builtin_ia32_mulpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_mul_pd (__mmask8 __U, __m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_mulpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_max_epi64 (__mmask8 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxsq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_max_epi64 (__m256i __W, __mmask8 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxsq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __W, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_min_epi64 (__m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pminsq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_min_epi64 (__m256i __W, __mmask8 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pminsq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __W, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_min_epi64 (__mmask8 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pminsq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_max_epu64 (__mmask8 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxuq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_max_epi64 (__m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxsq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_max_epu64 (__m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxuq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_max_epu64 (__m256i __W, __mmask8 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxuq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __W, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_min_epu64 (__m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pminuq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_min_epu64 (__m256i __W, __mmask8 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pminuq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __W, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_min_epu64 (__mmask8 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pminuq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_max_epi32 (__mmask8 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxsd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_max_epi32 (__m256i __W, __mmask8 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxsd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __W, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_min_epi32 (__mmask8 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pminsd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_min_epi32 (__m256i __W, __mmask8 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pminsd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __W, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_max_epu32 (__mmask8 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxud256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_max_epu32 (__m256i __W, __mmask8 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pmaxud256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __W, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_min_epu32 (__mmask8 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pminud256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_min_epu32 (__m256i __W, __mmask8 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pminud256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __W, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_max_epi64 (__mmask8 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxsq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_max_epi64 (__m128i __W, __mmask8 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxsq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_min_epi64 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pminsq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_min_epi64 (__m128i __W, __mmask8 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pminsq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_min_epi64 (__mmask8 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pminsq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_max_epu64 (__mmask8 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxuq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_max_epi64 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxsq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_max_epu64 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxuq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_max_epu64 (__m128i __W, __mmask8 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxuq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_min_epu64 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pminuq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_min_epu64 (__m128i __W, __mmask8 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pminuq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_min_epu64 (__mmask8 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pminuq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_max_epi32 (__mmask8 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxsd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_max_epi32 (__m128i __W, __mmask8 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxsd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_min_epi32 (__mmask8 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pminsd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_min_epi32 (__m128i __W, __mmask8 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pminsd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_max_epu32 (__mmask8 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxud128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_max_epu32 (__m128i __W, __mmask8 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pmaxud128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_min_epu32 (__mmask8 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pminud128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_min_epu32 (__m128i __W, __mmask8 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pminud128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, __M); +} + +#ifndef __AVX512CD__ +#pragma GCC push_options +#pragma GCC target("avx512vl,avx512cd") +#define __DISABLE_AVX512VLCD__ +#endif + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_broadcastmb_epi64 (__mmask8 __A) +{ + return (__m128i) __builtin_ia32_broadcastmb128 (__A); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcastmb_epi64 (__mmask8 __A) +{ + return (__m256i) __builtin_ia32_broadcastmb256 (__A); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_broadcastmw_epi32 (__mmask16 __A) +{ + return (__m128i) __builtin_ia32_broadcastmw128 (__A); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcastmw_epi32 (__mmask16 __A) +{ + return (__m256i) __builtin_ia32_broadcastmw256 (__A); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_lzcnt_epi32 (__m256i __A) +{ + return (__m256i) __builtin_ia32_vplzcntd_256_mask ((__v8si) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_lzcnt_epi32 (__m256i __W, __mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_vplzcntd_256_mask ((__v8si) __A, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_lzcnt_epi32 (__mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_vplzcntd_256_mask ((__v8si) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_lzcnt_epi64 (__m256i __A) +{ + return (__m256i) __builtin_ia32_vplzcntq_256_mask ((__v4di) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_lzcnt_epi64 (__m256i __W, __mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_vplzcntq_256_mask ((__v4di) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_lzcnt_epi64 (__mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_vplzcntq_256_mask ((__v4di) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_conflict_epi64 (__m256i __A) +{ + return (__m256i) __builtin_ia32_vpconflictdi_256_mask ((__v4di) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_conflict_epi64 (__m256i __W, __mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_vpconflictdi_256_mask ((__v4di) __A, + (__v4di) __W, + (__mmask8) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_conflict_epi64 (__mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_vpconflictdi_256_mask ((__v4di) __A, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_conflict_epi32 (__m256i __A) +{ + return (__m256i) __builtin_ia32_vpconflictsi_256_mask ((__v8si) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_conflict_epi32 (__m256i __W, __mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_vpconflictsi_256_mask ((__v8si) __A, + (__v8si) __W, + (__mmask8) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_conflict_epi32 (__mmask8 __U, __m256i __A) +{ + return (__m256i) __builtin_ia32_vpconflictsi_256_mask ((__v8si) __A, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_lzcnt_epi32 (__m128i __A) +{ + return (__m128i) __builtin_ia32_vplzcntd_128_mask ((__v4si) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_lzcnt_epi32 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_vplzcntd_128_mask ((__v4si) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_lzcnt_epi32 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_vplzcntd_128_mask ((__v4si) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_lzcnt_epi64 (__m128i __A) +{ + return (__m128i) __builtin_ia32_vplzcntq_128_mask ((__v2di) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_lzcnt_epi64 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_vplzcntq_128_mask ((__v2di) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_lzcnt_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_vplzcntq_128_mask ((__v2di) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_conflict_epi64 (__m128i __A) +{ + return (__m128i) __builtin_ia32_vpconflictdi_128_mask ((__v2di) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_conflict_epi64 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_vpconflictdi_128_mask ((__v2di) __A, + (__v2di) __W, + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_conflict_epi64 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_vpconflictdi_128_mask ((__v2di) __A, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_conflict_epi32 (__m128i __A) +{ + return (__m128i) __builtin_ia32_vpconflictsi_128_mask ((__v4si) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_conflict_epi32 (__m128i __W, __mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_vpconflictsi_128_mask ((__v4si) __A, + (__v4si) __W, + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_conflict_epi32 (__mmask8 __U, __m128i __A) +{ + return (__m128i) __builtin_ia32_vpconflictsi_128_mask ((__v4si) __A, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) + __U); +} + +#ifdef __DISABLE_AVX512VLCD__ +#pragma GCC pop_options +#endif + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_unpacklo_pd (__m256d __W, __mmask8 __U, __m256d __A, + __m256d __B) +{ + return (__m256d) __builtin_ia32_unpcklpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_unpacklo_pd (__mmask8 __U, __m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_unpcklpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_unpacklo_pd (__m128d __W, __mmask8 __U, __m128d __A, + __m128d __B) +{ + return (__m128d) __builtin_ia32_unpcklpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_unpacklo_pd (__mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_unpcklpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_unpacklo_ps (__m256 __W, __mmask8 __U, __m256 __A, + __m256 __B) +{ + return (__m256) __builtin_ia32_unpcklps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_unpackhi_pd (__m256d __W, __mmask8 __U, __m256d __A, + __m256d __B) +{ + return (__m256d) __builtin_ia32_unpckhpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_unpackhi_pd (__mmask8 __U, __m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_unpckhpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_unpackhi_pd (__m128d __W, __mmask8 __U, __m128d __A, + __m128d __B) +{ + return (__m128d) __builtin_ia32_unpckhpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_unpackhi_pd (__mmask8 __U, __m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_unpckhpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_unpackhi_ps (__m256 __W, __mmask8 __U, __m256 __A, + __m256 __B) +{ + return (__m256) __builtin_ia32_unpckhps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_unpackhi_ps (__mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_unpckhps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_unpackhi_ps (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_unpckhps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_unpackhi_ps (__mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_unpckhps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtph_ps (__m128 __W, __mmask8 __U, __m128i __A) +{ + return (__m128) __builtin_ia32_vcvtph2ps_mask ((__v8hi) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtph_ps (__mmask8 __U, __m128i __A) +{ + return (__m128) __builtin_ia32_vcvtph2ps_mask ((__v8hi) __A, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_unpacklo_ps (__mmask8 __U, __m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_unpcklps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtph_ps (__m256 __W, __mmask8 __U, __m128i __A) +{ + return (__m256) __builtin_ia32_vcvtph2ps256_mask ((__v8hi) __A, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtph_ps (__mmask8 __U, __m128i __A) +{ + return (__m256) __builtin_ia32_vcvtph2ps256_mask ((__v8hi) __A, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_unpacklo_ps (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_unpcklps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_unpacklo_ps (__mmask8 __U, __m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_unpcklps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sra_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + __m128i __B) +{ + return (__m256i) __builtin_ia32_psrad256_mask ((__v8si) __A, + (__v4si) __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sra_epi32 (__mmask8 __U, __m256i __A, __m128i __B) +{ + return (__m256i) __builtin_ia32_psrad256_mask ((__v8si) __A, + (__v4si) __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sra_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psrad128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sra_epi32 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psrad128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sra_epi64 (__m256i __A, __m128i __B) +{ + return (__m256i) __builtin_ia32_psraq256_mask ((__v4di) __A, + (__v2di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sra_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + __m128i __B) +{ + return (__m256i) __builtin_ia32_psraq256_mask ((__v4di) __A, + (__v2di) __B, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sra_epi64 (__mmask8 __U, __m256i __A, __m128i __B) +{ + return (__m256i) __builtin_ia32_psraq256_mask ((__v4di) __A, + (__v2di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sra_epi64 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psraq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sra_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psraq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sra_epi64 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psraq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sll_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pslld128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sll_epi32 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pslld128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_sll_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_psllq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_sll_epi64 (__mmask8 __U, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_psllq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sll_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + __m128i __B) +{ + return (__m256i) __builtin_ia32_pslld256_mask ((__v8si) __A, + (__v4si) __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sll_epi32 (__mmask8 __U, __m256i __A, __m128i __B) +{ + return (__m256i) __builtin_ia32_pslld256_mask ((__v8si) __A, + (__v4si) __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_sll_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + __m128i __B) +{ + return (__m256i) __builtin_ia32_psllq256_mask ((__v4di) __A, + (__v2di) __B, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_sll_epi64 (__mmask8 __U, __m256i __A, __m128i __B) +{ + return (__m256i) __builtin_ia32_psllq256_mask ((__v4di) __A, + (__v2di) __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permutexvar_ps (__m256 __W, __mmask8 __U, __m256i __X, + __m256 __Y) +{ + return (__m256) __builtin_ia32_permvarsf256_mask ((__v8sf) __Y, + (__v8si) __X, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permutexvar_ps (__mmask8 __U, __m256i __X, __m256 __Y) +{ + return (__m256) __builtin_ia32_permvarsf256_mask ((__v8sf) __Y, + (__v8si) __X, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permutexvar_pd (__m256i __X, __m256d __Y) +{ + return (__m256d) __builtin_ia32_permvardf256_mask ((__v4df) __Y, + (__v4di) __X, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permutexvar_pd (__m256d __W, __mmask8 __U, __m256i __X, + __m256d __Y) +{ + return (__m256d) __builtin_ia32_permvardf256_mask ((__v4df) __Y, + (__v4di) __X, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permutexvar_pd (__mmask8 __U, __m256i __X, __m256d __Y) +{ + return (__m256d) __builtin_ia32_permvardf256_mask ((__v4df) __Y, + (__v4di) __X, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permutevar_pd (__m256d __W, __mmask8 __U, __m256d __A, + __m256i __C) +{ + return (__m256d) __builtin_ia32_vpermilvarpd256_mask ((__v4df) __A, + (__v4di) __C, + (__v4df) __W, + (__mmask8) + __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permutevar_pd (__mmask8 __U, __m256d __A, __m256i __C) +{ + return (__m256d) __builtin_ia32_vpermilvarpd256_mask ((__v4df) __A, + (__v4di) __C, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) + __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permutevar_ps (__m256 __W, __mmask8 __U, __m256 __A, + __m256i __C) +{ + return (__m256) __builtin_ia32_vpermilvarps256_mask ((__v8sf) __A, + (__v8si) __C, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permutevar_ps (__mmask8 __U, __m256 __A, __m256i __C) +{ + return (__m256) __builtin_ia32_vpermilvarps256_mask ((__v8sf) __A, + (__v8si) __C, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_permutevar_pd (__m128d __W, __mmask8 __U, __m128d __A, + __m128i __C) +{ + return (__m128d) __builtin_ia32_vpermilvarpd_mask ((__v2df) __A, + (__v2di) __C, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_permutevar_pd (__mmask8 __U, __m128d __A, __m128i __C) +{ + return (__m128d) __builtin_ia32_vpermilvarpd_mask ((__v2df) __A, + (__v2di) __C, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_permutevar_ps (__m128 __W, __mmask8 __U, __m128 __A, + __m128i __C) +{ + return (__m128) __builtin_ia32_vpermilvarps_mask ((__v4sf) __A, + (__v4si) __C, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_permutevar_ps (__mmask8 __U, __m128 __A, __m128i __C) +{ + return (__m128) __builtin_ia32_vpermilvarps_mask ((__v4sf) __A, + (__v4si) __C, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_mullo_epi32 (__mmask8 __M, __m256i __A, __m256i __B) +{ + return (__m256i) __builtin_ia32_pmulld256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permutexvar_epi64 (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_permvardi256_mask ((__v4di) __Y, + (__v4di) __X, + (__v4di) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_mullo_epi32 (__m256i __W, __mmask8 __M, __m256i __A, + __m256i __B) +{ + return (__m256i) __builtin_ia32_pmulld256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __W, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_mullo_epi32 (__mmask8 __M, __m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_pmulld128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_mullo_epi32 (__m128i __W, __mmask16 __M, __m128i __A, + __m128i __B) +{ + return (__m128i) __builtin_ia32_pmulld128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __W, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_mul_epi32 (__m256i __W, __mmask8 __M, __m256i __X, + __m256i __Y) +{ + return (__m256i) __builtin_ia32_pmuldq256_mask ((__v8si) __X, + (__v8si) __Y, + (__v4di) __W, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_mul_epi32 (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_pmuldq256_mask ((__v8si) __X, + (__v8si) __Y, + (__v4di) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_mul_epi32 (__m128i __W, __mmask8 __M, __m128i __X, + __m128i __Y) +{ + return (__m128i) __builtin_ia32_pmuldq128_mask ((__v4si) __X, + (__v4si) __Y, + (__v2di) __W, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_mul_epi32 (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_pmuldq128_mask ((__v4si) __X, + (__v4si) __Y, + (__v2di) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permutexvar_epi64 (__m256i __W, __mmask8 __M, __m256i __X, + __m256i __Y) +{ + return (__m256i) __builtin_ia32_permvardi256_mask ((__v4di) __Y, + (__v4di) __X, + (__v4di) __W, + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_mul_epu32 (__m256i __W, __mmask8 __M, __m256i __X, + __m256i __Y) +{ + return (__m256i) __builtin_ia32_pmuludq256_mask ((__v8si) __X, + (__v8si) __Y, + (__v4di) __W, __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permutexvar_epi32 (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_permvarsi256_mask ((__v8si) __Y, + (__v8si) __X, + (__v8si) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_mul_epu32 (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__m256i) __builtin_ia32_pmuludq256_mask ((__v8si) __X, + (__v8si) __Y, + (__v4di) + _mm256_setzero_si256 (), + __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_mul_epu32 (__m128i __W, __mmask8 __M, __m128i __X, + __m128i __Y) +{ + return (__m128i) __builtin_ia32_pmuludq128_mask ((__v4si) __X, + (__v4si) __Y, + (__v2di) __W, __M); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_mul_epu32 (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_pmuludq128_mask ((__v4si) __X, + (__v4si) __Y, + (__v2di) + _mm_setzero_si128 (), + __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permutexvar_epi32 (__m256i __W, __mmask8 __M, __m256i __X, + __m256i __Y) +{ + return (__m256i) __builtin_ia32_permvarsi256_mask ((__v8si) __Y, + (__v8si) __X, + (__v8si) __W, + __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpneq_epu32_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpd256_mask ((__v8si) __X, + (__v8si) __Y, 4, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpneq_epu32_mask (__m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpd256_mask ((__v8si) __X, + (__v8si) __Y, 4, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmplt_epu32_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpd256_mask ((__v8si) __X, + (__v8si) __Y, 1, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmplt_epu32_mask (__m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpd256_mask ((__v8si) __X, + (__v8si) __Y, 1, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpge_epu32_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpd256_mask ((__v8si) __X, + (__v8si) __Y, 5, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpge_epu32_mask (__m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpd256_mask ((__v8si) __X, + (__v8si) __Y, 5, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmple_epu32_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpd256_mask ((__v8si) __X, + (__v8si) __Y, 2, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmple_epu32_mask (__m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpd256_mask ((__v8si) __X, + (__v8si) __Y, 2, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpneq_epu64_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq256_mask ((__v4di) __X, + (__v4di) __Y, 4, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpneq_epu64_mask (__m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq256_mask ((__v4di) __X, + (__v4di) __Y, 4, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmplt_epu64_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq256_mask ((__v4di) __X, + (__v4di) __Y, 1, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmplt_epu64_mask (__m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq256_mask ((__v4di) __X, + (__v4di) __Y, 1, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpge_epu64_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq256_mask ((__v4di) __X, + (__v4di) __Y, 5, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpge_epu64_mask (__m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq256_mask ((__v4di) __X, + (__v4di) __Y, 5, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmple_epu64_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq256_mask ((__v4di) __X, + (__v4di) __Y, 2, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmple_epu64_mask (__m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq256_mask ((__v4di) __X, + (__v4di) __Y, 2, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpneq_epi32_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpd256_mask ((__v8si) __X, + (__v8si) __Y, 4, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpneq_epi32_mask (__m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpd256_mask ((__v8si) __X, + (__v8si) __Y, 4, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmplt_epi32_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpd256_mask ((__v8si) __X, + (__v8si) __Y, 1, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmplt_epi32_mask (__m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpd256_mask ((__v8si) __X, + (__v8si) __Y, 1, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpge_epi32_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpd256_mask ((__v8si) __X, + (__v8si) __Y, 5, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpge_epi32_mask (__m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpd256_mask ((__v8si) __X, + (__v8si) __Y, 5, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmple_epi32_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpd256_mask ((__v8si) __X, + (__v8si) __Y, 2, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmple_epi32_mask (__m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpd256_mask ((__v8si) __X, + (__v8si) __Y, 2, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpneq_epi64_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq256_mask ((__v4di) __X, + (__v4di) __Y, 4, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpneq_epi64_mask (__m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq256_mask ((__v4di) __X, + (__v4di) __Y, 4, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmplt_epi64_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq256_mask ((__v4di) __X, + (__v4di) __Y, 1, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmplt_epi64_mask (__m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq256_mask ((__v4di) __X, + (__v4di) __Y, 1, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmpge_epi64_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq256_mask ((__v4di) __X, + (__v4di) __Y, 5, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmpge_epi64_mask (__m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq256_mask ((__v4di) __X, + (__v4di) __Y, 5, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmple_epi64_mask (__mmask8 __M, __m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq256_mask ((__v4di) __X, + (__v4di) __Y, 2, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmple_epi64_mask (__m256i __X, __m256i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq256_mask ((__v4di) __X, + (__v4di) __Y, 2, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpneq_epu32_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpd128_mask ((__v4si) __X, + (__v4si) __Y, 4, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpneq_epu32_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpd128_mask ((__v4si) __X, + (__v4si) __Y, 4, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmplt_epu32_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpd128_mask ((__v4si) __X, + (__v4si) __Y, 1, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmplt_epu32_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpd128_mask ((__v4si) __X, + (__v4si) __Y, 1, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpge_epu32_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpd128_mask ((__v4si) __X, + (__v4si) __Y, 5, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpge_epu32_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpd128_mask ((__v4si) __X, + (__v4si) __Y, 5, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmple_epu32_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpd128_mask ((__v4si) __X, + (__v4si) __Y, 2, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmple_epu32_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpd128_mask ((__v4si) __X, + (__v4si) __Y, 2, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpneq_epu64_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq128_mask ((__v2di) __X, + (__v2di) __Y, 4, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpneq_epu64_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq128_mask ((__v2di) __X, + (__v2di) __Y, 4, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmplt_epu64_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq128_mask ((__v2di) __X, + (__v2di) __Y, 1, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmplt_epu64_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq128_mask ((__v2di) __X, + (__v2di) __Y, 1, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpge_epu64_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq128_mask ((__v2di) __X, + (__v2di) __Y, 5, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpge_epu64_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq128_mask ((__v2di) __X, + (__v2di) __Y, 5, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmple_epu64_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq128_mask ((__v2di) __X, + (__v2di) __Y, 2, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmple_epu64_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_ucmpq128_mask ((__v2di) __X, + (__v2di) __Y, 2, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpneq_epi32_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpd128_mask ((__v4si) __X, + (__v4si) __Y, 4, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpneq_epi32_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpd128_mask ((__v4si) __X, + (__v4si) __Y, 4, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmplt_epi32_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpd128_mask ((__v4si) __X, + (__v4si) __Y, 1, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmplt_epi32_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpd128_mask ((__v4si) __X, + (__v4si) __Y, 1, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpge_epi32_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpd128_mask ((__v4si) __X, + (__v4si) __Y, 5, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpge_epi32_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpd128_mask ((__v4si) __X, + (__v4si) __Y, 5, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmple_epi32_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpd128_mask ((__v4si) __X, + (__v4si) __Y, 2, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmple_epi32_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpd128_mask ((__v4si) __X, + (__v4si) __Y, 2, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpneq_epi64_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq128_mask ((__v2di) __X, + (__v2di) __Y, 4, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpneq_epi64_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq128_mask ((__v2di) __X, + (__v2di) __Y, 4, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmplt_epi64_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq128_mask ((__v2di) __X, + (__v2di) __Y, 1, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmplt_epi64_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq128_mask ((__v2di) __X, + (__v2di) __Y, 1, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmpge_epi64_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq128_mask ((__v2di) __X, + (__v2di) __Y, 5, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpge_epi64_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq128_mask ((__v2di) __X, + (__v2di) __Y, 5, + (__mmask8) -1); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmple_epi64_mask (__mmask8 __M, __m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq128_mask ((__v2di) __X, + (__v2di) __Y, 2, + (__mmask8) __M); +} + +extern __inline __mmask8 + __attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmple_epi64_mask (__m128i __X, __m128i __Y) +{ + return (__mmask8) __builtin_ia32_cmpq128_mask ((__v2di) __X, + (__v2di) __Y, 2, + (__mmask8) -1); +} + +#ifdef __OPTIMIZE__ +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permutex_epi64 (__m256i __W, __mmask8 __M, + __m256i __X, const int __I) +{ + return (__m256i) __builtin_ia32_permdi256_mask ((__v4di) __X, + __I, + (__v4di) __W, + (__mmask8) __M); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permutex_epi64 (__mmask8 __M, __m256i __X, const int __I) +{ + return (__m256i) __builtin_ia32_permdi256_mask ((__v4di) __X, + __I, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __M); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_shuffle_pd (__m256d __W, __mmask8 __U, __m256d __A, + __m256d __B, const int __imm) +{ + return (__m256d) __builtin_ia32_shufpd256_mask ((__v4df) __A, + (__v4df) __B, __imm, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_shuffle_pd (__mmask8 __U, __m256d __A, __m256d __B, + const int __imm) +{ + return (__m256d) __builtin_ia32_shufpd256_mask ((__v4df) __A, + (__v4df) __B, __imm, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_shuffle_pd (__m128d __W, __mmask8 __U, __m128d __A, + __m128d __B, const int __imm) +{ + return (__m128d) __builtin_ia32_shufpd128_mask ((__v2df) __A, + (__v2df) __B, __imm, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_shuffle_pd (__mmask8 __U, __m128d __A, __m128d __B, + const int __imm) +{ + return (__m128d) __builtin_ia32_shufpd128_mask ((__v2df) __A, + (__v2df) __B, __imm, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_shuffle_ps (__m256 __W, __mmask8 __U, __m256 __A, + __m256 __B, const int __imm) +{ + return (__m256) __builtin_ia32_shufps256_mask ((__v8sf) __A, + (__v8sf) __B, __imm, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_shuffle_ps (__mmask8 __U, __m256 __A, __m256 __B, + const int __imm) +{ + return (__m256) __builtin_ia32_shufps256_mask ((__v8sf) __A, + (__v8sf) __B, __imm, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_shuffle_ps (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B, + const int __imm) +{ + return (__m128) __builtin_ia32_shufps128_mask ((__v4sf) __A, + (__v4sf) __B, __imm, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_shuffle_ps (__mmask8 __U, __m128 __A, __m128 __B, + const int __imm) +{ + return (__m128) __builtin_ia32_shufps128_mask ((__v4sf) __A, + (__v4sf) __B, __imm, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_inserti32x4 (__m256i __A, __m128i __B, const int __imm) +{ + return (__m256i) __builtin_ia32_inserti32x4_256_mask ((__v8si) __A, + (__v4si) __B, + __imm, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_inserti32x4 (__m256i __W, __mmask8 __U, __m256i __A, + __m128i __B, const int __imm) +{ + return (__m256i) __builtin_ia32_inserti32x4_256_mask ((__v8si) __A, + (__v4si) __B, + __imm, + (__v8si) __W, + (__mmask8) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_inserti32x4 (__mmask8 __U, __m256i __A, __m128i __B, + const int __imm) +{ + return (__m256i) __builtin_ia32_inserti32x4_256_mask ((__v8si) __A, + (__v4si) __B, + __imm, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) + __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_insertf32x4 (__m256 __A, __m128 __B, const int __imm) +{ + return (__m256) __builtin_ia32_insertf32x4_256_mask ((__v8sf) __A, + (__v4sf) __B, + __imm, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_insertf32x4 (__m256 __W, __mmask8 __U, __m256 __A, + __m128 __B, const int __imm) +{ + return (__m256) __builtin_ia32_insertf32x4_256_mask ((__v8sf) __A, + (__v4sf) __B, + __imm, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_insertf32x4 (__mmask8 __U, __m256 __A, __m128 __B, + const int __imm) +{ + return (__m256) __builtin_ia32_insertf32x4_256_mask ((__v8sf) __A, + (__v4sf) __B, + __imm, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_extracti32x4_epi32 (__m256i __A, const int __imm) +{ + return (__m128i) __builtin_ia32_extracti32x4_256_mask ((__v8si) __A, + __imm, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_extracti32x4_epi32 (__m128i __W, __mmask8 __U, __m256i __A, + const int __imm) +{ + return (__m128i) __builtin_ia32_extracti32x4_256_mask ((__v8si) __A, + __imm, + (__v4si) __W, + (__mmask8) + __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_extracti32x4_epi32 (__mmask8 __U, __m256i __A, + const int __imm) +{ + return (__m128i) __builtin_ia32_extracti32x4_256_mask ((__v8si) __A, + __imm, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) + __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_extractf32x4_ps (__m256 __A, const int __imm) +{ + return (__m128) __builtin_ia32_extractf32x4_256_mask ((__v8sf) __A, + __imm, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_extractf32x4_ps (__m128 __W, __mmask8 __U, __m256 __A, + const int __imm) +{ + return (__m128) __builtin_ia32_extractf32x4_256_mask ((__v8sf) __A, + __imm, + (__v4sf) __W, + (__mmask8) + __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_extractf32x4_ps (__mmask8 __U, __m256 __A, + const int __imm) +{ + return (__m128) __builtin_ia32_extractf32x4_256_mask ((__v8sf) __A, + __imm, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) + __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_shuffle_i64x2 (__m256i __A, __m256i __B, const int __imm) +{ + return (__m256i) __builtin_ia32_shuf_i64x2_256_mask ((__v4di) __A, + (__v4di) __B, + __imm, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_shuffle_i64x2 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B, const int __imm) +{ + return (__m256i) __builtin_ia32_shuf_i64x2_256_mask ((__v4di) __A, + (__v4di) __B, + __imm, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_shuffle_i64x2 (__mmask8 __U, __m256i __A, __m256i __B, + const int __imm) +{ + return (__m256i) __builtin_ia32_shuf_i64x2_256_mask ((__v4di) __A, + (__v4di) __B, + __imm, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_shuffle_i32x4 (__m256i __A, __m256i __B, const int __imm) +{ + return (__m256i) __builtin_ia32_shuf_i32x4_256_mask ((__v8si) __A, + (__v8si) __B, + __imm, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_shuffle_i32x4 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B, const int __imm) +{ + return (__m256i) __builtin_ia32_shuf_i32x4_256_mask ((__v8si) __A, + (__v8si) __B, + __imm, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_shuffle_i32x4 (__mmask8 __U, __m256i __A, __m256i __B, + const int __imm) +{ + return (__m256i) __builtin_ia32_shuf_i32x4_256_mask ((__v8si) __A, + (__v8si) __B, + __imm, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_shuffle_f64x2 (__m256d __A, __m256d __B, const int __imm) +{ + return (__m256d) __builtin_ia32_shuf_f64x2_256_mask ((__v4df) __A, + (__v4df) __B, + __imm, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_shuffle_f64x2 (__m256d __W, __mmask8 __U, __m256d __A, + __m256d __B, const int __imm) +{ + return (__m256d) __builtin_ia32_shuf_f64x2_256_mask ((__v4df) __A, + (__v4df) __B, + __imm, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_shuffle_f64x2 (__mmask8 __U, __m256d __A, __m256d __B, + const int __imm) +{ + return (__m256d) __builtin_ia32_shuf_f64x2_256_mask ((__v4df) __A, + (__v4df) __B, + __imm, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_shuffle_f32x4 (__m256 __A, __m256 __B, const int __imm) +{ + return (__m256) __builtin_ia32_shuf_f32x4_256_mask ((__v8sf) __A, + (__v8sf) __B, + __imm, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_shuffle_f32x4 (__m256 __W, __mmask8 __U, __m256 __A, + __m256 __B, const int __imm) +{ + return (__m256) __builtin_ia32_shuf_f32x4_256_mask ((__v8sf) __A, + (__v8sf) __B, + __imm, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_shuffle_f32x4 (__mmask8 __U, __m256 __A, __m256 __B, + const int __imm) +{ + return (__m256) __builtin_ia32_shuf_f32x4_256_mask ((__v8sf) __A, + (__v8sf) __B, + __imm, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_fixupimm_pd (__m256d __A, __m256d __B, __m256i __C, + const int __imm) +{ + return (__m256d) __builtin_ia32_fixupimmpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4di) __C, + __imm, + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_fixupimm_pd (__m256d __A, __mmask8 __U, __m256d __B, + __m256i __C, const int __imm) +{ + return (__m256d) __builtin_ia32_fixupimmpd256_mask ((__v4df) __A, + (__v4df) __B, + (__v4di) __C, + __imm, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_fixupimm_pd (__mmask8 __U, __m256d __A, __m256d __B, + __m256i __C, const int __imm) +{ + return (__m256d) __builtin_ia32_fixupimmpd256_maskz ((__v4df) __A, + (__v4df) __B, + (__v4di) __C, + __imm, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_fixupimm_ps (__m256 __A, __m256 __B, __m256i __C, + const int __imm) +{ + return (__m256) __builtin_ia32_fixupimmps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8si) __C, + __imm, + (__mmask8) -1); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_fixupimm_ps (__m256 __A, __mmask8 __U, __m256 __B, + __m256i __C, const int __imm) +{ + return (__m256) __builtin_ia32_fixupimmps256_mask ((__v8sf) __A, + (__v8sf) __B, + (__v8si) __C, + __imm, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_fixupimm_ps (__mmask8 __U, __m256 __A, __m256 __B, + __m256i __C, const int __imm) +{ + return (__m256) __builtin_ia32_fixupimmps256_maskz ((__v8sf) __A, + (__v8sf) __B, + (__v8si) __C, + __imm, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fixupimm_pd (__m128d __A, __m128d __B, __m128i __C, + const int __imm) +{ + return (__m128d) __builtin_ia32_fixupimmpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2di) __C, + __imm, + (__mmask8) -1); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fixupimm_pd (__m128d __A, __mmask8 __U, __m128d __B, + __m128i __C, const int __imm) +{ + return (__m128d) __builtin_ia32_fixupimmpd128_mask ((__v2df) __A, + (__v2df) __B, + (__v2di) __C, + __imm, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fixupimm_pd (__mmask8 __U, __m128d __A, __m128d __B, + __m128i __C, const int __imm) +{ + return (__m128d) __builtin_ia32_fixupimmpd128_maskz ((__v2df) __A, + (__v2df) __B, + (__v2di) __C, + __imm, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fixupimm_ps (__m128 __A, __m128 __B, __m128i __C, const int __imm) +{ + return (__m128) __builtin_ia32_fixupimmps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4si) __C, + __imm, + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_fixupimm_ps (__m128 __A, __mmask8 __U, __m128 __B, + __m128i __C, const int __imm) +{ + return (__m128) __builtin_ia32_fixupimmps128_mask ((__v4sf) __A, + (__v4sf) __B, + (__v4si) __C, + __imm, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_fixupimm_ps (__mmask8 __U, __m128 __A, __m128 __B, + __m128i __C, const int __imm) +{ + return (__m128) __builtin_ia32_fixupimmps128_maskz ((__v4sf) __A, + (__v4sf) __B, + (__v4si) __C, + __imm, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_srli_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + const int __imm) +{ + return (__m256i) __builtin_ia32_psrldi256_mask ((__v8si) __A, __imm, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_srli_epi32 (__mmask8 __U, __m256i __A, const int __imm) +{ + return (__m256i) __builtin_ia32_psrldi256_mask ((__v8si) __A, __imm, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_srli_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + const int __imm) +{ + return (__m128i) __builtin_ia32_psrldi128_mask ((__v4si) __A, __imm, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_srli_epi32 (__mmask8 __U, __m128i __A, const int __imm) +{ + return (__m128i) __builtin_ia32_psrldi128_mask ((__v4si) __A, __imm, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_srli_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + const int __imm) +{ + return (__m256i) __builtin_ia32_psrlqi256_mask ((__v4di) __A, __imm, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_srli_epi64 (__mmask8 __U, __m256i __A, const int __imm) +{ + return (__m256i) __builtin_ia32_psrlqi256_mask ((__v4di) __A, __imm, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_srli_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + const int __imm) +{ + return (__m128i) __builtin_ia32_psrlqi128_mask ((__v2di) __A, __imm, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_srli_epi64 (__mmask8 __U, __m128i __A, const int __imm) +{ + return (__m128i) __builtin_ia32_psrlqi128_mask ((__v2di) __A, __imm, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_ternarylogic_epi64 (__m256i __A, __m256i __B, __m256i __C, + const int __imm) +{ + return (__m256i) __builtin_ia32_pternlogq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __C, __imm, + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_ternarylogic_epi64 (__m256i __A, __mmask8 __U, + __m256i __B, __m256i __C, + const int __imm) +{ + return (__m256i) __builtin_ia32_pternlogq256_mask ((__v4di) __A, + (__v4di) __B, + (__v4di) __C, __imm, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_ternarylogic_epi64 (__mmask8 __U, __m256i __A, + __m256i __B, __m256i __C, + const int __imm) +{ + return (__m256i) __builtin_ia32_pternlogq256_maskz ((__v4di) __A, + (__v4di) __B, + (__v4di) __C, + __imm, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_ternarylogic_epi32 (__m256i __A, __m256i __B, __m256i __C, + const int __imm) +{ + return (__m256i) __builtin_ia32_pternlogd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __C, __imm, + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_ternarylogic_epi32 (__m256i __A, __mmask8 __U, + __m256i __B, __m256i __C, + const int __imm) +{ + return (__m256i) __builtin_ia32_pternlogd256_mask ((__v8si) __A, + (__v8si) __B, + (__v8si) __C, __imm, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_ternarylogic_epi32 (__mmask8 __U, __m256i __A, + __m256i __B, __m256i __C, + const int __imm) +{ + return (__m256i) __builtin_ia32_pternlogd256_maskz ((__v8si) __A, + (__v8si) __B, + (__v8si) __C, + __imm, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_ternarylogic_epi64 (__m128i __A, __m128i __B, __m128i __C, + const int __imm) +{ + return (__m128i) __builtin_ia32_pternlogq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __C, __imm, + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_ternarylogic_epi64 (__m128i __A, __mmask8 __U, + __m128i __B, __m128i __C, const int __imm) +{ + return (__m128i) __builtin_ia32_pternlogq128_mask ((__v2di) __A, + (__v2di) __B, + (__v2di) __C, __imm, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_ternarylogic_epi64 (__mmask8 __U, __m128i __A, + __m128i __B, __m128i __C, const int __imm) +{ + return (__m128i) __builtin_ia32_pternlogq128_maskz ((__v2di) __A, + (__v2di) __B, + (__v2di) __C, + __imm, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_ternarylogic_epi32 (__m128i __A, __m128i __B, __m128i __C, + const int __imm) +{ + return (__m128i) __builtin_ia32_pternlogd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __C, __imm, + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_ternarylogic_epi32 (__m128i __A, __mmask8 __U, + __m128i __B, __m128i __C, const int __imm) +{ + return (__m128i) __builtin_ia32_pternlogd128_mask ((__v4si) __A, + (__v4si) __B, + (__v4si) __C, __imm, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_ternarylogic_epi32 (__mmask8 __U, __m128i __A, + __m128i __B, __m128i __C, const int __imm) +{ + return (__m128i) __builtin_ia32_pternlogd128_maskz ((__v4si) __A, + (__v4si) __B, + (__v4si) __C, + __imm, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_roundscale_ps (__m256 __A, const int __imm) +{ + return (__m256) __builtin_ia32_rndscaleps_256_mask ((__v8sf) __A, + __imm, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_roundscale_ps (__m256 __W, __mmask8 __U, __m256 __A, + const int __imm) +{ + return (__m256) __builtin_ia32_rndscaleps_256_mask ((__v8sf) __A, + __imm, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_roundscale_ps (__mmask8 __U, __m256 __A, const int __imm) +{ + return (__m256) __builtin_ia32_rndscaleps_256_mask ((__v8sf) __A, + __imm, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_roundscale_pd (__m256d __A, const int __imm) +{ + return (__m256d) __builtin_ia32_rndscalepd_256_mask ((__v4df) __A, + __imm, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_roundscale_pd (__m256d __W, __mmask8 __U, __m256d __A, + const int __imm) +{ + return (__m256d) __builtin_ia32_rndscalepd_256_mask ((__v4df) __A, + __imm, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_roundscale_pd (__mmask8 __U, __m256d __A, const int __imm) +{ + return (__m256d) __builtin_ia32_rndscalepd_256_mask ((__v4df) __A, + __imm, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_roundscale_ps (__m128 __A, const int __imm) +{ + return (__m128) __builtin_ia32_rndscaleps_128_mask ((__v4sf) __A, + __imm, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_roundscale_ps (__m128 __W, __mmask8 __U, __m128 __A, + const int __imm) +{ + return (__m128) __builtin_ia32_rndscaleps_128_mask ((__v4sf) __A, + __imm, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_roundscale_ps (__mmask8 __U, __m128 __A, const int __imm) +{ + return (__m128) __builtin_ia32_rndscaleps_128_mask ((__v4sf) __A, + __imm, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_roundscale_pd (__m128d __A, const int __imm) +{ + return (__m128d) __builtin_ia32_rndscalepd_128_mask ((__v2df) __A, + __imm, + (__v2df) + _mm_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_roundscale_pd (__m128d __W, __mmask8 __U, __m128d __A, + const int __imm) +{ + return (__m128d) __builtin_ia32_rndscalepd_128_mask ((__v2df) __A, + __imm, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_roundscale_pd (__mmask8 __U, __m128d __A, const int __imm) +{ + return (__m128d) __builtin_ia32_rndscalepd_128_mask ((__v2df) __A, + __imm, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_getmant_ps (__m256 __A, _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m256) __builtin_ia32_getmantps256_mask ((__v8sf) __A, + (__C << 2) | __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_getmant_ps (__m256 __W, __mmask8 __U, __m256 __A, + _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m256) __builtin_ia32_getmantps256_mask ((__v8sf) __A, + (__C << 2) | __B, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_getmant_ps (__mmask8 __U, __m256 __A, + _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m256) __builtin_ia32_getmantps256_mask ((__v8sf) __A, + (__C << 2) | __B, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_getmant_ps (__m128 __A, _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m128) __builtin_ia32_getmantps128_mask ((__v4sf) __A, + (__C << 2) | __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) -1); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_getmant_ps (__m128 __W, __mmask8 __U, __m128 __A, + _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m128) __builtin_ia32_getmantps128_mask ((__v4sf) __A, + (__C << 2) | __B, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_getmant_ps (__mmask8 __U, __m128 __A, + _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m128) __builtin_ia32_getmantps128_mask ((__v4sf) __A, + (__C << 2) | __B, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_getmant_pd (__m256d __A, _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m256d) __builtin_ia32_getmantpd256_mask ((__v4df) __A, + (__C << 2) | __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_getmant_pd (__m256d __W, __mmask8 __U, __m256d __A, + _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m256d) __builtin_ia32_getmantpd256_mask ((__v4df) __A, + (__C << 2) | __B, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_getmant_pd (__mmask8 __U, __m256d __A, + _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m256d) __builtin_ia32_getmantpd256_mask ((__v4df) __A, + (__C << 2) | __B, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_getmant_pd (__m128d __A, _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m128d) __builtin_ia32_getmantpd128_mask ((__v2df) __A, + (__C << 2) | __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) -1); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_getmant_pd (__m128d __W, __mmask8 __U, __m128d __A, + _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m128d) __builtin_ia32_getmantpd128_mask ((__v2df) __A, + (__C << 2) | __B, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_getmant_pd (__mmask8 __U, __m128d __A, + _MM_MANTISSA_NORM_ENUM __B, + _MM_MANTISSA_SIGN_ENUM __C) +{ + return (__m128d) __builtin_ia32_getmantpd128_mask ((__v2df) __A, + (__C << 2) | __B, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mmask_i32gather_ps (__m256 __v1_old, __mmask8 __mask, + __m256i __index, void const *__addr, + int __scale) +{ + return (__m256) __builtin_ia32_gather3siv8sf ((__v8sf) __v1_old, + __addr, + (__v8si) __index, + __mask, __scale); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mmask_i32gather_ps (__m128 __v1_old, __mmask8 __mask, + __m128i __index, void const *__addr, + int __scale) +{ + return (__m128) __builtin_ia32_gather3siv4sf ((__v4sf) __v1_old, + __addr, + (__v4si) __index, + __mask, __scale); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mmask_i32gather_pd (__m256d __v1_old, __mmask8 __mask, + __m128i __index, void const *__addr, + int __scale) +{ + return (__m256d) __builtin_ia32_gather3siv4df ((__v4df) __v1_old, + __addr, + (__v4si) __index, + __mask, __scale); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mmask_i32gather_pd (__m128d __v1_old, __mmask8 __mask, + __m128i __index, void const *__addr, + int __scale) +{ + return (__m128d) __builtin_ia32_gather3siv2df ((__v2df) __v1_old, + __addr, + (__v4si) __index, + __mask, __scale); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mmask_i64gather_ps (__m128 __v1_old, __mmask8 __mask, + __m256i __index, void const *__addr, + int __scale) +{ + return (__m128) __builtin_ia32_gather3div8sf ((__v4sf) __v1_old, + __addr, + (__v4di) __index, + __mask, __scale); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mmask_i64gather_ps (__m128 __v1_old, __mmask8 __mask, + __m128i __index, void const *__addr, + int __scale) +{ + return (__m128) __builtin_ia32_gather3div4sf ((__v4sf) __v1_old, + __addr, + (__v2di) __index, + __mask, __scale); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mmask_i64gather_pd (__m256d __v1_old, __mmask8 __mask, + __m256i __index, void const *__addr, + int __scale) +{ + return (__m256d) __builtin_ia32_gather3div4df ((__v4df) __v1_old, + __addr, + (__v4di) __index, + __mask, __scale); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mmask_i64gather_pd (__m128d __v1_old, __mmask8 __mask, + __m128i __index, void const *__addr, + int __scale) +{ + return (__m128d) __builtin_ia32_gather3div2df ((__v2df) __v1_old, + __addr, + (__v2di) __index, + __mask, __scale); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mmask_i32gather_epi32 (__m256i __v1_old, __mmask8 __mask, + __m256i __index, void const *__addr, + int __scale) +{ + return (__m256i) __builtin_ia32_gather3siv8si ((__v8si) __v1_old, + __addr, + (__v8si) __index, + __mask, __scale); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mmask_i32gather_epi32 (__m128i __v1_old, __mmask8 __mask, + __m128i __index, void const *__addr, + int __scale) +{ + return (__m128i) __builtin_ia32_gather3siv4si ((__v4si) __v1_old, + __addr, + (__v4si) __index, + __mask, __scale); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mmask_i32gather_epi64 (__m256i __v1_old, __mmask8 __mask, + __m128i __index, void const *__addr, + int __scale) +{ + return (__m256i) __builtin_ia32_gather3siv4di ((__v4di) __v1_old, + __addr, + (__v4si) __index, + __mask, __scale); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mmask_i32gather_epi64 (__m128i __v1_old, __mmask8 __mask, + __m128i __index, void const *__addr, + int __scale) +{ + return (__m128i) __builtin_ia32_gather3siv2di ((__v2di) __v1_old, + __addr, + (__v4si) __index, + __mask, __scale); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mmask_i64gather_epi32 (__m128i __v1_old, __mmask8 __mask, + __m256i __index, void const *__addr, + int __scale) +{ + return (__m128i) __builtin_ia32_gather3div8si ((__v4si) __v1_old, + __addr, + (__v4di) __index, + __mask, __scale); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mmask_i64gather_epi32 (__m128i __v1_old, __mmask8 __mask, + __m128i __index, void const *__addr, + int __scale) +{ + return (__m128i) __builtin_ia32_gather3div4si ((__v4si) __v1_old, + __addr, + (__v2di) __index, + __mask, __scale); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mmask_i64gather_epi64 (__m256i __v1_old, __mmask8 __mask, + __m256i __index, void const *__addr, + int __scale) +{ + return (__m256i) __builtin_ia32_gather3div4di ((__v4di) __v1_old, + __addr, + (__v4di) __index, + __mask, __scale); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mmask_i64gather_epi64 (__m128i __v1_old, __mmask8 __mask, + __m128i __index, void const *__addr, + int __scale) +{ + return (__m128i) __builtin_ia32_gather3div2di ((__v2di) __v1_old, + __addr, + (__v2di) __index, + __mask, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_i32scatter_ps (void *__addr, __m256i __index, + __m256 __v1, const int __scale) +{ + __builtin_ia32_scattersiv8sf (__addr, (__mmask8) 0xFF, + (__v8si) __index, (__v8sf) __v1, + __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_i32scatter_ps (void *__addr, __mmask8 __mask, + __m256i __index, __m256 __v1, + const int __scale) +{ + __builtin_ia32_scattersiv8sf (__addr, __mask, (__v8si) __index, + (__v8sf) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_i32scatter_ps (void *__addr, __m128i __index, __m128 __v1, + const int __scale) +{ + __builtin_ia32_scattersiv4sf (__addr, (__mmask8) 0xFF, + (__v4si) __index, (__v4sf) __v1, + __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_i32scatter_ps (void *__addr, __mmask8 __mask, + __m128i __index, __m128 __v1, + const int __scale) +{ + __builtin_ia32_scattersiv4sf (__addr, __mask, (__v4si) __index, + (__v4sf) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_i32scatter_pd (void *__addr, __m128i __index, + __m256d __v1, const int __scale) +{ + __builtin_ia32_scattersiv4df (__addr, (__mmask8) 0xFF, + (__v4si) __index, (__v4df) __v1, + __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_i32scatter_pd (void *__addr, __mmask8 __mask, + __m128i __index, __m256d __v1, + const int __scale) +{ + __builtin_ia32_scattersiv4df (__addr, __mask, (__v4si) __index, + (__v4df) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_i32scatter_pd (void *__addr, __m128i __index, + __m128d __v1, const int __scale) +{ + __builtin_ia32_scattersiv2df (__addr, (__mmask8) 0xFF, + (__v4si) __index, (__v2df) __v1, + __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_i32scatter_pd (void *__addr, __mmask8 __mask, + __m128i __index, __m128d __v1, + const int __scale) +{ + __builtin_ia32_scattersiv2df (__addr, __mask, (__v4si) __index, + (__v2df) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_i64scatter_ps (void *__addr, __m256i __index, + __m128 __v1, const int __scale) +{ + __builtin_ia32_scatterdiv8sf (__addr, (__mmask8) 0xFF, + (__v4di) __index, (__v4sf) __v1, + __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_i64scatter_ps (void *__addr, __mmask8 __mask, + __m256i __index, __m128 __v1, + const int __scale) +{ + __builtin_ia32_scatterdiv8sf (__addr, __mask, (__v4di) __index, + (__v4sf) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_i64scatter_ps (void *__addr, __m128i __index, __m128 __v1, + const int __scale) +{ + __builtin_ia32_scatterdiv4sf (__addr, (__mmask8) 0xFF, + (__v2di) __index, (__v4sf) __v1, + __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_i64scatter_ps (void *__addr, __mmask8 __mask, + __m128i __index, __m128 __v1, + const int __scale) +{ + __builtin_ia32_scatterdiv4sf (__addr, __mask, (__v2di) __index, + (__v4sf) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_i64scatter_pd (void *__addr, __m256i __index, + __m256d __v1, const int __scale) +{ + __builtin_ia32_scatterdiv4df (__addr, (__mmask8) 0xFF, + (__v4di) __index, (__v4df) __v1, + __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_i64scatter_pd (void *__addr, __mmask8 __mask, + __m256i __index, __m256d __v1, + const int __scale) +{ + __builtin_ia32_scatterdiv4df (__addr, __mask, (__v4di) __index, + (__v4df) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_i64scatter_pd (void *__addr, __m128i __index, + __m128d __v1, const int __scale) +{ + __builtin_ia32_scatterdiv2df (__addr, (__mmask8) 0xFF, + (__v2di) __index, (__v2df) __v1, + __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_i64scatter_pd (void *__addr, __mmask8 __mask, + __m128i __index, __m128d __v1, + const int __scale) +{ + __builtin_ia32_scatterdiv2df (__addr, __mask, (__v2di) __index, + (__v2df) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_i32scatter_epi32 (void *__addr, __m256i __index, + __m256i __v1, const int __scale) +{ + __builtin_ia32_scattersiv8si (__addr, (__mmask8) 0xFF, + (__v8si) __index, (__v8si) __v1, + __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_i32scatter_epi32 (void *__addr, __mmask8 __mask, + __m256i __index, __m256i __v1, + const int __scale) +{ + __builtin_ia32_scattersiv8si (__addr, __mask, (__v8si) __index, + (__v8si) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_i32scatter_epi32 (void *__addr, __m128i __index, + __m128i __v1, const int __scale) +{ + __builtin_ia32_scattersiv4si (__addr, (__mmask8) 0xFF, + (__v4si) __index, (__v4si) __v1, + __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_i32scatter_epi32 (void *__addr, __mmask8 __mask, + __m128i __index, __m128i __v1, + const int __scale) +{ + __builtin_ia32_scattersiv4si (__addr, __mask, (__v4si) __index, + (__v4si) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_i32scatter_epi64 (void *__addr, __m128i __index, + __m256i __v1, const int __scale) +{ + __builtin_ia32_scattersiv4di (__addr, (__mmask8) 0xFF, + (__v4si) __index, (__v4di) __v1, + __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_i32scatter_epi64 (void *__addr, __mmask8 __mask, + __m128i __index, __m256i __v1, + const int __scale) +{ + __builtin_ia32_scattersiv4di (__addr, __mask, (__v4si) __index, + (__v4di) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_i32scatter_epi64 (void *__addr, __m128i __index, + __m128i __v1, const int __scale) +{ + __builtin_ia32_scattersiv2di (__addr, (__mmask8) 0xFF, + (__v4si) __index, (__v2di) __v1, + __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_i32scatter_epi64 (void *__addr, __mmask8 __mask, + __m128i __index, __m128i __v1, + const int __scale) +{ + __builtin_ia32_scattersiv2di (__addr, __mask, (__v4si) __index, + (__v2di) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_i64scatter_epi32 (void *__addr, __m256i __index, + __m128i __v1, const int __scale) +{ + __builtin_ia32_scatterdiv8si (__addr, (__mmask8) 0xFF, + (__v4di) __index, (__v4si) __v1, + __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_i64scatter_epi32 (void *__addr, __mmask8 __mask, + __m256i __index, __m128i __v1, + const int __scale) +{ + __builtin_ia32_scatterdiv8si (__addr, __mask, (__v4di) __index, + (__v4si) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_i64scatter_epi32 (void *__addr, __m128i __index, + __m128i __v1, const int __scale) +{ + __builtin_ia32_scatterdiv4si (__addr, (__mmask8) 0xFF, + (__v2di) __index, (__v4si) __v1, + __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_i64scatter_epi32 (void *__addr, __mmask8 __mask, + __m128i __index, __m128i __v1, + const int __scale) +{ + __builtin_ia32_scatterdiv4si (__addr, __mask, (__v2di) __index, + (__v4si) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_i64scatter_epi64 (void *__addr, __m256i __index, + __m256i __v1, const int __scale) +{ + __builtin_ia32_scatterdiv4di (__addr, (__mmask8) 0xFF, + (__v4di) __index, (__v4di) __v1, + __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_i64scatter_epi64 (void *__addr, __mmask8 __mask, + __m256i __index, __m256i __v1, + const int __scale) +{ + __builtin_ia32_scatterdiv4di (__addr, __mask, (__v4di) __index, + (__v4di) __v1, __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_i64scatter_epi64 (void *__addr, __m128i __index, + __m128i __v1, const int __scale) +{ + __builtin_ia32_scatterdiv2di (__addr, (__mmask8) 0xFF, + (__v2di) __index, (__v2di) __v1, + __scale); +} + +extern __inline void +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_i64scatter_epi64 (void *__addr, __mmask8 __mask, + __m128i __index, __m128i __v1, + const int __scale) +{ + __builtin_ia32_scatterdiv2di (__addr, __mask, (__v2di) __index, + (__v2di) __v1, __scale); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_shuffle_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + _MM_PERM_ENUM __mask) +{ + return (__m256i) __builtin_ia32_pshufd256_mask ((__v8si) __A, __mask, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_shuffle_epi32 (__mmask8 __U, __m256i __A, + _MM_PERM_ENUM __mask) +{ + return (__m256i) __builtin_ia32_pshufd256_mask ((__v8si) __A, __mask, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_shuffle_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + _MM_PERM_ENUM __mask) +{ + return (__m128i) __builtin_ia32_pshufd128_mask ((__v4si) __A, __mask, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_shuffle_epi32 (__mmask8 __U, __m128i __A, + _MM_PERM_ENUM __mask) +{ + return (__m128i) __builtin_ia32_pshufd128_mask ((__v4si) __A, __mask, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_rol_epi32 (__m256i __A, const int __B) +{ + return (__m256i) __builtin_ia32_prold256_mask ((__v8si) __A, __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_rol_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + const int __B) +{ + return (__m256i) __builtin_ia32_prold256_mask ((__v8si) __A, __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_rol_epi32 (__mmask8 __U, __m256i __A, const int __B) +{ + return (__m256i) __builtin_ia32_prold256_mask ((__v8si) __A, __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rol_epi32 (__m128i __A, const int __B) +{ + return (__m128i) __builtin_ia32_prold128_mask ((__v4si) __A, __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_rol_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + const int __B) +{ + return (__m128i) __builtin_ia32_prold128_mask ((__v4si) __A, __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_rol_epi32 (__mmask8 __U, __m128i __A, const int __B) +{ + return (__m128i) __builtin_ia32_prold128_mask ((__v4si) __A, __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_ror_epi32 (__m256i __A, const int __B) +{ + return (__m256i) __builtin_ia32_prord256_mask ((__v8si) __A, __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_ror_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + const int __B) +{ + return (__m256i) __builtin_ia32_prord256_mask ((__v8si) __A, __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_ror_epi32 (__mmask8 __U, __m256i __A, const int __B) +{ + return (__m256i) __builtin_ia32_prord256_mask ((__v8si) __A, __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_ror_epi32 (__m128i __A, const int __B) +{ + return (__m128i) __builtin_ia32_prord128_mask ((__v4si) __A, __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_ror_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + const int __B) +{ + return (__m128i) __builtin_ia32_prord128_mask ((__v4si) __A, __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_ror_epi32 (__mmask8 __U, __m128i __A, const int __B) +{ + return (__m128i) __builtin_ia32_prord128_mask ((__v4si) __A, __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_rol_epi64 (__m256i __A, const int __B) +{ + return (__m256i) __builtin_ia32_prolq256_mask ((__v4di) __A, __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_rol_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + const int __B) +{ + return (__m256i) __builtin_ia32_prolq256_mask ((__v4di) __A, __B, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_rol_epi64 (__mmask8 __U, __m256i __A, const int __B) +{ + return (__m256i) __builtin_ia32_prolq256_mask ((__v4di) __A, __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rol_epi64 (__m128i __A, const int __B) +{ + return (__m128i) __builtin_ia32_prolq128_mask ((__v2di) __A, __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_rol_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + const int __B) +{ + return (__m128i) __builtin_ia32_prolq128_mask ((__v2di) __A, __B, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_rol_epi64 (__mmask8 __U, __m128i __A, const int __B) +{ + return (__m128i) __builtin_ia32_prolq128_mask ((__v2di) __A, __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_ror_epi64 (__m256i __A, const int __B) +{ + return (__m256i) __builtin_ia32_prorq256_mask ((__v4di) __A, __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_ror_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + const int __B) +{ + return (__m256i) __builtin_ia32_prorq256_mask ((__v4di) __A, __B, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_ror_epi64 (__mmask8 __U, __m256i __A, const int __B) +{ + return (__m256i) __builtin_ia32_prorq256_mask ((__v4di) __A, __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_ror_epi64 (__m128i __A, const int __B) +{ + return (__m128i) __builtin_ia32_prorq128_mask ((__v2di) __A, __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_ror_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + const int __B) +{ + return (__m128i) __builtin_ia32_prorq128_mask ((__v2di) __A, __B, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_ror_epi64 (__mmask8 __U, __m128i __A, const int __B) +{ + return (__m128i) __builtin_ia32_prorq128_mask ((__v2di) __A, __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_alignr_epi32 (__m128i __A, __m128i __B, const int __imm) +{ + return (__m128i) __builtin_ia32_alignd128_mask ((__v4si) __A, + (__v4si) __B, __imm, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_alignr_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B, const int __imm) +{ + return (__m128i) __builtin_ia32_alignd128_mask ((__v4si) __A, + (__v4si) __B, __imm, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_alignr_epi32 (__mmask8 __U, __m128i __A, __m128i __B, + const int __imm) +{ + return (__m128i) __builtin_ia32_alignd128_mask ((__v4si) __A, + (__v4si) __B, __imm, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_alignr_epi64 (__m128i __A, __m128i __B, const int __imm) +{ + return (__m128i) __builtin_ia32_alignq128_mask ((__v2di) __A, + (__v2di) __B, __imm, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_alignr_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + __m128i __B, const int __imm) +{ + return (__m128i) __builtin_ia32_alignq128_mask ((__v2di) __A, + (__v2di) __B, __imm, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_alignr_epi64 (__mmask8 __U, __m128i __A, __m128i __B, + const int __imm) +{ + return (__m128i) __builtin_ia32_alignq128_mask ((__v2di) __A, + (__v2di) __B, __imm, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_alignr_epi32 (__m256i __A, __m256i __B, const int __imm) +{ + return (__m256i) __builtin_ia32_alignd256_mask ((__v8si) __A, + (__v8si) __B, __imm, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_alignr_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B, const int __imm) +{ + return (__m256i) __builtin_ia32_alignd256_mask ((__v8si) __A, + (__v8si) __B, __imm, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_alignr_epi32 (__mmask8 __U, __m256i __A, __m256i __B, + const int __imm) +{ + return (__m256i) __builtin_ia32_alignd256_mask ((__v8si) __A, + (__v8si) __B, __imm, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_alignr_epi64 (__m256i __A, __m256i __B, const int __imm) +{ + return (__m256i) __builtin_ia32_alignq256_mask ((__v4di) __A, + (__v4di) __B, __imm, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_alignr_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + __m256i __B, const int __imm) +{ + return (__m256i) __builtin_ia32_alignq256_mask ((__v4di) __A, + (__v4di) __B, __imm, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_alignr_epi64 (__mmask8 __U, __m256i __A, __m256i __B, + const int __imm) +{ + return (__m256i) __builtin_ia32_alignq256_mask ((__v4di) __A, + (__v4di) __B, __imm, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cvtps_ph (__m128i __W, __mmask8 __U, __m128 __A, + const int __I) +{ + return (__m128i) __builtin_ia32_vcvtps2ph_mask ((__v4sf) __A, __I, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_cvtps_ph (__mmask8 __U, __m128 __A, const int __I) +{ + return (__m128i) __builtin_ia32_vcvtps2ph_mask ((__v4sf) __A, __I, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cvtps_ph (__m128i __W, __mmask8 __U, __m256 __A, + const int __I) +{ + return (__m128i) __builtin_ia32_vcvtps2ph256_mask ((__v8sf) __A, __I, + (__v8hi) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_cvtps_ph (__mmask8 __U, __m256 __A, const int __I) +{ + return (__m128i) __builtin_ia32_vcvtps2ph256_mask ((__v8sf) __A, __I, + (__v8hi) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_srai_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + const int __imm) +{ + return (__m256i) __builtin_ia32_psradi256_mask ((__v8si) __A, __imm, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_srai_epi32 (__mmask8 __U, __m256i __A, const int __imm) +{ + return (__m256i) __builtin_ia32_psradi256_mask ((__v8si) __A, __imm, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_srai_epi32 (__m128i __W, __mmask8 __U, __m128i __A, + const int __imm) +{ + return (__m128i) __builtin_ia32_psradi128_mask ((__v4si) __A, __imm, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_srai_epi32 (__mmask8 __U, __m128i __A, const int __imm) +{ + return (__m128i) __builtin_ia32_psradi128_mask ((__v4si) __A, __imm, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_srai_epi64 (__m256i __A, const int __imm) +{ + return (__m256i) __builtin_ia32_psraqi256_mask ((__v4di) __A, __imm, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) -1); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_srai_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + const int __imm) +{ + return (__m256i) __builtin_ia32_psraqi256_mask ((__v4di) __A, __imm, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_srai_epi64 (__mmask8 __U, __m256i __A, const int __imm) +{ + return (__m256i) __builtin_ia32_psraqi256_mask ((__v4di) __A, __imm, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srai_epi64 (__m128i __A, const int __imm) +{ + return (__m128i) __builtin_ia32_psraqi128_mask ((__v2di) __A, __imm, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) -1); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_srai_epi64 (__m128i __W, __mmask8 __U, __m128i __A, + const int __imm) +{ + return (__m128i) __builtin_ia32_psraqi128_mask ((__v2di) __A, __imm, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_srai_epi64 (__mmask8 __U, __m128i __A, const int __imm) +{ + return (__m128i) __builtin_ia32_psraqi128_mask ((__v2di) __A, __imm, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_slli_epi32 (__m128i __W, __mmask8 __U, __m128i __A, int __B) +{ + return (__m128i) __builtin_ia32_pslldi128_mask ((__v4si) __A, __B, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_slli_epi32 (__mmask8 __U, __m128i __A, int __B) +{ + return (__m128i) __builtin_ia32_pslldi128_mask ((__v4si) __A, __B, + (__v4si) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_slli_epi64 (__m128i __W, __mmask8 __U, __m128i __A, int __B) +{ + return (__m128i) __builtin_ia32_psllqi128_mask ((__v2di) __A, __B, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_slli_epi64 (__mmask8 __U, __m128i __A, int __B) +{ + return (__m128i) __builtin_ia32_psllqi128_mask ((__v2di) __A, __B, + (__v2di) + _mm_setzero_si128 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_slli_epi32 (__m256i __W, __mmask8 __U, __m256i __A, + int __B) +{ + return (__m256i) __builtin_ia32_pslldi256_mask ((__v8si) __A, __B, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_slli_epi32 (__mmask8 __U, __m256i __A, int __B) +{ + return (__m256i) __builtin_ia32_pslldi256_mask ((__v8si) __A, __B, + (__v8si) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_slli_epi64 (__m256i __W, __mmask8 __U, __m256i __A, + int __B) +{ + return (__m256i) __builtin_ia32_psllqi256_mask ((__v4di) __A, __B, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_slli_epi64 (__mmask8 __U, __m256i __A, int __B) +{ + return (__m256i) __builtin_ia32_psllqi256_mask ((__v4di) __A, __B, + (__v4di) + _mm256_setzero_si256 (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permutex_pd (__m256d __W, __mmask8 __U, __m256d __X, + const int __imm) +{ + return (__m256d) __builtin_ia32_permdf256_mask ((__v4df) __X, __imm, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permutex_pd (__mmask8 __U, __m256d __X, const int __imm) +{ + return (__m256d) __builtin_ia32_permdf256_mask ((__v4df) __X, __imm, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permute_pd (__m256d __W, __mmask8 __U, __m256d __X, + const int __C) +{ + return (__m256d) __builtin_ia32_vpermilpd256_mask ((__v4df) __X, __C, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permute_pd (__mmask8 __U, __m256d __X, const int __C) +{ + return (__m256d) __builtin_ia32_vpermilpd256_mask ((__v4df) __X, __C, + (__v4df) + _mm256_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_permute_pd (__m128d __W, __mmask8 __U, __m128d __X, + const int __C) +{ + return (__m128d) __builtin_ia32_vpermilpd_mask ((__v2df) __X, __C, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_permute_pd (__mmask8 __U, __m128d __X, const int __C) +{ + return (__m128d) __builtin_ia32_vpermilpd_mask ((__v2df) __X, __C, + (__v2df) + _mm_setzero_pd (), + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_permute_ps (__m256 __W, __mmask8 __U, __m256 __X, + const int __C) +{ + return (__m256) __builtin_ia32_vpermilps256_mask ((__v8sf) __X, __C, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskz_permute_ps (__mmask8 __U, __m256 __X, const int __C) +{ + return (__m256) __builtin_ia32_vpermilps256_mask ((__v8sf) __X, __C, + (__v8sf) + _mm256_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_permute_ps (__m128 __W, __mmask8 __U, __m128 __X, + const int __C) +{ + return (__m128) __builtin_ia32_vpermilps_mask ((__v4sf) __X, __C, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskz_permute_ps (__mmask8 __U, __m128 __X, const int __C) +{ + return (__m128) __builtin_ia32_vpermilps_mask ((__v4sf) __X, __C, + (__v4sf) + _mm_setzero_ps (), + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_blend_pd (__mmask8 __U, __m256d __A, __m256d __W) +{ + return (__m256d) __builtin_ia32_blendmpd_256_mask ((__v4df) __A, + (__v4df) __W, + (__mmask8) __U); +} + +extern __inline __m256 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_blend_ps (__mmask8 __U, __m256 __A, __m256 __W) +{ + return (__m256) __builtin_ia32_blendmps_256_mask ((__v8sf) __A, + (__v8sf) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_blend_epi64 (__mmask8 __U, __m256i __A, __m256i __W) +{ + return (__m256i) __builtin_ia32_blendmq_256_mask ((__v4di) __A, + (__v4di) __W, + (__mmask8) __U); +} + +extern __inline __m256i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_blend_epi32 (__mmask8 __U, __m256i __A, __m256i __W) +{ + return (__m256i) __builtin_ia32_blendmd_256_mask ((__v8si) __A, + (__v8si) __W, + (__mmask8) __U); +} + +extern __inline __m128d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_blend_pd (__mmask8 __U, __m128d __A, __m128d __W) +{ + return (__m128d) __builtin_ia32_blendmpd_128_mask ((__v2df) __A, + (__v2df) __W, + (__mmask8) __U); +} + +extern __inline __m128 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_blend_ps (__mmask8 __U, __m128 __A, __m128 __W) +{ + return (__m128) __builtin_ia32_blendmps_128_mask ((__v4sf) __A, + (__v4sf) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_blend_epi64 (__mmask8 __U, __m128i __A, __m128i __W) +{ + return (__m128i) __builtin_ia32_blendmq_128_mask ((__v2di) __A, + (__v2di) __W, + (__mmask8) __U); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_blend_epi32 (__mmask8 __U, __m128i __A, __m128i __W) +{ + return (__m128i) __builtin_ia32_blendmd_128_mask ((__v4si) __A, + (__v4si) __W, + (__mmask8) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmp_epi64_mask (__m256i __X, __m256i __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_cmpq256_mask ((__v4di) __X, + (__v4di) __Y, __P, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmp_epi32_mask (__m256i __X, __m256i __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_cmpd256_mask ((__v8si) __X, + (__v8si) __Y, __P, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmp_epu64_mask (__m256i __X, __m256i __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_ucmpq256_mask ((__v4di) __X, + (__v4di) __Y, __P, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmp_epu32_mask (__m256i __X, __m256i __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_ucmpd256_mask ((__v8si) __X, + (__v8si) __Y, __P, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmp_pd_mask (__m256d __X, __m256d __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_cmppd256_mask ((__v4df) __X, + (__v4df) __Y, __P, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmp_ps_mask (__m256 __X, __m256 __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_cmpps256_mask ((__v8sf) __X, + (__v8sf) __Y, __P, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmp_epi64_mask (__mmask8 __U, __m256i __X, __m256i __Y, + const int __P) +{ + return (__mmask8) __builtin_ia32_cmpq256_mask ((__v4di) __X, + (__v4di) __Y, __P, + (__mmask8) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmp_epi32_mask (__mmask8 __U, __m256i __X, __m256i __Y, + const int __P) +{ + return (__mmask8) __builtin_ia32_cmpd256_mask ((__v8si) __X, + (__v8si) __Y, __P, + (__mmask8) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmp_epu64_mask (__mmask8 __U, __m256i __X, __m256i __Y, + const int __P) +{ + return (__mmask8) __builtin_ia32_ucmpq256_mask ((__v4di) __X, + (__v4di) __Y, __P, + (__mmask8) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmp_epu32_mask (__mmask8 __U, __m256i __X, __m256i __Y, + const int __P) +{ + return (__mmask8) __builtin_ia32_ucmpd256_mask ((__v8si) __X, + (__v8si) __Y, __P, + (__mmask8) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmp_pd_mask (__mmask8 __U, __m256d __X, __m256d __Y, + const int __P) +{ + return (__mmask8) __builtin_ia32_cmppd256_mask ((__v4df) __X, + (__v4df) __Y, __P, + (__mmask8) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mask_cmp_ps_mask (__mmask8 __U, __m256 __X, __m256 __Y, + const int __P) +{ + return (__mmask8) __builtin_ia32_cmpps256_mask ((__v8sf) __X, + (__v8sf) __Y, __P, + (__mmask8) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_epi64_mask (__m128i __X, __m128i __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_cmpq128_mask ((__v2di) __X, + (__v2di) __Y, __P, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_epi32_mask (__m128i __X, __m128i __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_cmpd128_mask ((__v4si) __X, + (__v4si) __Y, __P, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_epu64_mask (__m128i __X, __m128i __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_ucmpq128_mask ((__v2di) __X, + (__v2di) __Y, __P, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_epu32_mask (__m128i __X, __m128i __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_ucmpd128_mask ((__v4si) __X, + (__v4si) __Y, __P, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_pd_mask (__m128d __X, __m128d __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_cmppd128_mask ((__v2df) __X, + (__v2df) __Y, __P, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_ps_mask (__m128 __X, __m128 __Y, const int __P) +{ + return (__mmask8) __builtin_ia32_cmpps128_mask ((__v4sf) __X, + (__v4sf) __Y, __P, + (__mmask8) -1); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmp_epi64_mask (__mmask8 __U, __m128i __X, __m128i __Y, + const int __P) +{ + return (__mmask8) __builtin_ia32_cmpq128_mask ((__v2di) __X, + (__v2di) __Y, __P, + (__mmask8) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmp_epi32_mask (__mmask8 __U, __m128i __X, __m128i __Y, + const int __P) +{ + return (__mmask8) __builtin_ia32_cmpd128_mask ((__v4si) __X, + (__v4si) __Y, __P, + (__mmask8) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmp_epu64_mask (__mmask8 __U, __m128i __X, __m128i __Y, + const int __P) +{ + return (__mmask8) __builtin_ia32_ucmpq128_mask ((__v2di) __X, + (__v2di) __Y, __P, + (__mmask8) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmp_epu32_mask (__mmask8 __U, __m128i __X, __m128i __Y, + const int __P) +{ + return (__mmask8) __builtin_ia32_ucmpd128_mask ((__v4si) __X, + (__v4si) __Y, __P, + (__mmask8) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmp_pd_mask (__mmask8 __U, __m128d __X, __m128d __Y, + const int __P) +{ + return (__mmask8) __builtin_ia32_cmppd128_mask ((__v2df) __X, + (__v2df) __Y, __P, + (__mmask8) __U); +} + +extern __inline __mmask8 +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mask_cmp_ps_mask (__mmask8 __U, __m128 __X, __m128 __Y, + const int __P) +{ + return (__mmask8) __builtin_ia32_cmpps128_mask ((__v4sf) __X, + (__v4sf) __Y, __P, + (__mmask8) __U); +} + +extern __inline __m256d +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permutex_pd (__m256d __X, const int __M) +{ + return (__m256d) __builtin_ia32_permdf256_mask ((__v4df) __X, __M, + (__v4df) + _mm256_undefined_pd (), + (__mmask8) -1); +} + +#else +#define _mm256_permutex_pd(X, M) \ + ((__m256d) __builtin_ia32_permdf256_mask ((__v4df)(__m256d)(X), (int)(M), \ + (__v4df)(__m256d) \ + _mm256_undefined_pd (), \ + (__mmask8)-1)) + +#define _mm256_maskz_permutex_epi64(M, X, I) \ + ((__m256i) __builtin_ia32_permdi256_mask ((__v4di)(__m256i)(X), \ + (int)(I), \ + (__v4di)(__m256i) \ + (_mm256_setzero_si256 ()),\ + (__mmask8)(M))) + +#define _mm256_mask_permutex_epi64(W, M, X, I) \ + ((__m256i) __builtin_ia32_permdi256_mask ((__v4di)(__m256i)(X), \ + (int)(I), \ + (__v4di)(__m256i)(W), \ + (__mmask8)(M))) + +#define _mm256_insertf32x4(X, Y, C) \ + ((__m256) __builtin_ia32_insertf32x4_256_mask ((__v8sf)(__m256) (X), \ + (__v4sf)(__m128) (Y), (int) (C), \ + (__v8sf)(__m256)_mm256_setzero_ps (), \ + (__mmask8)-1)) + +#define _mm256_mask_insertf32x4(W, U, X, Y, C) \ + ((__m256) __builtin_ia32_insertf32x4_256_mask ((__v8sf)(__m256) (X), \ + (__v4sf)(__m128) (Y), (int) (C), \ + (__v8sf)(__m256)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_insertf32x4(U, X, Y, C) \ + ((__m256) __builtin_ia32_insertf32x4_256_mask ((__v8sf)(__m256) (X), \ + (__v4sf)(__m128) (Y), (int) (C), \ + (__v8sf)(__m256)_mm256_setzero_ps (), \ + (__mmask8)(U))) + +#define _mm256_inserti32x4(X, Y, C) \ + ((__m256i) __builtin_ia32_inserti32x4_256_mask ((__v8si)(__m256i) (X),\ + (__v4si)(__m128i) (Y), (int) (C), \ + (__v8si)(__m256i)_mm256_setzero_si256 (), \ + (__mmask8)-1)) + +#define _mm256_mask_inserti32x4(W, U, X, Y, C) \ + ((__m256i) __builtin_ia32_inserti32x4_256_mask ((__v8si)(__m256i) (X),\ + (__v4si)(__m128i) (Y), (int) (C), \ + (__v8si)(__m256i)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_inserti32x4(U, X, Y, C) \ + ((__m256i) __builtin_ia32_inserti32x4_256_mask ((__v8si)(__m256i) (X),\ + (__v4si)(__m128i) (Y), (int) (C), \ + (__v8si)(__m256i)_mm256_setzero_si256 (), \ + (__mmask8)(U))) + +#define _mm256_extractf32x4_ps(X, C) \ + ((__m128) __builtin_ia32_extractf32x4_256_mask ((__v8sf)(__m256) (X), \ + (int) (C), \ + (__v4sf)(__m128)_mm_setzero_ps (), \ + (__mmask8)-1)) + +#define _mm256_mask_extractf32x4_ps(W, U, X, C) \ + ((__m128) __builtin_ia32_extractf32x4_256_mask ((__v8sf)(__m256) (X), \ + (int) (C), \ + (__v4sf)(__m128)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_extractf32x4_ps(U, X, C) \ + ((__m128) __builtin_ia32_extractf32x4_256_mask ((__v8sf)(__m256) (X), \ + (int) (C), \ + (__v4sf)(__m128)_mm_setzero_ps (), \ + (__mmask8)(U))) + +#define _mm256_extracti32x4_epi32(X, C) \ + ((__m128i) __builtin_ia32_extracti32x4_256_mask ((__v8si)(__m256i) (X),\ + (int) (C), (__v4si)(__m128i)_mm_setzero_si128 (), (__mmask8)-1)) + +#define _mm256_mask_extracti32x4_epi32(W, U, X, C) \ + ((__m128i) __builtin_ia32_extracti32x4_256_mask ((__v8si)(__m256i) (X),\ + (int) (C), (__v4si)(__m128i)(W), (__mmask8)(U))) + +#define _mm256_maskz_extracti32x4_epi32(U, X, C) \ + ((__m128i) __builtin_ia32_extracti32x4_256_mask ((__v8si)(__m256i) (X),\ + (int) (C), (__v4si)(__m128i)_mm_setzero_si128 (), (__mmask8)(U))) + +#define _mm256_shuffle_i64x2(X, Y, C) \ + ((__m256i) __builtin_ia32_shuf_i64x2_256_mask ((__v4di)(__m256i)(X), \ + (__v4di)(__m256i)(Y), (int)(C), \ + (__v4di)(__m256i)_mm256_setzero_si256 (), \ + (__mmask8)-1)) + +#define _mm256_mask_shuffle_i64x2(W, U, X, Y, C) \ + ((__m256i) __builtin_ia32_shuf_i64x2_256_mask ((__v4di)(__m256i)(X), \ + (__v4di)(__m256i)(Y), (int)(C), \ + (__v4di)(__m256i)(W),\ + (__mmask8)(U))) + +#define _mm256_maskz_shuffle_i64x2(U, X, Y, C) \ + ((__m256i) __builtin_ia32_shuf_i64x2_256_mask ((__v4di)(__m256i)(X), \ + (__v4di)(__m256i)(Y), (int)(C), \ + (__v4di)(__m256i)_mm256_setzero_si256 (), \ + (__mmask8)(U))) + +#define _mm256_shuffle_i32x4(X, Y, C) \ + ((__m256i) __builtin_ia32_shuf_i32x4_256_mask ((__v8si)(__m256i)(X), \ + (__v8si)(__m256i)(Y), (int)(C), \ + (__v8si)(__m256i) \ + _mm256_setzero_si256 (), \ + (__mmask8)-1)) + +#define _mm256_mask_shuffle_i32x4(W, U, X, Y, C) \ + ((__m256i) __builtin_ia32_shuf_i32x4_256_mask ((__v8si)(__m256i)(X), \ + (__v8si)(__m256i)(Y), (int)(C), \ + (__v8si)(__m256i)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_shuffle_i32x4(U, X, Y, C) \ + ((__m256i) __builtin_ia32_shuf_i32x4_256_mask ((__v8si)(__m256i)(X), \ + (__v8si)(__m256i)(Y), (int)(C), \ + (__v8si)(__m256i) \ + _mm256_setzero_si256 (), \ + (__mmask8)(U))) + +#define _mm256_shuffle_f64x2(X, Y, C) \ + ((__m256d) __builtin_ia32_shuf_f64x2_256_mask ((__v4df)(__m256d)(X), \ + (__v4df)(__m256d)(Y), (int)(C), \ + (__v4df)(__m256d)_mm256_setzero_pd (),\ + (__mmask8)-1)) + +#define _mm256_mask_shuffle_f64x2(W, U, X, Y, C) \ + ((__m256d) __builtin_ia32_shuf_f64x2_256_mask ((__v4df)(__m256d)(X), \ + (__v4df)(__m256d)(Y), (int)(C), \ + (__v4df)(__m256d)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_shuffle_f64x2(U, X, Y, C) \ + ((__m256d) __builtin_ia32_shuf_f64x2_256_mask ((__v4df)(__m256d)(X), \ + (__v4df)(__m256d)(Y), (int)(C), \ + (__v4df)(__m256d)_mm256_setzero_pd( ),\ + (__mmask8)(U))) + +#define _mm256_shuffle_f32x4(X, Y, C) \ + ((__m256) __builtin_ia32_shuf_f32x4_256_mask ((__v8sf)(__m256)(X), \ + (__v8sf)(__m256)(Y), (int)(C), \ + (__v8sf)(__m256)_mm256_setzero_ps (), \ + (__mmask8)-1)) + +#define _mm256_mask_shuffle_f32x4(W, U, X, Y, C) \ + ((__m256) __builtin_ia32_shuf_f32x4_256_mask ((__v8sf)(__m256)(X), \ + (__v8sf)(__m256)(Y), (int)(C), \ + (__v8sf)(__m256)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_shuffle_f32x4(U, X, Y, C) \ + ((__m256) __builtin_ia32_shuf_f32x4_256_mask ((__v8sf)(__m256)(X), \ + (__v8sf)(__m256)(Y), (int)(C), \ + (__v8sf)(__m256)_mm256_setzero_ps (), \ + (__mmask8)(U))) + +#define _mm256_mask_shuffle_pd(W, U, A, B, C) \ + ((__m256d)__builtin_ia32_shufpd256_mask ((__v4df)(__m256d)(A), \ + (__v4df)(__m256d)(B), (int)(C), \ + (__v4df)(__m256d)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_shuffle_pd(U, A, B, C) \ + ((__m256d)__builtin_ia32_shufpd256_mask ((__v4df)(__m256d)(A), \ + (__v4df)(__m256d)(B), (int)(C), \ + (__v4df)(__m256d) \ + _mm256_setzero_pd (), \ + (__mmask8)(U))) + +#define _mm_mask_shuffle_pd(W, U, A, B, C) \ + ((__m128d)__builtin_ia32_shufpd128_mask ((__v2df)(__m128d)(A), \ + (__v2df)(__m128d)(B), (int)(C), \ + (__v2df)(__m128d)(W), \ + (__mmask8)(U))) + +#define _mm_maskz_shuffle_pd(U, A, B, C) \ + ((__m128d)__builtin_ia32_shufpd128_mask ((__v2df)(__m128d)(A), \ + (__v2df)(__m128d)(B), (int)(C), \ + (__v2df)(__m128d)_mm_setzero_pd (), \ + (__mmask8)(U))) + +#define _mm256_mask_shuffle_ps(W, U, A, B, C) \ + ((__m256) __builtin_ia32_shufps256_mask ((__v8sf)(__m256)(A), \ + (__v8sf)(__m256)(B), (int)(C), \ + (__v8sf)(__m256)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_shuffle_ps(U, A, B, C) \ + ((__m256) __builtin_ia32_shufps256_mask ((__v8sf)(__m256)(A), \ + (__v8sf)(__m256)(B), (int)(C), \ + (__v8sf)(__m256)_mm256_setzero_ps (),\ + (__mmask8)(U))) + +#define _mm_mask_shuffle_ps(W, U, A, B, C) \ + ((__m128) __builtin_ia32_shufps128_mask ((__v4sf)(__m128)(A), \ + (__v4sf)(__m128)(B), (int)(C), \ + (__v4sf)(__m128)(W), \ + (__mmask8)(U))) + +#define _mm_maskz_shuffle_ps(U, A, B, C) \ + ((__m128) __builtin_ia32_shufps128_mask ((__v4sf)(__m128)(A), \ + (__v4sf)(__m128)(B), (int)(C), \ + (__v4sf)(__m128)_mm_setzero_ps (), \ + (__mmask8)(U))) + +#define _mm256_fixupimm_pd(X, Y, Z, C) \ + ((__m256d)__builtin_ia32_fixupimmpd256_mask ((__v4df)(__m256d)(X), \ + (__v4df)(__m256d)(Y), \ + (__v4di)(__m256i)(Z), (int)(C), \ + (__mmask8)(-1))) + +#define _mm256_mask_fixupimm_pd(X, U, Y, Z, C) \ + ((__m256d)__builtin_ia32_fixupimmpd256_mask ((__v4df)(__m256d)(X), \ + (__v4df)(__m256d)(Y), \ + (__v4di)(__m256i)(Z), (int)(C), \ + (__mmask8)(U))) + +#define _mm256_maskz_fixupimm_pd(U, X, Y, Z, C) \ + ((__m256d)__builtin_ia32_fixupimmpd256_maskz ((__v4df)(__m256d)(X), \ + (__v4df)(__m256d)(Y), \ + (__v4di)(__m256i)(Z), (int)(C),\ + (__mmask8)(U))) + +#define _mm256_fixupimm_ps(X, Y, Z, C) \ + ((__m256)__builtin_ia32_fixupimmps256_mask ((__v8sf)(__m256)(X), \ + (__v8sf)(__m256)(Y), \ + (__v8si)(__m256i)(Z), (int)(C), \ + (__mmask8)(-1))) + + +#define _mm256_mask_fixupimm_ps(X, U, Y, Z, C) \ + ((__m256)__builtin_ia32_fixupimmps256_mask ((__v8sf)(__m256)(X), \ + (__v8sf)(__m256)(Y), \ + (__v8si)(__m256i)(Z), (int)(C), \ + (__mmask8)(U))) + +#define _mm256_maskz_fixupimm_ps(U, X, Y, Z, C) \ + ((__m256)__builtin_ia32_fixupimmps256_maskz ((__v8sf)(__m256)(X), \ + (__v8sf)(__m256)(Y), \ + (__v8si)(__m256i)(Z), (int)(C),\ + (__mmask8)(U))) + +#define _mm_fixupimm_pd(X, Y, Z, C) \ + ((__m128d)__builtin_ia32_fixupimmpd128_mask ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), \ + (__v2di)(__m128i)(Z), (int)(C), \ + (__mmask8)(-1))) + + +#define _mm_mask_fixupimm_pd(X, U, Y, Z, C) \ + ((__m128d)__builtin_ia32_fixupimmpd128_mask ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), \ + (__v2di)(__m128i)(Z), (int)(C), \ + (__mmask8)(U))) + +#define _mm_maskz_fixupimm_pd(U, X, Y, Z, C) \ + ((__m128d)__builtin_ia32_fixupimmpd128_maskz ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), \ + (__v2di)(__m128i)(Z), (int)(C),\ + (__mmask8)(U))) + +#define _mm_fixupimm_ps(X, Y, Z, C) \ + ((__m128)__builtin_ia32_fixupimmps128_mask ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), \ + (__v4si)(__m128i)(Z), (int)(C), \ + (__mmask8)(-1))) + +#define _mm_mask_fixupimm_ps(X, U, Y, Z, C) \ + ((__m128)__builtin_ia32_fixupimmps128_mask ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), \ + (__v4si)(__m128i)(Z), (int)(C),\ + (__mmask8)(U))) + +#define _mm_maskz_fixupimm_ps(U, X, Y, Z, C) \ + ((__m128)__builtin_ia32_fixupimmps128_maskz ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), \ + (__v4si)(__m128i)(Z), (int)(C),\ + (__mmask8)(U))) + +#define _mm256_mask_srli_epi32(W, U, A, B) \ + ((__m256i) __builtin_ia32_psrldi256_mask ((__v8si)(__m256i)(A), \ + (int)(B), (__v8si)(__m256i)(W), (__mmask8)(U))) + +#define _mm256_maskz_srli_epi32(U, A, B) \ + ((__m256i) __builtin_ia32_psrldi256_mask ((__v8si)(__m256i)(A), \ + (int)(B), (__v8si)_mm256_setzero_si256 (), (__mmask8)(U))) + +#define _mm_mask_srli_epi32(W, U, A, B) \ + ((__m128i) __builtin_ia32_psrldi128_mask ((__v4si)(__m128i)(A), \ + (int)(B), (__v4si)(__m128i)(W), (__mmask8)(U))) + +#define _mm_maskz_srli_epi32(U, A, B) \ + ((__m128i) __builtin_ia32_psrldi128_mask ((__v4si)(__m128i)(A), \ + (int)(B), (__v4si)_mm_setzero_si128 (), (__mmask8)(U))) + +#define _mm256_mask_srli_epi64(W, U, A, B) \ + ((__m256i) __builtin_ia32_psrlqi256_mask ((__v4di)(__m256i)(A), \ + (int)(B), (__v4di)(__m256i)(W), (__mmask8)(U))) + +#define _mm256_maskz_srli_epi64(U, A, B) \ + ((__m256i) __builtin_ia32_psrlqi256_mask ((__v4di)(__m256i)(A), \ + (int)(B), (__v4di)_mm256_setzero_si256 (), (__mmask8)(U))) + +#define _mm_mask_srli_epi64(W, U, A, B) \ + ((__m128i) __builtin_ia32_psrlqi128_mask ((__v2di)(__m128i)(A), \ + (int)(B), (__v2di)(__m128i)(W), (__mmask8)(U))) + +#define _mm_maskz_srli_epi64(U, A, B) \ + ((__m128i) __builtin_ia32_psrlqi128_mask ((__v2di)(__m128i)(A), \ + (int)(B), (__v2di)_mm_setzero_si128 (), (__mmask8)(U))) + +#define _mm256_mask_slli_epi32(W, U, X, C) \ + ((__m256i)__builtin_ia32_pslldi256_mask ((__v8si)(__m256i)(X), (int)(C),\ + (__v8si)(__m256i)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_slli_epi32(U, X, C) \ + ((__m256i)__builtin_ia32_pslldi256_mask ((__v8si)(__m256i)(X), (int)(C),\ + (__v8si)(__m256i)_mm256_setzero_si256 (), \ + (__mmask8)(U))) + +#define _mm256_mask_slli_epi64(W, U, X, C) \ + ((__m256i)__builtin_ia32_psllqi256_mask ((__v4di)(__m256i)(X), (int)(C),\ + (__v4di)(__m256i)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_slli_epi64(U, X, C) \ + ((__m256i)__builtin_ia32_psllqi256_mask ((__v4di)(__m256i)(X), (int)(C),\ + (__v4di)(__m256i)_mm256_setzero_si256 (), \ + (__mmask8)(U))) + +#define _mm_mask_slli_epi32(W, U, X, C) \ + ((__m128i)__builtin_ia32_pslldi128_mask ((__v4si)(__m128i)(X), (int)(C),\ + (__v4si)(__m128i)(W),\ + (__mmask8)(U))) + +#define _mm_maskz_slli_epi32(U, X, C) \ + ((__m128i)__builtin_ia32_pslldi128_mask ((__v4si)(__m128i)(X), (int)(C),\ + (__v4si)(__m128i)_mm_setzero_si128 (),\ + (__mmask8)(U))) + +#define _mm_mask_slli_epi64(W, U, X, C) \ + ((__m128i)__builtin_ia32_psllqi128_mask ((__v2di)(__m128i)(X), (int)(C),\ + (__v2di)(__m128i)(W),\ + (__mmask8)(U))) + +#define _mm_maskz_slli_epi64(U, X, C) \ + ((__m128i)__builtin_ia32_psllqi128_mask ((__v2di)(__m128i)(X), (int)(C),\ + (__v2di)(__m128i)_mm_setzero_si128 (),\ + (__mmask8)(U))) + +#define _mm256_ternarylogic_epi64(A, B, C, I) \ + ((__m256i) __builtin_ia32_pternlogq256_mask ((__v4di)(__m256i)(A), \ + (__v4di)(__m256i)(B), (__v4di)(__m256i)(C), (int)(I), (__mmask8)-1)) + +#define _mm256_mask_ternarylogic_epi64(A, U, B, C, I) \ + ((__m256i) __builtin_ia32_pternlogq256_mask ((__v4di)(__m256i)(A), \ + (__v4di)(__m256i)(B), (__v4di)(__m256i)(C), (int)(I), (__mmask8)(U))) + +#define _mm256_maskz_ternarylogic_epi64(U, A, B, C, I) \ + ((__m256i) __builtin_ia32_pternlogq256_maskz ((__v4di)(__m256i)(A), \ + (__v4di)(__m256i)(B), (__v4di)(__m256i)(C), (int)(I), (__mmask8)(U))) + +#define _mm256_ternarylogic_epi32(A, B, C, I) \ + ((__m256i) __builtin_ia32_pternlogd256_mask ((__v8si)(__m256i)(A), \ + (__v8si)(__m256i)(B), (__v8si)(__m256i)(C), (int)(I), (__mmask8)-1)) + +#define _mm256_mask_ternarylogic_epi32(A, U, B, C, I) \ + ((__m256i) __builtin_ia32_pternlogd256_mask ((__v8si)(__m256i)(A), \ + (__v8si)(__m256i)(B), (__v8si)(__m256i)(C), (int)(I), (__mmask8)(U))) + +#define _mm256_maskz_ternarylogic_epi32(U, A, B, C, I) \ + ((__m256i) __builtin_ia32_pternlogd256_maskz ((__v8si)(__m256i)(A), \ + (__v8si)(__m256i)(B), (__v8si)(__m256i)(C), (int)(I), (__mmask8)(U))) + +#define _mm_ternarylogic_epi64(A, B, C, I) \ + ((__m128i) __builtin_ia32_pternlogq128_mask ((__v2di)(__m128i)(A), \ + (__v2di)(__m128i)(B), (__v2di)(__m128i)(C), (int)(I), (__mmask8)-1)) + +#define _mm_mask_ternarylogic_epi64(A, U, B, C, I) \ + ((__m128i) __builtin_ia32_pternlogq128_mask ((__v2di)(__m128i)(A), \ + (__v2di)(__m128i)(B), (__v2di)(__m128i)(C), (int)(I), (__mmask8)(U))) + +#define _mm_maskz_ternarylogic_epi64(U, A, B, C, I) \ + ((__m128i) __builtin_ia32_pternlogq128_maskz ((__v2di)(__m128i)(A), \ + (__v2di)(__m128i)(B), (__v2di)(__m128i)(C), (int)(I), (__mmask8)(U))) + +#define _mm_ternarylogic_epi32(A, B, C, I) \ + ((__m128i) __builtin_ia32_pternlogd128_mask ((__v4si)(__m128i)(A), \ + (__v4si)(__m128i)(B), (__v4si)(__m128i)(C), (int)(I), (__mmask8)-1)) + +#define _mm_mask_ternarylogic_epi32(A, U, B, C, I) \ + ((__m128i) __builtin_ia32_pternlogd128_mask ((__v4si)(__m128i)(A), \ + (__v4si)(__m128i)(B), (__v4si)(__m128i)(C), (int)(I), (__mmask8)(U))) + +#define _mm_maskz_ternarylogic_epi32(U, A, B, C, I) \ + ((__m128i) __builtin_ia32_pternlogd128_maskz ((__v4si)(__m128i)(A), \ + (__v4si)(__m128i)(B), (__v4si)(__m128i)(C), (int)(I), (__mmask8)(U))) + +#define _mm256_roundscale_ps(A, B) \ + ((__m256) __builtin_ia32_rndscaleps_256_mask ((__v8sf)(__m256)(A), \ + (int)(B), (__v8sf)(__m256)_mm256_setzero_ps (), (__mmask8)-1)) + +#define _mm256_mask_roundscale_ps(W, U, A, B) \ + ((__m256) __builtin_ia32_rndscaleps_256_mask ((__v8sf)(__m256)(A), \ + (int)(B), (__v8sf)(__m256)(W), (__mmask8)(U))) + +#define _mm256_maskz_roundscale_ps(U, A, B) \ + ((__m256) __builtin_ia32_rndscaleps_256_mask ((__v8sf)(__m256)(A), \ + (int)(B), (__v8sf)(__m256)_mm256_setzero_ps (), (__mmask8)(U))) + +#define _mm256_roundscale_pd(A, B) \ + ((__m256d) __builtin_ia32_rndscalepd_256_mask ((__v4df)(__m256d)(A), \ + (int)(B), (__v4df)(__m256d)_mm256_setzero_pd (), (__mmask8)-1)) + +#define _mm256_mask_roundscale_pd(W, U, A, B) \ + ((__m256d) __builtin_ia32_rndscalepd_256_mask ((__v4df)(__m256d)(A), \ + (int)(B), (__v4df)(__m256d)(W), (__mmask8)(U))) + +#define _mm256_maskz_roundscale_pd(U, A, B) \ + ((__m256d) __builtin_ia32_rndscalepd_256_mask ((__v4df)(__m256d)(A), \ + (int)(B), (__v4df)(__m256d)_mm256_setzero_pd (), (__mmask8)(U))) + +#define _mm_roundscale_ps(A, B) \ + ((__m128) __builtin_ia32_rndscaleps_128_mask ((__v4sf)(__m128)(A), \ + (int)(B), (__v4sf)(__m128)_mm_setzero_ps (), (__mmask8)-1)) + +#define _mm_mask_roundscale_ps(W, U, A, B) \ + ((__m128) __builtin_ia32_rndscaleps_128_mask ((__v4sf)(__m128)(A), \ + (int)(B), (__v4sf)(__m128)(W), (__mmask8)(U))) + +#define _mm_maskz_roundscale_ps(U, A, B) \ + ((__m128) __builtin_ia32_rndscaleps_128_mask ((__v4sf)(__m128)(A), \ + (int)(B), (__v4sf)(__m128)_mm_setzero_ps (), (__mmask8)(U))) + +#define _mm_roundscale_pd(A, B) \ + ((__m128d) __builtin_ia32_rndscalepd_128_mask ((__v2df)(__m128d)(A), \ + (int)(B), (__v2df)(__m128d)_mm_setzero_pd (), (__mmask8)-1)) + +#define _mm_mask_roundscale_pd(W, U, A, B) \ + ((__m128d) __builtin_ia32_rndscalepd_128_mask ((__v2df)(__m128d)(A), \ + (int)(B), (__v2df)(__m128d)(W), (__mmask8)(U))) + +#define _mm_maskz_roundscale_pd(U, A, B) \ + ((__m128d) __builtin_ia32_rndscalepd_128_mask ((__v2df)(__m128d)(A), \ + (int)(B), (__v2df)(__m128d)_mm_setzero_pd (), (__mmask8)(U))) + +#define _mm256_getmant_ps(X, B, C) \ + ((__m256) __builtin_ia32_getmantps256_mask ((__v8sf)(__m256) (X), \ + (int)(((C)<<2) | (B)), \ + (__v8sf)(__m256)_mm256_setzero_ps (), \ + (__mmask8)-1)) + +#define _mm256_mask_getmant_ps(W, U, X, B, C) \ + ((__m256) __builtin_ia32_getmantps256_mask ((__v8sf)(__m256) (X), \ + (int)(((C)<<2) | (B)), \ + (__v8sf)(__m256)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_getmant_ps(U, X, B, C) \ + ((__m256) __builtin_ia32_getmantps256_mask ((__v8sf)(__m256) (X), \ + (int)(((C)<<2) | (B)), \ + (__v8sf)(__m256)_mm256_setzero_ps (), \ + (__mmask8)(U))) + +#define _mm_getmant_ps(X, B, C) \ + ((__m128) __builtin_ia32_getmantps128_mask ((__v4sf)(__m128) (X), \ + (int)(((C)<<2) | (B)), \ + (__v4sf)(__m128)_mm_setzero_ps (), \ + (__mmask8)-1)) + +#define _mm_mask_getmant_ps(W, U, X, B, C) \ + ((__m128) __builtin_ia32_getmantps128_mask ((__v4sf)(__m128) (X), \ + (int)(((C)<<2) | (B)), \ + (__v4sf)(__m128)(W), \ + (__mmask8)(U))) + +#define _mm_maskz_getmant_ps(U, X, B, C) \ + ((__m128) __builtin_ia32_getmantps128_mask ((__v4sf)(__m128) (X), \ + (int)(((C)<<2) | (B)), \ + (__v4sf)(__m128)_mm_setzero_ps (), \ + (__mmask8)(U))) + +#define _mm256_getmant_pd(X, B, C) \ + ((__m256d) __builtin_ia32_getmantpd256_mask ((__v4df)(__m256d) (X), \ + (int)(((C)<<2) | (B)), \ + (__v4df)(__m256d)_mm256_setzero_pd (),\ + (__mmask8)-1)) + +#define _mm256_mask_getmant_pd(W, U, X, B, C) \ + ((__m256d) __builtin_ia32_getmantpd256_mask ((__v4df)(__m256d) (X), \ + (int)(((C)<<2) | (B)), \ + (__v4df)(__m256d)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_getmant_pd(U, X, B, C) \ + ((__m256d) __builtin_ia32_getmantpd256_mask ((__v4df)(__m256d) (X), \ + (int)(((C)<<2) | (B)), \ + (__v4df)(__m256d)_mm256_setzero_pd (),\ + (__mmask8)(U))) + +#define _mm_getmant_pd(X, B, C) \ + ((__m128d) __builtin_ia32_getmantpd128_mask ((__v2df)(__m128d) (X), \ + (int)(((C)<<2) | (B)), \ + (__v2df)(__m128d)_mm_setzero_pd (), \ + (__mmask8)-1)) + +#define _mm_mask_getmant_pd(W, U, X, B, C) \ + ((__m128d) __builtin_ia32_getmantpd128_mask ((__v2df)(__m128d) (X), \ + (int)(((C)<<2) | (B)), \ + (__v2df)(__m128d)(W), \ + (__mmask8)(U))) + +#define _mm_maskz_getmant_pd(U, X, B, C) \ + ((__m128d) __builtin_ia32_getmantpd128_mask ((__v2df)(__m128d) (X), \ + (int)(((C)<<2) | (B)), \ + (__v2df)(__m128d)_mm_setzero_pd (), \ + (__mmask8)(U))) + +#define _mm256_mmask_i32gather_ps(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m256) __builtin_ia32_gather3siv8sf ((__v8sf)(__m256)V1OLD, \ + (void const *)ADDR, \ + (__v8si)(__m256i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm_mmask_i32gather_ps(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m128) __builtin_ia32_gather3siv4sf ((__v4sf)(__m128)V1OLD, \ + (void const *)ADDR, \ + (__v4si)(__m128i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm256_mmask_i32gather_pd(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m256d) __builtin_ia32_gather3siv4df ((__v4df)(__m256d)V1OLD, \ + (void const *)ADDR, \ + (__v4si)(__m128i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm_mmask_i32gather_pd(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m128d) __builtin_ia32_gather3siv2df ((__v2df)(__m128d)V1OLD, \ + (void const *)ADDR, \ + (__v4si)(__m128i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm256_mmask_i64gather_ps(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m128) __builtin_ia32_gather3div8sf ((__v4sf)(__m128)V1OLD, \ + (void const *)ADDR, \ + (__v4di)(__m256i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm_mmask_i64gather_ps(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m128) __builtin_ia32_gather3div4sf ((__v4sf)(__m128)V1OLD, \ + (void const *)ADDR, \ + (__v2di)(__m128i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm256_mmask_i64gather_pd(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m256d) __builtin_ia32_gather3div4df ((__v4df)(__m256d)V1OLD, \ + (void const *)ADDR, \ + (__v4di)(__m256i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm_mmask_i64gather_pd(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m128d) __builtin_ia32_gather3div2df ((__v2df)(__m128d)V1OLD, \ + (void const *)ADDR, \ + (__v2di)(__m128i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm256_mmask_i32gather_epi32(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m256i) __builtin_ia32_gather3siv8si ((__v8si)(__m256i)V1OLD, \ + (void const *)ADDR, \ + (__v8si)(__m256i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm_mmask_i32gather_epi32(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m128i) __builtin_ia32_gather3siv4si ((__v4si)(__m128i)V1OLD, \ + (void const *)ADDR, \ + (__v4si)(__m128i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm256_mmask_i32gather_epi64(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m256i) __builtin_ia32_gather3siv4di ((__v4di)(__m256i)V1OLD, \ + (void const *)ADDR, \ + (__v4si)(__m128i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm_mmask_i32gather_epi64(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m128i) __builtin_ia32_gather3siv2di ((__v2di)(__m128i)V1OLD, \ + (void const *)ADDR, \ + (__v4si)(__m128i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm256_mmask_i64gather_epi32(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m128i) __builtin_ia32_gather3div8si ((__v4si)(__m128i)V1OLD, \ + (void const *)ADDR, \ + (__v4di)(__m256i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm_mmask_i64gather_epi32(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m128i) __builtin_ia32_gather3div4si ((__v4si)(__m128i)V1OLD, \ + (void const *)ADDR, \ + (__v2di)(__m128i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm256_mmask_i64gather_epi64(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m256i) __builtin_ia32_gather3div4di ((__v4di)(__m256i)V1OLD, \ + (void const *)ADDR, \ + (__v4di)(__m256i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm_mmask_i64gather_epi64(V1OLD, MASK, INDEX, ADDR, SCALE) \ + (__m128i) __builtin_ia32_gather3div2di ((__v2di)(__m128i)V1OLD, \ + (void const *)ADDR, \ + (__v2di)(__m128i)INDEX, \ + (__mmask8)MASK, (int)SCALE) + +#define _mm256_i32scatter_ps(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv8sf ((void *)ADDR, (__mmask8)0xFF, \ + (__v8si)(__m256i)INDEX, \ + (__v8sf)(__m256)V1, (int)SCALE) + +#define _mm256_mask_i32scatter_ps(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv8sf ((void *)ADDR, (__mmask8)MASK, \ + (__v8si)(__m256i)INDEX, \ + (__v8sf)(__m256)V1, (int)SCALE) + +#define _mm_i32scatter_ps(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv4sf ((void *)ADDR, (__mmask8)0xFF, \ + (__v4si)(__m128i)INDEX, \ + (__v4sf)(__m128)V1, (int)SCALE) + +#define _mm_mask_i32scatter_ps(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv4sf ((void *)ADDR, (__mmask8)MASK, \ + (__v4si)(__m128i)INDEX, \ + (__v4sf)(__m128)V1, (int)SCALE) + +#define _mm256_i32scatter_pd(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv4df ((void *)ADDR, (__mmask8)0xFF, \ + (__v4si)(__m128i)INDEX, \ + (__v4df)(__m256d)V1, (int)SCALE) + +#define _mm256_mask_i32scatter_pd(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv4df ((void *)ADDR, (__mmask8)MASK, \ + (__v4si)(__m128i)INDEX, \ + (__v4df)(__m256d)V1, (int)SCALE) + +#define _mm_i32scatter_pd(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv2df ((void *)ADDR, (__mmask8)0xFF, \ + (__v4si)(__m128i)INDEX, \ + (__v2df)(__m128d)V1, (int)SCALE) + +#define _mm_mask_i32scatter_pd(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv2df ((void *)ADDR, (__mmask8)MASK, \ + (__v4si)(__m128i)INDEX, \ + (__v2df)(__m128d)V1, (int)SCALE) + +#define _mm256_i64scatter_ps(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv8sf ((void *)ADDR, (__mmask8)0xFF, \ + (__v4di)(__m256i)INDEX, \ + (__v4sf)(__m128)V1, (int)SCALE) + +#define _mm256_mask_i64scatter_ps(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv8sf ((void *)ADDR, (__mmask8)MASK, \ + (__v4di)(__m256i)INDEX, \ + (__v4sf)(__m128)V1, (int)SCALE) + +#define _mm_i64scatter_ps(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv4sf ((void *)ADDR, (__mmask8)0xFF, \ + (__v2di)(__m128i)INDEX, \ + (__v4sf)(__m128)V1, (int)SCALE) + +#define _mm_mask_i64scatter_ps(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv4sf ((void *)ADDR, (__mmask8)MASK, \ + (__v2di)(__m128i)INDEX, \ + (__v4sf)(__m128)V1, (int)SCALE) + +#define _mm256_i64scatter_pd(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv4df ((void *)ADDR, (__mmask8)0xFF, \ + (__v4di)(__m256i)INDEX, \ + (__v4df)(__m256d)V1, (int)SCALE) + +#define _mm256_mask_i64scatter_pd(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv4df ((void *)ADDR, (__mmask8)MASK, \ + (__v4di)(__m256i)INDEX, \ + (__v4df)(__m256d)V1, (int)SCALE) + +#define _mm_i64scatter_pd(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv2df ((void *)ADDR, (__mmask8)0xFF, \ + (__v2di)(__m128i)INDEX, \ + (__v2df)(__m128d)V1, (int)SCALE) + +#define _mm_mask_i64scatter_pd(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv2df ((void *)ADDR, (__mmask8)MASK, \ + (__v2di)(__m128i)INDEX, \ + (__v2df)(__m128d)V1, (int)SCALE) + +#define _mm256_i32scatter_epi32(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv8si ((void *)ADDR, (__mmask8)0xFF, \ + (__v8si)(__m256i)INDEX, \ + (__v8si)(__m256i)V1, (int)SCALE) + +#define _mm256_mask_i32scatter_epi32(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv8si ((void *)ADDR, (__mmask8)MASK, \ + (__v8si)(__m256i)INDEX, \ + (__v8si)(__m256i)V1, (int)SCALE) + +#define _mm_i32scatter_epi32(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv4si ((void *)ADDR, (__mmask8)0xFF, \ + (__v4si)(__m128i)INDEX, \ + (__v4si)(__m128i)V1, (int)SCALE) + +#define _mm_mask_i32scatter_epi32(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv4si ((void *)ADDR, (__mmask8)MASK, \ + (__v4si)(__m128i)INDEX, \ + (__v4si)(__m128i)V1, (int)SCALE) + +#define _mm256_i32scatter_epi64(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv4di ((void *)ADDR, (__mmask8)0xFF, \ + (__v4si)(__m128i)INDEX, \ + (__v4di)(__m256i)V1, (int)SCALE) + +#define _mm256_mask_i32scatter_epi64(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv4di ((void *)ADDR, (__mmask8)MASK, \ + (__v4si)(__m128i)INDEX, \ + (__v4di)(__m256i)V1, (int)SCALE) + +#define _mm_i32scatter_epi64(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv2di ((void *)ADDR, (__mmask8)0xFF, \ + (__v4si)(__m128i)INDEX, \ + (__v2di)(__m128i)V1, (int)SCALE) + +#define _mm_mask_i32scatter_epi64(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scattersiv2di ((void *)ADDR, (__mmask8)MASK, \ + (__v4si)(__m128i)INDEX, \ + (__v2di)(__m128i)V1, (int)SCALE) + +#define _mm256_i64scatter_epi32(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv8si ((void *)ADDR, (__mmask8)0xFF, \ + (__v4di)(__m256i)INDEX, \ + (__v4si)(__m128i)V1, (int)SCALE) + +#define _mm256_mask_i64scatter_epi32(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv8si ((void *)ADDR, (__mmask8)MASK, \ + (__v4di)(__m256i)INDEX, \ + (__v4si)(__m128i)V1, (int)SCALE) + +#define _mm_i64scatter_epi32(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv4si ((void *)ADDR, (__mmask8)0xFF, \ + (__v2di)(__m128i)INDEX, \ + (__v4si)(__m128i)V1, (int)SCALE) + +#define _mm_mask_i64scatter_epi32(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv4si ((void *)ADDR, (__mmask8)MASK, \ + (__v2di)(__m128i)INDEX, \ + (__v4si)(__m128i)V1, (int)SCALE) + +#define _mm256_i64scatter_epi64(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv4di ((void *)ADDR, (__mmask8)0xFF, \ + (__v4di)(__m256i)INDEX, \ + (__v4di)(__m256i)V1, (int)SCALE) + +#define _mm256_mask_i64scatter_epi64(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv4di ((void *)ADDR, (__mmask8)MASK, \ + (__v4di)(__m256i)INDEX, \ + (__v4di)(__m256i)V1, (int)SCALE) + +#define _mm_i64scatter_epi64(ADDR, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv2di ((void *)ADDR, (__mmask8)0xFF, \ + (__v2di)(__m128i)INDEX, \ + (__v2di)(__m128i)V1, (int)SCALE) + +#define _mm_mask_i64scatter_epi64(ADDR, MASK, INDEX, V1, SCALE) \ + __builtin_ia32_scatterdiv2di ((void *)ADDR, (__mmask8)MASK, \ + (__v2di)(__m128i)INDEX, \ + (__v2di)(__m128i)V1, (int)SCALE) + +#define _mm256_mask_shuffle_epi32(W, U, X, C) \ + ((__m256i) __builtin_ia32_pshufd256_mask ((__v8si)(__m256i)(X), (int)(C), \ + (__v8si)(__m256i)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_shuffle_epi32(U, X, C) \ + ((__m256i) __builtin_ia32_pshufd256_mask ((__v8si)(__m256i)(X), (int)(C), \ + (__v8si)(__m256i) \ + _mm256_setzero_si256 (), \ + (__mmask8)(U))) + +#define _mm_mask_shuffle_epi32(W, U, X, C) \ + ((__m128i) __builtin_ia32_pshufd128_mask ((__v4si)(__m128i)(X), (int)(C), \ + (__v4si)(__m128i)(W), \ + (__mmask8)(U))) + +#define _mm_maskz_shuffle_epi32(U, X, C) \ + ((__m128i) __builtin_ia32_pshufd128_mask ((__v4si)(__m128i)(X), (int)(C), \ + (__v4si)(__m128i)_mm_setzero_si128 (), \ + (__mmask8)(U))) + +#define _mm256_rol_epi64(A, B) \ + ((__m256i)__builtin_ia32_prolq256_mask ((__v4di)(__m256i)(A), (int)(B), \ + (__v4di)(__m256i)_mm256_setzero_si256 (),\ + (__mmask8)-1)) + +#define _mm256_mask_rol_epi64(W, U, A, B) \ + ((__m256i)__builtin_ia32_prolq256_mask ((__v4di)(__m256i)(A), (int)(B), \ + (__v4di)(__m256i)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_rol_epi64(U, A, B) \ + ((__m256i)__builtin_ia32_prolq256_mask ((__v4di)(__m256i)(A), (int)(B), \ + (__v4di)(__m256i)_mm256_setzero_si256 (),\ + (__mmask8)(U))) + +#define _mm_rol_epi64(A, B) \ + ((__m128i)__builtin_ia32_prolq128_mask ((__v2di)(__m128i)(A), (int)(B), \ + (__v2di)(__m128i)_mm_setzero_si128 (),\ + (__mmask8)-1)) + +#define _mm_mask_rol_epi64(W, U, A, B) \ + ((__m128i)__builtin_ia32_prolq128_mask ((__v2di)(__m128i)(A), (int)(B), \ + (__v2di)(__m128i)(W), \ + (__mmask8)(U))) + +#define _mm_maskz_rol_epi64(U, A, B) \ + ((__m128i)__builtin_ia32_prolq128_mask ((__v2di)(__m128i)(A), (int)(B), \ + (__v2di)(__m128i)_mm_setzero_si128 (),\ + (__mmask8)(U))) + +#define _mm256_ror_epi64(A, B) \ + ((__m256i)__builtin_ia32_prorq256_mask ((__v4di)(__m256i)(A), (int)(B), \ + (__v4di)(__m256i)_mm256_setzero_si256 (),\ + (__mmask8)-1)) + +#define _mm256_mask_ror_epi64(W, U, A, B) \ + ((__m256i)__builtin_ia32_prorq256_mask ((__v4di)(__m256i)(A), (int)(B), \ + (__v4di)(__m256i)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_ror_epi64(U, A, B) \ + ((__m256i)__builtin_ia32_prorq256_mask ((__v4di)(__m256i)(A), (int)(B), \ + (__v4di)(__m256i)_mm256_setzero_si256 (),\ + (__mmask8)(U))) + +#define _mm_ror_epi64(A, B) \ + ((__m128i)__builtin_ia32_prorq128_mask ((__v2di)(__m128i)(A), (int)(B), \ + (__v2di)(__m128i)_mm_setzero_si128 (),\ + (__mmask8)-1)) + +#define _mm_mask_ror_epi64(W, U, A, B) \ + ((__m128i)__builtin_ia32_prorq128_mask ((__v2di)(__m128i)(A), (int)(B), \ + (__v2di)(__m128i)(W), \ + (__mmask8)(U))) + +#define _mm_maskz_ror_epi64(U, A, B) \ + ((__m128i)__builtin_ia32_prorq128_mask ((__v2di)(__m128i)(A), (int)(B), \ + (__v2di)(__m128i)_mm_setzero_si128 (),\ + (__mmask8)(U))) + +#define _mm256_rol_epi32(A, B) \ + ((__m256i)__builtin_ia32_prold256_mask ((__v8si)(__m256i)(A), (int)(B), \ + (__v8si)(__m256i)_mm256_setzero_si256 (),\ + (__mmask8)-1)) + +#define _mm256_mask_rol_epi32(W, U, A, B) \ + ((__m256i)__builtin_ia32_prold256_mask ((__v8si)(__m256i)(A), (int)(B), \ + (__v8si)(__m256i)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_rol_epi32(U, A, B) \ + ((__m256i)__builtin_ia32_prold256_mask ((__v8si)(__m256i)(A), (int)(B), \ + (__v8si)(__m256i)_mm256_setzero_si256 (),\ + (__mmask8)(U))) + +#define _mm_rol_epi32(A, B) \ + ((__m128i)__builtin_ia32_prold128_mask ((__v4si)(__m128i)(A), (int)(B), \ + (__v4si)(__m128i)_mm_setzero_si128 (),\ + (__mmask8)-1)) + +#define _mm_mask_rol_epi32(W, U, A, B) \ + ((__m128i)__builtin_ia32_prold128_mask ((__v4si)(__m128i)(A), (int)(B), \ + (__v4si)(__m128i)(W), \ + (__mmask8)(U))) + +#define _mm_maskz_rol_epi32(U, A, B) \ + ((__m128i)__builtin_ia32_prold128_mask ((__v4si)(__m128i)(A), (int)(B), \ + (__v4si)(__m128i)_mm_setzero_si128 (),\ + (__mmask8)(U))) + +#define _mm256_ror_epi32(A, B) \ + ((__m256i)__builtin_ia32_prord256_mask ((__v8si)(__m256i)(A), (int)(B), \ + (__v8si)(__m256i)_mm256_setzero_si256 (),\ + (__mmask8)-1)) + +#define _mm256_mask_ror_epi32(W, U, A, B) \ + ((__m256i)__builtin_ia32_prord256_mask ((__v8si)(__m256i)(A), (int)(B), \ + (__v8si)(__m256i)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_ror_epi32(U, A, B) \ + ((__m256i)__builtin_ia32_prord256_mask ((__v8si)(__m256i)(A), (int)(B), \ + (__v8si)(__m256i) \ + _mm256_setzero_si256 (), \ + (__mmask8)(U))) + +#define _mm_ror_epi32(A, B) \ + ((__m128i)__builtin_ia32_prord128_mask ((__v4si)(__m128i)(A), (int)(B), \ + (__v4si)(__m128i)_mm_setzero_si128 (),\ + (__mmask8)-1)) + +#define _mm_mask_ror_epi32(W, U, A, B) \ + ((__m128i)__builtin_ia32_prord128_mask ((__v4si)(__m128i)(A), (int)(B), \ + (__v4si)(__m128i)(W), \ + (__mmask8)(U))) + +#define _mm_maskz_ror_epi32(U, A, B) \ + ((__m128i)__builtin_ia32_prord128_mask ((__v4si)(__m128i)(A), (int)(B), \ + (__v4si)(__m128i)_mm_setzero_si128 (),\ + (__mmask8)(U))) + +#define _mm256_alignr_epi32(X, Y, C) \ + ((__m256i)__builtin_ia32_alignd256_mask ((__v8si)(__m256i)(X), \ + (__v8si)(__m256i)(Y), (int)(C), (__v8si)(__m256i)(X), (__mmask8)-1)) + +#define _mm256_mask_alignr_epi32(W, U, X, Y, C) \ + ((__m256i)__builtin_ia32_alignd256_mask ((__v8si)(__m256i)(X), \ + (__v8si)(__m256i)(Y), (int)(C), (__v8si)(__m256i)(W), (__mmask8)(U))) + +#define _mm256_maskz_alignr_epi32(U, X, Y, C) \ + ((__m256i)__builtin_ia32_alignd256_mask ((__v8si)(__m256i)(X), \ + (__v8si)(__m256i)(Y), (int)(C), (__v8si)(__m256i)_mm256_setzero_si256 (),\ + (__mmask8)(U))) + +#define _mm256_alignr_epi64(X, Y, C) \ + ((__m256i)__builtin_ia32_alignq256_mask ((__v4di)(__m256i)(X), \ + (__v4di)(__m256i)(Y), (int)(C), (__v4di)(__m256i)(X), (__mmask8)-1)) + +#define _mm256_mask_alignr_epi64(W, U, X, Y, C) \ + ((__m256i)__builtin_ia32_alignq256_mask ((__v4di)(__m256i)(X), \ + (__v4di)(__m256i)(Y), (int)(C), (__v4di)(__m256i)(W), (__mmask8)(U))) + +#define _mm256_maskz_alignr_epi64(U, X, Y, C) \ + ((__m256i)__builtin_ia32_alignq256_mask ((__v4di)(__m256i)(X), \ + (__v4di)(__m256i)(Y), (int)(C), (__v4di)(__m256i)_mm256_setzero_si256 (),\ + (__mmask8)(U))) + +#define _mm_alignr_epi32(X, Y, C) \ + ((__m128i)__builtin_ia32_alignd128_mask ((__v4si)(__m128i)(X), \ + (__v4si)(__m128i)(Y), (int)(C), (__v4si)(__m128i)(X), (__mmask8)-1)) + +#define _mm_mask_alignr_epi32(W, U, X, Y, C) \ + ((__m128i)__builtin_ia32_alignd128_mask ((__v4si)(__m128i)(X), \ + (__v4si)(__m128i)(Y), (int)(C), (__v4si)(__m128i)(W), (__mmask8)(U))) + +#define _mm_maskz_alignr_epi32(U, X, Y, C) \ + ((__m128i)__builtin_ia32_alignd128_mask ((__v4si)(__m128i)(X), \ + (__v4si)(__m128i)(Y), (int)(C), (__v4si)(__m128i)_mm_setzero_si128 (),\ + (__mmask8)(U))) + +#define _mm_alignr_epi64(X, Y, C) \ + ((__m128i)__builtin_ia32_alignq128_mask ((__v2di)(__m128i)(X), \ + (__v2di)(__m128i)(Y), (int)(C), (__v2di)(__m128i)(X), (__mmask8)-1)) + +#define _mm_mask_alignr_epi64(W, U, X, Y, C) \ + ((__m128i)__builtin_ia32_alignq128_mask ((__v2di)(__m128i)(X), \ + (__v2di)(__m128i)(Y), (int)(C), (__v2di)(__m128i)(X), (__mmask8)-1)) + +#define _mm_maskz_alignr_epi64(U, X, Y, C) \ + ((__m128i)__builtin_ia32_alignq128_mask ((__v2di)(__m128i)(X), \ + (__v2di)(__m128i)(Y), (int)(C), (__v2di)(__m128i)_mm_setzero_si128 (),\ + (__mmask8)(U))) + +#define _mm_mask_cvtps_ph(W, U, A, I) \ + ((__m128i) __builtin_ia32_vcvtps2ph_mask ((__v4sf)(__m128) A, (int) (I), \ + (__v8hi)(__m128i) (W), (__mmask8) (U))) + +#define _mm_maskz_cvtps_ph(U, A, I) \ + ((__m128i) __builtin_ia32_vcvtps2ph_mask ((__v4sf)(__m128) A, (int) (I), \ + (__v8hi)(__m128i) _mm_setzero_si128 (), (__mmask8) (U))) + +#define _mm256_mask_cvtps_ph(W, U, A, I) \ + ((__m128i) __builtin_ia32_vcvtps2ph256_mask ((__v8sf)(__m256) A, (int) (I), \ + (__v8hi)(__m128i) (W), (__mmask8) (U))) + +#define _mm256_maskz_cvtps_ph(U, A, I) \ + ((__m128i) __builtin_ia32_vcvtps2ph256_mask ((__v8sf)(__m256) A, (int) (I), \ + (__v8hi)(__m128i) _mm_setzero_si128 (), (__mmask8) (U))) + +#define _mm256_mask_srai_epi32(W, U, A, B) \ + ((__m256i) __builtin_ia32_psradi256_mask ((__v8si)(__m256i)(A), \ + (int)(B), (__v8si)(__m256i)(W), (__mmask8)(U))) + +#define _mm256_maskz_srai_epi32(U, A, B) \ + ((__m256i) __builtin_ia32_psradi256_mask ((__v8si)(__m256i)(A), \ + (int)(B), (__v8si)_mm256_setzero_si256 (), (__mmask8)(U))) + +#define _mm_mask_srai_epi32(W, U, A, B) \ + ((__m128i) __builtin_ia32_psradi128_mask ((__v4si)(__m128i)(A), \ + (int)(B), (__v4si)(__m128i)(W), (__mmask8)(U))) + +#define _mm_maskz_srai_epi32(U, A, B) \ + ((__m128i) __builtin_ia32_psradi128_mask ((__v4si)(__m128i)(A), \ + (int)(B), (__v4si)_mm_setzero_si128 (), (__mmask8)(U))) + +#define _mm256_srai_epi64(A, B) \ + ((__m256i) __builtin_ia32_psraqi256_mask ((__v4di)(__m256i)(A), \ + (int)(B), (__v4di)_mm256_setzero_si256 (), (__mmask8)-1)) + +#define _mm256_mask_srai_epi64(W, U, A, B) \ + ((__m256i) __builtin_ia32_psraqi256_mask ((__v4di)(__m256i)(A), \ + (int)(B), (__v4di)(__m256i)(W), (__mmask8)(U))) + +#define _mm256_maskz_srai_epi64(U, A, B) \ + ((__m256i) __builtin_ia32_psraqi256_mask ((__v4di)(__m256i)(A), \ + (int)(B), (__v4di)_mm256_setzero_si256 (), (__mmask8)(U))) + +#define _mm_srai_epi64(A, B) \ + ((__m128i) __builtin_ia32_psraqi128_mask ((__v2di)(__m128i)(A), \ + (int)(B), (__v2di)_mm_setzero_si128 (), (__mmask8)-1)) + +#define _mm_mask_srai_epi64(W, U, A, B) \ + ((__m128i) __builtin_ia32_psraqi128_mask ((__v2di)(__m128i)(A), \ + (int)(B), (__v2di)(__m128i)(W), (__mmask8)(U))) + +#define _mm_maskz_srai_epi64(U, A, B) \ + ((__m128i) __builtin_ia32_psraqi128_mask ((__v2di)(__m128i)(A), \ + (int)(B), (__v2di)_mm_setzero_si128 (), (__mmask8)(U))) + +#define _mm256_mask_permutex_pd(W, U, A, B) \ + ((__m256d) __builtin_ia32_permdf256_mask ((__v4df)(__m256d)(A), \ + (int)(B), (__v4df)(__m256d)(W), (__mmask8)(U))) + +#define _mm256_maskz_permutex_pd(U, A, B) \ + ((__m256d) __builtin_ia32_permdf256_mask ((__v4df)(__m256d)(A), \ + (int)(B), (__v4df)(__m256d)_mm256_setzero_pd (), (__mmask8)(U))) + +#define _mm256_mask_permute_pd(W, U, X, C) \ + ((__m256d) __builtin_ia32_vpermilpd256_mask ((__v4df)(__m256d)(X), (int)(C), \ + (__v4df)(__m256d)(W), \ + (__mmask8)(U))) + +#define _mm256_maskz_permute_pd(U, X, C) \ + ((__m256d) __builtin_ia32_vpermilpd256_mask ((__v4df)(__m256d)(X), (int)(C), \ + (__v4df)(__m256d)_mm256_setzero_pd (),\ + (__mmask8)(U))) + +#define _mm256_mask_permute_ps(W, U, X, C) \ + ((__m256) __builtin_ia32_vpermilps256_mask ((__v8sf)(__m256)(X), (int)(C), \ + (__v8sf)(__m256)(W), (__mmask8)(U))) + +#define _mm256_maskz_permute_ps(U, X, C) \ + ((__m256) __builtin_ia32_vpermilps256_mask ((__v8sf)(__m256)(X), (int)(C), \ + (__v8sf)(__m256)_mm256_setzero_ps (), \ + (__mmask8)(U))) + +#define _mm_mask_permute_pd(W, U, X, C) \ + ((__m128d) __builtin_ia32_vpermilpd_mask ((__v2df)(__m128d)(X), (int)(C), \ + (__v2df)(__m128d)(W), (__mmask8)(U))) + +#define _mm_maskz_permute_pd(U, X, C) \ + ((__m128d) __builtin_ia32_vpermilpd_mask ((__v2df)(__m128d)(X), (int)(C), \ + (__v2df)(__m128d)_mm_setzero_pd (), \ + (__mmask8)(U))) + +#define _mm_mask_permute_ps(W, U, X, C) \ + ((__m128) __builtin_ia32_vpermilps_mask ((__v4sf)(__m128)(X), (int)(C), \ + (__v4sf)(__m128)(W), (__mmask8)(U))) + +#define _mm_maskz_permute_ps(U, X, C) \ + ((__m128) __builtin_ia32_vpermilps_mask ((__v4sf)(__m128)(X), (int)(C), \ + (__v4sf)(__m128)_mm_setzero_ps (), \ + (__mmask8)(U))) + +#define _mm256_mask_blend_pd(__U, __A, __W) \ + ((__m256d) __builtin_ia32_blendmpd_256_mask ((__v4df) (__A), \ + (__v4df) (__W), \ + (__mmask8) (__U))) + +#define _mm256_mask_blend_ps(__U, __A, __W) \ + ((__m256) __builtin_ia32_blendmps_256_mask ((__v8sf) (__A), \ + (__v8sf) (__W), \ + (__mmask8) (__U))) + +#define _mm256_mask_blend_epi64(__U, __A, __W) \ + ((__m256i) __builtin_ia32_blendmq_256_mask ((__v4di) (__A), \ + (__v4di) (__W), \ + (__mmask8) (__U))) + +#define _mm256_mask_blend_epi32(__U, __A, __W) \ + ((__m256i) __builtin_ia32_blendmd_256_mask ((__v8si) (__A), \ + (__v8si) (__W), \ + (__mmask8) (__U))) + +#define _mm_mask_blend_pd(__U, __A, __W) \ + ((__m128d) __builtin_ia32_blendmpd_128_mask ((__v2df) (__A), \ + (__v2df) (__W), \ + (__mmask8) (__U))) + +#define _mm_mask_blend_ps(__U, __A, __W) \ + ((__m128) __builtin_ia32_blendmps_128_mask ((__v4sf) (__A), \ + (__v4sf) (__W), \ + (__mmask8) (__U))) + +#define _mm_mask_blend_epi64(__U, __A, __W) \ + ((__m128i) __builtin_ia32_blendmq_128_mask ((__v2di) (__A), \ + (__v2di) (__W), \ + (__mmask8) (__U))) + +#define _mm_mask_blend_epi32(__U, __A, __W) \ + ((__m128i) __builtin_ia32_blendmd_128_mask ((__v4si) (__A), \ + (__v4si) (__W), \ + (__mmask8) (__U))) + +#define _mm256_cmp_epu32_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_ucmpd256_mask ((__v8si)(__m256i)(X), \ + (__v8si)(__m256i)(Y), (int)(P),\ + (__mmask8)-1)) + +#define _mm256_cmp_epi64_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpq256_mask ((__v4di)(__m256i)(X), \ + (__v4di)(__m256i)(Y), (int)(P),\ + (__mmask8)-1)) + +#define _mm256_cmp_epi32_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpd256_mask ((__v8si)(__m256i)(X), \ + (__v8si)(__m256i)(Y), (int)(P),\ + (__mmask8)-1)) + +#define _mm256_cmp_epu64_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_ucmpq256_mask ((__v4di)(__m256i)(X), \ + (__v4di)(__m256i)(Y), (int)(P),\ + (__mmask8)-1)) + +#define _mm256_cmp_pd_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_cmppd256_mask ((__v4df)(__m256d)(X), \ + (__v4df)(__m256d)(Y), (int)(P),\ + (__mmask8)-1)) + +#define _mm256_cmp_ps_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpps256_mask ((__v8sf)(__m256)(X), \ + (__v8sf)(__m256)(Y), (int)(P),\ + (__mmask8)-1)) + +#define _mm256_mask_cmp_epi64_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpq256_mask ((__v4di)(__m256i)(X), \ + (__v4di)(__m256i)(Y), (int)(P),\ + (__mmask8)(M))) + +#define _mm256_mask_cmp_epi32_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpd256_mask ((__v8si)(__m256i)(X), \ + (__v8si)(__m256i)(Y), (int)(P),\ + (__mmask8)(M))) + +#define _mm256_mask_cmp_epu64_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_ucmpq256_mask ((__v4di)(__m256i)(X), \ + (__v4di)(__m256i)(Y), (int)(P),\ + (__mmask8)(M))) + +#define _mm256_mask_cmp_epu32_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_ucmpd256_mask ((__v8si)(__m256i)(X), \ + (__v8si)(__m256i)(Y), (int)(P),\ + (__mmask8)(M))) + +#define _mm256_mask_cmp_pd_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_cmppd256_mask ((__v4df)(__m256d)(X), \ + (__v4df)(__m256d)(Y), (int)(P),\ + (__mmask8)(M))) + +#define _mm256_mask_cmp_ps_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpps256_mask ((__v8sf)(__m256)(X), \ + (__v8sf)(__m256)(Y), (int)(P),\ + (__mmask8)(M))) + +#define _mm_cmp_epi64_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpq128_mask ((__v2di)(__m128i)(X), \ + (__v2di)(__m128i)(Y), (int)(P),\ + (__mmask8)-1)) + +#define _mm_cmp_epi32_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpd128_mask ((__v4si)(__m128i)(X), \ + (__v4si)(__m128i)(Y), (int)(P),\ + (__mmask8)-1)) + +#define _mm_cmp_epu64_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_ucmpq128_mask ((__v2di)(__m128i)(X), \ + (__v2di)(__m128i)(Y), (int)(P),\ + (__mmask8)-1)) + +#define _mm_cmp_epu32_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_ucmpd128_mask ((__v4si)(__m128i)(X), \ + (__v4si)(__m128i)(Y), (int)(P),\ + (__mmask8)-1)) + +#define _mm_cmp_pd_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_cmppd128_mask ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (int)(P),\ + (__mmask8)-1)) + +#define _mm_cmp_ps_mask(X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpps128_mask ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (int)(P),\ + (__mmask8)-1)) + +#define _mm_mask_cmp_epi64_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpq128_mask ((__v2di)(__m128i)(X), \ + (__v2di)(__m128i)(Y), (int)(P),\ + (__mmask8)(M))) + +#define _mm_mask_cmp_epi32_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpd128_mask ((__v4si)(__m128i)(X), \ + (__v4si)(__m128i)(Y), (int)(P),\ + (__mmask8)(M))) + +#define _mm_mask_cmp_epu64_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_ucmpq128_mask ((__v2di)(__m128i)(X), \ + (__v2di)(__m128i)(Y), (int)(P),\ + (__mmask8)(M))) + +#define _mm_mask_cmp_epu32_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_ucmpd128_mask ((__v4si)(__m128i)(X), \ + (__v4si)(__m128i)(Y), (int)(P),\ + (__mmask8)(M))) + +#define _mm_mask_cmp_pd_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_cmppd128_mask ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (int)(P),\ + (__mmask8)(M))) + +#define _mm_mask_cmp_ps_mask(M, X, Y, P) \ + ((__mmask8) __builtin_ia32_cmpps128_mask ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (int)(P),\ + (__mmask8)(M))) + +#endif + +#define _mm256_permutexvar_ps(A, B) _mm256_permutevar8x32_ps ((B), (A)) + +#ifdef __DISABLE_AVX512VL__ +#undef __DISABLE_AVX512VL__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX512VL__ */ + +#endif /* _AVX512VLINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512vpopcntdqintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512vpopcntdqintrin.h new file mode 100644 index 0000000000..969a23785c --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avx512vpopcntdqintrin.h @@ -0,0 +1,94 @@ +/* Copyright (C) 2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#if !defined _IMMINTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _AVX512VPOPCNTDQINTRIN_H_INCLUDED +#define _AVX512VPOPCNTDQINTRIN_H_INCLUDED + +#ifndef __AVX512VPOPCNTDQ__ +#pragma GCC push_options +//#pragma GCC target("avx512vpopcntdq") +#define __DISABLE_AVX512VPOPCNTDQ__ +#endif /* __AVX512VPOPCNTDQ__ */ + +extern __inline __m512i +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_popcnt_epi32 (__m512i __A) +{ + return (__m512i) __builtin_ia32_vpopcountd_v16si ((__v16si) __A); +} + +extern __inline __m512i +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_popcnt_epi32 (__m512i __A, __mmask16 __U, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpopcountd_v16si_mask ((__v16si) __A, + (__v16si) __B, + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_popcnt_epi32 (__mmask16 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_vpopcountd_v16si_mask ((__v16si) __A, + (__v16si) + _mm512_setzero_si512 (), + (__mmask16) __U); +} + +extern __inline __m512i +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_popcnt_epi64 (__m512i __A) +{ + return (__m512i) __builtin_ia32_vpopcountq_v8di ((__v8di) __A); +} + +extern __inline __m512i +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_mask_popcnt_epi64 (__m512i __A, __mmask8 __U, __m512i __B) +{ + return (__m512i) __builtin_ia32_vpopcountq_v8di_mask ((__v8di) __A, + (__v8di) __B, + (__mmask8) __U); +} + +extern __inline __m512i +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm512_maskz_popcnt_epi64 (__mmask8 __U, __m512i __A) +{ + return (__m512i) __builtin_ia32_vpopcountq_v8di_mask ((__v8di) __A, + (__v8di) + _mm512_setzero_si512 (), + (__mmask8) __U); +} + +#ifdef __DISABLE_AVX512VPOPCNTDQ__ +#undef __DISABLE_AVX512VPOPCNTDQ__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX512VPOPCNTDQ__ */ + +#endif /* _AVX512VPOPCNTDQINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/avxintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avxintrin.h new file mode 100644 index 0000000000..20c5a078a9 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/avxintrin.h @@ -0,0 +1,1492 @@ +/* Copyright (C) 2008-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +/* Implemented from the specification included in the Intel C++ Compiler + User Guide and Reference, version 11.0. */ + +#ifndef _IMMINTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _AVXINTRIN_H_INCLUDED +#define _AVXINTRIN_H_INCLUDED + +#ifndef __AVX__ +#pragma GCC push_options +#pragma GCC target("avx") +#define __DISABLE_AVX__ +#endif /* __AVX__ */ + +/* Internal data types for implementing the intrinsics. */ +typedef double __v4df __attribute__ ((__vector_size__ (32))); +typedef float __v8sf __attribute__ ((__vector_size__ (32))); +typedef long long __v4di __attribute__ ((__vector_size__ (32))); +typedef unsigned long long __v4du __attribute__ ((__vector_size__ (32))); +typedef int __v8si __attribute__ ((__vector_size__ (32))); +typedef unsigned int __v8su __attribute__ ((__vector_size__ (32))); +typedef short __v16hi __attribute__ ((__vector_size__ (32))); +typedef unsigned short __v16hu __attribute__ ((__vector_size__ (32))); +typedef char __v32qi __attribute__ ((__vector_size__ (32))); +typedef unsigned char __v32qu __attribute__ ((__vector_size__ (32))); + +/* The Intel API is flexible enough that we must allow aliasing with other + vector types, and their scalar components. */ +typedef float __m256 __attribute__ ((__vector_size__ (32), + __may_alias__)); +typedef long long __m256i __attribute__ ((__vector_size__ (32), + __may_alias__)); +typedef double __m256d __attribute__ ((__vector_size__ (32), + __may_alias__)); + +/* Unaligned version of the same types. */ +typedef float __m256_u __attribute__ ((__vector_size__ (32), + __may_alias__, + __aligned__ (1))); +typedef long long __m256i_u __attribute__ ((__vector_size__ (32), + __may_alias__, + __aligned__ (1))); +typedef double __m256d_u __attribute__ ((__vector_size__ (32), + __may_alias__, + __aligned__ (1))); + +/* Compare predicates for scalar and packed compare intrinsics. */ + +/* Equal (ordered, non-signaling) */ +#define _CMP_EQ_OQ 0x00 +/* Less-than (ordered, signaling) */ +#define _CMP_LT_OS 0x01 +/* Less-than-or-equal (ordered, signaling) */ +#define _CMP_LE_OS 0x02 +/* Unordered (non-signaling) */ +#define _CMP_UNORD_Q 0x03 +/* Not-equal (unordered, non-signaling) */ +#define _CMP_NEQ_UQ 0x04 +/* Not-less-than (unordered, signaling) */ +#define _CMP_NLT_US 0x05 +/* Not-less-than-or-equal (unordered, signaling) */ +#define _CMP_NLE_US 0x06 +/* Ordered (nonsignaling) */ +#define _CMP_ORD_Q 0x07 +/* Equal (unordered, non-signaling) */ +#define _CMP_EQ_UQ 0x08 +/* Not-greater-than-or-equal (unordered, signaling) */ +#define _CMP_NGE_US 0x09 +/* Not-greater-than (unordered, signaling) */ +#define _CMP_NGT_US 0x0a +/* False (ordered, non-signaling) */ +#define _CMP_FALSE_OQ 0x0b +/* Not-equal (ordered, non-signaling) */ +#define _CMP_NEQ_OQ 0x0c +/* Greater-than-or-equal (ordered, signaling) */ +#define _CMP_GE_OS 0x0d +/* Greater-than (ordered, signaling) */ +#define _CMP_GT_OS 0x0e +/* True (unordered, non-signaling) */ +#define _CMP_TRUE_UQ 0x0f +/* Equal (ordered, signaling) */ +#define _CMP_EQ_OS 0x10 +/* Less-than (ordered, non-signaling) */ +#define _CMP_LT_OQ 0x11 +/* Less-than-or-equal (ordered, non-signaling) */ +#define _CMP_LE_OQ 0x12 +/* Unordered (signaling) */ +#define _CMP_UNORD_S 0x13 +/* Not-equal (unordered, signaling) */ +#define _CMP_NEQ_US 0x14 +/* Not-less-than (unordered, non-signaling) */ +#define _CMP_NLT_UQ 0x15 +/* Not-less-than-or-equal (unordered, non-signaling) */ +#define _CMP_NLE_UQ 0x16 +/* Ordered (signaling) */ +#define _CMP_ORD_S 0x17 +/* Equal (unordered, signaling) */ +#define _CMP_EQ_US 0x18 +/* Not-greater-than-or-equal (unordered, non-signaling) */ +#define _CMP_NGE_UQ 0x19 +/* Not-greater-than (unordered, non-signaling) */ +#define _CMP_NGT_UQ 0x1a +/* False (ordered, signaling) */ +#define _CMP_FALSE_OS 0x1b +/* Not-equal (ordered, signaling) */ +#define _CMP_NEQ_OS 0x1c +/* Greater-than-or-equal (ordered, non-signaling) */ +#define _CMP_GE_OQ 0x1d +/* Greater-than (ordered, non-signaling) */ +#define _CMP_GT_OQ 0x1e +/* True (unordered, signaling) */ +#define _CMP_TRUE_US 0x1f + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_add_pd (__m256d __A, __m256d __B) +{ + return (__m256d) ((__v4df)__A + (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_add_ps (__m256 __A, __m256 __B) +{ + return (__m256) ((__v8sf)__A + (__v8sf)__B); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_addsub_pd (__m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_addsubpd256 ((__v4df)__A, (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_addsub_ps (__m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_addsubps256 ((__v8sf)__A, (__v8sf)__B); +} + + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_and_pd (__m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_andpd256 ((__v4df)__A, (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_and_ps (__m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_andps256 ((__v8sf)__A, (__v8sf)__B); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_andnot_pd (__m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_andnpd256 ((__v4df)__A, (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_andnot_ps (__m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_andnps256 ((__v8sf)__A, (__v8sf)__B); +} + +/* Double/single precision floating point blend instructions - select + data from 2 sources using constant/variable mask. */ + +#ifdef __OPTIMIZE__ +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_blend_pd (__m256d __X, __m256d __Y, const int __M) +{ + return (__m256d) __builtin_ia32_blendpd256 ((__v4df)__X, + (__v4df)__Y, + __M); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_blend_ps (__m256 __X, __m256 __Y, const int __M) +{ + return (__m256) __builtin_ia32_blendps256 ((__v8sf)__X, + (__v8sf)__Y, + __M); +} +#else +#define _mm256_blend_pd(X, Y, M) \ + ((__m256d) __builtin_ia32_blendpd256 ((__v4df)(__m256d)(X), \ + (__v4df)(__m256d)(Y), (int)(M))) + +#define _mm256_blend_ps(X, Y, M) \ + ((__m256) __builtin_ia32_blendps256 ((__v8sf)(__m256)(X), \ + (__v8sf)(__m256)(Y), (int)(M))) +#endif + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_blendv_pd (__m256d __X, __m256d __Y, __m256d __M) +{ + return (__m256d) __builtin_ia32_blendvpd256 ((__v4df)__X, + (__v4df)__Y, + (__v4df)__M); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_blendv_ps (__m256 __X, __m256 __Y, __m256 __M) +{ + return (__m256) __builtin_ia32_blendvps256 ((__v8sf)__X, + (__v8sf)__Y, + (__v8sf)__M); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_div_pd (__m256d __A, __m256d __B) +{ + return (__m256d) ((__v4df)__A / (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_div_ps (__m256 __A, __m256 __B) +{ + return (__m256) ((__v8sf)__A / (__v8sf)__B); +} + +/* Dot product instructions with mask-defined summing and zeroing parts + of result. */ + +#ifdef __OPTIMIZE__ +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_dp_ps (__m256 __X, __m256 __Y, const int __M) +{ + return (__m256) __builtin_ia32_dpps256 ((__v8sf)__X, + (__v8sf)__Y, + __M); +} +#else +#define _mm256_dp_ps(X, Y, M) \ + ((__m256) __builtin_ia32_dpps256 ((__v8sf)(__m256)(X), \ + (__v8sf)(__m256)(Y), (int)(M))) +#endif + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_hadd_pd (__m256d __X, __m256d __Y) +{ + return (__m256d) __builtin_ia32_haddpd256 ((__v4df)__X, (__v4df)__Y); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_hadd_ps (__m256 __X, __m256 __Y) +{ + return (__m256) __builtin_ia32_haddps256 ((__v8sf)__X, (__v8sf)__Y); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_hsub_pd (__m256d __X, __m256d __Y) +{ + return (__m256d) __builtin_ia32_hsubpd256 ((__v4df)__X, (__v4df)__Y); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_hsub_ps (__m256 __X, __m256 __Y) +{ + return (__m256) __builtin_ia32_hsubps256 ((__v8sf)__X, (__v8sf)__Y); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_max_pd (__m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_maxpd256 ((__v4df)__A, (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_max_ps (__m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_maxps256 ((__v8sf)__A, (__v8sf)__B); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_min_pd (__m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_minpd256 ((__v4df)__A, (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_min_ps (__m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_minps256 ((__v8sf)__A, (__v8sf)__B); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mul_pd (__m256d __A, __m256d __B) +{ + return (__m256d) ((__v4df)__A * (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_mul_ps (__m256 __A, __m256 __B) +{ + return (__m256) ((__v8sf)__A * (__v8sf)__B); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_or_pd (__m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_orpd256 ((__v4df)__A, (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_or_ps (__m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_orps256 ((__v8sf)__A, (__v8sf)__B); +} + +#ifdef __OPTIMIZE__ +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_shuffle_pd (__m256d __A, __m256d __B, const int __mask) +{ + return (__m256d) __builtin_ia32_shufpd256 ((__v4df)__A, (__v4df)__B, + __mask); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_shuffle_ps (__m256 __A, __m256 __B, const int __mask) +{ + return (__m256) __builtin_ia32_shufps256 ((__v8sf)__A, (__v8sf)__B, + __mask); +} +#else +#define _mm256_shuffle_pd(A, B, N) \ + ((__m256d)__builtin_ia32_shufpd256 ((__v4df)(__m256d)(A), \ + (__v4df)(__m256d)(B), (int)(N))) + +#define _mm256_shuffle_ps(A, B, N) \ + ((__m256) __builtin_ia32_shufps256 ((__v8sf)(__m256)(A), \ + (__v8sf)(__m256)(B), (int)(N))) +#endif + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sub_pd (__m256d __A, __m256d __B) +{ + return (__m256d) ((__v4df)__A - (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sub_ps (__m256 __A, __m256 __B) +{ + return (__m256) ((__v8sf)__A - (__v8sf)__B); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_xor_pd (__m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_xorpd256 ((__v4df)__A, (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_xor_ps (__m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_xorps256 ((__v8sf)__A, (__v8sf)__B); +} + +#ifdef __OPTIMIZE__ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_pd (__m128d __X, __m128d __Y, const int __P) +{ + return (__m128d) __builtin_ia32_cmppd ((__v2df)__X, (__v2df)__Y, __P); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_ps (__m128 __X, __m128 __Y, const int __P) +{ + return (__m128) __builtin_ia32_cmpps ((__v4sf)__X, (__v4sf)__Y, __P); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmp_pd (__m256d __X, __m256d __Y, const int __P) +{ + return (__m256d) __builtin_ia32_cmppd256 ((__v4df)__X, (__v4df)__Y, + __P); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cmp_ps (__m256 __X, __m256 __Y, const int __P) +{ + return (__m256) __builtin_ia32_cmpps256 ((__v8sf)__X, (__v8sf)__Y, + __P); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_sd (__m128d __X, __m128d __Y, const int __P) +{ + return (__m128d) __builtin_ia32_cmpsd ((__v2df)__X, (__v2df)__Y, __P); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmp_ss (__m128 __X, __m128 __Y, const int __P) +{ + return (__m128) __builtin_ia32_cmpss ((__v4sf)__X, (__v4sf)__Y, __P); +} +#else +#define _mm_cmp_pd(X, Y, P) \ + ((__m128d) __builtin_ia32_cmppd ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (int)(P))) + +#define _mm_cmp_ps(X, Y, P) \ + ((__m128) __builtin_ia32_cmpps ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (int)(P))) + +#define _mm256_cmp_pd(X, Y, P) \ + ((__m256d) __builtin_ia32_cmppd256 ((__v4df)(__m256d)(X), \ + (__v4df)(__m256d)(Y), (int)(P))) + +#define _mm256_cmp_ps(X, Y, P) \ + ((__m256) __builtin_ia32_cmpps256 ((__v8sf)(__m256)(X), \ + (__v8sf)(__m256)(Y), (int)(P))) + +#define _mm_cmp_sd(X, Y, P) \ + ((__m128d) __builtin_ia32_cmpsd ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (int)(P))) + +#define _mm_cmp_ss(X, Y, P) \ + ((__m128) __builtin_ia32_cmpss ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (int)(P))) +#endif + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi32_pd (__m128i __A) +{ + return (__m256d)__builtin_ia32_cvtdq2pd256 ((__v4si) __A); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtepi32_ps (__m256i __A) +{ + return (__m256)__builtin_ia32_cvtdq2ps256 ((__v8si) __A); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtpd_ps (__m256d __A) +{ + return (__m128)__builtin_ia32_cvtpd2ps256 ((__v4df) __A); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtps_epi32 (__m256 __A) +{ + return (__m256i)__builtin_ia32_cvtps2dq256 ((__v8sf) __A); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtps_pd (__m128 __A) +{ + return (__m256d)__builtin_ia32_cvtps2pd256 ((__v4sf) __A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvttpd_epi32 (__m256d __A) +{ + return (__m128i)__builtin_ia32_cvttpd2dq256 ((__v4df) __A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtpd_epi32 (__m256d __A) +{ + return (__m128i)__builtin_ia32_cvtpd2dq256 ((__v4df) __A); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvttps_epi32 (__m256 __A) +{ + return (__m256i)__builtin_ia32_cvttps2dq256 ((__v8sf) __A); +} + +extern __inline double +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtsd_f64 (__m256d __A) +{ + return __A[0]; +} + +extern __inline float +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtss_f32 (__m256 __A) +{ + return __A[0]; +} + +#ifdef __OPTIMIZE__ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_extractf128_pd (__m256d __X, const int __N) +{ + return (__m128d) __builtin_ia32_vextractf128_pd256 ((__v4df)__X, __N); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_extractf128_ps (__m256 __X, const int __N) +{ + return (__m128) __builtin_ia32_vextractf128_ps256 ((__v8sf)__X, __N); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_extractf128_si256 (__m256i __X, const int __N) +{ + return (__m128i) __builtin_ia32_vextractf128_si256 ((__v8si)__X, __N); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_extract_epi32 (__m256i __X, int const __N) +{ + __m128i __Y = _mm256_extractf128_si256 (__X, __N >> 2); + return _mm_extract_epi32 (__Y, __N % 4); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_extract_epi16 (__m256i __X, int const __N) +{ + __m128i __Y = _mm256_extractf128_si256 (__X, __N >> 3); + return _mm_extract_epi16 (__Y, __N % 8); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_extract_epi8 (__m256i __X, int const __N) +{ + __m128i __Y = _mm256_extractf128_si256 (__X, __N >> 4); + return _mm_extract_epi8 (__Y, __N % 16); +} + +#ifdef __x86_64__ +extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_extract_epi64 (__m256i __X, const int __N) +{ + __m128i __Y = _mm256_extractf128_si256 (__X, __N >> 1); + return _mm_extract_epi64 (__Y, __N % 2); +} +#endif +#else +#define _mm256_extractf128_pd(X, N) \ + ((__m128d) __builtin_ia32_vextractf128_pd256 ((__v4df)(__m256d)(X), \ + (int)(N))) + +#define _mm256_extractf128_ps(X, N) \ + ((__m128) __builtin_ia32_vextractf128_ps256 ((__v8sf)(__m256)(X), \ + (int)(N))) + +#define _mm256_extractf128_si256(X, N) \ + ((__m128i) __builtin_ia32_vextractf128_si256 ((__v8si)(__m256i)(X), \ + (int)(N))) + +#define _mm256_extract_epi32(X, N) \ + (__extension__ \ + ({ \ + __m128i __Y = _mm256_extractf128_si256 ((X), (N) >> 2); \ + _mm_extract_epi32 (__Y, (N) % 4); \ + })) + +#define _mm256_extract_epi16(X, N) \ + (__extension__ \ + ({ \ + __m128i __Y = _mm256_extractf128_si256 ((X), (N) >> 3); \ + _mm_extract_epi16 (__Y, (N) % 8); \ + })) + +#define _mm256_extract_epi8(X, N) \ + (__extension__ \ + ({ \ + __m128i __Y = _mm256_extractf128_si256 ((X), (N) >> 4); \ + _mm_extract_epi8 (__Y, (N) % 16); \ + })) + +#ifdef __x86_64__ +#define _mm256_extract_epi64(X, N) \ + (__extension__ \ + ({ \ + __m128i __Y = _mm256_extractf128_si256 ((X), (N) >> 1); \ + _mm_extract_epi64 (__Y, (N) % 2); \ + })) +#endif +#endif + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_zeroall (void) +{ + __builtin_ia32_vzeroall (); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_zeroupper (void) +{ + __builtin_ia32_vzeroupper (); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permutevar_pd (__m128d __A, __m128i __C) +{ + return (__m128d) __builtin_ia32_vpermilvarpd ((__v2df)__A, + (__v2di)__C); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permutevar_pd (__m256d __A, __m256i __C) +{ + return (__m256d) __builtin_ia32_vpermilvarpd256 ((__v4df)__A, + (__v4di)__C); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permutevar_ps (__m128 __A, __m128i __C) +{ + return (__m128) __builtin_ia32_vpermilvarps ((__v4sf)__A, + (__v4si)__C); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permutevar_ps (__m256 __A, __m256i __C) +{ + return (__m256) __builtin_ia32_vpermilvarps256 ((__v8sf)__A, + (__v8si)__C); +} + +#ifdef __OPTIMIZE__ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permute_pd (__m128d __X, const int __C) +{ + return (__m128d) __builtin_ia32_vpermilpd ((__v2df)__X, __C); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permute_pd (__m256d __X, const int __C) +{ + return (__m256d) __builtin_ia32_vpermilpd256 ((__v4df)__X, __C); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permute_ps (__m128 __X, const int __C) +{ + return (__m128) __builtin_ia32_vpermilps ((__v4sf)__X, __C); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permute_ps (__m256 __X, const int __C) +{ + return (__m256) __builtin_ia32_vpermilps256 ((__v8sf)__X, __C); +} +#else +#define _mm_permute_pd(X, C) \ + ((__m128d) __builtin_ia32_vpermilpd ((__v2df)(__m128d)(X), (int)(C))) + +#define _mm256_permute_pd(X, C) \ + ((__m256d) __builtin_ia32_vpermilpd256 ((__v4df)(__m256d)(X), (int)(C))) + +#define _mm_permute_ps(X, C) \ + ((__m128) __builtin_ia32_vpermilps ((__v4sf)(__m128)(X), (int)(C))) + +#define _mm256_permute_ps(X, C) \ + ((__m256) __builtin_ia32_vpermilps256 ((__v8sf)(__m256)(X), (int)(C))) +#endif + +#ifdef __OPTIMIZE__ +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permute2f128_pd (__m256d __X, __m256d __Y, const int __C) +{ + return (__m256d) __builtin_ia32_vperm2f128_pd256 ((__v4df)__X, + (__v4df)__Y, + __C); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permute2f128_ps (__m256 __X, __m256 __Y, const int __C) +{ + return (__m256) __builtin_ia32_vperm2f128_ps256 ((__v8sf)__X, + (__v8sf)__Y, + __C); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permute2f128_si256 (__m256i __X, __m256i __Y, const int __C) +{ + return (__m256i) __builtin_ia32_vperm2f128_si256 ((__v8si)__X, + (__v8si)__Y, + __C); +} +#else +#define _mm256_permute2f128_pd(X, Y, C) \ + ((__m256d) __builtin_ia32_vperm2f128_pd256 ((__v4df)(__m256d)(X), \ + (__v4df)(__m256d)(Y), \ + (int)(C))) + +#define _mm256_permute2f128_ps(X, Y, C) \ + ((__m256) __builtin_ia32_vperm2f128_ps256 ((__v8sf)(__m256)(X), \ + (__v8sf)(__m256)(Y), \ + (int)(C))) + +#define _mm256_permute2f128_si256(X, Y, C) \ + ((__m256i) __builtin_ia32_vperm2f128_si256 ((__v8si)(__m256i)(X), \ + (__v8si)(__m256i)(Y), \ + (int)(C))) +#endif + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_broadcast_ss (float const *__X) +{ + return (__m128) __builtin_ia32_vbroadcastss (__X); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcast_sd (double const *__X) +{ + return (__m256d) __builtin_ia32_vbroadcastsd256 (__X); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcast_ss (float const *__X) +{ + return (__m256) __builtin_ia32_vbroadcastss256 (__X); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcast_pd (__m128d const *__X) +{ + return (__m256d) __builtin_ia32_vbroadcastf128_pd256 (__X); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_broadcast_ps (__m128 const *__X) +{ + return (__m256) __builtin_ia32_vbroadcastf128_ps256 (__X); +} + +#ifdef __OPTIMIZE__ +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_insertf128_pd (__m256d __X, __m128d __Y, const int __O) +{ + return (__m256d) __builtin_ia32_vinsertf128_pd256 ((__v4df)__X, + (__v2df)__Y, + __O); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_insertf128_ps (__m256 __X, __m128 __Y, const int __O) +{ + return (__m256) __builtin_ia32_vinsertf128_ps256 ((__v8sf)__X, + (__v4sf)__Y, + __O); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_insertf128_si256 (__m256i __X, __m128i __Y, const int __O) +{ + return (__m256i) __builtin_ia32_vinsertf128_si256 ((__v8si)__X, + (__v4si)__Y, + __O); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_insert_epi32 (__m256i __X, int __D, int const __N) +{ + __m128i __Y = _mm256_extractf128_si256 (__X, __N >> 2); + __Y = _mm_insert_epi32 (__Y, __D, __N % 4); + return _mm256_insertf128_si256 (__X, __Y, __N >> 2); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_insert_epi16 (__m256i __X, int __D, int const __N) +{ + __m128i __Y = _mm256_extractf128_si256 (__X, __N >> 3); + __Y = _mm_insert_epi16 (__Y, __D, __N % 8); + return _mm256_insertf128_si256 (__X, __Y, __N >> 3); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_insert_epi8 (__m256i __X, int __D, int const __N) +{ + __m128i __Y = _mm256_extractf128_si256 (__X, __N >> 4); + __Y = _mm_insert_epi8 (__Y, __D, __N % 16); + return _mm256_insertf128_si256 (__X, __Y, __N >> 4); +} + +#ifdef __x86_64__ +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_insert_epi64 (__m256i __X, long long __D, int const __N) +{ + __m128i __Y = _mm256_extractf128_si256 (__X, __N >> 1); + __Y = _mm_insert_epi64 (__Y, __D, __N % 2); + return _mm256_insertf128_si256 (__X, __Y, __N >> 1); +} +#endif +#else +#define _mm256_insertf128_pd(X, Y, O) \ + ((__m256d) __builtin_ia32_vinsertf128_pd256 ((__v4df)(__m256d)(X), \ + (__v2df)(__m128d)(Y), \ + (int)(O))) + +#define _mm256_insertf128_ps(X, Y, O) \ + ((__m256) __builtin_ia32_vinsertf128_ps256 ((__v8sf)(__m256)(X), \ + (__v4sf)(__m128)(Y), \ + (int)(O))) + +#define _mm256_insertf128_si256(X, Y, O) \ + ((__m256i) __builtin_ia32_vinsertf128_si256 ((__v8si)(__m256i)(X), \ + (__v4si)(__m128i)(Y), \ + (int)(O))) + +#define _mm256_insert_epi32(X, D, N) \ + (__extension__ \ + ({ \ + __m128i __Y = _mm256_extractf128_si256 ((X), (N) >> 2); \ + __Y = _mm_insert_epi32 (__Y, (D), (N) % 4); \ + _mm256_insertf128_si256 ((X), __Y, (N) >> 2); \ + })) + +#define _mm256_insert_epi16(X, D, N) \ + (__extension__ \ + ({ \ + __m128i __Y = _mm256_extractf128_si256 ((X), (N) >> 3); \ + __Y = _mm_insert_epi16 (__Y, (D), (N) % 8); \ + _mm256_insertf128_si256 ((X), __Y, (N) >> 3); \ + })) + +#define _mm256_insert_epi8(X, D, N) \ + (__extension__ \ + ({ \ + __m128i __Y = _mm256_extractf128_si256 ((X), (N) >> 4); \ + __Y = _mm_insert_epi8 (__Y, (D), (N) % 16); \ + _mm256_insertf128_si256 ((X), __Y, (N) >> 4); \ + })) + +#ifdef __x86_64__ +#define _mm256_insert_epi64(X, D, N) \ + (__extension__ \ + ({ \ + __m128i __Y = _mm256_extractf128_si256 ((X), (N) >> 1); \ + __Y = _mm_insert_epi64 (__Y, (D), (N) % 2); \ + _mm256_insertf128_si256 ((X), __Y, (N) >> 1); \ + })) +#endif +#endif + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_load_pd (double const *__P) +{ + return *(__m256d *)__P; +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_store_pd (double *__P, __m256d __A) +{ + *(__m256d *)__P = __A; +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_load_ps (float const *__P) +{ + return *(__m256 *)__P; +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_store_ps (float *__P, __m256 __A) +{ + *(__m256 *)__P = __A; +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_loadu_pd (double const *__P) +{ + return *(__m256d_u *)__P; +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_storeu_pd (double *__P, __m256d __A) +{ + *(__m256d_u *)__P = __A; +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_loadu_ps (float const *__P) +{ + return *(__m256_u *)__P; +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_storeu_ps (float *__P, __m256 __A) +{ + *(__m256_u *)__P = __A; +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_load_si256 (__m256i const *__P) +{ + return *__P; +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_store_si256 (__m256i *__P, __m256i __A) +{ + *__P = __A; +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_loadu_si256 (__m256i_u const *__P) +{ + return *__P; +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_storeu_si256 (__m256i_u *__P, __m256i __A) +{ + *__P = __A; +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskload_pd (double const *__P, __m128i __M) +{ + return (__m128d) __builtin_ia32_maskloadpd ((const __v2df *)__P, + (__v2di)__M); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskstore_pd (double *__P, __m128i __M, __m128d __A) +{ + __builtin_ia32_maskstorepd ((__v2df *)__P, (__v2di)__M, (__v2df)__A); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskload_pd (double const *__P, __m256i __M) +{ + return (__m256d) __builtin_ia32_maskloadpd256 ((const __v4df *)__P, + (__v4di)__M); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskstore_pd (double *__P, __m256i __M, __m256d __A) +{ + __builtin_ia32_maskstorepd256 ((__v4df *)__P, (__v4di)__M, (__v4df)__A); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskload_ps (float const *__P, __m128i __M) +{ + return (__m128) __builtin_ia32_maskloadps ((const __v4sf *)__P, + (__v4si)__M); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskstore_ps (float *__P, __m128i __M, __m128 __A) +{ + __builtin_ia32_maskstoreps ((__v4sf *)__P, (__v4si)__M, (__v4sf)__A); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskload_ps (float const *__P, __m256i __M) +{ + return (__m256) __builtin_ia32_maskloadps256 ((const __v8sf *)__P, + (__v8si)__M); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maskstore_ps (float *__P, __m256i __M, __m256 __A) +{ + __builtin_ia32_maskstoreps256 ((__v8sf *)__P, (__v8si)__M, (__v8sf)__A); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_movehdup_ps (__m256 __X) +{ + return (__m256) __builtin_ia32_movshdup256 ((__v8sf)__X); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_moveldup_ps (__m256 __X) +{ + return (__m256) __builtin_ia32_movsldup256 ((__v8sf)__X); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_movedup_pd (__m256d __X) +{ + return (__m256d) __builtin_ia32_movddup256 ((__v4df)__X); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_lddqu_si256 (__m256i const *__P) +{ + return (__m256i) __builtin_ia32_lddqu256 ((char const *)__P); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_stream_si256 (__m256i *__A, __m256i __B) +{ + __builtin_ia32_movntdq256 ((__v4di *)__A, (__v4di)__B); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_stream_pd (double *__A, __m256d __B) +{ + __builtin_ia32_movntpd256 (__A, (__v4df)__B); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_stream_ps (float *__P, __m256 __A) +{ + __builtin_ia32_movntps256 (__P, (__v8sf)__A); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_rcp_ps (__m256 __A) +{ + return (__m256) __builtin_ia32_rcpps256 ((__v8sf)__A); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_rsqrt_ps (__m256 __A) +{ + return (__m256) __builtin_ia32_rsqrtps256 ((__v8sf)__A); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sqrt_pd (__m256d __A) +{ + return (__m256d) __builtin_ia32_sqrtpd256 ((__v4df)__A); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_sqrt_ps (__m256 __A) +{ + return (__m256) __builtin_ia32_sqrtps256 ((__v8sf)__A); +} + +#ifdef __OPTIMIZE__ +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_round_pd (__m256d __V, const int __M) +{ + return (__m256d) __builtin_ia32_roundpd256 ((__v4df)__V, __M); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_round_ps (__m256 __V, const int __M) +{ + return (__m256) __builtin_ia32_roundps256 ((__v8sf)__V, __M); +} +#else +#define _mm256_round_pd(V, M) \ + ((__m256d) __builtin_ia32_roundpd256 ((__v4df)(__m256d)(V), (int)(M))) + +#define _mm256_round_ps(V, M) \ + ((__m256) __builtin_ia32_roundps256 ((__v8sf)(__m256)(V), (int)(M))) +#endif + +#define _mm256_ceil_pd(V) _mm256_round_pd ((V), _MM_FROUND_CEIL) +#define _mm256_floor_pd(V) _mm256_round_pd ((V), _MM_FROUND_FLOOR) +#define _mm256_ceil_ps(V) _mm256_round_ps ((V), _MM_FROUND_CEIL) +#define _mm256_floor_ps(V) _mm256_round_ps ((V), _MM_FROUND_FLOOR) + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_unpackhi_pd (__m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_unpckhpd256 ((__v4df)__A, (__v4df)__B); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_unpacklo_pd (__m256d __A, __m256d __B) +{ + return (__m256d) __builtin_ia32_unpcklpd256 ((__v4df)__A, (__v4df)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_unpackhi_ps (__m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_unpckhps256 ((__v8sf)__A, (__v8sf)__B); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_unpacklo_ps (__m256 __A, __m256 __B) +{ + return (__m256) __builtin_ia32_unpcklps256 ((__v8sf)__A, (__v8sf)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_testz_pd (__m128d __M, __m128d __V) +{ + return __builtin_ia32_vtestzpd ((__v2df)__M, (__v2df)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_testc_pd (__m128d __M, __m128d __V) +{ + return __builtin_ia32_vtestcpd ((__v2df)__M, (__v2df)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_testnzc_pd (__m128d __M, __m128d __V) +{ + return __builtin_ia32_vtestnzcpd ((__v2df)__M, (__v2df)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_testz_ps (__m128 __M, __m128 __V) +{ + return __builtin_ia32_vtestzps ((__v4sf)__M, (__v4sf)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_testc_ps (__m128 __M, __m128 __V) +{ + return __builtin_ia32_vtestcps ((__v4sf)__M, (__v4sf)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_testnzc_ps (__m128 __M, __m128 __V) +{ + return __builtin_ia32_vtestnzcps ((__v4sf)__M, (__v4sf)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testz_pd (__m256d __M, __m256d __V) +{ + return __builtin_ia32_vtestzpd256 ((__v4df)__M, (__v4df)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testc_pd (__m256d __M, __m256d __V) +{ + return __builtin_ia32_vtestcpd256 ((__v4df)__M, (__v4df)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testnzc_pd (__m256d __M, __m256d __V) +{ + return __builtin_ia32_vtestnzcpd256 ((__v4df)__M, (__v4df)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testz_ps (__m256 __M, __m256 __V) +{ + return __builtin_ia32_vtestzps256 ((__v8sf)__M, (__v8sf)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testc_ps (__m256 __M, __m256 __V) +{ + return __builtin_ia32_vtestcps256 ((__v8sf)__M, (__v8sf)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testnzc_ps (__m256 __M, __m256 __V) +{ + return __builtin_ia32_vtestnzcps256 ((__v8sf)__M, (__v8sf)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testz_si256 (__m256i __M, __m256i __V) +{ + return __builtin_ia32_ptestz256 ((__v4di)__M, (__v4di)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testc_si256 (__m256i __M, __m256i __V) +{ + return __builtin_ia32_ptestc256 ((__v4di)__M, (__v4di)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_testnzc_si256 (__m256i __M, __m256i __V) +{ + return __builtin_ia32_ptestnzc256 ((__v4di)__M, (__v4di)__V); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_movemask_pd (__m256d __A) +{ + return __builtin_ia32_movmskpd256 ((__v4df)__A); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_movemask_ps (__m256 __A) +{ + return __builtin_ia32_movmskps256 ((__v8sf)__A); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_undefined_pd (void) +{ + __m256d __Y = __Y; + return __Y; +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_undefined_ps (void) +{ + __m256 __Y = __Y; + return __Y; +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_undefined_si256 (void) +{ + __m256i __Y = __Y; + return __Y; +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_setzero_pd (void) +{ + return __extension__ (__m256d){ 0.0, 0.0, 0.0, 0.0 }; +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_setzero_ps (void) +{ + return __extension__ (__m256){ 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0 }; +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_setzero_si256 (void) +{ + return __extension__ (__m256i)(__v4di){ 0, 0, 0, 0 }; +} + +/* Create the vector [A B C D]. */ +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set_pd (double __A, double __B, double __C, double __D) +{ + return __extension__ (__m256d){ __D, __C, __B, __A }; +} + +/* Create the vector [A B C D E F G H]. */ +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set_ps (float __A, float __B, float __C, float __D, + float __E, float __F, float __G, float __H) +{ + return __extension__ (__m256){ __H, __G, __F, __E, + __D, __C, __B, __A }; +} + +/* Create the vector [A B C D E F G H]. */ +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set_epi32 (int __A, int __B, int __C, int __D, + int __E, int __F, int __G, int __H) +{ + return __extension__ (__m256i)(__v8si){ __H, __G, __F, __E, + __D, __C, __B, __A }; +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set_epi16 (short __q15, short __q14, short __q13, short __q12, + short __q11, short __q10, short __q09, short __q08, + short __q07, short __q06, short __q05, short __q04, + short __q03, short __q02, short __q01, short __q00) +{ + return __extension__ (__m256i)(__v16hi){ + __q00, __q01, __q02, __q03, __q04, __q05, __q06, __q07, + __q08, __q09, __q10, __q11, __q12, __q13, __q14, __q15 + }; +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set_epi8 (char __q31, char __q30, char __q29, char __q28, + char __q27, char __q26, char __q25, char __q24, + char __q23, char __q22, char __q21, char __q20, + char __q19, char __q18, char __q17, char __q16, + char __q15, char __q14, char __q13, char __q12, + char __q11, char __q10, char __q09, char __q08, + char __q07, char __q06, char __q05, char __q04, + char __q03, char __q02, char __q01, char __q00) +{ + return __extension__ (__m256i)(__v32qi){ + __q00, __q01, __q02, __q03, __q04, __q05, __q06, __q07, + __q08, __q09, __q10, __q11, __q12, __q13, __q14, __q15, + __q16, __q17, __q18, __q19, __q20, __q21, __q22, __q23, + __q24, __q25, __q26, __q27, __q28, __q29, __q30, __q31 + }; +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set_epi64x (long long __A, long long __B, long long __C, + long long __D) +{ + return __extension__ (__m256i)(__v4di){ __D, __C, __B, __A }; +} + +/* Create a vector with all elements equal to A. */ +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set1_pd (double __A) +{ + return __extension__ (__m256d){ __A, __A, __A, __A }; +} + +/* Create a vector with all elements equal to A. */ +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set1_ps (float __A) +{ + return __extension__ (__m256){ __A, __A, __A, __A, + __A, __A, __A, __A }; +} + +/* Create a vector with all elements equal to A. */ +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set1_epi32 (int __A) +{ + return __extension__ (__m256i)(__v8si){ __A, __A, __A, __A, + __A, __A, __A, __A }; +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set1_epi16 (short __A) +{ + return _mm256_set_epi16 (__A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set1_epi8 (char __A) +{ + return _mm256_set_epi8 (__A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_set1_epi64x (long long __A) +{ + return __extension__ (__m256i)(__v4di){ __A, __A, __A, __A }; +} + +/* Create vectors of elements in the reversed order from the + _mm256_set_XXX functions. */ + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_setr_pd (double __A, double __B, double __C, double __D) +{ + return _mm256_set_pd (__D, __C, __B, __A); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_setr_ps (float __A, float __B, float __C, float __D, + float __E, float __F, float __G, float __H) +{ + return _mm256_set_ps (__H, __G, __F, __E, __D, __C, __B, __A); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_setr_epi32 (int __A, int __B, int __C, int __D, + int __E, int __F, int __G, int __H) +{ + return _mm256_set_epi32 (__H, __G, __F, __E, __D, __C, __B, __A); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_setr_epi16 (short __q15, short __q14, short __q13, short __q12, + short __q11, short __q10, short __q09, short __q08, + short __q07, short __q06, short __q05, short __q04, + short __q03, short __q02, short __q01, short __q00) +{ + return _mm256_set_epi16 (__q00, __q01, __q02, __q03, + __q04, __q05, __q06, __q07, + __q08, __q09, __q10, __q11, + __q12, __q13, __q14, __q15); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_setr_epi8 (char __q31, char __q30, char __q29, char __q28, + char __q27, char __q26, char __q25, char __q24, + char __q23, char __q22, char __q21, char __q20, + char __q19, char __q18, char __q17, char __q16, + char __q15, char __q14, char __q13, char __q12, + char __q11, char __q10, char __q09, char __q08, + char __q07, char __q06, char __q05, char __q04, + char __q03, char __q02, char __q01, char __q00) +{ + return _mm256_set_epi8 (__q00, __q01, __q02, __q03, + __q04, __q05, __q06, __q07, + __q08, __q09, __q10, __q11, + __q12, __q13, __q14, __q15, + __q16, __q17, __q18, __q19, + __q20, __q21, __q22, __q23, + __q24, __q25, __q26, __q27, + __q28, __q29, __q30, __q31); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_setr_epi64x (long long __A, long long __B, long long __C, + long long __D) +{ + return _mm256_set_epi64x (__D, __C, __B, __A); +} + +/* Casts between various SP, DP, INT vector types. Note that these do no + conversion of values, they just change the type. */ +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castpd_ps (__m256d __A) +{ + return (__m256) __A; +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castpd_si256 (__m256d __A) +{ + return (__m256i) __A; +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castps_pd (__m256 __A) +{ + return (__m256d) __A; +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castps_si256(__m256 __A) +{ + return (__m256i) __A; +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castsi256_ps (__m256i __A) +{ + return (__m256) __A; +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castsi256_pd (__m256i __A) +{ + return (__m256d) __A; +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castpd256_pd128 (__m256d __A) +{ + return (__m128d) __builtin_ia32_pd_pd256 ((__v4df)__A); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castps256_ps128 (__m256 __A) +{ + return (__m128) __builtin_ia32_ps_ps256 ((__v8sf)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castsi256_si128 (__m256i __A) +{ + return (__m128i) __builtin_ia32_si_si256 ((__v8si)__A); +} + +/* When cast is done from a 128 to 256-bit type, the low 128 bits of + the 256-bit result contain source parameter value and the upper 128 + bits of the result are undefined. Those intrinsics shouldn't + generate any extra moves. */ + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castpd128_pd256 (__m128d __A) +{ + return (__m256d) __builtin_ia32_pd256_pd ((__v2df)__A); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castps128_ps256 (__m128 __A) +{ + return (__m256) __builtin_ia32_ps256_ps ((__v4sf)__A); +} + +extern __inline __m256i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_castsi128_si256 (__m128i __A) +{ + return (__m256i) __builtin_ia32_si256_si ((__v4si)__A); +} + +#ifdef __DISABLE_AVX__ +#undef __DISABLE_AVX__ +#pragma GCC pop_options +#endif /* __DISABLE_AVX__ */ + +#endif /* _AVXINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/bmi2intrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/bmi2intrin.h new file mode 100644 index 0000000000..29569ae8de --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/bmi2intrin.h @@ -0,0 +1,109 @@ +/* Copyright (C) 2011-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#if !defined _X86INTRIN_H_INCLUDED && !defined _IMMINTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _BMI2INTRIN_H_INCLUDED +#define _BMI2INTRIN_H_INCLUDED + +#ifndef __BMI2__ +#pragma GCC push_options +#pragma GCC target("bmi2") +#define __DISABLE_BMI2__ +#endif /* __BMI2__ */ + +extern __inline unsigned int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_bzhi_u32 (unsigned int __X, unsigned int __Y) +{ + return __builtin_ia32_bzhi_si (__X, __Y); +} + +extern __inline unsigned int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_pdep_u32 (unsigned int __X, unsigned int __Y) +{ + return __builtin_ia32_pdep_si (__X, __Y); +} + +extern __inline unsigned int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_pext_u32 (unsigned int __X, unsigned int __Y) +{ + return __builtin_ia32_pext_si (__X, __Y); +} + +#ifdef __x86_64__ + +extern __inline unsigned long long +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_bzhi_u64 (unsigned long long __X, unsigned long long __Y) +{ + return __builtin_ia32_bzhi_di (__X, __Y); +} + +extern __inline unsigned long long +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_pdep_u64 (unsigned long long __X, unsigned long long __Y) +{ + return __builtin_ia32_pdep_di (__X, __Y); +} + +extern __inline unsigned long long +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_pext_u64 (unsigned long long __X, unsigned long long __Y) +{ + return __builtin_ia32_pext_di (__X, __Y); +} + +extern __inline unsigned long long +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mulx_u64 (unsigned long long __X, unsigned long long __Y, + unsigned long long *__P) +{ + unsigned __int128 __res = (unsigned __int128) __X * __Y; + *__P = (unsigned long long) (__res >> 64); + return (unsigned long long) __res; +} + +#else /* !__x86_64__ */ + +extern __inline unsigned int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mulx_u32 (unsigned int __X, unsigned int __Y, unsigned int *__P) +{ + unsigned long long __res = (unsigned long long) __X * __Y; + *__P = (unsigned int) (__res >> 32); + return (unsigned int) __res; +} + +#endif /* !__x86_64__ */ + +#ifdef __DISABLE_BMI2__ +#undef __DISABLE_BMI2__ +#pragma GCC pop_options +#endif /* __DISABLE_BMI2__ */ + +#endif /* _BMI2INTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/bmiintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/bmiintrin.h new file mode 100644 index 0000000000..55c6219e97 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/bmiintrin.h @@ -0,0 +1,184 @@ +/* Copyright (C) 2010-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#if !defined _X86INTRIN_H_INCLUDED && !defined _IMMINTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _BMIINTRIN_H_INCLUDED +#define _BMIINTRIN_H_INCLUDED + +#ifndef __BMI__ +#pragma GCC push_options +#pragma GCC target("bmi") +#define __DISABLE_BMI__ +#endif /* __BMI__ */ + +extern __inline unsigned short __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__tzcnt_u16 (unsigned short __X) +{ + return __builtin_ia32_tzcnt_u16 (__X); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__andn_u32 (unsigned int __X, unsigned int __Y) +{ + return ~__X & __Y; +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__bextr_u32 (unsigned int __X, unsigned int __Y) +{ + return __builtin_ia32_bextr_u32 (__X, __Y); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_bextr_u32 (unsigned int __X, unsigned int __Y, unsigned __Z) +{ + return __builtin_ia32_bextr_u32 (__X, ((__Y & 0xff) | ((__Z & 0xff) << 8))); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blsi_u32 (unsigned int __X) +{ + return __X & -__X; +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_blsi_u32 (unsigned int __X) +{ + return __blsi_u32 (__X); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blsmsk_u32 (unsigned int __X) +{ + return __X ^ (__X - 1); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_blsmsk_u32 (unsigned int __X) +{ + return __blsmsk_u32 (__X); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blsr_u32 (unsigned int __X) +{ + return __X & (__X - 1); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_blsr_u32 (unsigned int __X) +{ + return __blsr_u32 (__X); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__tzcnt_u32 (unsigned int __X) +{ + return __builtin_ia32_tzcnt_u32 (__X); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_tzcnt_u32 (unsigned int __X) +{ + return __builtin_ia32_tzcnt_u32 (__X); +} + + +#ifdef __x86_64__ +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__andn_u64 (unsigned long long __X, unsigned long long __Y) +{ + return ~__X & __Y; +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__bextr_u64 (unsigned long long __X, unsigned long long __Y) +{ + return __builtin_ia32_bextr_u64 (__X, __Y); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_bextr_u64 (unsigned long long __X, unsigned int __Y, unsigned int __Z) +{ + return __builtin_ia32_bextr_u64 (__X, ((__Y & 0xff) | ((__Z & 0xff) << 8))); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blsi_u64 (unsigned long long __X) +{ + return __X & -__X; +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_blsi_u64 (unsigned long long __X) +{ + return __blsi_u64 (__X); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blsmsk_u64 (unsigned long long __X) +{ + return __X ^ (__X - 1); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_blsmsk_u64 (unsigned long long __X) +{ + return __blsmsk_u64 (__X); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blsr_u64 (unsigned long long __X) +{ + return __X & (__X - 1); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_blsr_u64 (unsigned long long __X) +{ + return __blsr_u64 (__X); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__tzcnt_u64 (unsigned long long __X) +{ + return __builtin_ia32_tzcnt_u64 (__X); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_tzcnt_u64 (unsigned long long __X) +{ + return __builtin_ia32_tzcnt_u64 (__X); +} + +#endif /* __x86_64__ */ + +#ifdef __DISABLE_BMI__ +#undef __DISABLE_BMI__ +#pragma GCC pop_options +#endif /* __DISABLE_BMI__ */ + +#endif /* _BMIINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/bmmintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/bmmintrin.h new file mode 100644 index 0000000000..f460393508 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/bmmintrin.h @@ -0,0 +1,29 @@ +/* Copyright (C) 2007-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _BMMINTRIN_H_INCLUDED +#define _BMMINTRIN_H_INCLUDED + +# error "SSE5 instruction set removed from compiler" + +#endif /* _BMMINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/cilk.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/cilk.h new file mode 100644 index 0000000000..86038ac1ad --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/cilk.h @@ -0,0 +1,82 @@ +/* cilk.h -*-C++-*- + * + * Copyright (C) 2010-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + */ + +/** @file cilk.h + * + * @brief Provides convenient aliases for Intel(R) Cilk(TM) language keywords. + * + * @details + * Since Intel Cilk Plus is a nonstandard extension to both C and C++, the Intel + * Cilk language keywords all begin with "`_Cilk_`", which guarantees that they + * will not conflict with user-defined identifiers in properly written + * programs. This way, a Cilk-enabled C or C++ compiler can safely compile + * "standard" C and C++ programs. + * + * However, this means that the keywords _look_ like something grafted on to + * the base language. Therefore, you can include this header: + * + * #include "cilk/cilk.h" + * + * and then write the Intel Cilk keywords with a "`cilk_`" prefix instead of + * "`_Cilk_`". + * + * @ingroup language + */ + + +/** @defgroup language Language Keywords + * Definitions for the Intel Cilk language. + * @{ + */ + +#ifndef cilk_spawn +# define cilk_spawn _Cilk_spawn ///< Spawn a task that can execute in parallel. +# define cilk_sync _Cilk_sync ///< Wait for spawned tasks to complete. +# define cilk_for _Cilk_for ///< Execute iterations of a `for` loop in parallel. +#endif + +/// @} diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/cilk_api.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/cilk_api.h new file mode 100644 index 0000000000..6cc62c994b --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/cilk_api.h @@ -0,0 +1,436 @@ +/* cilk_api.h + * + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + */ + +/** @file cilk_api.h + * + * @brief Defines the Intel(R) Cilk(TM) Plus API for use by applications. + * + * @ingroup api + */ + +#ifndef INCLUDED_CILK_API_H +#define INCLUDED_CILK_API_H + + +/** @defgroup api Runtime API +* API to interact with the Intel Cilk Plus runtime. +* @{ +*/ + +#ifndef CILK_STUB /* Real (non-stub) definitions */ + +#if ! defined(__cilk) && ! defined(USE_CILK_API) +# ifdef _WIN32 +# error Cilk API is being used with non-Cilk compiler (or Cilk is disabled) +# else +# warning Cilk API is being used with non-Cilk compiler (or Cilk is disabled) +# endif +#endif + +#include + +#ifdef __cplusplus +# include /* Defines size_t */ +#else +# include /* Defines size_t */ +#endif + +#ifdef _WIN32 +# ifndef IN_CILK_RUNTIME +/* Ensure the library is brought if any of these functions are being called. */ +# pragma comment(lib, "cilkrts") +# endif + +# ifndef __cplusplus +# include +# endif +#endif /* _WIN32 */ + +__CILKRTS_BEGIN_EXTERN_C + +/** Return values from `__cilkrts_set_param()` and `__cilkrts_set_param_w()` + */ +enum __cilkrts_set_param_status { + __CILKRTS_SET_PARAM_SUCCESS = 0, /**< Success - parameter set */ + __CILKRTS_SET_PARAM_UNIMP = 1, /**< Unimplemented parameter */ + __CILKRTS_SET_PARAM_XRANGE = 2, /**< Parameter value out of range */ + __CILKRTS_SET_PARAM_INVALID = 3, /**< Invalid parameter value */ + __CILKRTS_SET_PARAM_LATE = 4 /**< Too late to change parameter value */ +}; + +/** Sets user controllable runtime parameters + * + * Call this function to set runtime parameters that control the behavior + * of the Intel Cilk Plus scheduler. + * + * @param param A string specifying the parameter to be set. One of: + * - `"nworkers"` + * - `"force reduce"` + * @param value A string specifying the parameter value. + * @returns A value from the @ref __cilkrts_set_param_status + * enumeration indicating the result of the operation. + * + * @par The "nworkers" parameter + * + * This parameter specifies the number of worker threads to be created by the + * Intel Cilk Plus runtime. @a Value must be a string of digits to be parsed by + * `strtol()` as a decimal number. + * + * The number of worker threads is: + * 1. the value set with `__cilkrts_set_param("nworkers")`, if it is + * positive; otherwise, + * 2. the value of the CILK_NWORKERS environment variable, if it is + * defined; otherwise + * 3. the number of cores available, as reported by the operating system. + * + * @note + * Technically, Intel Cilk Plus distinguishes between the _user thread_ (the thread + * that the user code was executing on when the Intel Cilk Plus runtime started), + * and _worker threads_ (new threads created by the Intel Cilk Plus runtime to + * support Intel Cilk Plus parallelism). `nworkers` actually includes both the user + * thread and the worker threads; that is, it is one greater than the number of + * true "worker threads". + * + * @note + * Setting `nworkers = 1` produces serial behavior. Intel Cilk Plus spawns and syncs + * will be executed, but with only one worker, continuations will never be + * stolen, so all code will execute in serial. + * + * @warning + * The number of worker threads can only be set *before* the runtime has + * started. Attempting to set it when the runtime is running will have no + * effect, and will return an error code. You can call __cilkrts_end_cilk() + * to shut down the runtime to change the number of workers. + * + * @warning + * The default Intel Cilk scheduler behavior is usually pretty good. The + * ability to override `nworkers` can be useful for experimentation, but it + * won't usually be necessary for getting good performance. + * + * @par The "force reduce" parameter + * + * This parameter controls whether the runtime should allocate a new view + * for a reducer for every parallel strand that it is accessed on. (See + * @ref pagereducers.) @a Value must be `"1"` or `"true"` to enable the + * "force reduce" behavior, or `"0"` or `"false"` to disable it. + * + * "Force reduce" behavior will also be enabled if + * `__cilkrts_set_param("force reduce")` is not called, but the + * `CILK_FORCE_REDUCE` environment variable is defined. + * + * @warning + * When this option is enabled, `nworkers` should be set to `1`. Using "force + * reduce" with more than one worker may result in runtime errors. + * + * @warning + * Enabling this option can significantly reduce performance. Use it + * _only_ as a debugging tool. + */ +CILK_API(int) __cilkrts_set_param(const char *param, const char *value); + +#ifdef _WIN32 +/** + * Sets user controllable parameters using wide strings + * + * @note This variant of __cilkrts_set_param() is only available + * on Windows. + * + * @copydetails __cilkrts_set_param + */ +CILK_API(int) __cilkrts_set_param_w(const wchar_t *param, const wchar_t *value); +#endif + +/** Shuts down and deallocates all Intel Cilk Plus states. If Intel Cilk Plus is still in + * use by the calling thread, the runtime aborts the application. Otherwise, the + * runtime waits for all other threads using Intel Cilk Plus to exit. + */ +CILK_API(void) __cilkrts_end_cilk(void); + +/** Initializes Intel Cilk Plus data structures and start the runtime. + */ +CILK_API(void) __cilkrts_init(void); + +/** Returns the runtime `nworkers` parameter. (See the discussion of `nworkers` + * in the documentation for __cilkrts_set_param().) + */ +CILK_API(int) __cilkrts_get_nworkers(void); + +/** Returns the number of thread data structures. + * + * This function returns the number of data structures that have been allocated + * by the runtime to hold information about user and worker threads. + * + * If you don't already know what this is good for, then you probably don't + * need it. :) + */ +CILK_API(int) __cilkrts_get_total_workers(void); + +/** Returns a small integer identifying the current thread. + * + * What thread is the function running on? Each worker thread + * started by the Intel Cilk Plus runtime library has a unique worker number in the + * range `1 .. nworkers - 1`. + * + * All _user_ threads (threads started by the user, or by other libraries) are + * identified as worker number 0. Therefore, the worker number is not unique + * across multiple user threads. + */ +CILK_API(int) __cilkrts_get_worker_number(void); + +/** Tests whether "force reduce" behavior is enabled. + * + * @return Non-zero if force-reduce mode is on, zero if it is off. + */ +CILK_API(int) __cilkrts_get_force_reduce(void); + +/** Interacts with tools + */ +CILK_API(void) + __cilkrts_metacall(unsigned int tool, unsigned int code, void *data); + +#ifdef _WIN32 +/// Windows exception description record. +typedef struct _EXCEPTION_RECORD _EXCEPTION_RECORD; + +/** Function signature for Windows exception notification callbacks. + */ +typedef void (*__cilkrts_pfn_seh_callback)(const _EXCEPTION_RECORD *exception); + +/** Specifies a function to call when a non-C++ exception is caught. + * + * Intel Cilk Plus parallelism plays nicely with C++ exception handling, but + * the Intel Cilk Plus runtime has no way to unwind the stack across a strand + * boundary for Microsoft SEH ("Structured Exception Handling") exceptions. + * Therefore, when the runtime catches such an exception, it must abort the + * application. + * + * If an SEH callback has been set, the runtime will call it before aborting. + * + * @param pfn A pointer to a callback function to be called before the + * runtime aborts the program because of an SEH exception. + */ +CILK_API(int) __cilkrts_set_seh_callback(__cilkrts_pfn_seh_callback pfn); +#endif /* _WIN32 */ + +#if __CILKRTS_ABI_VERSION >= 1 +/* Pedigree API is available only for compilers that use ABI version >= 1. */ + + +/** @name Pedigrees + */ +//@{ + +// @cond internal + +/** Support for __cilkrts_get_pedigree. + */ +CILK_API(__cilkrts_pedigree) +__cilkrts_get_pedigree_internal(__cilkrts_worker *w); + +/** Support for __cilkrts_bump_worker_rank. + */ +CILK_API(int) +__cilkrts_bump_worker_rank_internal(__cilkrts_worker* w); + +/// @endcond + + +/** Gets the current pedigree in a linked list representation. + * + * This routine returns a copy of the last node in the pedigree list. + * For example, if the current pedigree (in order) is <1, 2, 3, 4>, + * then this method returns a node with rank == 4, and whose parent + * field points to the node with rank of 3. In summary, following the + * nodes in the chain visits the terms of the pedigree in reverse. + * + * The returned node is guaranteed to be valid only until the caller + * of this routine has returned. + */ +__CILKRTS_INLINE +__cilkrts_pedigree __cilkrts_get_pedigree(void) +{ + return __cilkrts_get_pedigree_internal(__cilkrts_get_tls_worker()); +} + +/** Context used by __cilkrts_get_pedigree_info. + * + * @deprecated + * This data structure is only used by the deprecated + * __cilkrts_get_pedigree_info function. + * + * Callers should initialize the `data` array to NULL and set the `size` + * field to `sizeof(__cilkrts_pedigree_context_t)` before the first call + * to `__cilkrts_get_pedigree_info()`. Also, callers should not examine or + * modify `data` thereafter. + */ +typedef struct +{ + __STDNS size_t size; /**< Size of the struct in bytes */ + void *data[3]; /**< Opaque context data */ +} __cilkrts_pedigree_context_t; + +/** Gets pedigree information. + * + * @deprecated + * Use __cilkrts_get_pedigree() instead. + * + * This routine allows code to walk up the stack of Intel Cilk Plus frames to gather + * the pedigree. + * + * Initialize the pedigree walk by filling the pedigree context with NULLs + * and setting the size field to `sizeof(__cilkrts_pedigree_context)`. + * Other than initialization to NULL to start the walk, user coder should + * consider the pedigree context data opaque and should not examine or + * modify it. + * + * @returns 0 - Success - birthrank is valid + * @returns >0 - End of pedigree walk + * @returns -1 - Failure - No worker bound to thread + * @returns -2 - Failure - Sanity check failed, + * @returns -3 - Failure - Invalid context size + * @returns -4 - Failure - Internal error - walked off end of chain of frames + */ +CILK_API(int) +__cilkrts_get_pedigree_info(/* In/Out */ __cilkrts_pedigree_context_t *context, + /* Out */ uint64_t *sf_birthrank); + +/** Gets the rank of the currently executing worker. + * + * @deprecated + * Use `__cilkrts_get_pedigree().rank` instead. + * + * @returns 0 - Success - *rank is valid + * @returns <0 - Failure - *rank is not changed + */ +CILK_EXPORT_AND_INLINE +int __cilkrts_get_worker_rank(uint64_t *rank) +{ + *rank = __cilkrts_get_pedigree().rank; + return 0; +} + +/** Increments the pedigree rank of the currently executing worker. + * + * @returns 0 - Success - rank was incremented + * @returns -1 - Failure + */ +CILK_EXPORT_AND_INLINE +int __cilkrts_bump_worker_rank(void) +{ + return __cilkrts_bump_worker_rank_internal(__cilkrts_get_tls_worker()); +} + +/** Increments the pedigree rank for a `cilk_for` loop. + * Obsolete. + * + * @deprecated + * This function was provided to allow the user to manipulate the pedigree + * rank of a `cilk_for` loop. The compiler now generates code to do that + * manipulation automatically, so this function is now unnecessary. It may + * be called, but will have no effect. + */ +CILK_EXPORT_AND_INLINE +int __cilkrts_bump_loop_rank(void) +{ + return 0; +} + +//@} + +#endif /* __CILKRTS_ABI_VERSION >= 1 */ + +__CILKRTS_END_EXTERN_C + +#else /* CILK_STUB */ + +// Programs compiled with CILK_STUB are not linked with the Intel Cilk Plus runtime +// library, so they should not have external references to runtime functions. +// Therefore, the functions are replaced with stubs. + +#ifdef _WIN32 +#define __cilkrts_set_param_w(name,value) ((value), 0) +#define __cilkrts_set_seh_callback(pfn) (0) +#endif +#define __cilkrts_set_param(name,value) ((value), 0) +#define __cilkrts_end_cilk() ((void) 0) +#define __cilkrts_init() ((void) 0) +#define __cilkrts_get_nworkers() (1) +#define __cilkrts_get_total_workers() (1) +#define __cilkrts_get_worker_number() (0) +#define __cilkrts_get_force_reduce() (0) +#define __cilkrts_metacall(tool,code,data) ((tool), (code), (data), 0) + +#if __CILKRTS_ABI_VERSION >= 1 +/* Pedigree stubs */ +#define __cilkrts_get_pedigree_info(context, sf_birthrank) (-1) +#define __cilkrts_get_worker_rank(rank) (*(rank) = 0) +#define __cilkrts_bump_worker_rank() (-1) +#define __cilkrts_bump_loop_rank() (-1) + +/* + * A stub method for __cilkrts_get_pedigree. + * Returns an empty __cilkrts_pedigree. + */ +__CILKRTS_INLINE +__cilkrts_pedigree __cilkrts_get_pedigree_stub(void) +{ + __cilkrts_pedigree ans; + ans.rank = 0; + ans.parent = NULL; + return ans; +} + +/* Renamed to an actual stub method. */ +#define __cilkrts_get_pedigree() __cilkrts_get_pedigree_stub() + +#endif /* __CILKRTS_ABI_VERSION >= 1 */ + +#endif /* CILK_STUB */ + +//@} + +#endif /* INCLUDED_CILK_API_H */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/cilk_api_linux.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/cilk_api_linux.h new file mode 100644 index 0000000000..0ebd57cba2 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/cilk_api_linux.h @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + * + */ + +/* THIS FILE IS DEPRECATED. USE cilk_api.h INSTEAD. */ +#include diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/cilk_stub.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/cilk_stub.h new file mode 100644 index 0000000000..b4a54f37c9 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/cilk_stub.h @@ -0,0 +1,71 @@ +/* cilk_stub.h -*-C++-*- + * + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + * + */ + +#ifndef INCLUDED_CILK_STUB_DOT_H +#define INCLUDED_CILK_STUB_DOT_H + +/* Definitions for creating a serialization from an Intel(R) Cilk(TM) Plus program. + * These definitions are suitable for use by a compiler that is not + * Cilk-enabled. + */ + +/* Pretend we are a non-Cilk compiler */ +#undef __cilk +#define CILK_STUB + +/* Replace Intel Cilk keywords with serial equivalents */ +#define _Cilk_spawn +#define _Cilk_sync +#define _Cilk_for for + +/* Replace simd-loop keywords with serial equivalents */ +#define _Simd +#define _Safelen(...) +#define _Reduction(...) + +#endif /* ! defined(INCLUDED_CILK_STUB_DOT_H) */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/cilk_undocumented.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/cilk_undocumented.h new file mode 100644 index 0000000000..71a51ec52d --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/cilk_undocumented.h @@ -0,0 +1,139 @@ +/* + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + * + ****************************************************************************** + * + * cilk_undocumented.h + * + * This file defines exported functions that are not included in the standard + * documentation. + */ + +#ifndef INCLUDED_CILK_UNDOCUMENTED_H +#define INCLUDED_CILK_UNDOCUMENTED_H + +#include + +#ifndef CILK_STUB + +__CILKRTS_BEGIN_EXTERN_C + +/* + * __cilkrts_synched + * + * Allows an application to determine if there are any outstanding children at + * this instant. This function will examine the current full frame to + * determine this. This function will return a valid result only when called + * within a spawn continuation, within the stack frame of the continuation + * itself. + */ + +CILK_EXPORT __CILKRTS_NOTHROW +int __cilkrts_synched(void); + +/* + * __cilkrts_cilkscreen_puts + * + * Allows an application to write a string to the Cilkscreen log. + * The standard error stream will be flushed after the write. + */ + +CILK_EXPORT __CILKRTS_NOTHROW +void __cilkrts_cilkscreen_puts(const char *); + +/* + * __cilkrts_get_sf + * + * A debugging aid that allows an application to get the __cilkrts_stack_frame + * for the current function. Only compiled into the DLL in debug builds. + */ + +CILK_EXPORT __CILKRTS_NOTHROW +void *__cilkrts_get_sf(void); + +/** + * Returns the size of stacks created by Intel(R) Cilk(TM) Plus. + */ +CILK_EXPORT __CILKRTS_NOTHROW +size_t __cilkrts_get_stack_size(void); + +/** + * Dumps runtime statistics to stderr. + * Undocumented API for debugging. + */ +CILK_EXPORT __CILKRTS_NOTHROW +void __cilkrts_dump_stats(void); + +struct __cilk_tbb_unwatch_thunk; +struct __cilk_tbb_stack_op_thunk; + +CILK_EXPORT __CILKRTS_NOTHROW +int __cilkrts_watch_stack(struct __cilk_tbb_unwatch_thunk *u, + struct __cilk_tbb_stack_op_thunk o); + +#ifndef IN_CILK_RUNTIME +#ifdef _WIN32 +/* Do not use CILK_API because __cilkrts_worker_stub must be __stdcall */ +CILK_EXPORT unsigned __CILKRTS_NOTHROW __stdcall +__cilkrts_worker_stub(void *arg); +#else +/* Do not use CILK_API because __cilkrts_worker_stub have default visibility */ +CILK_EXPORT void* __CILKRTS_NOTHROW +__cilkrts_worker_stub(void *arg); +#endif /* _WIN32 */ +#endif /* IN_CILK_RUNTIME */ + +__CILKRTS_END_EXTERN_C + +#else /* CILK_STUB */ + +/* Stubs for the api functions */ + +#define __cilkrts_get_stack_size() (0) +#define __cilkrts_synched() (1) + +#endif /* CILK_STUB */ + +#endif /* INCLUDED_CILK_UNDOCUMENTED_H */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/common.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/common.h new file mode 100644 index 0000000000..91b2928e7e --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/common.h @@ -0,0 +1,385 @@ +/** common.h + * + * Copyright (C) 2010-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + */ + +/** @file common.h + * + * @brief Defines common macros and structures used by the Intel(R) Cilk(TM) Plus runtime. + * + * @ingroup common + */ + +/** @defgroup common Common Definitions + * Definitions for runtime macros, structures, and classes. + * @{ + */ + +#ifndef INCLUDED_CILK_COMMON +#define INCLUDED_CILK_COMMON + +#ifdef __cplusplus +/** Namespace for all Intel Cilk Plus definitions that can be included in user code. + */ +namespace cilk { + + /** Namespace for definitions re-used in other Intel Cilk Plus definitions. + */ + namespace internal {} +} +#endif + +/** Intel Cilk Plus library version = 1.02 + */ +#define CILK_LIBRARY_VERSION 102 + +#ifdef __cplusplus +# include +#else +# include +#endif + +/** + * Prefix standard library function and type names with __STDNS to + * get correct lookup in both C and C++. + */ +#ifdef __cplusplus +# define __STDNS std:: +#else +# define __STDNS +#endif + +/** + * @def CILK_EXPORT + * Define export of runtime functions from shared library. + * Should be exported only from cilkrts*.dll/cilkrts*.so + * @def CILK_EXPORT_DATA + * Define export of runtime data from shared library. + */ +#ifdef _WIN32 +# ifdef IN_CILK_RUNTIME +# define CILK_EXPORT __declspec(dllexport) +# define CILK_EXPORT_DATA __declspec(dllexport) +# else +# define CILK_EXPORT __declspec(dllimport) +# define CILK_EXPORT_DATA __declspec(dllimport) +# endif /* IN_CILK_RUNTIME */ +#elif defined(__CYGWIN__) || defined(__APPLE__) || defined(_DARWIN_C_SOURCE) +# define CILK_EXPORT /* nothing */ +# define CILK_EXPORT_DATA /* nothing */ +#else /* Unix/gcc */ +# if defined(IN_CILK_RUNTIME) && defined(HAVE_ATTRIBUTE_VISIBILITY) +# define CILK_EXPORT __attribute__((visibility("protected"))) +# define CILK_EXPORT_DATA __attribute__((visibility("protected"))) +# else +# define CILK_EXPORT /* nothing */ +# define CILK_EXPORT_DATA /* nothing */ +# endif /* IN_CILK_RUNTIME */ +#endif /* Unix/gcc */ + +/** + * @def __CILKRTS_BEGIN_EXTERN_C + * Macro to denote the start of a section in which all names have "C" linkage. + * That is, none of the names are to be mangled. + * @see __CILKRTS_END_EXTERN_C + * @see __CILKRTS_EXTERN_C + * + * @def __CILKRTS_END_EXTERN_C + * Macro to denote the end of a section in which all names have "C" linkage. + * That is, none of the names are to be mangled. + * @see __CILKRTS_BEGIN_EXTERN_C + * @see __CILKRTS_EXTERN_C + * + * @def __CILKRTS_EXTERN_C + * Macro to prefix a single definition which has "C" linkage. + * That is, the defined name is not to be mangled. + * @see __CILKRTS_BEGIN_EXTERN_C + * @see __CILKRTS_END_EXTERN_C + */ +#ifdef __cplusplus +# define __CILKRTS_BEGIN_EXTERN_C extern "C" { +# define __CILKRTS_END_EXTERN_C } +# define __CILKRTS_EXTERN_C extern "C" +#else +# define __CILKRTS_BEGIN_EXTERN_C +# define __CILKRTS_END_EXTERN_C +# define __CILKRTS_EXTERN_C +#endif + +/** + * OS-independent macro to specify a function which is known to not throw + * an exception. + */ +#ifdef __cplusplus +# ifdef _WIN32 +# define __CILKRTS_NOTHROW __declspec(nothrow) +# else /* Unix/gcc */ +# define __CILKRTS_NOTHROW __attribute__((nothrow)) +# endif /* Unix/gcc */ +#else +# define __CILKRTS_NOTHROW /* nothing */ +#endif /* __cplusplus */ + +/** Cache alignment. (Good enough for most architectures.) + */ +#define __CILKRTS_CACHE_LINE__ 64 + +/** + * Macro to specify alignment of a data member in a structure. + * Because of the way that gcc's alignment attribute is defined, @a n must + * be a numeric literal, not just a compile-time constant expression. + */ +#ifdef _WIN32 +# define CILK_ALIGNAS(n) __declspec(align(n)) +#else /* Unix/gcc */ +# define CILK_ALIGNAS(n) __attribute__((__aligned__(n))) +#endif + +/** + * Macro to specify cache-line alignment of a data member in a structure. + */ +#define __CILKRTS_CACHE_ALIGN CILK_ALIGNAS(__CILKRTS_CACHE_LINE__) + +/** + * Macro to specify a class as being at least as strictly aligned as some + * type on Windows. gcc does not provide a way of doing this, so on Unix, + * this just specifies the largest natural type alignment. Put the macro + * between the `class` keyword and the class name: + * + * class CILK_ALIGNAS_TYPE(foo) bar { ... }; + */ +#ifdef _WIN32 +# define CILK_ALIGNAS_TYPE(t) __declspec(align(__alignof(t))) +#else /* Unix/gcc */ +# define CILK_ALIGNAS_TYPE(t) __attribute__((__aligned__)) +#endif + +/** + * @def CILK_API(RET_TYPE) + * A function called explicitly by the programmer. + * @def CILK_ABI(RET_TYPE) + * A function called by compiler-generated code. + * @def CILK_ABI_THROWS(RET_TYPE) + * An ABI function that may throw an exception + * + * Even when these are the same definitions, they should be separate macros so + * that they can be easily found in the code. + */ + +#ifdef _WIN32 +# define CILK_API(RET_TYPE) CILK_EXPORT RET_TYPE __CILKRTS_NOTHROW __cdecl +# define CILK_ABI(RET_TYPE) CILK_EXPORT RET_TYPE __CILKRTS_NOTHROW __cdecl +# define CILK_ABI_THROWS(RET_TYPE) CILK_EXPORT RET_TYPE __cdecl +#else +# define CILK_API(RET_TYPE) CILK_EXPORT RET_TYPE __CILKRTS_NOTHROW +# define CILK_ABI(RET_TYPE) CILK_EXPORT RET_TYPE __CILKRTS_NOTHROW +# define CILK_ABI_THROWS(RET_TYPE) CILK_EXPORT RET_TYPE +#endif + +/** + * __CILKRTS_ASSERT should be defined for debugging only, otherwise it + * interferes with vectorization. Since NDEBUG is not reliable (it must be + * set by the user), we must use a platform-specific detection of debug mode. + */ +#if defined(_WIN32) && defined(_DEBUG) + /* Windows debug */ +# define __CILKRTS_ASSERT(e) assert(e) +#elif (! defined(_WIN32)) && ! defined(__OPTIMIZE__) + /* Unix non-optimized */ +# define __CILKRTS_ASSERT(e) assert(e) +#elif defined __cplusplus + /* C++ non-debug */ +# define __CILKRTS_ASSERT(e) static_cast(0) +#else + /* C non-debug */ +# define __CILKRTS_ASSERT(e) ((void) 0) +#endif + +/** + * OS-independent macro to specify a function that should be inlined + */ +#ifdef __cplusplus + // C++ +# define __CILKRTS_INLINE inline +#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + // C99 +# define __CILKRTS_INLINE static inline +#elif defined(_MSC_VER) + // C89 on Windows +# define __CILKRTS_INLINE __inline +#else + // C89 on GCC-compatible systems +# define __CILKRTS_INLINE extern __inline__ +#endif + +/** + * Functions marked as CILK_EXPORT_AND_INLINE have both + * inline versions defined in the Intel Cilk Plus API, as well as + * non-inlined versions that are exported (for + * compatibility with previous versions that did not + * inline the functions). + */ +#ifdef COMPILING_CILK_API_FUNCTIONS +# define CILK_EXPORT_AND_INLINE CILK_EXPORT +#else +# define CILK_EXPORT_AND_INLINE __CILKRTS_INLINE +#endif + +/** + * Try to determine if compiler supports rvalue references. + */ +#if defined(__cplusplus) && !defined(__CILKRTS_RVALUE_REFERENCES) +# if __cplusplus >= 201103L // C++11 +# define __CILKRTS_RVALUE_REFERENCES 1 +# elif defined(__GXX_EXPERIMENTAL_CXX0X__) +# define __CILKRTS_RVALUE_REFERENCES 1 +# elif __cplusplus >= 199711L && __cplusplus < 201103L + // Compiler recognizes a language version prior to C++11 +# elif __INTEL_COMPILER == 1200 && defined(__STDC_HOSTED__) + // Intel compiler version 12.0 + // __cplusplus has a non-standard definition. In the absence of a + // proper definition, look for the C++0x macro, __STDC_HOSTED__. +# define __CILKRTS_RVALUE_REFERENCES 1 +# elif __INTEL_COMPILER > 1200 && defined(CHAR16T) + // Intel compiler version >= 12.1 + // __cplusplus has a non-standard definition. In the absence of a + // proper definition, look for the Intel macro, CHAR16T +# define __CILKRTS_RVALUE_REFERENCES 1 +# endif +#endif + +/* + * Include stdint.h to define the standard integer types. + * + * Unfortunately Microsoft doesn't provide stdint.h until Visual Studio 2010, + * so use our own definitions until those are available + */ + +#if ! defined(_MSC_VER) || (_MSC_VER >= 1600) +# include +#else +# ifndef __MS_STDINT_TYPES_DEFINED__ +# define __MS_STDINT_TYPES_DEFINED__ + typedef signed char int8_t; + typedef short int16_t; + typedef int int32_t; + typedef __int64 int64_t; + + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; + typedef unsigned __int64 uint64_t; +# endif /* __MS_STDINT_TYPES_DEFINED__ */ +#endif /* ! defined(_MSC_VER) || (_MSC_VER >= 1600) */ + +/** + * @brief Application Binary Interface (ABI) version of the Intel Cilk Plus runtime + * library. + * + * The compiler determines the ABI version used for compilation. Object files + * compiled with higher ABI versions are not compatible with libraries compiled + * with lower ABI versions. However, an object file compiled with a lower ABI + * version can be used with a library compiled with a higher ABI version + * (unless otherwise stated.) + */ +#ifndef __CILKRTS_ABI_VERSION +# ifdef IN_CILK_RUNTIME +# define __CILKRTS_ABI_VERSION 1 +# elif defined(__INTEL_COMPILER) && (__INTEL_COMPILER <= 1200) + // Intel compilers prior to version 12.1 support only ABI 0 +# define __CILKRTS_ABI_VERSION 0 +# else + // Non-Intel compiler or Intel compiler after version 12.0. +# define __CILKRTS_ABI_VERSION 1 +# endif +#endif + +// These structs are exported because the inlining of +// the internal version of API methods require a worker +// structure as parameter. +__CILKRTS_BEGIN_EXTERN_C + /// Worker struct, exported for inlined API methods + /// @ingroup api + struct __cilkrts_worker; + + /// Worker struct, exported for inlined API methods + /// @ingroup api + typedef struct __cilkrts_worker __cilkrts_worker; + + /// Worker struct pointer, exported for inlined API methods + /// @ingroup api + typedef struct __cilkrts_worker *__cilkrts_worker_ptr; + + + /// Fetch the worker out of TLS. + CILK_ABI(__cilkrts_worker_ptr) __cilkrts_get_tls_worker(void); + + /// void *, defined to work around complaints from the compiler + /// about using __declspec(nothrow) after the "void *" return type + typedef void * __cilkrts_void_ptr; + +__CILKRTS_END_EXTERN_C + + +#if __CILKRTS_ABI_VERSION >= 1 +// Pedigree API is available only for compilers that use ABI version >= 1. + +/** Pedigree information kept in the worker and stack frame. + * @ingroup api + */ +typedef struct __cilkrts_pedigree +{ + /** Rank at start of spawn helper. Saved rank for spawning functions */ + uint64_t rank; + + /** Link to next in chain */ + const struct __cilkrts_pedigree *parent; +} __cilkrts_pedigree; + +#endif // __CILKRTS_ABI_VERSION >= 1 + +/// @} + +#endif /* INCLUDED_CILK_COMMON */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/holder.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/holder.h new file mode 100644 index 0000000000..66899a25bc --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/holder.h @@ -0,0 +1,1011 @@ +/* + * Copyright (C) 2011-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + * + */ + +/* + * holder.h + * + * Purpose: hyperobject to provide different views of an object to each + * parallel strand. + */ + +#ifndef HOLDER_H_INCLUDED +#define HOLDER_H_INCLUDED + +#include +#include +#include + +#ifdef __cplusplus + +/* C++ Interface + * + * Classes: holder + * + * Description: + * ============ + * This component provides a hyperobject that isolates a parallel uses of a + * common variable where it is not necessary to preserve changes from + * different parallel strands. In effect, a holder acts a bit like + * thread-local storage, but has qualities that work better with the + * fork-join structure of Intel(R) Cilk(TM) Plus. In particular, a holder has the + * following qualities: + * + * - The view of a holder before the first spawn within a function is the same + * as the view after each sync (as in the case of a reducer). + * - The view of a holder within the first spawned child of a function (or the + * first child spawned after a sync) is the same as the view on entry to the + * function. + * - The view of a holder before entering a _Cilk_for loop is the same as the + * view during the first iteration of the loop and the view at the end of + * the loop. + * - The view of a holder in the continuation of a spawn or in an arbitrary + * iteration of a _Cilk_for loop is *non-deterministic*. It is generally + * recommended that the holder be explicitly put into a known state in these + * situations. + * + * A holder can be used as an alternative to parameter-passing. They are most + * useful for replacing non-local variables without massive refactoring. A + * holder takes advantage of the fact that, most of the time, a holder view + * does not change after a spawn or from one iteration of a parallel for loop + * to the next (i.e., stealing is the exception, not the rule). When the + * holder view is a large object that is expensive to construct, this + * optimization can save significant time versus creating a separate local + * object for each view. In addition, a holder using the "keep last" policy + * will have the same value after a sync as the serialization of the same + * program. The last quality will often allow the program to avoid + * recomputing a value. + * + * Usage Example: + * ============== + * Function 'compute()' is a complex function that computes a value using a + * memoized algorithm, storing intermediate results in a hash table. Compute + * calls several other functions, each of which calls several other functions, + * all of which share a global hash table. In all, there are over a dozen + * functions with a total of about 60 references to the hash table. + *.. + * hash_table memos; + * + * void h(const X& x); // Uses memos + * + * double compute(const X& x) + * { + * memos.clear(); + * // ... + * memos[i] = x; + * ... + * g(i); // Uses memos + * // ... + * std::for_each(c.begin(), c.end(), h); // Call h for each element of c + * } + * + * int main() + * { + * const std::size_t ARRAY_SIZE = 1000000; + * extern X myArray[ARRAY_SIZE]; + * + * for (std::size_t i = 0; i < ARRAY_SIZE; ++i) + * { + * compute(myArray[i]); + * } + * } + *.. + * We would like to replace the 'for' loop in 'main' with a 'cilk_for'. + * Although the hash table is cleared on entry to each call to 'compute()', + * and although the values stored in the hash table are no longer used after + * 'compute()' returns, the use of the hash table as a global variable + * prevents 'compute()' from being called safely in parallel. One way to do + * this would be to make 'memos' a private variable within the cilk_for loop + * and pass it down to the actual computation, so that each loop iteration has + * its own private copy: + *.. + * cilk_for (std::size_t i = 0; i < ARRAY_SIZE; ++i) + * { + * hash_table memos; + * compute(myArray[i], memos); + * } + *.. + * The problem with this approach is that it requires changing the signature + * of 'compute', 'h', 'g', and every one of the dozen or so functions that + * reference 'memos' as well as any function that calls those functions. This + * may break the abstraction of 'compute' and other functions, exposing an + * implementation detail that was not part of the interface. In addition, the + * function 'h' is called through a templated algorithm, 'for_each', which + * requires a fixed interface. Finally, there is constructor and destructor + * overhead for 'hash_table' each time through the loop. + * + * The alternative approach is to replace 'memos' with a holder. The holder + * would be available to all of the functions involved, but would not cause a + * race between parallel loop iterations. In order to make this work, each + * use of the 'memos' variable must be (mechanically) replaced by a use of the + * holder: + *.. + * cilk::holder > memos_h; + * + * void h(const X& x); // Uses memos_h + * + * double compute(const X& x) + * { + * memos_h().clear(); // operator() used to "dereference" the holder + * // ... + * memos_h()[i] = x; // operator() used to "dereference" the holder + * ... + * g(i); // Uses memos_h + * // ... + * std::for_each(c.begin(), c.end(), h); // Call h for each element of c + * } + *.. + * Note that each reference to the holder must be modified with an empty pair + * of parenthesis. This syntax is needed because there is no facility in C++ + * for a "smart reference" that would allow 'memos_h' to be a perfect + * replacement for 'memos'. One way that a user can avoid this syntax change + * is to wrap the holder in a class that has the same inteface as + * 'hash_table' but redirects all calls to the holder: + *.. + * template + * class hash_table_holder + * { + * private: + * cilk::holder > m_holder; + * public: + * void clear() { m_holder().clear(); } + * V& operator[](const K& x) { return m_holder()[x]; } + * std::size_t size() const { return m_holder().size(); } + * // etc. ... + * }; + *.. + * Using the above wrapper, the original code can be left unchanged except for + * replacing 'hash_table' with 'hash_table_holder' and replacing 'for' with + * 'cilk_for': + *.. + * hash_table_holder memos; + * + * void h(const X& x); // Uses memos + * + * double compute(const X& x) + * { + * memos.clear(); // Calls hash_table_holder::clear(). + * // ... + * } + *.. + * The above changes have no benefit over the use of thread-local storage. + * What if one of the functions has a 'cilk_spawn', however? + *.. + * void h(const X& x) + * { + * Y y = x.nested(); + * double d, w; + * if (y) + * { + * w = cilk_spawn compute_width(y); // May use 'memos' + * d = compute_depth(y); // Does not use 'memos' + * cilk_sync; + * compute(y); // recursive call. Uses 'memos'. + * } + * } + *.. + * In the above example, the view of the holder within 'compute_width' is the + * same as the view on entry to 'h'. More importantly, the view of the holder + * within the recursive call to 'compute' is the same as the view on entry to + * 'h', even if a different worker is executing the recursive call. Thus, the + * holder view within a Intel Cilk Plus program has useful qualities not found in + * thread-local storage. + */ + +namespace cilk { + + /** + * After a sync, the value stored in a holder matches the most recent + * value stored into the holder by one of the starnds entering the sync. + * The holder policy used to instantiate the holder determines which of + * the entering strands determines the final value of the holder. A policy + * of 'holder_keep_indeterminate' (the default) is the most efficient, and + * results in an indeterminate value depending on the runtime schedule + * (see below for more specifics). An indeterminate value after a sync is + * often acceptable, especially if the value of the holder is not reused + * after the sync. All of the remaining policies retain the value of the + * last strand that would be executed in the serialization of the program. + * They differ in the mechanism used to move the value from one view to + * another. A policy of 'holder_keep_last_copy' moves values by + * copy-assignment. A policy of 'holder_keep_last_swap' moves values by + * calling 'swap'. A policy of 'holder_keep_last_move' is available only + * for compilers that support C++0x rvalue references and moves values by + * move-assignment. A policy of 'holder_keep_last' attempts to choose the + * most efficient mechanism: member-function 'swap' if the view type + * supports it, otherwise move-assignment if supported, otherwise + * copy-assignment. (The swap member function for a class that provides + * one is almost always as fast or faster than move-assignment or + * copy-assignment.) + * + * The behavior of 'holder_keep_indeterminate', while indeterminate, is + * not random and can be used for advanced programming or debugging. With + * a policy of 'holder_keep_intermediate', values are never copied or + * moved between views. The value of the view after a sync is the same as + * the value set in the last spawned child before a steal occurs or the + * last value set in the continuation if no steal occurs. Using this + * knowledge, a programmer can use a holder to detect the earliest steal + * in a piece of code. An indeterminate holder is also useful for keeping + * cached data similar to the way some applications might use thread-local + * storage. + */ + enum holder_policy { + holder_keep_indeterminate, + holder_keep_last, + holder_keep_last_copy, + holder_keep_last_swap, +#ifdef __CILKRTS_RVALUE_REFERENCES + holder_keep_last_move +#endif + }; + + namespace internal { + + // Private special-case holder policy using the swap member-function + const holder_policy holder_keep_last_member_swap = + (holder_policy) (holder_keep_last_swap | 0x10); + + /* The constant, 'has_member_swap::value', will be 'true' if 'T' + * has a non-static member function with prototype 'void swap(T&)'. + * The mechanism used to detect 'swap' is the most portable among + * present-day compilers, but is not the most robust. Specifically, + * the prototype for 'swap' must exactly match 'void swap(T&)'. + * Near-matches like a 'swap' function that returns 'int' instead of + * 'void' will not be detected. Detection will also fail if 'T' + * inherits 'swap' from a base class. + */ + template + class has_member_swap + { + // This technique for detecting member functions was described by + // Rani Sharoni in comp.lang.c++.moderated: + // http://groups.google.com/group/comp.lang.c++.moderated/msg/2b06b2432fddfb60 + + // sizeof(notchar) is guaranteed larger than 1 + struct notchar { char x[2]; }; + + // Instantiationg Q will fail unless U contains a + // non-static member with prototype 'void swap(U&)'. + template struct Q { }; + + // First 'test' is preferred overload if U::swap exists with the + // correct prototype. Second 'test' is preferred overload + // otherwise. + template static char test(Q*); + template static notchar test(...); + + public: + /// 'value' will be true if T has a non-static member function + /// with prototype 'void swap(T&)'. + static const bool value = (1 == sizeof(test(0))); + }; + + template const bool has_member_swap::value; + + /** + * @brief Utility class for exception safety. + * + * The constuctor for this class takes a pointer and an allocator and + * holds on to them. The destructor deallocates the pointed-to + * object, without calling its destructor, typically to recover memory + * in case an exception is thrown. The release member clears the + * pointer so that the deallocation is prevented, i.e., when the + * exception danger has passed. The behavior of this class is similar + * to auto_ptr and unique_ptr. + */ + template > + class auto_deallocator + { + Allocator m_alloc; + Type* m_ptr; + + // Non-copiable + auto_deallocator(const auto_deallocator&); + auto_deallocator& operator=(const auto_deallocator&); + + public: + /// Constructor + explicit auto_deallocator(Type* p, const Allocator& a = Allocator()) + : m_alloc(a), m_ptr(p) { } + + /// Destructor - free allocated resources + ~auto_deallocator() { if (m_ptr) m_alloc.deallocate(m_ptr, 1); } + + /// Remove reference to resource + void release() { m_ptr = 0; } + }; + + /** + * Pure-abstract base class to initialize holder views + */ + template + class init_base + { + public: + virtual ~init_base() { } + virtual init_base* clone_self(Allocator& a) const = 0; + virtual void delete_self(Allocator& a) = 0; + virtual void construct_view(Type* p, Allocator& a) const = 0; + }; + + /** + * Class to default-initialize a holder view + */ + template + class default_init : public init_base + { + typedef init_base base; + + /// Private constructor (called from static make() function). + default_init() { } + + // Non-copiable + default_init(const default_init&); + default_init& operator=(const default_init&); + + public: + // Static factory function + static default_init* make(Allocator& a); + + // Virtual function overrides + virtual ~default_init(); + virtual base* clone_self(Allocator& a) const; + virtual void delete_self(Allocator& a); + virtual void construct_view(Type* p, Allocator& a) const; + }; + + template + default_init* + default_init::make(Allocator&) + { + // Return a pointer to a singleton. All instances of this class + // are identical, so we need only one. + static default_init self; + return &self; + } + + template + default_init::~default_init() + { + } + + template + init_base* + default_init::clone_self(Allocator& a) const + { + return make(a); + } + + template + void default_init::delete_self(Allocator&) + { + // Since make() returned a shared singleton, there is nothing to + // delete here. + } + + template + void + default_init::construct_view(Type* p, + Allocator&) const + { + ::new((void*) p) Type(); + // TBD: In a C++0x library, this should be rewritten + // std::allocator_traits::construct(a, p); + } + + /** + * Class to copy-construct a view from a stored exemplar. + */ + template + class exemplar_init : public init_base + { + typedef init_base base; + + Type* m_exemplar; + + // Private constructors (called from make() functions). + exemplar_init(const Type& val, Allocator& a); +#ifdef __CILKRTS_RVALUE_REFERENCES + exemplar_init(Type&& val, Allocator& a); +#endif + + // Non-copyiable + exemplar_init(const exemplar_init&); + exemplar_init& operator=(const exemplar_init&); + + public: + // Static factory functions + static exemplar_init* make(const Type& val, + Allocator& a = Allocator()); +#ifdef __CILKRTS_RVALUE_REFERENCES + static exemplar_init* make(Type&& val, + Allocator& a = Allocator()); +#endif + + // Virtual function overrides + virtual ~exemplar_init(); + virtual base* clone_self(Allocator& a) const; + virtual void delete_self(Allocator& a); + virtual void construct_view(Type* p, Allocator& a) const; + }; + + template + exemplar_init::exemplar_init(const Type& val, + Allocator& a) + { + m_exemplar = a.allocate(1); + auto_deallocator guard(m_exemplar, a); + a.construct(m_exemplar, val); + guard.release(); + } + +#ifdef __CILKRTS_RVALUE_REFERENCES + template + exemplar_init::exemplar_init(Type&& val, + Allocator& a) + { + m_exemplar = a.allocate(1); + auto_deallocator guard(m_exemplar, a); + a.construct(m_exemplar, std::forward(val)); + guard.release(); + } +#endif + + template + exemplar_init* + exemplar_init::make(const Type& val, + Allocator& a) + { + typedef typename Allocator::template rebind::other + self_alloc_t; + self_alloc_t alloc(a); + + exemplar_init *self = alloc.allocate(1); + auto_deallocator guard(self, alloc); + + // Don't use allocator to construct self. Allocator should be + // used only on elements of type 'Type'. + ::new((void*) self) exemplar_init(val, a); + + guard.release(); + + return self; + } + +#ifdef __CILKRTS_RVALUE_REFERENCES + template + exemplar_init* + exemplar_init::make(Type&& val, + Allocator& a) + { + typedef typename Allocator::template rebind::other + self_alloc_t; + self_alloc_t alloc(a); + + exemplar_init *self = alloc.allocate(1); + auto_deallocator guard(self, alloc); + + // Don't use allocator to construct self. Allocator should be + // used only on elements of type 'Type'. + ::new((void*) self) exemplar_init(std::forward(val), a); + + guard.release(); + + return self; + } +#endif + + template + exemplar_init::~exemplar_init() + { + // Called only by delete_self, which deleted the exemplar using an + // allocator. + __CILKRTS_ASSERT(0 == m_exemplar); + } + + template + init_base* + exemplar_init::clone_self(Allocator& a) const + { + return make(*m_exemplar, a); + } + + template + void exemplar_init::delete_self(Allocator& a) + { + typename Allocator::template rebind::other alloc(a); + + a.destroy(m_exemplar); + a.deallocate(m_exemplar, 1); + m_exemplar = 0; + + this->~exemplar_init(); + alloc.deallocate(this, 1); + } + + template + void + exemplar_init::construct_view(Type* p, + Allocator& a) const + { + a.construct(p, *m_exemplar); + // TBD: In a C++0x library, this should be rewritten + // std::allocator_traits::construct(a, p, *m_exemplar); + } + + /** + * Class to construct a view using a stored functor. The functor, + * 'f', must be be invokable using the expression 'Type x = f()'. + */ + template + class functor_init : + public init_base + { + typedef typename Allocator::value_type value_type; + typedef init_base base; + typedef typename Allocator::template rebind::other f_alloc; + + Func *m_functor; + + /// Private constructors (called from make() functions + functor_init(const Func& f, Allocator& a); +#ifdef __CILKRTS_RVALUE_REFERENCES + functor_init(Func&& f, Allocator& a); +#endif + + // Non-copiable + functor_init(const functor_init&); + functor_init& operator=(const functor_init&); + + public: + // Static factory functions + static functor_init* make(const Func& val, + Allocator& a = Allocator()); +#ifdef __CILKRTS_RVALUE_REFERENCES + static functor_init* make(Func&& val, + Allocator& a = Allocator()); +#endif + + // Virtual function overrides + virtual ~functor_init(); + virtual base* clone_self(Allocator& a) const; + virtual void delete_self(Allocator& a); + virtual void + construct_view(value_type* p, Allocator& a) const; + }; + + /// Specialization to strip off reference from 'Func&'. + template + struct functor_init + : functor_init { }; + + /// Specialization to strip off reference and cvq from 'const Func&'. + template + struct functor_init + : functor_init { }; + + template + functor_init::functor_init(const Func& f, + Allocator& a) + { + f_alloc alloc(a); + + m_functor = alloc.allocate(1); + auto_deallocator guard(m_functor, alloc); + alloc.construct(m_functor, f); + guard.release(); + } + +#ifdef __CILKRTS_RVALUE_REFERENCES + template + functor_init::functor_init(Func&& f, + Allocator& a) + { + f_alloc alloc(a); + + m_functor = alloc.allocate(1); + auto_deallocator guard(m_functor, alloc); + alloc.construct(m_functor, std::forward(f)); + guard.release(); + } +#endif + + template + functor_init* + functor_init::make(const Func& f, Allocator& a) + { + typedef typename Allocator::template rebind::other + self_alloc_t; + self_alloc_t alloc(a); + + functor_init *self = alloc.allocate(1); + auto_deallocator guard(self, alloc); + + // Don't use allocator to construct self. Allocator should be + // used only on elements of type 'Func'. + ::new((void*) self) functor_init(f, a); + + guard.release(); + + return self; + } + +#ifdef __CILKRTS_RVALUE_REFERENCES + template + functor_init* + functor_init::make(Func&& f, Allocator& a) + { + typedef typename Allocator::template rebind::other + self_alloc_t; + self_alloc_t alloc(a); + + functor_init *self = alloc.allocate(1); + auto_deallocator guard(self, alloc); + + // Don't use allocator to construct self. Allocator should be + // used only on elements of type 'Func'. + ::new((void*) self) functor_init(std::forward(f), a); + + guard.release(); + + return self; + } +#endif + + template + functor_init::~functor_init() + { + // Called only by delete_self, which deleted the functor using an + // allocator. + __CILKRTS_ASSERT(0 == m_functor); + } + + template + init_base* + functor_init::clone_self(Allocator& a) const + { + return make(*m_functor, a); + } + + template + inline + void functor_init::delete_self(Allocator& a) + { + typename Allocator::template rebind::other alloc(a); + f_alloc fa(a); + + fa.destroy(m_functor); + fa.deallocate(m_functor, 1); + m_functor = 0; + + this->~functor_init(); + alloc.deallocate(this, 1); + } + + template + void functor_init::construct_view(value_type* p, + Allocator& a) const + { + a.construct(p, (*m_functor)()); + // In C++0x, the above should be written + // std::allocator_traits::construct(a, p, m_functor()); + } + + /** + * Functor called to reduce a holder + */ + template + struct holder_reduce_functor; + + /** + * Specialization to keep the left (first) value. + */ + template + struct holder_reduce_functor + { + void operator()(Type* left, Type* right) const { } + }; + + /** + * Specialization to copy-assign from the right (last) value. + */ + template + struct holder_reduce_functor + { + void operator()(Type* left, Type* right) const { + *left = *right; + } + }; + + /* + * Specialization to keep the right (last) value via swap. + */ + template + struct holder_reduce_functor + { + void operator()(Type* left, Type* right) const { + using std::swap; + swap(*left, *right); + } + }; + +#ifdef __CILKRTS_RVALUE_REFERENCES + /* + * Specialization to move-assign from the right (last) value. + */ + template + struct holder_reduce_functor + { + void operator()(Type* left, Type* right) const { + *left = std::move(*right); + } + }; +#endif + + /* + * Specialization to keep the right (last) value via the swap member + * function. + */ + template + struct holder_reduce_functor + { + void operator()(Type* left, Type* right) const { + left->swap(*right); + } + }; + + /* + * Specialization to keep the right (last) value by the most efficient + * means detectable. + */ + template + struct holder_reduce_functor : + holder_reduce_functor::value ? + holder_keep_last_member_swap : +#ifdef __CILKRTS_RVALUE_REFERENCES + holder_keep_last_move +#else + holder_keep_last_copy +#endif + )> + { + }; + } // end namespace internal + + /** + * Monoid for holders. + * Allocator type is required to be thread-safe. + */ + template > + class holder_monoid : public monoid_base + { + // Allocator is mutable because the copy of the monoid inside the + // reducer is const (to avoid races on the shared state). However, + // the allocator is required to be thread-safe, so it is ok (and + // necessary) to modify. + mutable Allocator m_allocator; + internal::init_base *m_initializer; + + public: + /// This constructor uses default-initialization for both the leftmost + /// view and each identity view. + holder_monoid(const Allocator& a = Allocator()) + : m_allocator(a) + , m_initializer( + internal::default_init::make(m_allocator)) + { } + + /// These constructors use 'val' as an exemplar to copy-construct both + /// the leftmost view and each identity view. + holder_monoid(const Type& val, const Allocator& a = Allocator()) + : m_allocator(a) + , m_initializer(internal::exemplar_init::make( + val, m_allocator)) { } + /// This constructor uses 'f' as a functor to construct both + /// the leftmost view and each identity view. + template + holder_monoid(const Func& f, const Allocator& a = Allocator()) + : m_allocator(a) + , m_initializer( + internal::functor_init::make(f,m_allocator)) + { } + + /// Copy constructor + holder_monoid(const holder_monoid& rhs) + : m_allocator(rhs.m_allocator) + , m_initializer(rhs.m_initializer->clone_self(m_allocator)) { } + + /// "Extended" copy constructor with allocator + holder_monoid(const holder_monoid& rhs, const Allocator& a) + : m_allocator(a) + , m_initializer(rhs.m_initializer->clone_self(m_allocator)) { } + +#ifdef __CILKRTS_RVALUE_REFERENCES + /// Move constructor + holder_monoid(holder_monoid&& rhs) + : m_allocator(rhs.m_allocator) + , m_initializer(rhs.m_initializer) { + rhs.m_initializer = + internal::default_init::make(m_allocator); + } + + /// "Extended" move constructor with allocator + holder_monoid(holder_monoid&& rhs, const Allocator& a) + : m_allocator(a) + , m_initializer(0) { + if (a != rhs.m_allocator) + m_initializer = rhs.m_initializer->clone_self(a); + else { + m_initializer = rhs.m_initializer; + rhs.m_initializer = + internal::default_init::make(m_allocator); + } + } +#endif + /// Destructor + ~holder_monoid() { m_initializer->delete_self(m_allocator); } + + holder_monoid& operator=(const holder_monoid& rhs) { + if (this == &rhs) return *this; + m_initializer->delete_self(m_allocator); + m_initializer = rhs.m_initializer->clone_self(m_allocator); + } + +#ifdef __CILKRTS_RVALUE_REFERENCES + holder_monoid& operator=(holder_monoid&& rhs) { + if (m_allocator != rhs.m_allocator) + // Delegate to copy-assignment on unequal allocators + return operator=(static_cast(rhs)); + std::swap(m_initializer, rhs.m_initializer); + return *this; + } +#endif + + /// Constructs IDENTITY value into the uninitilized '*p' + void identity(Type* p) const + { m_initializer->construct_view(p, m_allocator); } + + /// Calls the destructor on the object pointed-to by 'p' + void destroy(Type* p) const + { m_allocator.destroy(p); } + + /// Return a pointer to size bytes of raw memory + void* allocate(std::size_t s) const { + __CILKRTS_ASSERT(sizeof(Type) == s); + return m_allocator.allocate(1); + } + + /// Deallocate the raw memory at p + void deallocate(void* p) const { + m_allocator.deallocate(static_cast(p), sizeof(Type)); + } + + void reduce(Type* left, Type* right) const { + internal::holder_reduce_functor()(left, right); + } + + void swap(holder_monoid& other) { + __CILKRTS_ASSERT(m_allocator == other.m_allocator); + std::swap(m_initializer, other.m_initializer); + } + + Allocator get_allocator() const { + return m_allocator; + } + }; + + // Namespace-scope swap + template + inline void swap(holder_monoid& a, + holder_monoid& b) + { + a.swap(b); + } + + /** + * Hyperobject to provide different views of an object to each + * parallel strand. + */ + template > + class holder : public reducer > + { + typedef holder_monoid monoid_type; + typedef reducer imp; + + // Return a value of Type constructed using the functor Func. + template + Type make_value(const Func& f) const { + struct obj { + union { + char buf[sizeof(Type)]; + void* align1; + double align2; + }; + + obj(const Func& f) { f(static_cast(buf)); } + ~obj() { static_cast(buf)->~Type(); } + + operator Type&() { return *static_cast(buf); } + }; + + return obj(f); + } + + public: + /// Default constructor uses default-initialization for both the + /// leftmost view and each identity view. + holder(const Allocator& alloc = Allocator()) + : imp(monoid_type(alloc)) { } + + /// Construct from an exemplar that is used to initialize both the + /// leftmost view and each identity view. + holder(const Type& v, const Allocator& alloc = Allocator()) + // Alas, cannot use an rvalue reference for 'v' because it is used + // twice in the same expression for initializing imp. + : imp(monoid_type(v, alloc), v) { } + + /// Construct from a functor that is used to initialize both the + /// leftmost view and each identity view. The functor, 'f', must be be + /// invokable using the expression 'Type x = f()'. + template + holder(const Func& f, const Allocator& alloc = Allocator()) + // Alas, cannot use an rvalue for 'f' because it is used twice in + // the same expression for initializing imp. + : imp(monoid_type(f, alloc), make_value(f)) { } + }; + +} // end namespace cilk + +#else /* C */ +# error Holders are currently available only for C++ +#endif /* __cplusplus */ + +#endif /* HOLDER_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/hyperobject_base.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/hyperobject_base.h new file mode 100644 index 0000000000..dd7ccfd902 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/hyperobject_base.h @@ -0,0 +1,183 @@ +/* + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + * + */ + +#ifndef INCLUDED_CILK_HYPEROBJECT_BASE +#define INCLUDED_CILK_HYPEROBJECT_BASE + +#ifdef __cplusplus +# include +# include +#else +# include +# include +#endif + +#include + +#if defined _WIN32 || defined _WIN64 +# if !defined CILK_STUB && !defined IN_CILK_RUNTIME + /* bring in the Intel(R) Cilk(TM) Plus library, which has definitions for some of these + * functions. */ +# pragma comment(lib, "cilkrts") +# endif +#endif + +/* The __CILKRTS_STRAND_PURE attribute tells the compiler that the value + * returned by 'func' for a given argument to 'func' will remain valid until + * the next strand boundary (spawn or sync) or until the next call to a + * function with the __CILKRTS_STRAND_STALE attribute using the same function + * argument. + */ +#if 0 && defined __cilk && (defined __GNUC__ && !defined _WIN32) && defined __cilkartsrev +# define __CILKRTS_STRAND_PURE(func) \ + func __attribute__((__cilk_hyper__("lookup"))) +# define __CILKRTS_STRAND_STALE(func) \ + func __attribute__((__cilk_hyper__("flush"))) +#else +# define __CILKRTS_STRAND_PURE(func) func +# define __CILKRTS_STRAND_STALE(func) func +#endif + +/***************************************************************************** + * C runtime interface to the hyperobject subsystem + *****************************************************************************/ + +__CILKRTS_BEGIN_EXTERN_C + +/* Callback function signatures. The 'r' argument always points to the + * reducer itself and is commonly ignored. */ +typedef void (*cilk_c_reducer_reduce_fn_t)(void* r, void* lhs, void* rhs); +typedef void (*cilk_c_reducer_identity_fn_t)(void* r, void* view); +typedef void (*cilk_c_reducer_destroy_fn_t)(void* r, void* view); +typedef void* (*cilk_c_reducer_allocate_fn_t)(void* r, __STDNS size_t bytes); +typedef void (*cilk_c_reducer_deallocate_fn_t)(void* r, void* view); + +/** Representation of the monoid */ +typedef struct cilk_c_monoid { + cilk_c_reducer_reduce_fn_t reduce_fn; + cilk_c_reducer_identity_fn_t identity_fn; + cilk_c_reducer_destroy_fn_t destroy_fn; + cilk_c_reducer_allocate_fn_t allocate_fn; + cilk_c_reducer_deallocate_fn_t deallocate_fn; +} cilk_c_monoid; + +/** Base of the hyperobject */ +typedef struct __cilkrts_hyperobject_base +{ + cilk_c_monoid __c_monoid; + unsigned long long __flags; + __STDNS ptrdiff_t __view_offset; /* offset (in bytes) to leftmost view */ + __STDNS size_t __view_size; /* Size of each view */ +} __cilkrts_hyperobject_base; + + +#ifndef CILK_STUB + +/* Library functions. */ +CILK_EXPORT + void __cilkrts_hyper_create(__cilkrts_hyperobject_base *key); +CILK_EXPORT void __CILKRTS_STRAND_STALE( + __cilkrts_hyper_destroy(__cilkrts_hyperobject_base *key)); +CILK_EXPORT void* __CILKRTS_STRAND_PURE( + __cilkrts_hyper_lookup(__cilkrts_hyperobject_base *key)); + +CILK_EXPORT + void* __cilkrts_hyperobject_alloc(void* ignore, __STDNS size_t bytes); +CILK_EXPORT + void __cilkrts_hyperobject_dealloc(void* ignore, void* view); + +/* No-op destroy function */ +CILK_EXPORT + void __cilkrts_hyperobject_noop_destroy(void* ignore, void* ignore2); + + +#else // CILK_STUB + +// Programs compiled with CILK_STUB are not linked with the Intel Cilk Plus runtime +// library, so they should not have external references to cilkrts functions. +// Furthermore, they don't need the hyperobject functionality, so the +// functions can be stubbed. + +#define __cilkrts_hyperobject_create __cilkrts_hyperobject_create__stub +__CILKRTS_INLINE + void __cilkrts_hyper_create(__cilkrts_hyperobject_base *key) + {} + +#define __cilkrts_hyperobject_destroy __cilkrts_hyperobject_destroy__stub +__CILKRTS_INLINE + void __cilkrts_hyper_destroy(__cilkrts_hyperobject_base *key) + {} + +#define __cilkrts_hyperobject_lookup __cilkrts_hyperobject_lookup__stub +__CILKRTS_INLINE + void* __cilkrts_hyper_lookup(__cilkrts_hyperobject_base *key) + { return (char*)(key) + key->__view_offset; } + +// Pointers to these functions are stored into monoids, so real functions +// are needed. + +#define __cilkrts_hyperobject_alloc __cilkrts_hyperobject_alloc__stub +__CILKRTS_INLINE + void* __cilkrts_hyperobject_alloc(void* ignore, __STDNS size_t bytes) + { assert(0); return __STDNS malloc(bytes); } + +#define __cilkrts_hyperobject_dealloc __cilkrts_hyperobject_dealloc__stub +__CILKRTS_INLINE + void __cilkrts_hyperobject_dealloc(void* ignore, void* view) + { assert(0); __STDNS free(view); } + +#define __cilkrts_hyperobject_noop_destroy \ + __cilkrts_hyperobject_noop_destroy__stub +__CILKRTS_INLINE + void __cilkrts_hyperobject_noop_destroy(void* ignore, void* ignore2) + {} + +#endif + +__CILKRTS_END_EXTERN_C + +#endif /* INCLUDED_CILK_HYPEROBJECT_BASE */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/metaprogramming.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/metaprogramming.h new file mode 100644 index 0000000000..2df7cf6467 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/metaprogramming.h @@ -0,0 +1,621 @@ +/* metaprogramming.h -*- C++ -*- + * + * Copyright (C) 2012-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + */ + +/** @file metaprogramming.h + * + * @brief Defines metaprogramming utility classes used in the Intel(R) Cilk(TM) Plus library. + * + * @ingroup common + */ + +#ifndef METAPROGRAMMING_H_INCLUDED +#define METAPROGRAMMING_H_INCLUDED + +#ifdef __cplusplus + +#include +#include +#include +#ifdef _WIN32 +#include +#endif +#include + +namespace cilk { + +namespace internal { + +/** Test if a class is empty. + * + * If @a Class is an empty (and therefore necessarily stateless) class, then + * the "empty base-class optimization" guarantees that + * `sizeof(check_for_empty_class) == sizeof(char)`. Conversely, if + * `sizeof(check_for_empty_class) > sizeof(char)`, then @a Class is not + * empty, and we must discriminate distinct instances of @a Class. + * + * Typical usage: + * + * // General definition of A for non-empty B: + * template ::value> > + * class A { ... }; + * + * // Specialized definition of A for empty B: + * template + * class A { ... }; + * + * @tparam Class The class to be tested for emptiness. + * + * @result The `value` member will be `true` if @a Class is empty, + * `false` otherwise. + * + * @ingroup common + */ +template +class class_is_empty { + class check_for_empty_class : public Class + { + char m_data; + public: + // Declared but not defined + check_for_empty_class(); + check_for_empty_class(const check_for_empty_class&); + check_for_empty_class& operator=(const check_for_empty_class&); + ~check_for_empty_class(); + }; +public: + + /** Constant is true if and only if @a Class is empty. + */ + static const bool value = (sizeof(check_for_empty_class) == sizeof(char)); +}; + + +/** Get the alignment of a type. + * + * For example: + * + * align_of::value == 8 + * + * @tparam Tp The type whose alignment is to be computed. + * + * @result The `value` member of an instantiation of this class template + * will hold the integral alignment requirement of @a Tp. + * + * @pre @a Tp shall be a complete type. + * + * @ingroup common + */ +template +struct align_of +{ +private: + struct imp { + char m_padding; + Tp m_val; + + // The following declarations exist to suppress compiler-generated + // definitions, in case @a Tp does not have a public default + // constructor, copy constructor, or destructor. + imp(const imp&); // Declared but not defined + ~imp(); // Declared but not defined + }; + +public: + /// The integral alignment requirement of @a Tp. + static const std::size_t value = (sizeof(imp) - sizeof(Tp)); +}; + + +/** A class containing raw bytes with a specified alignment and size. + * + * An object of type `aligned_storage` will have alignment `A` and + * size at least `S`. Its contents will be uninitialized bytes. + * + * @tparam Size The required minimum size of the resulting class. + * @tparam Alignment The required alignment of the resulting class. + * + * @pre @a Alignment shall be a power of 2 no greater than 64. + * + * @note This is implemented using the `CILK_ALIGNAS` macro, which uses + * the non-standard, implementation-specific features + * `__declspec(align(N))` on Windows, and + * `__attribute__((__aligned__(N)))` on Unix. The `gcc` implementation + * of `__attribute__((__aligned__(N)))` requires a numeric literal `N` + * (_not_ an arbitrary compile-time constant expression). Therefore, + * this class is implemented using specialization on the required + * alignment. + * + * @note The template class is specialized only for the supported + * alignments. An attempt to instantiate it for an unsupported + * alignment will result in a compilation error. + */ +template +struct aligned_storage; + +/// @cond +template class aligned_storage + { CILK_ALIGNAS( 1) char m_bytes[Size]; }; +template class aligned_storage + { CILK_ALIGNAS( 2) char m_bytes[Size]; }; +template class aligned_storage + { CILK_ALIGNAS( 4) char m_bytes[Size]; }; +template class aligned_storage + { CILK_ALIGNAS( 8) char m_bytes[Size]; }; +template class aligned_storage + { CILK_ALIGNAS(16) char m_bytes[Size]; }; +template class aligned_storage + { CILK_ALIGNAS(32) char m_bytes[Size]; }; +template class aligned_storage + { CILK_ALIGNAS(64) char m_bytes[Size]; }; +/// @endcond + +/** A buffer of uninitialized bytes with the same size and alignment as a + * specified type. + * + * The class `storage_for_object` will have the same size and alignment + * properties as `Type`, but it will contain only raw (uninitialized) bytes. + * This allows the definition of a data member which can contain a `Type` + * object which is initialized explicitly under program control, rather + * than implicitly as part of the initialization of the containing class. + * For example: + * + * class C { + * storage_for_object _member; + * public: + * C() ... // Does NOT initialize _member + * void initialize(args) + * { new (_member.pointer()) MemberClass(args); } + * const MemberClass& member() const { return _member.object(); } + * MemberClass& member() { return _member.object(); } + * + * @tparam Type The type whose size and alignment are to be reflected + * by this class. + */ +template +class storage_for_object : + aligned_storage< sizeof(Type), align_of::value > +{ +public: + /// Return a typed reference to the buffer. + const Type& object() const { return *reinterpret_cast(this); } + /// Return a typed reference to the buffer. + Type& object() { return *reinterpret_cast(this); } +}; + + +/** Get the functor class corresponding to a binary function type. + * + * The `binary_functor` template class can be instantiated with a binary + * functor class or with a real binary function, and will yield an equivalent + * binary functor class in either case. + * + * @tparam F A binary functor class, a binary function type, or a pointer to + * binary function type. + * + * @result `binary_functor::%type` will be the same as @a F if @a F is + * a class. It will be a `std::pointer_to_binary_function` wrapper + * if @a F is a binary function or binary function pointer type. + * (It will _not_ necessarily be an `Adaptable Binary Function` + * class, since @a F might be a non-adaptable binary functor + * class.) + * + * @ingroup common + */ +template +struct binary_functor { + /// The binary functor class equivalent to @a F. + typedef F type; +}; + +/// @copydoc binary_functor +/// Specialization for binary function. +template +struct binary_functor { + /// The binary functor class equivalent to @a F. + typedef std::pointer_to_binary_function type; +}; + +/// @copydoc binary_functor +/// Specialization for pointer to binary function. +template +struct binary_functor { + /// The binary functor class equivalent to @a F. + typedef std::pointer_to_binary_function type; +}; + + +/** Indirect binary function class with specified types. + * + * `typed_indirect_binary_function` is an `Adaptable Binary Function` class + * based on an existing binary functor class or binary function type @a F. If + * @a F is a stateless class, then this class will be empty, and its + * `operator()` will invoke @a F's `operator()`. Otherwise, an object of this + * class will hold a pointer to an object of type @a F, and will refer its + * `operator()` calls to the pointed-to @a F object. + * + * That is, suppose that we have the declarations: + * + * F *p; + * typed_indirect_binary_function ibf(p); + * + * Then: + * + * - `ibf(x, y) == (*p)(x, y)`. + * - `ibf(x, y)` will not do a pointer dereference if `F` is an empty class. + * + * @note Just to repeat: if `F` is an empty class, then + * `typed_indirect_binary_function\' is also an empty class. + * This is critical for its use in the + * @ref cilk::cilk_lib_1_1::min_max_internal::view_base + * "min/max reducer view classes", where it allows the view to + * call a comparison functor in the monoid without actually + * having to allocate a pointer in the view class when the + * comparison class is empty. + * + * @note If you have an `Adaptable Binary Function` class or a binary + * function type, then you can use the + * @ref indirect_binary_function class, which derives the + * argument and result types parameter type instead of requiring + * you to specify them as template arguments. + * + * @tparam F A binary functor class, a binary function type, or a pointer to + * binary function type. + * @param A1 The first argument type. + * @param A2 The second argument type. + * @param R The result type. + * + * @see min_max::comparator_base + * @see indirect_binary_function + * + * @ingroup common + */ +template < typename F + , typename A1 + , typename A2 + , typename R + , typename Functor = typename binary_functor::type + , bool FunctorIsEmpty = class_is_empty::value + > +class typed_indirect_binary_function : std::binary_function +{ + const F* f; +public: + /// Constructor captures a pointer to the wrapped function. + typed_indirect_binary_function(const F* f) : f(f) {} + + /// Return the comparator pointer, or `NULL` if the comparator is stateless. + const F* pointer() const { return f; } + + /// Apply the pointed-to functor to the arguments. + R operator()(const A1& a1, const A2& a2) const { return (*f)(a1, a2); } +}; + + +/// @copydoc typed_indirect_binary_function +/// Specialization for an empty functor class. (This is only possible if @a F +/// itself is an empty class. If @a F is a function or pointer-to-function +/// type, then the functor will contain a pointer.) +template +class typed_indirect_binary_function : + std::binary_function +{ +public: + /// Return `NULL` for the comparator pointer of a stateless comparator. + const F* pointer() const { return 0; } + + /// Constructor discards the pointer to a stateless functor class. + typed_indirect_binary_function(const F* f) {} + + /// Create an instance of the stateless functor class and apply it to the arguments. + R operator()(const A1& a1, const A2& a2) const { return F()(a1, a2); } +}; + + +/** Indirect binary function class with inferred types. + * + * This is identical to @ref cilk::internal::typed_indirect_binary_function, + * except that it derives the binary function argument and result types from + * the parameter type @a F instead of taking them as additional template + * parameters. If @a F is a class type, then it must be an `Adaptable Binary + * Function`. + * + * @see typed_indirect_binary_function + * + * @ingroup common + */ +template ::type> +class indirect_binary_function : + typed_indirect_binary_function< F + , typename Functor::first_argument_type + , typename Functor::second_argument_type + , typename Functor::result_type + > +{ + typedef typed_indirect_binary_function< F + , typename Functor::first_argument_type + , typename Functor::second_argument_type + , typename Functor::result_type + > + base; +public: + indirect_binary_function(const F* f) : base(f) {} ///< Constructor +}; + + +/** Choose a type based on a boolean constant. + * + * This metafunction is identical to C++11's condition metafunction. + * It needs to be here until we can reasonably assume that users will be + * compiling with C++11. + * + * @tparam Cond A boolean constant. + * @tparam IfTrue A type. + * @tparam IfFalse A type. + * @result The `type` member will be a typedef of @a IfTrue if @a Cond + * is true, and a typedef of @a IfFalse if @a Cond is false. + * + * @ingroup common + */ +template +struct condition +{ + typedef IfTrue type; ///< The type selected by the condition. +}; + +/// @copydoc condition +/// Specialization for @a Cond == `false`. +template +struct condition +{ + typedef IfFalse type; ///< The type selected by the condition. +}; + + +/** @def __CILKRTS_STATIC_ASSERT + * + * @brief Compile-time assertion. + * + * Causes a compilation error if a compile-time constant expression is false. + * + * @par Usage example. + * This assertion is used in reducer_min_max.h to avoid defining + * legacy reducer classes that would not be binary-compatible with the + * same classes compiled with earlier versions of the reducer library. + * + * __CILKRTS_STATIC_ASSERT( + * internal::class_is_empty< internal::binary_functor >::value, + * "cilk::reducer_max only works with an empty Compare class"); + * + * @note In a C++11 compiler, this is just the language predefined + * `static_assert` macro. + * + * @note In a non-C++11 compiler, the @a Msg string is not directly included + * in the compiler error message, but it may appear if the compiler + * prints the source line that the error occurred on. + * + * @param Cond The expression to test. + * @param Msg A string explaining the failure. + * + * @ingroup common + */ +#if defined(__INTEL_CXX11_MODE__) || defined(__GXX_EXPERIMENTAL_CXX0X__) +# define __CILKRTS_STATIC_ASSERT(Cond, Msg) static_assert(Cond, Msg) +#else +# define __CILKRTS_STATIC_ASSERT(Cond, Msg) \ + typedef int __CILKRTS_STATIC_ASSERT_DUMMY_TYPE \ + [::cilk::internal::static_assert_failure<(Cond)>::Success] + +/// @cond internal + template struct static_assert_failure { }; + template <> struct static_assert_failure { enum { Success = 1 }; }; + +# define __CILKRTS_STATIC_ASSERT_DUMMY_TYPE \ + __CILKRTS_STATIC_ASSERT_DUMMY_TYPE1(__cilkrts_static_assert_, __LINE__) +# define __CILKRTS_STATIC_ASSERT_DUMMY_TYPE1(a, b) \ + __CILKRTS_STATIC_ASSERT_DUMMY_TYPE2(a, b) +# define __CILKRTS_STATIC_ASSERT_DUMMY_TYPE2(a, b) a ## b +/// @endcond + +#endif + +/// @cond internal + +/** @name Aligned heap management. + */ +//@{ + +/** Implementation-specific aligned memory allocation function. + * + * @param size The minimum number of bytes to allocate. + * @param alignment The required alignment (must be a power of 2). + * @return The address of a block of memory of at least @a size + * bytes. The address will be a multiple of @a alignment. + * `NULL` if the allocation fails. + * + * @see deallocate_aligned() + */ +inline void* allocate_aligned(std::size_t size, std::size_t alignment) +{ +#ifdef _WIN32 + return _aligned_malloc(size, alignment); +#else +#if defined(__ANDROID__) || defined(__VXWORKS__) + return memalign(std::max(alignment, sizeof(void*)), size); +#else + void* ptr; + return (posix_memalign(&ptr, std::max(alignment, sizeof(void*)), size) == 0) ? ptr : 0; +#endif +#endif +} + +/** Implementation-specific aligned memory deallocation function. + * + * @param ptr A pointer which was returned by a call to alloc_aligned(). + */ +inline void deallocate_aligned(void* ptr) +{ +#ifdef _WIN32 + _aligned_free(ptr); +#else + std::free(ptr); +#endif +} + +/** Class to allocate and guard an aligned pointer. + * + * A new_aligned_pointer object allocates aligned heap-allocated memory when + * it is created, and automatically deallocates it when it is destroyed + * unless its `ok()` function is called. + * + * @tparam T The type of the object to allocate on the heap. The allocated + * will have the size and alignment of an object of type T. + */ +template +class new_aligned_pointer { + void* m_ptr; +public: + /// Constructor allocates the pointer. + new_aligned_pointer() : + m_ptr(allocate_aligned(sizeof(T), internal::align_of::value)) {} + /// Destructor deallocates the pointer. + ~new_aligned_pointer() { if (m_ptr) deallocate_aligned(m_ptr); } + /// Get the pointer. + operator void*() { return m_ptr; } + /// Return the pointer and release the guard. + T* ok() { + T* ptr = static_cast(m_ptr); + m_ptr = 0; + return ptr; + } +}; + +//@} + +/// @endcond + +} // namespace internal + +//@{ + +/** Allocate an aligned data structure on the heap. + * + * `cilk::aligned_new([args])` is equivalent to `new T([args])`, except + * that it guarantees that the returned pointer will be at least as aligned + * as the alignment requirements of type `T`. + * + * @ingroup common + */ +template +T* aligned_new() +{ + internal::new_aligned_pointer ptr; + new (ptr) T(); + return ptr.ok(); +} + +template +T* aligned_new(const T1& x1) +{ + internal::new_aligned_pointer ptr; + new (ptr) T(x1); + return ptr.ok(); +} + +template +T* aligned_new(const T1& x1, const T2& x2) +{ + internal::new_aligned_pointer ptr; + new (ptr) T(x1, x2); + return ptr.ok(); +} + +template +T* aligned_new(const T1& x1, const T2& x2, const T3& x3) +{ + internal::new_aligned_pointer ptr; + new (ptr) T(x1, x2, x3); + return ptr.ok(); +} + +template +T* aligned_new(const T1& x1, const T2& x2, const T3& x3, const T4& x4) +{ + internal::new_aligned_pointer ptr; + new (ptr) T(x1, x2, x3, x4); + return ptr.ok(); +} + +template +T* aligned_new(const T1& x1, const T2& x2, const T3& x3, const T4& x4, const T5& x5) +{ + internal::new_aligned_pointer ptr; + new (ptr) T(x1, x2, x3, x4, x5); + return ptr.ok(); +} + +//@} + + +/** Deallocate an aligned data structure on the heap. + * + * `cilk::aligned_delete(ptr)` is equivalent to `delete ptr`, except that it + * operates on a pointer that was allocated by aligned_new(). + * + * @ingroup common + */ +template +void aligned_delete(const T* ptr) +{ + ptr->~T(); + internal::deallocate_aligned((void*)ptr); +} + +} // namespace cilk + +#endif // __cplusplus + +#endif // METAPROGRAMMING_H_INCLUDED diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer.h new file mode 100644 index 0000000000..09c2e19690 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer.h @@ -0,0 +1,2002 @@ +/* reducer.h -*- C++ -*- + * + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + */ + +/** @file reducer.h + * + * @brief Defines foundation classes for creating Intel(R) Cilk(TM) Plus reducers. + * + * @ingroup Reducers + * + * @see @ref pagereducers + * + * @defgroup Reducers Reducers + */ + +#ifndef REDUCER_H_INCLUDED +#define REDUCER_H_INCLUDED + +#include "cilk/hyperobject_base.h" +#include "cilk/metaprogramming.h" + +#ifdef __cplusplus + +//===================== C++ interfaces =================================== + +#include + +namespace cilk { + +/** Class for provisionally constructed objects. + * + * The monoid_base::construct() functions manually construct both a + * monoid and a view. If one of these is constructed successfully, and the + * construction of the other (or some other initialization) fails, then the + * first one must be destroyed to avoid a memory leak. Because the + * construction is explicit, the destruction must be explicit, too. + * + * A provisional_guard object wraps a pointer to a newly constructed + * object. A call to its confirm() function confirms that the object is + * really going to be used. If the guard is destroyed without being + * confirmed, then the pointed-to object is destroyed (but not + * deallocated). + * + * Expected usage: + * + * provisional_guard x1_provisional( new (x1) T1 ); + * … more initialization … + * x1_provisional.confirm(); + * + * or + * + * provisional_guard x1_provisional( new (x1) T1 ); + * x1_provisional.confirm_if( new (x2) T2 ); + * + * If an exception is thrown in the "more initialization" code in the + * first example, or in the `T2` constructor in the second example, then + * `x1_provisional` will not be confirmed, so when its destructor is + * called during exception unwinding, the `T1` object that was constructed + * in `x1` will be destroyed. + * + * **NOTE**: Do *not* be tempted to chain a `provisional_guard` + * constructor with `confirm_if` as in this example: + * + * // BAD IDEA + * provisional_guard( new (x1) T1 ).confirm_if( new (x2) T2 ); + * + * The code above is problematic because the evaluation of the T2 + * constructor is unsequenced with respect to the call to the + * `provisional_guard` constructor (and with respect the T1 constructor). + * Thus, the compiler may choose to evaluate `new (x2) T2` before + * constructing the guard and leak the T1 object if the `T2` constructor + * throws. + * + * @tparam Type The type of the provisionally constructed object. + */ +template +class provisional_guard { + Type* m_ptr; + +public: + + /** Constructor. Creates a guard for a provisionally constructed object. + * + * @param ptr A pointer to the provisionally constructed object. + */ + provisional_guard(Type* ptr) : m_ptr(ptr) {} + + /** Destructor. Destroy the object pointed to by the contained pointer + * if it has not been confirmed. + */ + ~provisional_guard() { if (m_ptr) m_ptr->~Type(); } + + /** Confirm the provisional construction. Do *not* delete the contained + * pointer when the guard is destroyed. + */ + void confirm() { m_ptr = 0; } + + /** Confirm provisional construction if argument is non-null. Note that + * if an exception is thrown during evaluation of the argument + * expression, then this function will not be called, and the + * provisional object will not be confirmed. This allows the usage: + * + * x1_provisional.confirm_if( new (x2) T2() ); + * + * @param cond An arbitrary pointer. The provisional object will be + * confirmed if @a cond is not null. + * + * @returns The value of the @a cond argument. + */ + template + Cond* confirm_if(Cond* cond) { if (cond) m_ptr = 0; return cond; } +}; + +/** Base class for defining monoids. + * + * The monoid_base class template is useful for creating classes that model + * the monoid concept. It provides the core type and memory management + * functionality. A subclass of monoid_base need only declare and implement + * the `identity` and `reduce` functions. + * + * The monoid_base class also manages the integration between the monoid, the + * reducer class that is based on it, and an optional view class which wraps + * value objects and restricts access to their operations. + * + * @tparam Value The value type for the monoid. + * @tparam View An optional view class that serves as a proxy for the value + * type. + * + * @see monoid_with_view + */ +template +class monoid_base +{ + +public: + + /** Value type of the monoid. + */ + typedef Value value_type; + + /** View type of the monoid. Defaults to be the same as the value type. + * @see monoid_with_view + */ + typedef View view_type; + + enum { + /** Should reducers created with this monoid be aligned? + * + * @details + * "Aligned" means that the view is allocated at a cache-line aligned + * offset in the reducer, and the reducer must be cache-line aligned. + * "Unaligned" means that the reducer as a whole is just naturally + * aligned, but it contains a large enough block of uninitialized + * storage for a cache-line aligned view to be allocated in it at + * reducer construction time. + * + * Since the standard heap allocator (new reducer) does not allocate + * cache-line aligned storage, only unaligned reducers can be safely + * allocated on the heap. + * + * Default is false (unaligned) unless overridden in a subclass. + * + * @since 1.02 + * (In Intel Cilk Plus library versions 1.0 and 1.01, the default was true. + * In Intel Cilk Plus library versions prior to 1.0, reducers were always + * aligned, and this data member did not exist.) + */ + align_reducer = false + }; + + /** Destroys a view. Destroys (without deallocating) the @a View object + * pointed to by @a p. + * + * @param p The address of the @a View object to be destroyed. + */ + void destroy(view_type* p) const { p->~view_type(); } + + /** Allocates raw memory. Allocate @a s bytes of memory with no + * initialization. + * + * @param s The number of bytes of memory to allocate. + * @return An untyped pointer to the allocated memory. + */ + void* allocate(size_t s) const { return operator new(s); } + + /** Deallocates raw memory pointed to by @a p + * without doing any destruction. + * + * @param p Pointer to the memory to be deallocated. + * + * @pre @a p points to a block of memory that was allocated by a + * call to allocate(). + */ + void deallocate(void* p) const { operator delete(p); } + + /** Creates the identity value. Constructs (without allocating) a @a View + * object representing the default value of the @a Value type. + * + * @param p A pointer to a block of raw memory large enough to hold a + * @a View object. + * + * @post The memory pointed to by @a p contains a @a View object that + * represents the default value of the @a View type. + * + * @deprecated This function constructs the @a View object with its default + * constructor, which will often, but not always, yield the + * appropriate identity value. Monoid classes should declare + * their identity function explicitly, rather than relying on + * this default definition. + */ + void identity(View* p) const { new ((void*) p) View(); } + + + /** @name Constructs the monoid and the view with arbitrary arguments. + * + * A @ref reducer object contains monoid and view data members, which are + * declared as raw storage (byte arrays), so that they are not implicitly + * constructed when the reducer is constructed. Instead, a reducer + * constructor calls one of the monoid class's static construct() + * functions with the addresses of the monoid and the view, and the + * construct() function uses placement `new` to construct them. + * This allows the monoid to determine the order in which the monoid and + * view are constructed, and to make one of them dependent on the other. + * + * Any arguments to the reducer constructor are just passed on as + * additional arguments to the construct() function (after the monoid + * and view addresses are set). + * + * A monoid whose needs are satisfied by the suite of construct() + * functions below, such as @ref monoid_with_view, can just inherit them + * from monoid_base. Other monoids will need to provide their own versions + * to override the monoid_base functions. + */ + //@{ + + /** Default-constructs the monoid, identity-constructs the view. + * + * @param monoid Address of uninitialized monoid object. + * @param view Address of uninitialized initial view object. + */ + //@{ + template + static void construct(Monoid* monoid, View* view) + { + provisional_guard guard( new((void*) monoid) Monoid() ); + monoid->identity(view); + guard.confirm(); + } + //@} + + /** Default-constructs the monoid, and passes one to five const reference + * arguments to the view constructor. + */ + //@{ + + template + static void construct(Monoid* monoid, View* view, const T1& x1) + { + provisional_guard guard( new((void*) monoid) Monoid() ); + guard.confirm_if( new((void*) view) View(x1) ); + } + + template + static void construct(Monoid* monoid, View* view, + const T1& x1, const T2& x2) + { + provisional_guard guard( new((void*) monoid) Monoid() ); + guard.confirm_if( new((void*) view) View(x1, x2) ); + } + + template + static void construct(Monoid* monoid, View* view, + const T1& x1, const T2& x2, const T3& x3) + { + provisional_guard guard( new((void*) monoid) Monoid() ); + guard.confirm_if( new((void*) view) View(x1, x2, x3) ); + } + + template + static void construct(Monoid* monoid, View* view, + const T1& x1, const T2& x2, const T3& x3, + const T4& x4) + { + provisional_guard guard( new((void*) monoid) Monoid() ); + guard.confirm_if( new((void*) view) View(x1, x2, x3, x4) ); + } + + template + static void construct(Monoid* monoid, View* view, + const T1& x1, const T2& x2, const T3& x3, + const T4& x4, const T5& x5) + { + provisional_guard guard( new((void*) monoid) Monoid() ); + guard.confirm_if( new((void*) view) View(x1, x2, x3, x4, x5) ); + } + + //@} + + /** Default-constructs the monoid, and passes one non-const reference + * argument to the view constructor. + */ + //@{ + template + static void construct(Monoid* monoid, View* view, T1& x1) + { + provisional_guard guard( new((void*) monoid) Monoid() ); + guard.confirm_if( new((void*) view) View(x1) ); + } + //@} + + /** Copy-constructs the monoid, and identity-constructs the view + * constructor. + * + * @param monoid Address of uninitialized monoid object. + * @param view Address of uninitialized initial view object. + * @param m Object to be copied into `*monoid` + */ + //@{ + template + static void construct(Monoid* monoid, View* view, const Monoid& m) + { + provisional_guard guard( new((void*) monoid) Monoid(m) ); + monoid->identity(view); + guard.confirm(); + } + //@} + + /** Copy-constructs the monoid, and passes one to four const reference + * arguments to the view constructor. + */ + //@{ + + template + static void construct(Monoid* monoid, View* view, const Monoid& m, + const T1& x1) + { + provisional_guard guard( new((void*) monoid) Monoid(m) ); + guard.confirm_if( new((void*) view) View(x1) ); + } + + template + static void construct(Monoid* monoid, View* view, const Monoid& m, + const T1& x1, const T2& x2) + { + provisional_guard guard( new((void*) monoid) Monoid(m) ); + guard.confirm_if( new((void*) view) View(x1, x2) ); + } + + template + static void construct(Monoid* monoid, View* view, const Monoid& m, + const T1& x1, const T2& x2, const T3& x3) + { + provisional_guard guard( new((void*) monoid) Monoid(m) ); + guard.confirm_if( new((void*) view) View(x1, x2, x3) ); + } + + template + static void construct(Monoid* monoid, View* view, const Monoid& m, + const T1& x1, const T2& x2, const T3& x3, + const T4& x4) + { + provisional_guard guard( new((void*) monoid) Monoid(m) ); + guard.confirm_if( new((void*) view) View(x1, x2, x3, x4) ); + } + + //@} + + //@} +}; + + +/** Monoid class that gets its value type and identity and reduce operations + * from its view. + * + * A simple implementation of the monoid-view-reducer architecture would + * distribute knowledge about the type and operations for the reduction + * between the monoid and the view - the identity and reduction operations are + * specified in the monoid, the reduction operations are implemented in the + * view, and the value type is specified in both the monoid and the view. + * This is inelegant. + * + * monoid_with_view is a subclass of @ref monoid_base that gets its value type + * and its identity and reduction operations from its view class. No + * customization of the monoid_with_view class itself is needed beyond + * instantiating it with an appropriate view class. (Customized subclasses of + * monoid_with_view may be needed for other reasons, such as to keep some + * state for the reducer.) All of the Intel Cilk Plus predefined reducers use + * monoid_with_view or one of its subclasses. + * + * The view class `View` of a monoid_with_view must provide the following + * public definitions: + * + * Definition | Meaning + * ---------------------------------|-------- + * `value_type` | a typedef of the value type for the reduction + * `View()` | a default constructor which constructs the identity value for the reduction + * `void reduce(const View* other)` | a member function which applies the reduction operation to the values of `this` view and the `other` view, leaving the result as the value of `this` view, and leaving the value of the `other` view undefined (but valid) + * + * @tparam View The view class for the monoid. + * @tparam Align If true, reducers instantiated on this monoid will be + * cache-aligned. By default, library reducers (unlike legacy + * library reducer _wrappers_) are aligned only as required by + * contents. + */ +template +class monoid_with_view : public monoid_base +{ +public: + /** Should reducers created with this monoid be aligned? + */ + enum { align_reducer = Align }; + + /** Create the identity value. + * + * Implements the monoid `identity` operation by using the @a View class's + * default constructor. + * + * @param p A pointer to a block of raw memory large enough to hold a + * @p View object. + */ + void identity(View* p) const { new((void*) p) View(); } + + /** Reduce the values of two views. + * + * Implements the monoid `reduce` operation by calling the left view's + * `%reduce()` function with the right view as an operand. + * + * @param left The left operand of the reduce operation. + * @param right The right operand of the reduce operation. + * @post The left view contains the result of the reduce + * operation, and the right view is undefined. + */ + void reduce(View* left, View* right) const { left->reduce(right); } +}; + + +/** Base class for simple views with (usually) scalar values. + * + * The scalar_view class is intended as a base class which provides about half + * of the required definitions for simple views. It defines the `value_type` + * required by a @ref monoid_with_view (but not the identity constructor and + * reduce operation, which are inherently specific to a particular kind of + * reduction). It also defines the value access functions which will be called + * by the corresponding @ref reducer functions. (It uses copy semantics for + * the view_move_in() and view_move_out() functions, which is appropriate + * for simple scalar types, but not necessarily for more complex types like + * STL containers. + * + * @tparam Type The type of value wrapped by the view. + */ +template +class scalar_view +{ +protected: + Type m_value; ///< The wrapped accumulator variable. + +public: + /** Value type definition required by @ref monoid_with_view. + */ + typedef Type value_type; + + /** Default constructor. + */ + scalar_view() : m_value() {} + + /** Value constructor. + */ + scalar_view(const Type& v) : m_value(v) {} + + /** @name Value functions required by the reducer class. + * + * Note that the move in/out functions use simple assignment semantics. + */ + //@{ + + /** Set the value of the view. + */ + void view_move_in(Type& v) { m_value = v; } + + /** Get the value of the view. + */ + void view_move_out(Type& v) { v = m_value; } + + /** Set the value of the view. + */ + void view_set_value(const Type& v) { m_value = v; } + + /** Get the value of the view. + */ + Type const& view_get_value() const { return m_value; } + + /** Type returned by view_get_value. + */ + typedef Type const& return_type_for_get_value; + + /** Get a reference to the value contained in the view. For legacy + * reducer support only. + */ + Type & view_get_reference() { return m_value; } + + /** Get a reference to the value contained in the view. For legacy + * reducer support only. + */ + Type const& view_get_reference() const { return m_value; } + //@} +}; + + +/** Wrapper class for move-in construction. + * + * Some types allow their values to be _moved_ as an alternative to copying. + * Moving a value may be much faster than copying it, but may leave the value + * of the move's source undefined. Consider the `swap` operation provided by + * many STL container classes: + * + * list x, y; + * x = y; // Copy + * x.swap(y); // Move + * + * The assignment _copies_ the value of `y` into `x` in time linear in the + * size of `y`, leaving `y` unchanged. The `swap` _moves_ the value of `y` + * into `x` in constant time, but it also moves the value of `x` into `y`, + * potentially leaving `y` undefined. + * + * A move_in_wrapper simply wraps a pointer to an object. It is created by a + * call to cilk::move_in(). Passing a move_in_wrapper to a view constructor + * (actually, passing it to a reducer constructor, which passes it to the + * monoid `construct()` function, which passes it to the view constructor) + * allows, but does not require, the value pointed to by the wrapper to be + * moved into the view instead of copied. + * + * A view class exercises this option by defining a _move-in constructor_, + * i.e., a constructor with a move_in_wrapper parameter. The constructor calls + * the wrapper's `value()` function to get a reference to its pointed-to + * value, and can then use that reference in a move operation. + * + * A move_in_wrapper also has an implicit conversion to its pointed-to value, + * so if a view class does not define a move-in constructor, its ordinary + * value constructor will be called with the wrapped value. For example, an + * @ref ReducersAdd "op_add" view does not have a move-in constructor, so + * + * int x; + * reducer< op_add > xr(move_in(x)); + * + * will simply call the `op_add_view(const int &)` constructor. But an + * @ref ReducersList "op_list_append" view does have a move-in constructor, + * so + * + * list x; + * reducer< op_list_append > xr(move_in(x)); + * + * will call the `op_list_append_view(move_in_wrapper)` constructor, + * which can `swap` the value of `x` into the view. + * + * @note Remember that passing the value of a variable to a reducer + * constructor using a move_in_wrapper leaves the variable undefined. + * You cannot assume that the constructor either will or will not copy + * or move the value. + * + * @tparam Type The type of the wrapped value. + * + * @see cilk::move_in() + */ +template +class move_in_wrapper +{ + Type *m_pointer; +public: + + /** Constructor that captures the address of its argument. This is almost + * always called from the @ref move_in function. + */ + explicit move_in_wrapper(Type& ref) : m_pointer(&ref) { } + + /** Implicit conversion to the wrapped value. This allows a move_in_wrapper + * to be used where a value of the wrapped type is expected, in which case + * the wrapper is completely transparent. + */ + operator Type&() const { return *m_pointer; } + + /** Get a reference to the pointed-to value. This has the same effect as + * the implicit conversion, but makes the intent clearer in a move-in + * constructor. + */ + Type& value() const { return *m_pointer; } +}; + +/** Function to create a move_in_wrapper for a value. + * + * @tparam Type The type of the argument, which will be the `type` of the + * created wrapper. + * + * @see move_in_wrapper + */ +template +inline +move_in_wrapper move_in(Type& ref) + { return move_in_wrapper(ref); } + + +/** @copydoc move_in(Type&) + * + * @note Applying a function that is explicitly specified as modifying its + * argument to a const argument is obviously an irrational thing to + * do. This move_in() variant is just provided to allow calling a + * move-in constructor with a function return value, which the + * language treats as a const. Using it for any other purpose will + * probably end in tears. + */ +template +inline +move_in_wrapper move_in(const Type& ref) + { return move_in_wrapper(ref); } + + +/** Wrapper class to allow implicit downcasts to reducer subclasses. + * + * The Intel Cilk Plus library contains a collection of reducer wrapper classes which + * were created before the `cilk::reducer` style was developed. For + * example, `cilk::reducer_opadd` provided essentially the same + * functionality that is now provided by + * `cilk::reducer< cilk::op_add >`. These legacy reducer classes are + * deprecated, but still supported, and they have been reimplemented as + * subclasses of the corresponding `cilk::reducer` classes. For example: + * + * template + * reducer_opadd : public reducer< op_add > { ... }; + * + * This reimplementation allows transparent conversion between legacy and + * new reducers. That is, a `reducer*` or `reducer&` can be + * used anywhere that a `reducer_opadd*` or `reducer_opadd&` is expected, + * and vice versa. + * + * The conversion from the legacy reducer to the new reducer is just an + * up-cast, which is provided for free by C++. The conversion from the new + * reducer to the legacy reducer is a down-cast, though, which requires an + * explicit conversion member function in the `reducer` class. The challenge + * is to define a function in the reducer template class which will convert + * each cilk::reducer specialization to the corresponding legacy reducer, + * if there is one. + * + * The trick is in the legacy_reducer_downcast template class, which provides + * a mapping from `cilk::reducer` specializations to legacy reducer classes. + * `reducer` has a conversion function to convert itself to + * `legacy_reducer_downcast< reducer >::%type`. By default, + * `legacy_reducer_downcast::%type` is just a trivial subclass of + * `Reducer`, which is uninteresting, but a reducer with a legacy counterpart + * will have a specialization of `legacy_reducer_downcast` whose `type` is + * the corresponding legacy reducer. For example: + * + * template + * struct legacy_reducer_downcast< reducer< op_add > > + * { + * typedef reducer_opadd type; + * }; + * + * + * @tparam Reducer The new-style reducer class whose corresponding legacy + * reducer class is `type`, if there is such a legacy reducer + * class. + */ +template +struct legacy_reducer_downcast +{ + /** The related legacy reducer class. + * + * By default, this is just a trivial subclass of Reducer, but it can be + * overridden in the specialization of legacy_reducer_downcast for + * a reducer that has a corresponding legacy reducers. + */ + struct type : Reducer { }; +}; + + +namespace internal { +/// @cond internal + +template +struct reducer_set_get +{ + // sizeof(notchar) != sizeof(char) + struct notchar { char x[2]; }; + + // `does_view_define_return_type_for_get_value(View*)` returns `char` if + // `View` defines `return_type_for_get_value`, and `notchar` if it doesn't. + + template + struct using_type {}; + + template + static char does_view_define_return_type_for_get_value( + using_type*); + + template + static notchar does_view_define_return_type_for_get_value(...); + + // `VIEW_DOES_DEFINE_RETURN_TYPE_FOR_GET_VALUE` is true if `View` defines + // `return_type_for_get_value`. + + enum { VIEW_DOES_DEFINE_RETURN_TYPE_FOR_GET_VALUE = + sizeof( does_view_define_return_type_for_get_value(0) ) + == sizeof(char) } ; + + // `return_type_for_get_value` is `View::return_type_for_get_value` + // if it is defined, and just `Value` otherwise. + + template + struct return_type_for_view_get_value { + typedef Value type; + }; + + template + struct return_type_for_view_get_value { + typedef typename InnerView::return_type_for_get_value type; + }; + +public: + + typedef + typename + return_type_for_view_get_value< + View, + VIEW_DOES_DEFINE_RETURN_TYPE_FOR_GET_VALUE + >::type + return_type_for_get_value; + + static void move_in(View& view, Value& v) { view.view_move_in(v); } + static void move_out(View& view, Value& v) { view.view_move_out(v); } + + static void set_value(View& view, const Value& v) + { view.view_set_value(v); } + + static return_type_for_get_value get_value(const View& view) + { return view.view_get_value(); } +}; + +template +struct reducer_set_get +{ + typedef const Value& return_type_for_get_value; + + static void move_in(Value& view, Value& v) { view = v; } + static void move_out(Value& view, Value& v) { v = view; } + + static void set_value(Value& view, const Value& v) + { view = v; } + + static return_type_for_get_value get_value(const Value& view) + { return view; } +}; + +/// @endcond + + +/** Base class defining the data layout that is common to all reducers. + */ +template +class reducer_base { + typedef typename Monoid::view_type view_type; + + // This makes the reducer a hyper-object. (Partially initialized in + // the derived reducer_content class.) + // + __cilkrts_hyperobject_base m_base; + + // The monoid is allocated here as raw bytes, and is constructed explicitly + // by a call to the monoid_type::construct() function in the constructor of + // the `reducer` subclass. + // + storage_for_object m_monoid; + + // Used for sanity checking at destruction. + // + void* m_initialThis; + + // The leftmost view comes next. It is defined in the derived + // reducer_content class. + + /** @name C-callable wrappers for the C++-coded monoid dispatch functions. + */ + //@{ + + static void reduce_wrapper(void* r, void* lhs, void* rhs); + static void identity_wrapper(void* r, void* view); + static void destroy_wrapper(void* r, void* view); + static void* allocate_wrapper(void* r, __STDNS size_t bytes); + static void deallocate_wrapper(void* r, void* view); + + //@} + +protected: + + /** Constructor. + * + * @param leftmost The address of the leftmost view in the reducer. + */ + reducer_base(char* leftmost) + { + static const cilk_c_monoid c_monoid_initializer = { + (cilk_c_reducer_reduce_fn_t) &reduce_wrapper, + (cilk_c_reducer_identity_fn_t) &identity_wrapper, + (cilk_c_reducer_destroy_fn_t) &destroy_wrapper, + (cilk_c_reducer_allocate_fn_t) &allocate_wrapper, + (cilk_c_reducer_deallocate_fn_t) &deallocate_wrapper + }; + + m_base.__c_monoid = c_monoid_initializer; + m_base.__flags = 0; + m_base.__view_offset = (char*)leftmost - (char*)this; + m_base.__view_size = sizeof(view_type); + m_initialThis = this; + + __cilkrts_hyper_create(&m_base); + } + + /** Destructor. + */ + __CILKRTS_STRAND_STALE(~reducer_base()) + { + // Make sure we haven't been memcopy'd or corrupted + __CILKRTS_ASSERT( + this == m_initialThis || + // Allow for a layout bug that may put the initialThis field one + // word later in 1.0 reducers than in 0.9 and 1.1 reducers. + this == *(&m_initialThis + 1) + ); + __cilkrts_hyper_destroy(&m_base); + } + + /** Monoid data member. + * + * @return A pointer to the reducer's monoid data member. + */ + Monoid* monoid_ptr() { return &m_monoid.object(); } + + /** Leftmost view data member. + * + * @return A pointer to the reducer's leftmost view data member. + * + * @note This function returns the address of the *leftmost* view, + * which is unique for the lifetime of the reducer. It is + * intended to be used in constructors and destructors. + * Use the reducer::view() function to access the per-strand + * view instance. + */ + view_type* leftmost_ptr() + { + char* view_addr = (char*)this + m_base.__view_offset; + return reinterpret_cast(view_addr); + } + +public: + + /** @name Access the current view. + * + * These functions return a reference to the instance of the reducer's + * view that was created for the current strand of a parallel computation + * (and create it if it doesn't already exist). Note the difference from + * the (private) leftmost_ptr() function, which returns a pointer to the + * _leftmost_ view, which is the same in all strands. + */ + //@{ + + /** Per-strand view instance. + * + * @return A reference to the per-strand view instance. + */ + view_type& view() + { + return *static_cast(__cilkrts_hyper_lookup(&m_base)); + } + + /** @copydoc view() + */ + const view_type& view() const + { + return const_cast(this)->view(); + } + + //@} + + /** Initial view pointer field. + * + * @internal + * + * @return a reference to the m_initialThis field. + * + * @note This function is provided for "white-box" testing of the + * reducer layout code. There is never any reason for user code + * to call it. + */ + const void* const & initial_this() const { return m_initialThis; } +}; + +template +void reducer_base::reduce_wrapper(void* r, void* lhs, void* rhs) +{ + Monoid* monoid = static_cast(r)->monoid_ptr(); + monoid->reduce(static_cast(lhs), + static_cast(rhs)); +} + +template +void reducer_base::identity_wrapper(void* r, void* view) +{ + Monoid* monoid = static_cast(r)->monoid_ptr(); + monoid->identity(static_cast(view)); +} + +template +void reducer_base::destroy_wrapper(void* r, void* view) +{ + Monoid* monoid = static_cast(r)->monoid_ptr(); + monoid->destroy(static_cast(view)); +} + +template +void* reducer_base::allocate_wrapper(void* r, __STDNS size_t bytes) +{ + Monoid* monoid = static_cast(r)->monoid_ptr(); + return monoid->allocate(bytes); +} + +template +void reducer_base::deallocate_wrapper(void* r, void* view) +{ + Monoid* monoid = static_cast(r)->monoid_ptr(); + monoid->deallocate(static_cast(view)); +} + + +/** Base class defining the data members of a reducer. + * + * @tparam Aligned The `m_view` data member, and therefore the entire + * structure, are cache-line aligned if this parameter + * is `true'. + */ +template +class reducer_content; + +/** Base class defining the data members of an aligned reducer. + */ +template +class reducer_content : public reducer_base +{ + typedef typename Monoid::view_type view_type; + + // The leftmost view is defined as raw bytes. It will be constructed + // by the monoid `construct` function. It is cache-aligned, which + // will push it into a new cache line. Furthermore, its alignment causes + // the reducer as a whole to be cache-aligned, which makes the reducer + // size a multiple of a cache line. Since there is nothing in the reducer + // after the view, all this means that the leftmost view gets one or more + // cache lines all to itself, which prevents false sharing. + // + __CILKRTS_CACHE_ALIGN + char m_leftmost[sizeof(view_type)]; + + /** Test if the reducer is cache-line-aligned. + * + * Used in assertions. + */ + bool reducer_is_cache_aligned() const + { return 0 == ((std::size_t) this & (__CILKRTS_CACHE_LINE__ - 1)); } + +protected: + + /** Constructor. + */ + reducer_content() : reducer_base((char*)&m_leftmost) + { +#ifndef CILK_IGNORE_REDUCER_ALIGNMENT + assert(reducer_is_cache_aligned() && + "Reducer should be cache aligned. Please see comments following " + "this assertion for explanation and fixes."); +#endif + /* "REDUCER SHOULD BE CACHE ALIGNED" ASSERTION. + * + * This Reducer class instantiation specifies cache-line alignment of the + * leftmost view field (and, implicitly, of the reducer itself). You got + * this assertion because a reducer with this class was allocated at a + * non-cache-aligned address, probably because it was allocated on the + * heap with `new`. This can be a problem for two reasons: + * + * 1. If the leftmost view is not on a cache line by itself, there might + * be a slowdown resulting from accesses to the same cache line from + * different threads. + * + * 2. The compiler thinks that reducer is cache-line aligned, but it + * really isn't. If the reducer is contained in a structure, then the + * compiler will believe that the containing structure, and other + * fields contained in it, are also more aligned than they really + * are. In particular, if the structure contains a numeric array that + * is used in a vectorizable loop, then the compiler might generate + * invalid vector instructions, resulting in a runtime error. + * + * The compiler will always allocate reducer variables, and structure + * variables containing reducers, with their required alignment. + * Reducers, and structures containing a reducer, which are allocated + * on the heap with `new` will _not_ be properly aligned. + * + * There are three ways that you can fix this assertion failure. + * + * A. Rewrite your code to use the new-style `reducer< op_XXX >` + * instead of the legacy `reducer_XXX`. The new-style reducers + * are not declared to be cache-aligned, and will work properly if + * they are not cache-aligned. + * + * B. If you must allocate an old-style reducer or a structure containing + * a reducer on the heap, figure out how to align it correctly. The + * suggested fix is to use `cilk::aligned_new()` and + * `cilk::aligned_delete()` instead of `new` and `delete`, as follows: + * + * Type* ptr = cilk::aligned_new(constructor-arguments); + * cilk::aligned_delete(ptr); + * + * C. Define the macro CILK_IGNORE_REDUCER_ALIGNMENT, which will suppress + * the assertion check. Do this only if you are comfortable that + * problem (2) above will not occur. + */ + } +}; + +/** Base class defining the data members of an unaligned reducer. + */ +template +class reducer_content : public reducer_base +{ + typedef typename Monoid::view_type view_type; ///< The view type. + + // Reserve space for the leftmost view. The view will be allocated at an + // aligned offset in this space at runtime, to guarantee that the view + // will get one or more cache lines all to itself, to prevent false + // sharing. + // + // The number of bytes to reserve is determined as follows: + // * Start with the view size. + // * Round up to a multiple of the cache line size, to get the total size + // of the cache lines that will be dedicated to the view. + // * Add (cache line size - 1) filler bytes to guarantee that the reserved + // area will contain a cache-aligned block of the required cache lines, + // no matter where the reserved area starts. + // + char m_leftmost[ + // View size rounded up to multiple cache lines + ( (sizeof(view_type) + __CILKRTS_CACHE_LINE__ - 1) + & ~ (__CILKRTS_CACHE_LINE__ - 1) + ) + // plus filler to allow alignment. + + __CILKRTS_CACHE_LINE__ - 1 + ]; + +protected: + + /** Constructor. Find the first cache-aligned position in the reserved + * area, and pass it to the base constructor as the leftmost view + * address. + */ + reducer_content() : + reducer_base( + (char*)( ((std::size_t)&m_leftmost + __CILKRTS_CACHE_LINE__ - 1) + & ~ (__CILKRTS_CACHE_LINE__ - 1) ) ) + {} +}; + + +} // namespace internal + + +// The __cilkrts_hyperobject_ functions are defined differently depending on +// whether a file is compiled with or without the CILK_STUB option. Therefore, +// reducers compiled in the two modes should be link-time incompatible, so that +// object files compiled with stubbed reducers won't be linked into an +// unstubbed program, or vice versa. We achieve this by putting the reducer +// class definition into the cilk::stub namespace in a stubbed compilation. + +#ifdef CILK_STUB +namespace stub { +#endif + +/** Reducer class. + * + * A reducer is instantiated on a Monoid. The Monoid provides the value + * type, associative reduce function, and identity for the reducer. + * + * @tparam Monoid The monoid class that the reducer is instantiated on. It + * must model the @ref reducers_monoid_concept "monoid + * concept". + * + * @see @ref pagereducers + */ +template +class reducer : public internal::reducer_content +{ + typedef internal::reducer_content base; + using base::monoid_ptr; + using base::leftmost_ptr; + public: + typedef Monoid monoid_type; ///< The monoid type. + typedef typename Monoid::value_type value_type; ///< The value type. + typedef typename Monoid::view_type view_type; ///< The view type. + + private: + typedef internal::reducer_set_get set_get; + + reducer(const reducer&); ///< Disallow copying. + reducer& operator=(const reducer&); ///< Disallow assignment. + + public: + + /** @name Constructors + * + * All reducer constructors call the static `construct()` function of the + * monoid class to construct the reducer's monoid and leftmost view. + * + * The reducer constructor arguments are simply passed through to the + * construct() function. Thus, the constructor parameters accepted by a + * particular reducer class are determined by its monoid class. + */ + //@{ + + /** 0 – 6 const reference parameters. + */ + //@{ + + reducer() + { + monoid_type::construct(monoid_ptr(), leftmost_ptr()); + } + + template + reducer(const T1& x1) + { + monoid_type::construct(monoid_ptr(), leftmost_ptr(), x1); + } + + template + reducer(const T1& x1, const T2& x2) + { + monoid_type::construct(monoid_ptr(), leftmost_ptr(), x1, x2); + } + + template + reducer(const T1& x1, const T2& x2, const T3& x3) + { + monoid_type::construct(monoid_ptr(), leftmost_ptr(), x1, x2, x3); + } + + template + reducer(const T1& x1, const T2& x2, const T3& x3, const T4& x4) + { + monoid_type::construct(monoid_ptr(), leftmost_ptr(), x1, x2, x3, x4); + } + + template + reducer(const T1& x1, const T2& x2, const T3& x3, const T4& x4, + const T5& x5) + { + monoid_type::construct(monoid_ptr(), leftmost_ptr(), + x1, x2, x3, x4, x5); + } + + template + reducer(const T1& x1, const T2& x2, const T3& x3, const T4& x4, + const T5& x5, const T6& x6) + { + monoid_type::construct(monoid_ptr(), leftmost_ptr(), + x1, x2, x3, x4, x5, x6); + } + + //@} + + /** 1 non-const reference parameter. + */ + //@{ + + template + reducer(T1& x1) + { + monoid_type::construct(monoid_ptr(), leftmost_ptr(), x1); + } + + //@} + + /** Destructor. + */ + __CILKRTS_STRAND_STALE(~reducer()) + { + leftmost_ptr()->~view_type(); + monoid_ptr()->~monoid_type(); + } + + //@{ + /** Get the monoid. + * + * @return A reference to the monoid object belonging to this reducer. + */ + Monoid& monoid() { return *monoid_ptr(); } + + const Monoid& monoid() const + { return const_cast(this)->monoid(); } + //@} + + //@{ + /** Access the current view. + * + * Return a reference to the instance of the reducer's view that was + * created for the current strand of a parallel computation (and create + * it if it doesn't already exist). + */ + view_type& view() { return base::view(); } + const view_type& view() const { return base::view(); } + //@} + + + /** @name Dereference the reducer to get the view. + * + * "Dereferencing" a reducer yields the view for the current strand. The + * view, in turn, acts as a proxy for its contained value, exposing only + * those operations which are consistent with the reducer's monoid. Thus, + * all modifications of the reducer's accumulator variable are written as + * + * *reducer OP ... + * + * or + * + * reducer->func(...) + * + * (The permitted operations on a reducer's accumulator are listed in the + * documentation for that particular kind of reducer.) + * + * @note `*r` is a synonym for `r.view()`. Recommended style is to use + * `*r` (or `r->`) in the common case where code is simply + * updating the accumulator variable wrapped in the view, and to + * use `r.view()` in the unusual case where it is desirable to + * call attention to the view itself. + */ + //@{ + + //@{ + /** Dereference operator. + * + * @return A reference to the per-strand view instance. + */ + view_type& operator*() { return view(); } + view_type const& operator*() const { return view(); } + //@} + + //@{ + /** Pointer operator. + * + * @return A pointer to the per-strand view instance. + */ + view_type* operator->() { return &view(); } + view_type const* operator->() const { return &view(); } + //@} + + //@{ + /** Deprecated view access. + * + * `r()` is a synonym for `*r` which was used with early versions of + * Intel Cilk Plus reducers. `*r` is now the preferred usage. + * + * @deprecated Use operator*() instead of operator()(). + * + * @return A reference to the per-strand view instance. + */ + view_type& operator()() { return view(); } + view_type const& operator()() const { return view(); } + //@} + + //@} + + /** @name Set and get the value. + * + * These functions are used to set an initial value for the reducer before + * starting the reduction, or to get the final value after the reduction + * is complete. + * + * @note These functions are completely different from the view + * operations that are made available via operator*() and + * operator->(), which are used to _modify_ the reducer's value + * _during_ the reduction. + * + * @warning These functions _can_ be called at any time, and in + * general, they will refer to the value contained in the view + * for the current strand. However, using them other than to + * set the reduction's initial value or get its final value + * will almost always result in undefined behavior. + */ + //@{ + + /** Move a value into the reducer. + * + * This function is used to set the initial value of the reducer's + * accumulator variable by either copying or _moving_ the value of @a obj + * into it. Moving a value can often be performed in constant time, even + * for large container objects, but has the side effect of leaving the + * value of @a obj undefined. (See the description of the + * @ref move_in_wrapper class for a discussion of moving values.) + * + * @par Usage + * A move_in() call to initialize a reducer is often paired with a + * move_out() call to get its final value: + * + * reducer xr; + * xr.move_in(x); + * … do the reduction … + * xr.move_out(x); + * + * @par Assumptions + * - You cannot assume either that this will function will copy its + * value or that it will move it. + * - You must assume that the value of @a obj will be undefined + * after the call to move_in(). + * - You can assume that move_in() will be at least as efficient as + * set_value(), and you should therefore prefer move_in() unless + * you need the value of @a obj to be unchanged after the call. + * (But you should usually prefer the move-in constructor over a + * move_in() call - see the note below.) + * + * @note The behavior of a default constructor followed by move-in + * initialization: + * + * reducer xr; + * xr.move_in(x); + * + * @note is not necessarily the same as a move-in constructor: + * + * reducer xr(move_in(x)); + * + * @note In particular, when @a Type is a container type with a + * non-empty allocator, the move-in constructor will create the + * accumulator variable with the same allocator as the input + * argument @a x, while the default constructor will create the + * accumulator variable with a default allocator. The mismatch of + * allocators in the latter case means that the input argument + * @a x may have to be copied in linear time instead of being + * moved in constant time. + * + * @note Best practice is to prefer the move-in constructor over the + * move-in function unless the move-in function is required for + * some specific reason. + * + * @warning Calling this function other than to set the initial value + * for a reduction will almost always result in undefined + * behavior. + * + * @param obj The object containing the value that will be moved into the + * reducer. + * + * @post The reducer contains the value that was initially in @a obj. + * @post The value of @a obj is undefined. + * + * @see set_value() + */ + void move_in(value_type& obj) { set_get::move_in(view(), obj);} + + /** Move the value out of the reducer. + * + * This function is used to retrieve the final value of the reducer's + * accumulator variable by either copying or _moving_ the value of @a obj + * into it. Moving a value can often be performed in constant time, even + * for large container objects, but has the side effect of leaving the + * value of the reducer's accumulator variable undefined. (See the + * description of the @ref move_in_wrapper class for a discussion of + * moving values.) + * + * @par Usage + * A move_in() call to initialize a reducer is often paired with a + * move_out() call to get its final value: + * + * reducer xr; + * xr.move_in(x); + * … do the reduction … + * xr.move_out(x); + * + * @par Assumptions + * - You cannot assume either that this will function will copy its + * value or that it will move it. + * - You must assume that the value of the reducer's accumulator + * variable will be undefined after the call to move_out(). + * - You can assume that move_out() will be at least as efficient as + * get_value(), and you should therefore prefer move_out() unless + * you need the accumulator variable to be preserved after the + * call. + * + * @warning Calling this function other than to retrieve the final + * value of a reduction will almost always result in undefined + * behavior. + * + * @param obj The object that the value of the reducer will be moved into. + * + * @post @a obj contains the value that was initially in the reducer. + * @post The value of the reducer is undefined. + * + * @see get_value() + */ + void move_out(value_type& obj) { set_get::move_out(view(), obj); } + + /** Set the value of the reducer. + * + * This function sets the initial value of the reducer's accumulator + * variable to the value of @a obj. + * + * @note The behavior of a default constructor followed by + * initialization: + * + * reducer xr; + * xr.set_value(x); + * + * @note is not necessarily the same as a value constructor: + * + * reducer xr(x); + * + * @note In particular, when @a Type is a container type with a + * non-empty allocator, the value constructor will create the + * accumulator variable with the same allocator as the input + * argument @a x, while the default constructor will create the + * accumulator variable with a default allocator. + * + * @warning Calling this function other than to set the initial value + * for a reduction will almost always result in undefined + * behavior. + * + * @param obj The object containing the value that will be copied into + * the reducer. + * + * @post The reducer contains a copy of the value in @a obj. + * + * @see move_in() + */ + void set_value(const value_type& obj) { set_get::set_value(view(), obj); } + + /** Get the value of the reducer. + * + * This function gets the final value of the reducer's accumulator + * variable. + * + * @warning Calling this function other than to retrieve the final + * value of a reduction will almost always result in undefined + * behavior. + * + * @return A reference to the value contained in the reducer. + * + * @see move_out() + */ + typename set_get::return_type_for_get_value get_value() const + { return set_get::get_value(view()); } + + //@} + + /** Implicit downcast to legacy reducer wrapper, if any. + * + * @see legacy_reducer_downcast + */ + operator typename legacy_reducer_downcast::type& () + { + typedef typename legacy_reducer_downcast::type downcast_type; + return *reinterpret_cast(this); + } + + + /** Implicit downcast to legacy reducer wrapper, if any. + * + * @see legacy_reducer_downcast + */ + operator const typename legacy_reducer_downcast::type& () const + { + typedef typename legacy_reducer_downcast::type downcast_type; + return *reinterpret_cast(this); + } +}; + +#ifdef CILK_STUB +} // namespace stub +using stub::reducer; +#endif + +} // end namespace cilk + +#endif /* __cplusplus */ + +/** @page page_reducers_in_c Creating and Using Reducers in C + * + * @tableofcontents + * + * The Intel Cilk Plus runtime supports reducers written in C as well as in C++. The + * basic logic is the same, but the implementation details are very + * different. The C++ reducer implementation uses templates heavily to create + * very generic components. The C reducer implementation uses macros, which + * are a much blunter instrument. The most immediate consequence is that the + * monoid/view/reducer architecture is mostly implicit rather than explicit + * in C reducers. + * + * @section reducers_c_overview Overview of Using Reducers in C + * + * The basic usage pattern for C reducers is: + * + * 1. Create and initialize a reducer object. + * 2. Tell the Intel Cilk Plus runtime about the reducer. + * 3. Update the value contained in the reducer in a parallel computation. + * 4. Tell the Intel Cilk Plus runtime that you are done with the reducer. + * 5. Retrieve the value from the reducer. + * + * @subsection reducers_c_creation Creating and Initializing a C Reducer + * + * The basic pattern for creating and initializing a reducer object in C is + * + * CILK_C_DECLARE_REDUCER(value-type) reducer-name = + * CILK_C_INIT_REDUCER(value-type, + * reduce-function, + * identity-function, + * destroy-function, + * initial-value); + * + * This is simply an initialized definition of a variable named + * _reducer-name_. The @ref CILK_C_DECLARE_REDUCER macro expands to an + * anonymous `struct` declaration for a reducer object containing a view of + * type _value-type_, and the @ref CILK_C_INIT_REDUCER macro expands to a + * struct initializer. + * + * @subsection reducers_c_reduce_func Reduce Functions + * + * The reduce function for a reducer is called when a parallel execution + * strand terminates, to combine the values computed by the terminating + * strand and the strand to its left. It takes three arguments: + * + * - `void* reducer` - the address of the reducer. + * - `void* left` - the address of the value for the left strand. + * - `void* right` - the address of the value for the right (terminating) + * strand. + * + * It must apply the reducer's reduction operation to the `left` and `right` + * values, leaving the result in the `left` value. The `right` value is + * undefined after the reduce function call. + * + * @subsection reducers_c_identity_func Identity Functions + * + * The identity function for a reducer is called when a parallel execution + * strand begins, to initialize its value to the reducer's identity value. It + * takes two arguments: + * + * - `void* reducer` - the address of the reducer. + * - `void* v` - the address of a freshly allocated block of memory of size + * `sizeof(value-type)`. + * + * It must initialize the memory pointed to by `v` so that it contains the + * reducer's identity value. + * + * @subsection reducers_c_destroy_func Destroy Functions + * + * The destroy function for a reducer is called when a parallel execution + * strand terminates, to do any necessary cleanup before its value is + * deallocated. It takes two arguments: + * + * - `void* reducer` - the address of the reducer. + * - `void* p` - the address of the value for the terminating strand. + * + * It must release any resources belonging to the value pointed to by `p`, to + * avoid a resource leak when the memory containing the value is deallocated. + * + * The runtime function `__cilkrts_hyperobject_noop_destroy` can be used for + * the destructor function if the reducer's values do not need any cleanup. + * + * @subsection reducers_c_register Tell the Intel Cilk Plus Runtime About the + * Reducer + * + * Call the @ref CILK_C_REGISTER_REDUCER macro to register the reducer with + * the Intel Cilk Plus runtime: + * + * CILK_C_REGISTER_REDUCER(reducer-name); + * + * The runtime will manage reducer values for all registered reducers when + * parallel execution strands begin and end. + * + * @subsection reducers_c_update Update the Value Contained in the Reducer + * + * The @ref REDUCER_VIEW macro returns a reference to the reducer's value for + * the current parallel strand: + * + * REDUCER_VIEW(reducer-name) = REDUCER_VIEW(reducer-name) OP x; + * + * C++ reducer views restrict access to the wrapped value so that it can only + * be modified in ways consistent with the reducer's operation. No such + * protection is provided for C reducers. It is entirely the responsibility + * of the user to avoid modifying the value in any inappropriate way. + * + * @subsection c_reducers_unregister Tell the Intel Cilk Plus Runtime That You Are + * Done with the Reducer + * + * When the parallel computation is complete, call the @ref + * CILK_C_UNREGISTER_REDUCER macro to unregister the reducer with the + * Intel Cilk Plus runtime: + * + * CILK_C_UNREGISTER_REDUCER(reducer-name); + * + * The runtime will stop managing reducer values for the reducer. + * + * @subsection c_reducers_retrieve Retrieve the Value from the Reducer + * + * When the parallel computation is complete, use the @ref REDUCER_VIEW macro + * to retrieve the final value computed by the reducer. + * + * @subsection reducers_c_example_custom Example - Creating and Using a + * Custom C Reducer + * + * The `IntList` type represents a simple list of integers. + * + * struct _intListNode { + * int value; + * _intListNode* next; + * } IntListNode; + * typedef struct { IntListNode* head; IntListNode* tail; } IntList; + * + * // Initialize a list to be empty + * void IntList_init(IntList* list) { list->head = list->tail = 0; } + * + * // Append an integer to the list + * void IntList_append(IntList* list, int x) + * { + * IntListNode* node = (IntListNode*) malloc(sizeof(IntListNode)); + * if (list->tail) list->tail->next = node; else list->head = node; + * list->tail = node; + * } + * + * // Append the right list to the left list, and leave the right list + * // empty + * void IntList_concat(IntList* left, IntList* right) + * { + * if (left->head) { + * left->tail->next = right->head; + * if (right->tail) left->tail = right->tail; + * } + * else { + * *left = *right; + * } + * IntList_init(*right); + * } + * + * This code creates a reducer that supports creating an `IntList` by + * appending values to it. + * + * void identity_IntList(void* reducer, void* list) + * { + * IntList_init((IntList*)list); + * } + * + * void reduce_IntList(void* reducer, void* left, void* right) + * { + * IntList_concat((IntList*)left, (IntList*)right); + * } + * + * CILK_C_DECLARE_REDUCER(IntList) my_list_int_reducer = + * CILK_C_INIT_REDUCER(IntList, + * reduce_int_list, + * identity_int_list, + * __cilkrts_hyperobject_noop_destroy); + * // Initial value omitted // + * ListInt_init(&REDUCER_VIEW(my_int_list_reducer)); + * + * CILK_C_REGISTER_REDUCER(my_int_list_reducer); + * cilk_for (int i = 0; i != n; ++i) { + * IntList_append(&REDUCER_VIEW(my_int_list_reducer), a[i]); + * } + * CILK_C_UNREGISTER_REDUCER(my_int_list_reducer); + * + * IntList result = REDUCER_VIEW(my_int_list_reducer); + * + * @section reducers_c_predefined Predefined C Reducers + * + * Some of the predefined reducer classes in the Intel Cilk Plus library come with + * a set of predefined macros to provide the same capabilities in C. + * In general, two macros are provided for each predefined reducer family: + * + * - `CILK_C_REDUCER_operation(reducer-name, type-name, initial-value)` - + * Declares a reducer object named _reducer-name_ with initial value + * _initial-value_ to perform a reduction using the _operation_ on values + * of the type specified by _type-name_. This is the equivalent of the + * general code described in @ref reducers_c_creation : + * + * CILK_C_DECLARE_REDUCER(type) reducer-name = + * CILK_C_INIT_REDUCER(type, ..., initial-value); + * + * where _type_ is the C type corresponding to _type_name_. See @ref + * reducers_c_type_names below for the _type-names_ that you can use. + * + * - `CILK_C_REDUCER_operation_TYPE(type-name)` - Expands to the `typedef` + * name for the type of the reducer object declared by + * `CILK_C_REDUCER_operation(reducer-name, type-name, initial-value)`. + * + * See @ref reducers_c_example_predefined. + * + * The predefined C reducers are: + * + * | Operation | Name | Documentation | + * |-------------------|---------------|-------------------------------| + * | addition | `OPADD` | @ref ReducersAdd | + * | bitwise AND | `OPAND` | @ref ReducersAnd | + * | bitwise OR | `OPOR` | @ref ReducersOr | + * | bitwise XOR | `OPXOR` | @ref ReducersXor | + * | multiplication | `OPMUL` | @ref ReducersMul | + * | minimum | `MIN` | @ref ReducersMinMax | + * | minimum & index | `MIN_INDEX` | @ref ReducersMinMax | + * | maximum | `MAX` | @ref ReducersMinMax | + * | maximum & index | `MAX_INDEX` | @ref ReducersMinMax | + * + * @subsection reducers_c_type_names Numeric Type Names + * + * The type and function names created by the C reducer definition macros + * incorporate both the reducer kind (`opadd`, `opxor`, etc.) and the value + * type of the reducer (`int`, `double`, etc.). The value type is represented + * by a _numeric type name_ string. The types supported in C reducers, and + * their corresponding numeric type names, are given in the following table: + * + * | Type | Numeric Type Name | + * |-----------------------|-------------------------------| + * | `char` | `char` | + * | `unsigned char` | `uchar` | + * | `signed char` | `schar` | + * | `wchar_t` | `wchar_t` | + * | `short` | `short` | + * | `unsigned short` | `ushort` | + * | `int` | `int` | + * | `unsigned int` | `uint` | + * | `unsigned int` | `unsigned` (alternate name) | + * | `long` | `long` | + * | `unsigned long` | `ulong` | + * | `long long` | `longlong` | + * | `unsigned long long` | `ulonglong` | + * | `float` | `float` | + * | `double` | `double` | + * | `long double` | `longdouble` | + * + * @subsection reducers_c_example_predefined Example - Using a Predefined C + * Reducer + * + * To compute the sum of all the values in an array of `unsigned int`: + * + * CILK_C_REDUCER_OPADD(sum, uint, 0); + * CILK_C_REGISTER_REDUCER(sum); + * cilk_for(int i = 0; i != n; ++i) { + * REDUCER_VIEW(sum) += a[i]; + * } + * CILK_C_UNREGISTER_REDUCER(sum); + * printf("The sum is %u\n", REDUCER_VIEW(sum)); + */ + + + /** @name C language reducer macros + * + * These macros are used to declare and work with reducers in C code. + * + * @see @ref page_reducers_in_c + */ + //@{ + +/// @cond internal + +/** @name Compound identifier macros. + * + * These macros are used to construct an identifier by concatenating two or + * three identifiers. + */ +//@{ + +/** Expand to an identifier formed by concatenating two identifiers. + */ +#define __CILKRTS_MKIDENT(a,b) __CILKRTS_MKIDENT_IMP(a,b,) + +/** Expand to an identifier formed by concatenating three identifiers. + */ +#define __CILKRTS_MKIDENT3(a,b,c) __CILKRTS_MKIDENT_IMP(a,b,c) + +/** Helper macro to do the concatenation. + */ +#define __CILKRTS_MKIDENT_IMP(a,b,c) a ## b ## c + +//@} + +/** Compiler-specific keyword for the "type of" operator. + */ +#if defined(__GNUC__) && !defined(__INTEL_COMPILER) +# define _Typeof __typeof__ +#endif + +/** @name Predefined reducer function declaration macros. + * + * These macros are used to create the function headers for the identity, + * reduction, and destructor functions for a builtin reducer family. The + * macro can be followed by a semicolon to create a declaration, or by a + * brace-enclosed body to create a definition. + */ +//@{ + +/** Create an identity function header. + * + * @note The name of the function's value pointer parameter will always be `v`. + * + * @param name The reducer family name. + * @param tn The type name. + */ +#define __CILKRTS_DECLARE_REDUCER_IDENTITY(name,tn) CILK_EXPORT \ + void __CILKRTS_MKIDENT3(name,_identity_,tn)(void* key, void* v) + +/** Create a reduction function header. + * + * @param name The reducer family name. + * @param tn The type name. + * @param l The name to use for the function's left value pointer parameter. + * @param r The name to use for the function's right value pointer + * parameter. + */ +#define __CILKRTS_DECLARE_REDUCER_REDUCE(name,tn,l,r) CILK_EXPORT \ + void __CILKRTS_MKIDENT3(name,_reduce_,tn)(void* key, void* l, void* r) + +/** Create a destructor function header. + * + * @param name The reducer family name. + * @param tn The type name. + * @param p The name to use for the function's value pointer parameter. + */ +#define __CILKRTS_DECLARE_REDUCER_DESTROY(name,tn,p) CILK_EXPORT \ + void __CILKRTS_MKIDENT3(name,_destroy_,tn)(void* key, void* p) + +//@} + +/// @endcond + + +/*************************************************************************** + * Real implementation + ***************************************************************************/ + +/** Declaration of a C reducer structure type. + * + * This macro expands into an anonymous structure declaration for a C reducer + * structure which contains a @a Type value. For example: + * + * CILK_C_DECLARE_REDUCER(int) my_add_int_reducer = + * CILK_C_INIT_REDUCER(int, …); + * + * @param Type The type of the value contained in the reducer object. + * + * @see @ref reducers_c_creation + */ +#define CILK_C_DECLARE_REDUCER(Type) struct { \ + __cilkrts_hyperobject_base __cilkrts_hyperbase; \ + __CILKRTS_CACHE_ALIGN Type value; \ + } + +/** Initializer for a C reducer structure. + * + * This macro expands into a brace-enclosed structure initializer for a C + * reducer structure that was declared with + * `CILK_C_DECLARE_REDUCER(Type)`. For example: + * + * CILK_C_DECLARE_REDUCER(int) my_add_int_reducer = + * CILK_C_INIT_REDUCER(int, + * add_int_reduce, + * add_int_identity, + * __cilkrts_hyperobject_noop_destroy, + * 0); + * + * @param Type The type of the value contained in the reducer object. Must + * be the same as the @a Type argument of the + * CILK_C_DECLARE_REDUCER macro call that created the + * reducer. + * @param Reduce The address of the @ref reducers_c_reduce_func + * "reduce function" for the reducer. + * @param Identity The address of the @ref reducers_c_identity_func + * "identity function" for the reducer. + * @param Destroy The address of the @ref reducers_c_destroy_func + * "destroy function" for the reducer. + * @param ... The initial value for the reducer. (A single expression if + * @a Type is a scalar type; a list of values if @a Type is a + * struct or array type.) + * + * @see @ref reducers_c_creation + */ + +#define CILK_C_INIT_REDUCER(Type, Reduce, Identity, Destroy, ...) \ + { { { Reduce \ + , Identity \ + , Destroy \ + , __cilkrts_hyperobject_alloc \ + , __cilkrts_hyperobject_dealloc \ + } \ + , 0 \ + , __CILKRTS_CACHE_LINE__ \ + , sizeof(Type) \ + } \ + , __VA_ARGS__ \ + } + +/** Register a reducer with the Intel Cilk Plus runtime. + * + * The runtime will manage reducer values for all registered reducers when + * parallel execution strands begin and end. For example: + * + * CILK_C_REGISTER_REDUCER(my_add_int_reducer); + * cilk_for (int i = 0; i != n; ++i) { + * … + * } + * + * @param Expr The reducer to be registered. + * + * @see @ref page_reducers_in_c + */ +#define CILK_C_REGISTER_REDUCER(Expr) \ + __cilkrts_hyper_create(&(Expr).__cilkrts_hyperbase) + +/** Unregister a reducer with the Intel Cilk Plus runtime. + * + * The runtime will stop managing reducer values for a reducer after it is + * unregistered. For example: + * + * cilk_for (int i = 0; i != n; ++i) { + * … + * } + * CILK_C_UNREGISTER_REDUCER(my_add_int_reducer); + * + * @param Expr The reducer to be unregistered. + * + * @see @ref page_reducers_in_c + */ +#define CILK_C_UNREGISTER_REDUCER(Expr) \ + __cilkrts_hyper_destroy(&(Expr).__cilkrts_hyperbase) + +/** Get the current view for a reducer. + * + * The `REDUCER_VIEW(reducer-name)` returns a reference to the reducer's + * value for the current parallel strand. This can be used to initialize the + * value of the reducer before it is used, to modify the value of the reducer + * on the current parallel strand, or to retrieve the final value of the + * reducer at the end of the parallel computation. + * + * REDUCER_VIEW(my_add_int_reducer) = REDUCER_VIEW(my_add_int_reducer) + x; + * + * @note C++ reducer views restrict access to the wrapped value so that it + * can only be modified in ways consistent with the reducer's operation. No + * such protection is provided for C reducers. It is entirely the + * responsibility of the user to refrain from modifying the value in any + * inappropriate way. + * + * @param Expr The reducer whose value is to be returned. + * + * @see @ref page_reducers_in_c + */ +#define REDUCER_VIEW(Expr) (*(_Typeof((Expr).value)*) \ + __cilkrts_hyper_lookup(&(Expr).__cilkrts_hyperbase)) + +//@} C language reducer macros + +#endif // CILK_REDUCER_H_INCLUDED diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_file.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_file.h new file mode 100644 index 0000000000..a372cee7cd --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_file.h @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + * + */ + + diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_list.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_list.h new file mode 100644 index 0000000000..80204af1d9 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_list.h @@ -0,0 +1,1150 @@ +/* reducer_list.h -*- C++ -*- + * + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + */ + +/** @file reducer_list.h + * + * @brief Defines classes for parallel list creation by appending or + * prepending reducers. + * + * @ingroup ReducersList + * + * @see ReducersList + */ + +#ifndef REDUCER_LIST_H_INCLUDED +#define REDUCER_LIST_H_INCLUDED + +#include +#include + +/** @defgroup ReducersList List Reducers + * + * List-append and list-prepend reducers create standard lists by + * concatenating a set of lists or values in parallel. + * + * @ingroup Reducers + * + * You should be familiar with @ref pagereducers "Intel(R) Cilk(TM) Plus reducers" + * (from file `reducers.md`) and particularly with @ref reducers_using, before + * trying to use the information in this file. + * + * @section redlist_usage Usage Example + * + * // Create a list containing the labels of the nodes of a tree in + * // "inorder" (left subtree, root, right subtree). + * + * struct Tree { Tree* left; Tree* right; string label; ... }; + * + * list x; + * cilk::reducer< cilk::op_list_append > xr(cilk::move_in(x)); + * collect_labels(tree, xr); + * xr.move_out(x); + * + * void collect_labels(Tree* node, + * cilk::reducer< cilk::op_list_append >& xr) + * { + * if (node) { + * cilk_spawn collect_labels(node->left, xr); + * xr->push_back(node->label); + * collect_labels(node->right, xr); + * cilk_sync; + * } + * } + * + * @section redlist_monoid The Monoid + * + * @subsection redlist_monoid_values Value Set + * + * The __value set__ of a list reducer is the set of values of the class + * `std::list`, which we refer to as the reducer's _list + * type_. + * + * @subsection redlist_monoid_operator Operator + * + * The operator of a list-append reducer is defined as + * + * x CAT y == (every element of x, followed by every element of y) + * + * The operator of a list-prepend reducer is defined as + * + * x RCAT y == (every element of y, followed by every element of x) + * + * @subsection redlist_monoid_identity Identity + * + * The identity value of a list reducer is the empty list, which is the value + * of the expression `std::list([allocator])`. + * + * @section redlist_operations Operations + * + * In the operation descriptions below, the type name `List` refers to the + * reducer's string type, `std::list`. + * + * @subsection redlist_constructors Constructors + * + * Any argument list which is valid for a `std::list` constructor is valid for + * a list reducer constructor. The usual move-in constructor is also provided: + * + * reducer(move_in(List& variable)) + * + * A list reducer with no constructor arguments, or with only an allocator + * argument, will initially contain the identity value, an empty list. + * + * @subsection redlist_get_set Set and Get + * + * r.set_value(const List& value) + * const List& = r.get_value() const + * r.move_in(List& variable) + * r.move_out(List& variable) + * + * @subsection redlist_view_ops View Operations + * + * The view of a list-append reducer provides the following member functions: + * + * void push_back(const Type& element) + * void insert_back(List::size_type n, const Type& element) + * template void insert_back(Iter first, Iter last) + * void splice_back(List& x) + * void splice_back(List& x, List::iterator i) + * void splice_back(List& x, List::iterator first, List::iterator last) + * + * The view of a list-prepend reducer provides the following member functions: + * + * void push_front(const Type& element) + * void insert_front(List::size_type n, const Type& element) + * template void insert_front(Iter first, Iter last) + * void splice_front(List& x) + * void splice_front(List& x, List::iterator i) + * void splice_front(List& x, List::iterator first, List::iterator last) + * + * The `push_back` and `push_front` functions are the same as the + * corresponding `std::list` functions. The `insert_back`, `splice_back`, + * `insert_front`, and `splice_front` functions are the same as the + * `std::list` `insert` and `splice` functions, with the first parameter + * fixed to the end or beginning of the list, respectively. + * + * @section redlist_performance Performance Considerations + * + * An efficient reducer requires that combining the values of two views (using + * the view `reduce()` function) be a constant-time operations. Two lists can + * be merged in constant time using the `splice()` function if they have the + * same allocator. Therefore, the lists for new views are created (by the view + * identity constructor) using the same allocator as the list that was created + * when the reducer was constructed. + * + * The performance of adding elements to a list reducer depends on the view + * operations that are used: + * + * * The `push` functions add a single element to the list, and therefore + * take constant time. + * * An `insert` function that inserts _N_ elements adds each of them + * individually, and therefore takes _O(N)_ time. + * * A `splice` function that inserts _N_ elements just adjusts a couple of + * pointers, and therefore takes constant time, _if the splice is from a + * list with the same allocator as the reducer_. Otherwise, it is + * equivalent to an `insert`, and takes _O(N)_ time. + * + * This means that for best performance, if you will be adding elements to a + * list reducer in batches, you should `splice` them from a list having the + * same allocator as the reducer. + * + * The reducer `move_in` and `move_out` functions do a constant-time `swap` if + * the variable has the same allocator as the reducer, and a linear-time copy + * otherwise. + * + * Note that the allocator of a list reducer is determined when the reducer is + * constructed. The following two examples may have very different behavior: + * + * list a_list; + * + * reducer< list_append reducer1(move_in(a_list)); + * ... parallel computation ... + * reducer1.move_out(a_list); + * + * reducer< list_append reducer2; + * reducer2.move_in(a_list); + * ... parallel computation ... + * reducer2.move_out(a_list); + * + * * `reducer1` will be constructed with the same allocator as `a_list`, + * because the list was specified in the constructor. The `move_in` + * and `move_out` can therefore be done with a `swap` in constant time. + * * `reducer2` will be constructed with a _default_ allocator, + * "`Allocator()`", which may or may not be the same as the allocator of + * `a_list`. Therefore, the `move_in` and `move_out` may have to be done + * with a copy in _O(N)_ time. + * + * (All instances of an allocator type with no internal state (like + * `std::allocator`) are "the same". You only need to worry about the "same + * allocator" issue when you create list reducers with custom allocator types.) + * + * @section redlist_types Type and Operator Requirements + * + * `std::list` must be a valid type. + */ + + +namespace cilk { + +namespace internal { + +/** @ingroup ReducersList */ +//@{ + +/** Base class for list-append and prepend view classes. + * + * @note This class provides the definitions that are required for a class + * that will be used as the parameter of a @ref list_monoid_base + * specialization. + * + * @tparam Type The list element type (not the list type). + * @tparam Allocator The list's allocator class. + * + * @see ReducersList + * @see list_monoid_base + */ +template +class list_view_base +{ +protected: + /// The type of the contained list. + typedef std::list list_type; + + /// The list accumulator variable. + list_type m_value; + +public: + + /** @name Monoid support. + */ + //@{ + + /// Required by @ref monoid_with_view + typedef list_type value_type; + + /// Required by @ref list_monoid_base + Allocator get_allocator() const + { + return m_value.get_allocator(); + } + + //@} + + + /** @name Constructors. + */ + //@{ + + /// Standard list constructor. + explicit list_view_base(const Allocator& a = Allocator()) : m_value(a) {} + explicit list_view_base( + typename list_type::size_type n, + const Type& value = Type(), + const Allocator& a = Allocator() ) : m_value(n, value, a) {} + template + list_view_base(Iter first, Iter last, const Allocator& a = Allocator()) : + m_value(first, last, a) {} + list_view_base(const list_type& list) : m_value(list) {} + + /// Move-in constructor. + explicit list_view_base(move_in_wrapper w) + : m_value(w.value().get_allocator()) + { + m_value.swap(w.value()); + } + + //@} + + /** @name Reducer support. + */ + //@{ + + /// Required by reducer::move_in() + void view_move_in(value_type& v) + { + if (m_value.get_allocator() == v.get_allocator()) + // Equal allocators. Do a (fast) swap. + m_value.swap(v); + else + // Unequal allocators. Do a (slow) copy. + m_value = v; + v.clear(); + } + + /// Required by reducer::move_out() + void view_move_out(value_type& v) + { + if (m_value.get_allocator() == v.get_allocator()) + // Equal allocators. Do a (fast) swap. + m_value.swap(v); + else + // Unequal allocators. Do a (slow) copy. + v = m_value; + m_value.clear(); + } + + /// Required by reducer::set_value() + void view_set_value(const value_type& v) { m_value = v; } + + /// Required by reducer::get_value() + value_type const& view_get_value() const { return m_value; } + + /// Type returned by view_get_value. + typedef value_type const& return_type_for_get_value; + + // Required by legacy wrapper get_reference() + value_type & view_get_reference() { return m_value; } + value_type const& view_get_reference() const { return m_value; } + + //@} +}; + + +/** Base class for list-append and prepend monoid classes. + * + * The key to efficient reducers is that the `identity` operation, which + * creates a new per-strand view, and the `reduce` operation, which combines + * two per-strand views, must be constant-time operations. Two lists can be + * concatenated in constant time only if they have the same allocator. + * Therefore, all the per-strand list accumulator variables must be created + * with the same allocator as the leftmost view list. + * + * This means that a list reduction monoid must have a copy of the allocator + * of the leftmost view's list, so that it can use it in the `identity` + * operation. This, in turn, requires that list reduction monoids have a + * specialized `construct()` function, which constructs the leftmost view + * before the monoid, and then passes the leftmost view's allocator to the + * monoid constructor. + * + * @tparam View The list-append or prepend view class. + * @tparam Align If `false` (the default), reducers instantiated on this + * monoid will be naturally aligned (the Intel Cilk Plus library 1.0 + * behavior). If `true`, reducers instantiated on this monoid + * will be cache-aligned for binary compatibility with + * reducers in Intel Cilk Plus library version 0.9. + * + * @see ReducersList + * @see list_view_base + */ +template +class list_monoid_base : public monoid_with_view +{ + typedef typename View::value_type list_type; + typedef typename list_type::allocator_type allocator_type; + typedef provisional_guard view_guard; + + allocator_type m_allocator; + +public: + + /** Constructor. + * + * There is no default constructor for list monoids, because the allocator + * must always be specified. + * + * @param allocator The list allocator to be used when + * identity-constructing new views. + */ + list_monoid_base(const allocator_type& allocator = allocator_type()) : + m_allocator(allocator) {} + + /** Creates an identity view. + * + * List view identity constructors take the list allocator as an argument. + * + * @param v The address of the uninitialized memory in which the view + * will be constructed. + */ + void identity(View *v) const { ::new((void*) v) View(m_allocator); } + + /** @name construct functions + * + * All `construct()` functions first construct the leftmost view, using + * the optional @a x1, @a x2, and @a x3 arguments that were passed in from + * the reducer constructor. They then call the view's `get_allocator()` + * function to get the list allocator from its contained list, and pass it + * to the monoid constructor. + */ + //@{ + + template + static void construct(Monoid* monoid, View* view) + { + view_guard vg( new((void*) view) View() ); + vg.confirm_if( new((void*) monoid) Monoid(view->get_allocator()) ); + } + + template + static void construct(Monoid* monoid, View* view, const T1& x1) + { + view_guard vg( new((void*) view) View(x1) ); + vg.confirm_if( new((void*) monoid) Monoid(view->get_allocator()) ); + } + + template + static void construct(Monoid* monoid, View* view, + const T1& x1, const T2& x2) + { + view_guard vg( new((void*) view) View(x1, x2) ); + vg.confirm_if( new((void*) monoid) Monoid(view->get_allocator()) ); + } + + template + static void construct(Monoid* monoid, View* view, + const T1& x1, const T2& x2, const T3& x3) + { + view_guard vg( new((void*) view) View(x1, x2, x3) ); + vg.confirm_if( new((void*) monoid) Monoid(view->get_allocator()) ); + } + + //@} +}; + +//@} + +} // namespace internal + + +/** @ingroup ReducersList */ +//@{ + +/** The list-append reducer view class. + * + * This is the view class for reducers created with + * `cilk::reducer< cilk::op_list_append >`. It holds the + * accumulator variable for the reduction, and allows only append operations + * to be performed on it. + * + * @note The reducer "dereference" operation (`reducer::operator *()`) + * yields a reference to the view. Thus, for example, the view class's + * `push_back` operation would be used in an expression like + * `r->push_back(a)`, where `r` is a list-append reducer variable. + * + * @tparam Type The list element type (not the list type). + * @tparam Allocator The list allocator type. + * + * @see ReducersList + * @see op_list_append + */ +template ::allocator_type> +class op_list_append_view : public internal::list_view_base +{ + typedef internal::list_view_base base; + typedef std::list list_type; + typedef typename list_type::iterator iterator; + + iterator end() { return this->m_value.end(); } + +public: + + /** @name Constructors. + * + * All op_list_append_view constructors simply pass their arguments on to + * the @ref internal::list_view_base base class constructor. + * + * @ref internal::list_view_base supports all the std::list constructor + * forms, as well as the reducer move_in constructor form. + */ + //@{ + + op_list_append_view() : base() {} + + template + op_list_append_view(const T1& x1) : base(x1) {} + + template + op_list_append_view(const T1& x1, const T2& x2) : base(x1, x2) {} + + template + op_list_append_view(const T1& x1, const T2& x2, const T3& x3) : + base(x1, x2, x3) {} + + //@} + + /** @name View modifier operations. + */ + //@{ + + /** Adds an element at the end of the list. + * + * This is equivalent to `list.push_back(element)` + */ + void push_back(const Type& element) + { this->m_value.push_back(element); } + + /** Inserts elements at the end of the list. + * + * This is equivalent to `list.insert(list.end(), n, element)` + */ + void insert_back(typename list_type::size_type n, const Type& element) + { this->m_value.insert(end(), n, element); } + + /** Inserts elements at the end of the list. + * + * This is equivalent to `list.insert(list.end(), first, last)` + */ + template + void insert_back(Iter first, Iter last) + { this->m_value.insert(end(), first, last); } + + /** Splices elements at the end of the list. + * + * This is equivalent to `list.splice(list.end(), x)` + */ + void splice_back(list_type& x) { + if (x.get_allocator() == this->m_value.get_allocator()) + this->m_value.splice(end(), x); + else { + insert_back(x.begin(), x.end()); + x.clear(); + } + } + + /** Splices elements at the end of the list. + * + * This is equivalent to `list.splice(list.end(), x, i)` + */ + void splice_back(list_type& x, iterator i) { + if (x.get_allocator() == this->m_value.get_allocator()) + this->m_value.splice(end(), x, i); + else { + push_back(*i); + x.erase(i); + } + } + + /** Splices elements at the end of the list. + * + * This is equivalent to `list.splice(list.end(), x, first, last)` + */ + void splice_back(list_type& x, iterator first, iterator last) { + if (x.get_allocator() == this->m_value.get_allocator()) + this->m_value.splice(end(), x, first, last); + else { + insert_back(first, last); + x.erase(first, last); + } + } + + //@} + + /** Reduces the views of two strands. + * + * This function is invoked by the @ref op_list_append monoid to combine + * the views of two strands when the right strand merges with the left + * one. It appends the value contained in the right-strand view to the + * value contained in the left-strand view, and leaves the value in the + * right-strand view undefined. + * + * @param right A pointer to the right-strand view. (`this` points to + * the left-strand view.) + * + * @note Used only by the @ref op_list_append monoid to implement the + * monoid reduce operation. + */ + void reduce(op_list_append_view* right) + { + __CILKRTS_ASSERT( + this->m_value.get_allocator() == right->m_value.get_allocator()); + this->m_value.splice(end(), right->m_value); + } +}; + + +/** The list-prepend reducer view class. + * + * This is the view class for reducers created with + * `cilk::reducer< cilk::op_list_prepend >`. It holds the + * accumulator variable for the reduction, and allows only prepend operations + * to be performed on it. + * + * @note The reducer "dereference" operation (`reducer::operator *()`) + * yields a reference to the view. Thus, for example, the view class's + * `push_front` operation would be used in an expression like + * `r->push_front(a)`, where `r` is a list-prepend reducer variable. + * + * @tparam Type The list element type (not the list type). + * @tparam Allocator The list allocator type. + * + * @see ReducersList + * @see op_list_prepend + */ +template ::allocator_type> +class op_list_prepend_view : public internal::list_view_base +{ + typedef internal::list_view_base base; + typedef std::list list_type; + typedef typename list_type::iterator iterator; + + iterator begin() { return this->m_value.begin(); } + +public: + + /** @name Constructors. + * + * All op_list_prepend_view constructors simply pass their arguments on to + * the @ref internal::list_view_base base class constructor. + * + * @ref internal::list_view_base supports all the std::list constructor + * forms, as well as the reducer move_in constructor form. + * + */ + //@{ + + op_list_prepend_view() : base() {} + + template + op_list_prepend_view(const T1& x1) : base(x1) {} + + template + op_list_prepend_view(const T1& x1, const T2& x2) : base(x1, x2) {} + + template + op_list_prepend_view(const T1& x1, const T2& x2, const T3& x3) : + base(x1, x2, x3) {} + + //@} + + /** @name View modifier operations. + */ + //@{ + + /** Adds an element at the beginning of the list. + * + * This is equivalent to `list.push_front(element)` + */ + void push_front(const Type& element) + { this->m_value.push_front(element); } + + /** Inserts elements at the beginning of the list. + * + * This is equivalent to `list.insert(list.begin(), n, element)` + */ + void insert_front(typename list_type::size_type n, const Type& element) + { this->m_value.insert(begin(), n, element); } + + /** Inserts elements at the beginning of the list. + * + * This is equivalent to `list.insert(list.begin(), first, last)` + */ + template + void insert_front(Iter first, Iter last) + { this->m_value.insert(begin(), first, last); } + + /** Splices elements at the beginning of the list. + * + * This is equivalent to `list.splice(list.begin(), x)` + */ + void splice_front(list_type& x) { + if (x.get_allocator() == this->m_value.get_allocator()) + this->m_value.splice(begin(), x); + else { + insert_front(x.begin(), x.begin()); + x.clear(); + } + } + + /** Splices elements at the beginning of the list. + * + * This is equivalent to `list.splice(list.begin(), x, i)` + */ + void splice_front(list_type& x, iterator i) { + if (x.get_allocator() == this->m_value.get_allocator()) + this->m_value.splice(begin(), x, i); + else { + push_front(*i); + x.erase(i); + } + } + + /** Splices elements at the beginning of the list. + * + * This is equivalent to `list.splice(list.begin(), x, first, last)` + */ + void splice_front(list_type& x, iterator first, iterator last) { + if (x.get_allocator() == this->m_value.get_allocator()) + this->m_value.splice(begin(), x, first, last); + else { + insert_front(first, last); + x.erase(first, last); + } + } + + //@} + + /** Reduces the views of two strands. + * + * This function is invoked by the @ref op_list_prepend monoid to combine + * the views of two strands when the right strand merges with the left + * one. It prepends the value contained in the right-strand view to the + * value contained in the left-strand view, and leaves the value in the + * right-strand view undefined. + * + * @param right A pointer to the right-strand view. (`this` points to + * the left-strand view.) + * + * @note Used only by the @ref op_list_prepend monoid to implement the + * monoid reduce operation. + */ + /** Reduce operation. + * + * Required by @ref monoid_base. + */ + void reduce(op_list_prepend_view* right) + { + __CILKRTS_ASSERT( + this->m_value.get_allocator() == right->m_value.get_allocator()); + this->m_value.splice(begin(), right->m_value); + } +}; + + + +/** Monoid class for list-append reductions. Instantiate the cilk::reducer + * template class with a op_list_append monoid to create a list-append reducer + * class. For example, to create a list of strings: + * + * cilk::reducer< cilk::op_list_append > r; + * + * @tparam Type The list element type (not the list type). + * @tparam Alloc The list allocator type. + * @tparam Align If `false` (the default), reducers instantiated on this + * monoid will be naturally aligned (the Intel Cilk Plus library 1.0 + * behavior). If `true`, reducers instantiated on this monoid + * will be cache-aligned for binary compatibility with + * reducers in Intel Cilk Plus library version 0.9. + * + * @see ReducersList + * @see op_list_append_view + */ +template ::allocator_type, + bool Align = false> +struct op_list_append : + public internal::list_monoid_base, Align> +{ + /// Construct with default allocator. + op_list_append() {} + /// Construct with specified allocator. + op_list_append(const Allocator& alloc) : + internal::list_monoid_base, Align>(alloc) {} +}; + +/** Monoid class for list-prepend reductions. Instantiate the cilk::reducer + * template class with a op_list_prepend monoid to create a list-prepend + * reducer class. For example, to create a list of strings: + * + * cilk::reducer< cilk::op_list_prepend > r; + * + * @tparam Type The list element type (not the list type). + * @tparam Alloc The list allocator type. + * @tparam Align If `false` (the default), reducers instantiated on this + * monoid will be naturally aligned (the Intel Cilk Plus library 1.0 + * behavior). If `true`, reducers instantiated on this monoid + * will be cache-aligned for binary compatibility with + * reducers in Intel Cilk Plus library version 0.9. + * + * @see ReducersList + * @see op_list_prepend_view + */ +template ::allocator_type, + bool Align = false> +struct op_list_prepend : + public internal::list_monoid_base, Align> +{ + /// Construct with default allocator. + op_list_prepend() {} + /// Construct with specified allocator. + op_list_prepend(const Allocator& alloc) : + internal::list_monoid_base, Align>(alloc) {} +}; + + +/** Deprecated list-append reducer wrapper class. + * + * reducer_list_append is the same as + * @ref reducer<@ref op_list_append>, except that reducer_list_append is a + * proxy for the contained view, so that accumulator variable update + * operations can be applied directly to the reducer. For example, an element + * is appended to a `reducer<%op_list_append>` with `r->push_back(a)`, but an + * element can be appended to a `%reducer_list_append` with `r.push_back(a)`. + * + * @deprecated Users are strongly encouraged to use `reducer` + * reducers rather than the old wrappers like reducer_list_append. + * The `reducer` reducers show the reducer/monoid/view + * architecture more clearly, are more consistent in their + * implementation, and present a simpler model for new + * user-implemented reducers. + * + * @note Implicit conversions are provided between `%reducer_list_append` + * and `reducer<%op_list_append>`. This allows incremental code + * conversion: old code that used `%reducer_list_append` can pass a + * `%reducer_list_append` to a converted function that now expects a + * pointer or reference to a `reducer<%op_list_append>`, and vice + * versa. + * + * @tparam Type The value type of the list. + * @tparam Allocator The allocator type of the list. + * + * @see op_list_append + * @see reducer + * @see ReducersList + */ +template > +class reducer_list_append : + public reducer > +{ + typedef reducer > base; + using base::view; +public: + + /// The reducer's list type. + typedef typename base::value_type list_type; + + /// The list's element type. + typedef Type list_value_type; + + /// The reducer's primitive component type. + typedef Type basic_value_type; + + /// The monoid type. + typedef typename base::monoid_type Monoid; + + /** @name Constructors + */ + //@{ + + /** Constructs a reducer with an empty list. + */ + reducer_list_append() {} + + /** Constructs a reducer with a specified initial list value. + */ + reducer_list_append(const std::list &initial_value) : + base(initial_value) {} + + //@} + + + /** @name Forwarded functions + * @details Functions that update the contained accumulator variable are + * simply forwarded to the contained @ref op_and_view. */ + //@{ + + /// @copydoc op_list_append_view::push_back(const Type&) + void push_back(const Type& element) { view().push_back(element); } + + //@} + + /** Allows mutable access to the list within the current view. + * + * @warning If this method is called before the parallel calculation is + * complete, the list returned by this method will be a partial + * result. + * + * @returns A mutable reference to the list within the current view. + */ + list_type &get_reference() { return view().view_get_reference(); } + + /** Allows read-only access to the list within the current view. + * + * @warning If this method is called before the parallel calculation is + * complete, the list returned by this method will be a partial + * result. + * + * @returns A const reference to the list within the current view. + */ + list_type const &get_reference() const { return view().view_get_reference(); } + + /// @name Dereference + //@{ + /** Dereferencing a wrapper is a no-op. It simply returns the wrapper. + * Combined with the rule that a wrapper forwards view operations to the + * view, this means that view operations can be written the same way on + * reducers and wrappers, which is convenient for incrementally + * converting code using wrappers to code using reducers. That is: + * + * reducer< op_list_append > r; + * r->push_back(a); // *r returns the view + * // push_back is a view member function + * + * reducer_list_append w; + * w->push_back(a); // *w returns the wrapper + * // push_back is a wrapper member function that + * // calls the corresponding view function + */ + //@{ + reducer_list_append& operator*() { return *this; } + reducer_list_append const& operator*() const { return *this; } + + reducer_list_append* operator->() { return this; } + reducer_list_append const* operator->() const { return this; } + //@} + + /** @name Upcast + * @details In Intel Cilk Plus library 0.9, reducers were always cache-aligned. + * In library 1.0, reducer cache alignment is optional. By default, + * reducers are unaligned (i.e., just naturally aligned), but legacy + * wrappers inherit from cache-aligned reducers for binary compatibility. + * + * This means that a wrapper will automatically be upcast to its aligned + * reducer base class. The following conversion operators provide + * pseudo-upcasts to the corresponding unaligned reducer class. + */ + //@{ + operator reducer< op_list_append >& () + { + return *reinterpret_cast< + reducer< op_list_append >* + >(this); + } + operator const reducer< op_list_append >& () const + { + return *reinterpret_cast< + const reducer< op_list_append >* + >(this); + } + //@} + +}; + + +/** Deprecated list-prepend reducer wrapper class. + * + * reducer_list_prepend is the same as + * @ref reducer<@ref op_list_prepend>, except that reducer_list_prepend is a + * proxy for the contained view, so that accumulator variable update operations + * can be applied directly to the reducer. For example, an element is prepended + * to a `reducer` with `r->push_back(a)`, but an element is + * prepended to a `reducer_list_prepend` with `r.push_back(a)`. + * + * @deprecated Users are strongly encouraged to use `reducer` + * reducers rather than the old wrappers like reducer_list_prepend. + * The `reducer` reducers show the reducer/monoid/view + * architecture more clearly, are more consistent in their + * implementation, and present a simpler model for new + * user-implemented reducers. + * + * @note Implicit conversions are provided between `%reducer_list_prepend` + * and `reducer<%op_list_prepend>`. This allows incremental code + * conversion: old code that used `%reducer_list_prepend` can pass a + * `%reducer_list_prepend` to a converted function that now expects a + * pointer or reference to a `reducer<%op_list_prepend>`, and vice + * versa. + * + * @tparam Type The value type of the list. + * @tparam Allocator The allocator type of the list. + * + * @see op_list_prepend + * @see reducer + * @see ReducersList + */ +template > +class reducer_list_prepend : + public reducer > +{ + typedef reducer > base; + using base::view; +public: + + /** The reducer's list type. + */ + typedef typename base::value_type list_type; + + /** The list's element type. + */ + typedef Type list_value_type; + + /** The reducer's primitive component type. + */ + typedef Type basic_value_type; + + /** The monoid type. + */ + typedef typename base::monoid_type Monoid; + + /** @name Constructors + */ + //@{ + + /** Constructs a reducer with an empty list. + */ + reducer_list_prepend() {} + + /** Constructs a reducer with a specified initial list value. + */ + reducer_list_prepend(const std::list &initial_value) : + base(initial_value) {} + + //@} + + /** @name Forwarded functions + * @details Functions that update the contained accumulator variable are + * simply forwarded to the contained @ref op_and_view. + */ + //@{ + + /// @copydoc op_list_prepend_view::push_front(const Type&) + void push_front(const Type& element) { view().push_front(element); } + + //@} + + /** Allows mutable access to the list within the current view. + * + * @warning If this method is called before the parallel calculation is + * complete, the list returned by this method will be a partial + * result. + * + * @returns A mutable reference to the list within the current view. + */ + list_type &get_reference() { return view().view_get_reference(); } + + /** Allows read-only access to the list within the current view. + * + * @warning If this method is called before the parallel calculation is + * complete, the list returned by this method will be a partial + * result. + * + * @returns A const reference to the list within the current view. + */ + list_type const &get_reference() const { return view().view_get_reference(); } + + /// @name Dereference + /** Dereferencing a wrapper is a no-op. It simply returns the wrapper. + * Combined with the rule that a wrapper forwards view operations to the + * view, this means that view operations can be written the same way on + * reducers and wrappers, which is convenient for incrementally + * converting code using wrappers to code using reducers. That is: + * + * reducer< op_list_prepend > r; + * r->push_front(a); // *r returns the view + * // push_front is a view member function + * + * reducer_list_prepend w; + * w->push_front(a); // *w returns the wrapper + * // push_front is a wrapper member function that + * // calls the corresponding view function + */ + //@{ + reducer_list_prepend& operator*() { return *this; } + reducer_list_prepend const& operator*() const { return *this; } + + reducer_list_prepend* operator->() { return this; } + reducer_list_prepend const* operator->() const { return this; } + //@} + + /** @name Upcast + * @details In Intel Cilk Plus library 0.9, reducers were always cache-aligned. + * In library 1.0, reducer cache alignment is optional. By default, + * reducers are unaligned (i.e., just naturally aligned), but legacy + * wrappers inherit from cache-aligned reducers for binary compatibility. + * + * This means that a wrapper will automatically be upcast to its aligned + * reducer base class. The following conversion operators provide + * pseudo-upcasts to the corresponding unaligned reducer class. + */ + //@{ + operator reducer< op_list_prepend >& () + { + return *reinterpret_cast< + reducer< op_list_prepend >* + >(this); + } + operator const reducer< op_list_prepend >& () const + { + return *reinterpret_cast< + const reducer< op_list_prepend >* + >(this); + } + //@} + +}; + +/// @cond internal + +/** Metafunction specialization for reducer conversion. + * + * This specialization of the @ref legacy_reducer_downcast template class + * defined in reducer.h causes the `reducer< op_list_append >` + * class to have an `operator reducer_list_append& ()` + * conversion operator that statically downcasts the `reducer` + * to the corresponding `reducer_list_append` type. (The reverse conversion, + * from `reducer_list_append` to `reducer`, is just an upcast, + * which is provided for free by the language.) + */ +template +struct legacy_reducer_downcast > > +{ + typedef reducer_list_append type; +}; + +/** Metafunction specialization for reducer conversion. + * + * This specialization of the @ref legacy_reducer_downcast template class + * defined in reducer.h causes the + * `reducer< op_list_prepend >` class to have an + * `operator reducer_list_prepend& ()` conversion operator + * that statically downcasts the `reducer` to the + * corresponding `reducer_list_prepend` type. (The reverse conversion, from + * `reducer_list_prepend` to `reducer`, is just an upcast, + * which is provided for free by the language.) + */ +template +struct legacy_reducer_downcast > > +{ + typedef reducer_list_prepend type; +}; + +/// @endcond + +//@} + +} // Close namespace cilk + +#endif // REDUCER_LIST_H_INCLUDED diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_max.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_max.h new file mode 100644 index 0000000000..3982cb11c2 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_max.h @@ -0,0 +1,57 @@ +/* reducer_max.h -*- C++ -*- + * + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + */ + +/** @file reducer_max.h + * + * @brief Defines classes for doing parallel maximum reductions. + * + * @ingroup ReducersMinMax + * + * @see ReducersMinMax + */ + +#include "reducer_min_max.h" diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_min.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_min.h new file mode 100644 index 0000000000..912979d722 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_min.h @@ -0,0 +1,57 @@ +/* reducer_min.h -*- C++ -*- + * + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + */ + +/** @file reducer_min.h + * + * @brief Defines classes for doing parallel minimum reductions. + * + * @ingroup ReducersMinMax + * + * @see ReducersMinMax + */ + +#include "reducer_min_max.h" diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_min_max.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_min_max.h new file mode 100644 index 0000000000..641aa82390 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_min_max.h @@ -0,0 +1,3741 @@ +/* reducer_min_max.h -*- C++ -*- + * + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + */ + +/** @file reducer_min_max.h + * + * @brief Defines classes for doing parallel minimum and maximum reductions. + * + * @ingroup ReducersMinMax + * + * @see ReducersMinMax + */ + +#ifndef REDUCER_MIN_MAX_H_INCLUDED +#define REDUCER_MIN_MAX_H_INCLUDED + +#include + +#ifdef __cplusplus + +#include +#include + +/** @defgroup ReducersMinMax Minimum and Maximum Reducers + * + * Minimum and maximum reducers allow the computation of the minimum or + * maximum of a set of values in parallel. + * + * @ingroup Reducers + * + * You should be familiar with @ref pagereducers "Intel(R) Cilk(TM) Plus reducers", + * described in file `reducers.md`, and particularly with @ref reducers_using, + * before trying to use the information in this file. + * + * @section redminmax_usage Usage Examples + * + * cilk::reducer< cilk::op_max > rm; + * cilk_for (int i = 0; i < ARRAY_SIZE; ++i) + * { + * rm->calc_max(a[i]); // or *rm = cilk::max_of(*max, a[i]) + * } + * std::cout << "maximum value is " << rm.get_value() << std::endl; + * + * and + * + * cilk::reducer< cilk::op_min_index > rmi; + * cilk_for (int i = 0; i < ARRAY_SIZE; ++i) + * { + * rmi->calc_min(i, a[i]) // or *rmi = cilk::min_of(*rmi, i, a[i]); + * } + * std::cout << "minimum value a[" << rmi.get_value().first << "] = " + * << rmi.get_value().second << std::endl; + * + * @section redminmax_monoid The Monoid + * + * @subsection redminmax_monoid_values Value Set + * + * The value set of a minimum or maximum reducer is the set of values of + * `Type`, augmented with a "special identity value" which is not a value of + * `Type`, but which is defined to be greater than (less than) any value of + * `Type`. + * + * @subsection redminmax_monoid_operator Operator + * + * By default, the operator of a minimum reducer is defined as + * + * x MIN y == (x < y) ? x : y + * + * Thus, `a1 MIN a2 MIN … an` is the first `ai` which is not greater than any + * other `ai`. + * + * The operator of a maximum reducer is defined as + * + * x MAX y == (x > y) ? x : y + * + * Thus, `a1 MAX a2 MAX … an` is the first `ai` which is not less than any + * other `ai`. + * + * @subsection redminmax_monoid_comparators Comparators + * + * Min/max reducers are not limited to finding the minimum or maximum value + * determined by the `<` or `>` operator. In fact, all min/max reducers use a + * _comparator_, which is either a function or an object of a function class + * that defines a [strict weak ordering] + * (http://en.wikipedia.org/wiki/Strict_weak_ordering#Strict_weak_orderings) + * on a set of values. (This is exactly the same as the requirement for the + * comparison predicate for STL associative containers and sorting + * algorithms.) + * + * Just as with STL algorithms and containers, the comparator type parameter + * for min/max reducers is optional. If it is omitted, it defaults to + * `std::less`, which gives the behavior described in the previous section. + * Using non-default comparators (anything other than `std::less`) with + * min/max reducers is just like using them with STL containers and + * algorithms. + * + * Taking comparator objects into account, the reduction operation `MIN` for a + * minimum reducer is defined as + * + * x MIN y == compare(x, y) ? x : y + * + * where `compare()` is the reducer's comparator. Similarly, the reduction + * operation MAX for a maximum reducer is defined as + * + * x MAX y == compare(y, x) ? x : y + * + * (If `compare(x, y) == x < y`, then `compare(y, x) == x > y`.) + * + * @subsection redminmax_monoid_identity Identity + * + * The identity value of a min/max reducer is its monoid's + * ["special identity value"](#redminmax_monoid_values), which is not a value + * of the reducer's data type. (See @ref redminmax_initial.) + * + * @section redminmax_index Value and Index Reducers + * + * Min/max reducers come in two families. The _value_ reducers, with the + * `op_min` and `op_max` monoids, simply find the smallest or largest value + * from a set of values. The _index_ reducers, with the `op_min_index` and + * `op_max_index` monoids, also record an index value associated with the + * first occurrence of the smallest or largest value. + * + * In the `%op_min_index` usage example [above](#redminmax_usage), the values + * are taken from an array, and the index of a value is the index of the array + * element it comes from. More generally, though, an index can be any sort of + * key which identifies a particular value in a collection of values. For + * example, if the values were taken from the nodes of a tree, then the + * "index" of a value might be a pointer to the node containing that value. + * + * A min/max index reducer is essentially the same as a min/max value reducer + * whose value type is an (index, value) pair, and whose comparator ignores + * the index part of the pair. (index, value) pairs are represented by + * `std::pair` objects. This has the consequence that wherever + * the interface of a min/max value reducer has a `Type`, the interface of a + * min/max index reducer has a `std::pair`. (There are + * convenience variants of the `reducer(Type)` constructor and the + * `calc_min()`, `calc_max()`, `%min_of()`, and `%max_of()` functions that + * take an index argument and a value argument instead of a single index/value + * pair argument.) + * + * @section redminmax_operations Operations + * + * @subsection redminmax_constructors Constructors + * + * @subsubsection redminmax_constructors_value Min/Max Value Reducers + * + * reducer() // identity + * reducer(const Compare& compare) // identity + * reducer(const Type& value) + * reducer(move_in(Type& variable)) + * reducer(const Type& value, const Compare& compare) + * reducer(move_in(Type& variable), const Compare& compare) + * + * @subsubsection redminmax_constructors_index Min/Max Index Reducers + * + * reducer() // identity + * reducer(const Compare& compare) // identity + * reducer(const std::pair& pair) + * reducer(const Index& index, const Type& value) + * reducer(move_in(std::pair& variable)) + * reducer(const std::pair& pair, const Compare& compare) + * reducer(const Index& index, const Type& value, const Compare& compare) + * reducer(move_in(std::pair& variable), const Compare& compare) + * + * See the explanation of the following two constructors in + * @ref redminmax_index_vector. + * + * reducer(const Index& index) + * reducer(const Index& index, const Compare& compare) + * + * @subsection redminmax_get_set Set and Get + * + * r.set_value(const Type& value) + * Type = r.get_value() const + * r.move_in(Type& variable) + * r.move_out(Type& variable) + * + * Note that for an index reducer, the `Type` in these operations is actually a + * `std::pair`. (See @ref redminmax_index.) There is _not_ a + * `set_value(value, index)` operation. + * + * @subsection redminmax_initial Initial Values and is_set() + * + * The initial value of the leftmost view of a default-initialized min/max + * reducer, or of a non-leftmost view (created for a stolen parallel strand) + * is the special identity value, which is not a value of the reducer's value + * type. + * + * A view will have a real (non-identity) value if: + * + * - it is the leftmost view of a reducer that was constructed with an + * initial value, or + * - it was assigned a value with a call to `reducer.set_value()` or + * `reducer.move_in()`, or + * - it has been updated with a call to `reducer->calc_min()` or + * `reducer->calc_max()`, or + * - it has been updated with an assignment `*reducer = min_of(*reducer, x)` + * or `*reducer = max_of(*reducer, x)`. + * + * Calling `get_value()` or `move_out()` on a reducer whose view has the + * special identity value will yield an undefined result. The `is_set()` + * function can be used to test whether a view has the special identity value + * or a real value. If a reducer's current view has the special identity + * value, then `reducer()->is_set()` will return `false` (and + * `reducer.get_value()` will return an undefined value); if the view has a + * real value, them `reducer->is_set()` will return `true` and + * `reducer.get_value()` will return the value. + * + * @subsubsection redminmax_index_vector Special Issues with Min/Max Index Reducers + * + * The index portion of the computed index/value pair will be wrong in the + * following special case: + * + * - The reducer's value type is a simple numeric type. + * - The reducer uses the default comparator (`std::less`). + * - The reducer is updated at least once with a call to `calc_min()` or + * `calc_max()` or an assignment with `min_of()` or `max_of()`. + * - The value in _every_ update to the reducer is the maximum value of the + * value type (for a min_index reducer) or the minimum value of the value + * type (for a max_index reducer). + * + * In this case, `reducer.get_value().first` should be the index argument from + * the first reducer update, but it will actually be the default value of the + * `Index` type. Now, in the common case where the index type is an integer + * type and the reducer is finding the smallest or largest element in an + * array, the default value of the index type will be zero, which is the + * index of the first element in the array, so everything will work out: + * + * unsigned a[3] = {0, 0, 0}; + * reducer< op_max_index > r; + * for (int i = 0; i < 3; ++i) r->calc_max(i, a[i]); + * // r.get_value() = (0, 0) + * + * However, it doesn't always work out so well: + * + * typedef std::map my_map; + * my_map a; + * a["first"] = 0; + * a["second"] = 0; + * a["third"] = 0; + * reducer< op_max_index > r; + * for (typename my_map::iterator i = a.begin(); i != a.end(); ++i) + * r.calc_max(i->first, i->second); + * // r.get_value() = ("", 0), should be ("first", 0) + * + * If you know that no data value is associated with the default index value, + * then you can treat the default index value as a flag meaning "use the index + * of the first data value." But suppose that you don't know whether there is + * an element in the map with index `""`. Then you won't know what to do when + * `r.get_value().first == ""`. + * + * As a workaround for this conundrum, you can specify an alternative + * "default" index value. Either provide an index argument, _but not a + * value argument_, to the reducer constructor: + * + * reducer< op_max_index > + * r(a.empty() ? std::string() : a.begin()->first); + * + * or specify the default index with the view `set_default_index()` function: + * + * reducer< op_max_index > r; + * if (!a.empty()) r->set_default_index(a.begin()->first); + * + * Note that setting a default index, unlike setting an initial value, does + * not mark the view as having a non-identity value: + * + * reducer< op_min_index > r; + * r->set_default_index(-1); + * // r->is_set() = false + * // r.get_value() is undefined + * + * @subsection redminmax_view_ops View Operations + * + * The basic reduction operation is `x = x MIN a` for a minimum reducer, or + * `x = x MAX a` for a maximum reducer. The basic syntax for these operations + * uses the `calc_min()` and `calc_max()` member functions of the view class. + * An assignment syntax is also provided, using the `%cilk::min_of()` and + * `%cilk::max_of()` global functions: + * + * Class | Modifier | Assignment + * ---------------|---------------------|----------- + * `op_min` | `r->calc_min(x)` | `*r = min_of(*r, x)` or `*r = min_of(x, *r)` + * `op_max` | `r->calc_max(x)` | `*r = max_of(*r, x)` or `*r = max_of(x, *r)` + * `op_min_index` | `r->calc_min(i, x)` | `*r = min_of(*r, i, x)` or `*r = min_of(i, x, *r)` + * `op_max_index` | `r->calc_max(i, x)` | `*r = max_of(*r, i, x)` or `*r = max_of(i, x, *r)` + * + * Wherever an "`i`, `x`" argument pair is shown in the table above, a single + * pair argument may be passed instead. For example: + * + * Index index; + * Type value; + * std::pair ind_val(index, value); + * // The following statements are all equivalent. + * r->calc_min(index, value); + * r->calc_min(ind_val); + * *r = min_of(*r, index, value); + * *r = min_of(*r, ind_val); + * + * The `calc_min()` and `calc_max()` member functions return a reference to + * the view, so they can be chained: + * + * r->calc_max(x).calc_max(y).calc_max(z); + * + * In a `%min_of()` or `%max_of()` assignment, the view on the left-hand side + * of the assignment must be the same as the view argument in the call. + * Otherwise, the behavior is undefined (but an assertion error will occur if + * the code is compiled with debugging enabled). + * + * *r = max_of(*r, x); // OK + * *r1 = max_of(*r2, y); // ERROR + * + * `%min_of()` and `%max_of()` calls can be nested: + * + * *r = max_of(max_of(max_of(*r, x), y), z); + * *r = min_of(i, a[i], min_of(j, a[j], min_of(k, a[k], *r))); + * + * @section redminmax_compatibility Binary Compatibility Issues + * + * Most Intel Cilk Plus library reducers provide binary compatibility between + * `reducer_KIND` reducers compiled with Intel Cilk Plus library version 0.9 + * (distributed with Intel® C++ Composer XE version 13.0 and earlier) and the + * ame reducers compiled with Intel Cilk Plus library version 1.0 and later. + * + * Because of implementation changes that were needed to allow vectorization + * of loops containing min/max reducers, this binary compatibility is _not_ + * generally available for min/max reducers, either between Intel Cilk Plus library + * versions 0.9 and 1.0, or between versions 1.0 and 1.1. (Code compiled with + * different versions can be linked together safely, but min/max reducers in + * different library versions are in different namespaces, so reducer objects + * cannot be shared between them.) + * + * If this is an inconvenience, the simplest solution is just to recompile any + * existing code you may have that uses min/max reducers. If that is + * impossible, you can define the `CILK_LIBRARY_0_9_REDUCER_MINMAX` macro (on + * the compiler command line, or in your source code before including + * `reducer_min_max.h`) when compiling with the new library. This will cause + * it to generate numeric reducers that will be link-time and run-time + * compatible with the 0.9 library. + * + * @subsection redminmax_compatibility_stateful Non-empty Comparators + * + * The representation of min/max reducers with non-empty comparator objects or + * with comparator functions is so different in between the 0.9 and 1.1 + * libraries that there is no way to make them binary compatible, even when + * compiling with `CILK_LIBRARY_0_9_REDUCER_MINMAX`. Therefore, the + * `reducer_{min|max}[_index]` wrapper classes have been coded in the 1.0 and + * later library so that they will not even compile when instantiated with a + * non-empty comparator class. + * + * This is not a problem when using an empty comparator class, such as the + * default `std::less`. + * + * @section redminmax_types Type Requirements + * + * `Type` and `Index` must be `Copy Constructible`, `Default Constructible`, + * and `Assignable`. + * + * `Compare` must be `Copy Constructible` if the reducer is constructed with a + * `compare` argument, and `Default Constructible` otherwise. + * + * The `Compare` function must induce a strict weak ordering on the elements + * of `Type`. + * + * @section redminmax_in_c Minimum and Maximum Reducers in C + * + * These macros can be used to do minimum and maximum reductions in C: + * + * Declaration | Type | Operation + * -----------------------------|-----------------------------------|---------- + * @ref CILK_C_REDUCER_MIN |@ref CILK_C_REDUCER_MIN_TYPE |@ref CILK_C_REDUCER_MIN_CALC + * @ref CILK_C_REDUCER_MAX |@ref CILK_C_REDUCER_MAX_TYPE |@ref CILK_C_REDUCER_MAX_CALC + * @ref CILK_C_REDUCER_MIN_INDEX |@ref CILK_C_REDUCER_MIN_INDEX_TYPE |@ref CILK_C_REDUCER_MIN_INDEX_CALC + * @ref CILK_C_REDUCER_MAX_INDEX |@ref CILK_C_REDUCER_MAX_INDEX_TYPE |@ref CILK_C_REDUCER_MAX_INDEX_CALC + * + * For example: + * + * CILK_C_REDUCER_MIN(r, int, INT_MAX); + * CILK_C_REGISTER_REDUCER(r); + * cilk_for(int i = 0; i != n; ++i) { + * CILK_C_REDUCER_MIN_CALC(r, a[i]); + * } + * CILK_C_UNREGISTER_REDUCER(r); + * printf("The smallest value in a is %d\n", REDUCER_VIEW(r)); + * + * + * CILK_C_REDUCER_MAX_INDEX(r, uint, 0); + * CILK_C_REGISTER_REDUCER(r); + * cilk_for(int i = 0; i != n; ++i) { + * CILK_C_REDUCER_MAX_INDEX_CALC(r, i, a[i]); + * } + * CILK_C_UNREGISTER_REDUCER(r); + * printf("The largest value in a is %u at %d\n", + * REDUCER_VIEW (r).value, REDUCER_VIEW(r).index); + * + * See @ref reducers_c_predefined. + */ + +namespace cilk { + +/** @defgroup ReducersMinMaxBinComp Binary compatibility + * + * If the macro `CILK_LIBRARY_0_9_REDUCER_MINMAX` is defined, then we generate + * reducer code and data structures which are binary-compatible with code that + * was compiled with the old min/max wrapper definitions, so we want the + * mangled names of the legacy min/max reducer wrapper classes to be the + * same as the names produced by the old definitions. + * + * Conversely, if the macro is not defined, then we generate binary- + * incompatible code, so we want different mangled names, to make sure that + * the linker does not allow new and old compiled legacy wrappers to be passed + * to one another. (Global variables are a different, and probably insoluble, + * problem.) + * + * Similarly, min/max classes compiled with and without + * CILK_LIBRARY_0_9_REDUCER_MINMAX are binary-incompatible, and must get + * different mangled names. + * + * The trick is, when compiling in normal (non-compatibility) mode, wrap + * everything in an extra namespace, and then `use` it into the top-level cilk + * namespace. Then + * + * * Classes and functions compiled in normal mode will be in + * different namespaces from the same classes and functions compiled in + * compatibility mode. + * * The legacy wrapper classes and functions will be in the same namespace + * as the same classes and functions compiled with the 0.9 library if and + * only if they are compiled in compatibility mode. + * + * @ingroup ReducersMinMax + */ + +#ifndef CILK_LIBRARY_0_9_REDUCER_MINMAX +/** Namespace to wrap min/max reducer definitions when not compiling in "binary + * compatibility" mode. + * + * By default, all of the min/max reducer definitions are defined in this + * namespace and then imported into namespace ::cilk, so that they do not + * clash with the legacy definitions with the same names. However, if the + * macro `CILK_LIBRARY_0_9_REDUCER_MINMAX` is defined, then the min/max + * definitions go directly into namespace ::cilk, so that, for example, + * cilk::reducer_max defined with the 1.0 library is equivalent (to the + * linker) to cilk::reducer_max defined with the 0.9 library. + * + * @ingroup ReducersMinMaxBinComp + * @ingroup ReducersMinMax + */ +namespace cilk_lib_1_1 { +#endif + +/** Namespace containing internal implementation classes and functions for + * min/max reducers. + * + * @ingroup ReducersMinMax + */ +namespace min_max_internal { + +using ::cilk::internal::binary_functor; +using ::cilk::internal::typed_indirect_binary_function; +using ::cilk::internal::class_is_empty; + +/** @defgroup ReducersMinMaxIsSet The "is_set optimization" + * + * The obvious definition of the identity value for a max or min reducer is as + * the smallest (or largest) value of the value type. However, for an + * arbitrary comparator and/or an arbitrary value type, the largest / smallest + * value may not be known. It may not even be defined - what is the largest + * string? + * + * Therefore, min/max reducers represent their value internally as a pair + * `(value, is_set)`. When `is_set` is true, the pair represents the known + * value `value`; when `is_set` is false, the pair represents the identity + * value. + * + * This is an effective solution, but the most common use of min/max reducers + * is probably with numeric types and the default definition of minimum or + * maximum (using `std::less`), in which case there are well-defined, knowable + * smallest and largest values. Testing `is_set` for every comparison is then + * unnecessary and wasteful. + * + * The "is_set optimization" just means generating code that doesn't use + * `is_set` when it isn't needed. It is implemented using two metaprogramming + * classes: + * + * - do_is_set_optimization tests whether the optimization is applicable. + * - identity_value gets the appropriate identity value for a type. + * + * The is_set optimization is the reason that min/max reducers compiled with + * Intel Cilk Plus library 1.0 are binary-incompatible with the same reducers + * compiled with library 0.9, and therefore the optimization is suppressed when + * compiling in + * ReducersMinMaxBinComp "binary compatibility mode". + * + * @ingroup ReducersMinMax + */ + +/** Tests whether the ReducersMinMaxIsSet "is_set optimization" is + * applicable. + * + * The @ref do_is_set_optimization class is used to test whether the is_set + * optimization should be applied for a particular reducer. It is instantiated + * with a value type and a comparator, and defines a boolean constant, + * `value`. Then `%do_is_set_optimization::%value` can be used as + * a boolean template parameter to control the specialization of another + * class. + * + * In ReducersMinMaxBinComp "binary compatibility mode" (i.e., when the + * `CILK_LIBRARY_0_9_REDUCER_MINMAX` macro is defined), `value` will always + * be false. + * + * @tparam Type The value type for the reducer. + * @tparam Compare The comparator type for the reducer. + * + * @result The `value` data member will be `true` if @a Type is a numeric + * type, @a Compare is `std::less`, and + * `CILK_LIBRARY_0_9_REDUCER_MINMAX` is not defined. + * + * @see ReducersMinMaxIsSet + * @see @ref view_content + * + * @ingroup ReducersMinMaxIsSet + */ +template < typename Type, + typename Compare > +struct do_is_set_optimization +{ + /// `True` if the is_set optimization should be applied to min/max reducers + /// with this value type and comparator; `false` otherwise. + static const bool value = false; +}; + +#ifndef CILK_LIBRARY_0_9_REDUCER_MINMAX +/// @cond +template +struct do_is_set_optimization > +{ + /// True in the special case where optimization is possible. + static const bool value = std::numeric_limits::is_specialized; +}; +/// @endcond +#endif + + +/** Gets the identity value when using the ReducersMinMaxIsSet + * "is_set optimization". + * + * This class defines a function which assigns the appropriate identity value + * to a variable when the is_set optimization is applicable. + * + * @tparam Type The value type for the reducer. + * @tparam Compare The comparator type for the reducer. + * @tparam ForMax `true` to get the identity value for a max reducer (i.e., + * the smallest value of @a Type), `false` to get the identity + * value for a min reducer (i.e., the largest value of + * @a Type). + * + * @result If @a Type and @a Compare qualify for the is_set optimization, the + * `set_identity()' function will set its argument variable to the + * smallest or largest value of @a Type, depending on @a ForMax. + * Otherwise, `set_identity()` will be a no-op. + * + * @see ReducersMinMaxIsSet + * + * @ingroup ReducersMinMaxIsSet + * @see @ref view_content + */ +template < typename Type, + typename Compare, + bool ForMax, + bool = std::numeric_limits::is_specialized, + bool = std::numeric_limits::has_infinity > +struct identity_value { + /// Assign the identity value to the reference parameter. + static void set_identity(Type&) {} +}; + +/// @cond +template +struct identity_value, true, true, true> { + /// Floating max identity is negative infinity. + static void set_identity(Type& id) + { id = -std::numeric_limits::infinity(); } +}; + +template +struct identity_value, true, true, false> { + /// Integer max identity is minimum value of type. + static void set_identity(Type& id) + { id = std::numeric_limits::min(); } +}; + +template +struct identity_value, false, true, true> { + /// Floating min identity is positive infinity. + static void set_identity(Type& id) + { id = std::numeric_limits::infinity(); } +}; + +template +struct identity_value, false, true, false> { + /// Integer min identity is maximum value of type. + static void set_identity(Type& id) + { id = std::numeric_limits::max(); } +}; + +/// @endcond + + +/** Adapter class to reverse the arguments of a predicate. + * + * Observe that: + * + * (x < y) == (y > x) + * max(x, y) == (x < y) ? y : x + * min(x, y) == (y < x) ? y : x == (x > y) ? y : x + * + * More generally, if `c` is a predicate defining a `Strict Weak Ordering`, + * and `c*(x, y) == c(y, x)`, then + * + * max(x, y, c) == c(x, y) ? y : x + * min(x, y, c) == c(y, x) ? y : x == c*(x, y) ? y : x == max(x, y, c*) + * + * For any predicate `C` with argument type `T`, the template class + * `%reverse_predicate` defines a predicate which is identical to `C`, + * except that its arguments are reversed. Thus, for example, we could + * implement `%op_min_view` as + * `%op_max_view >`. + * (Actually, op_min_view and op_max_view are both implemented as subclasses + * of a common base class, view_base.) + * + * @note If `C` is an empty functor class, then `reverse_predicate(C)` will + * also be an empty functor class. + * + * @tparam Predicate The predicate whose arguments are to be reversed. + * @tparam Argument @a Predicate's argument type. + * + * @ingroup ReducersMinMax + */ +template +class reverse_predicate : private binary_functor::type { + typedef typename binary_functor::type base; +public: + /// Default constructor + reverse_predicate() : base() {} + /// Constructor with predicate object + reverse_predicate(const Predicate& p) : base(p) {} + /// The reversed predicate operation + bool operator()(const Argument& x, const Argument& y) const + { return base::operator()(y, x); } +}; + + +/** Class to represent the comparator for a min/max view class. + * + * This class is intended to accomplish two objectives in the implementation + * of min/max views. + * + * 1. To minimize data bloat, when we have a reducer with a non-stateless + * comparator, we want to keep a single instance of the comparator object + * in the monoid, and just call it from the views. + * 2. In ReducersMinMaxBinComp "binary compatibility mode", views for + * reducers with a stateless comparator must have the same content as in + * Intel Cilk Plus library 0.9 - that is, they must contain only `value` and + * `is_set` data members. + * + * To achieve the first objective, we use the + * @ref internal::typed_indirect_binary_function class defined in + * metaprogramming.h to wrap a pointer to the actual comparator. If no + * pointer is needed because the actual comparator is stateless, the + * `typed_indirect_binary_function` class will be empty, too. + * + * To achieve the second objective, we make the + * `typed_indirect_binary_function` class a base class of the view rather than + * a data member, so the "empty base class" rule will ensure no that no + * additional space is allocated in the view unless it is needed. + * + * We could simply use typed_indirect_binary_function as the base class of the + * view, but this would mean writing comparisons as `(*this)(x, y)`, which is + * just weird. So, instead, we comparator_base as a subclass of + * typed_indirect_binary_function which provides function `compare()` + * as a synonym for `operator()`. + * + * @tparam Type The value type of the comparator class. + * @tparam Compare A predicate class. + * + * @see internal::typed_indirect_binary_function + * + * @ingroup ReducersMinMax + */ +template +class comparator_base : private typed_indirect_binary_function +{ + typedef typed_indirect_binary_function base; +protected: + comparator_base(const Compare* f) : base(f) {} ///< Constructor. + + /// Comparison function. + bool compare(const Type& a, const Type& b) const + { + return base::operator()(a, b); + } + + /// Get the comparator pointer. + const Compare* compare_pointer() const { return base::pointer(); } +}; + + +/** @defgroup ReducersMinMaxViewContent Content classes for min/max views + * + * @ingroup ReducersMinMax + * + * Minimum and maximum reducer view classes inherit from a "view content" + * class. The content class defines the actual data members for the view, + * and provides typedefs and member functions for accessing the data members + * as needed to support the view functionality. + * + * There are two content classes, which encapsulate the differences between + * simple min/max reducers and min/max with index reducers: + * + * - view_content + * - index_view_content + * + * @note An obvious, and arguably simpler, encapsulation strategy would be + * to just let the `Type` of a min/max view be an (index, value) pair + * structure for min_index and max_index reducers. Then all views + * would just have a `Type` data member and an `is_set` data member, + * and the comparator for min_index and max_index views could be + * customized to consider only the value component of the (index, + * value) `Type` pair. Unfortunately, this would break binary + * compatibility with reducer_max_index and reducer_min_index in + * Intel Cilk Plus library 0.9, because the memory layout of an + * (index, value) pair followed by a `bool` is different from the + * memory layout of an index data member followed by a value data + * member followed by a `bool` data member. The content class is + * designed to exactly replicate the layout of the views in library 0.9 + * reducers. + * + * A content class `C`, and its objects `c`, must define the following: + * + * Definition | Meaning + * ------------------------------------|-------- + * `C::value_type` | A typedef for `Type` of the view. (A `std::pair` for min_index and max_index views). + * `C::comp_value_type` | A typedef for the type of value compared by the view's `compare()` function. + * `C()` | Constructs the content with the identity value. + * `C(const value_type&)` | Constructs the content with a specified value. + * `c.is_set()` | Returns true if the content has a known value. + * `c.value()` | Returns the content's value. + * `c.set_value(const value_type&)` | Sets the content's value. (The value becomes known.) + * `c.comp_value()` | Returns a const reference to the value or component of the value that is to be compared by the view's comparator. + * `C::comp_value(const value_type&)` | Returns a const reference to a value or component of a value that is to be compared by the view's comparator. + * + * @see view_base + */ + +/** Content class for op_min_view and op_max_view. + * + * @tparam Type The value type of the op_min_view or op_max_view. + * @tparam Compare The comparator class specified for the op_min_view or + * op_max_view. (_Not_ the derived comparator class actually + * used by the view_base. For example, the view_content of an + * `op_min_view` will have `Compare = std::less`, + * but its comparator_base will have + * `Compare = reverse_predicate< std::less >`.) + * @tparam ForMax `true` if this is the content class for an op_max_view, + * `false` if it is for an op_min_view. + * + * @note The general implementation of view_content uses an `is_set` data + * member. There is also a specialization which implements the + * ReducersMinMaxIsSet "is_set optimization". View classes that + * inherit from view_content do not need to know anything about the + * difference, though; the details are abstracted away in the + * view_content interface. + * + * @see ReducersMinMaxViewContent + * + * @ingroup ReducersMinMaxViewContent + * @ingroup ReducersMinMax + */ +template < typename Type + , typename Compare + , bool ForMax + , bool = do_is_set_optimization::value + > +class view_content { +protected: +/// @cond + Type m_value; + bool m_is_set; +/// @endcond +public: + /// The value type of the view. + typedef Type value_type; + + /// The type compared by the view's `compare()` function (which is the same + /// as the value type for view_content). + typedef Type comp_value_type; + + /// Construct with the identity value. + view_content() : m_value(), m_is_set(false) {} + + /// Construct with a defined value. + view_content(const value_type& value) : m_value(value), m_is_set(true) {} + + /// Gets the value. + value_type value() const { return m_value; } + + /// Sets the value. + void set_value(const value_type& value) + { + m_value = value; + } + + /// Sets the is_set flag. + void set_is_set() + { + m_is_set = true; + } + + /// Sets the index part of the value (which is meaningless for non-index + ///reducers, but required for view_base). + void set_default_index(const value_type&) {} + + /// Gets the comparison value (which, for view_content, is the same as the + /// value). + const comp_value_type& comp_value() const { return m_value; } + + /// Given an arbitrary value, gets the corresponding comparison value + /// (which, for view_content, is the same as the value). + static const comp_value_type& comp_value(const value_type& value) + { + return value; + } + + /// Gets a const reference to value part of the value (which is the same as + /// the value for view_content). + const Type& get_reference() const { return m_value; } + + /// Gets a const reference to the index part of the value (which is + /// meaningless for non-index reducers, but required for view_base. + const Type& get_index_reference() const { return m_value; } + + /// Tests if the value is defined. + bool is_set() const { return m_is_set; } + + /// Tests if the view has a comparable value. + bool has_value() const { return is_set(); } +}; + +/// @cond + +/* This is the specialization of the view_content class for cases where + * the is_set optimization is applicable). + */ +template < typename Type + , typename Compare + , bool ForMax + > +class view_content : + public view_content +{ + typedef view_content base; + typedef identity_value Identity; + +public: + typedef typename base::value_type value_type;; + typedef typename base::comp_value_type comp_value_type;; + + view_content() : base() { Identity::set_identity(this->m_value); } + + view_content(const value_type& value) : base(value) {} + + bool has_value() const { return true; } +}; + +/// @endcond + + +/** Content class for op_min_index_view and op_max_index_view. + * + * @tparam Index The index type of the op_min_index_view or + op_max_index_view. + * @tparam Type The value type of the op_min_view or op_max_view. (_Not_ + * the value type of the view, which will be + * `std::pair`.) + * @tparam Compare The comparator class specified for the op_min_index_view or + * op_max_index_view. (_Not_ the derived comparator class + * actually used by the view_base. For example, the + * index_view_content of an `op_min_index_view` will have + * `Compare = std::less`, but its comparator_base will + * have `Compare = reverse_predicate< std::less >`.) + * @tparam ForMax `true` if this is the content class for an + * op_max_index_view, `false` if it is for an + * op_min_index_view. + * + * @see ReducersMinMaxViewContent + * + * @ingroup ReducersMinMaxViewContent + * @ingroup ReducersMinMax + */ +template < typename Index + , typename Type + , typename Compare + , bool ForMax + , bool = do_is_set_optimization::value + > +class index_view_content { +protected: +/// @cond + Index m_index; + Type m_value; + bool m_is_set; +/// @endcond +public: + /// The value type of the view (which is an pair for + /// index_view_content). + typedef std::pair value_type; + + /// The type compared by the view's `compare()` function (which is the data + /// value type for index_view_content). + typedef Type comp_value_type; + + /// Construct with the identity value. + index_view_content() : m_index(), m_value(), m_is_set(false) {} + + /// Construct with an index/value pair. + index_view_content(const value_type& value) : + m_index(value.first), m_value(value.second), m_is_set(true) {} + + /// Construct with an index and a value. + index_view_content(const Index& index, const Type& value) : + m_index(index), m_value(value), m_is_set(true) {} + + /// Construct with just an index. + index_view_content(const Index& index) : + m_index(index), m_value(), m_is_set(false) {} + + /// Gets the value. + value_type value() const { return value_type(m_index, m_value); } + + /// Sets the value. + void set_value(const value_type& value) + { + m_index = value.first; + m_value = value.second; + } + + /// Sets the is_set flag. + void set_is_set() + { + m_is_set = true; + } + + /// Sets the (initial) index, without marking the view as set. + void set_default_index(const Index& index) + { + m_index = index; + } + + /// Gets the comparison value (which, for index_view_content, is the value + /// component of the index/value pair). + const comp_value_type& comp_value() const { return m_value; } + + /// Given an arbitrary value (i.e., index/value pair), gets the + /// corresponding comparison value (which, for index_view_content, is the + /// value component of the index/value pair). + static const comp_value_type& comp_value(const value_type& value) + { return value.second; } + + /// Gets a const reference to the value part of the value. + const Type& get_reference() const { return m_value; } + + /// Gets a const reference to the index part of the value. + const Index& get_index_reference() const { return m_index; } + + /// Tests if the value is defined. + bool is_set() const { return m_is_set; } + + /// Tests if the view has a comparable value. + bool has_value() const { return is_set(); } +}; + + +/// @cond + +/* This is the specialization of the index_view_content class for cases where + * the is_set optimization is applicable). + */ +template < typename Index + , typename Type + , typename Compare + , bool ForMax + > +class index_view_content : + public index_view_content +{ + typedef index_view_content base; + typedef identity_value Identity; +public: + typedef typename base::value_type value_type;; + typedef typename base::comp_value_type comp_value_type;; + + index_view_content() : base() { Identity::set_identity(this->m_value); } + + index_view_content(const value_type& value) : base(value) {} + + index_view_content(const Index& index, const Type& value) : + base(index, value) {} + + index_view_content(const Index& index) : base() { + Identity::set_identity(this->m_value); + this->m_index = index; + } + + /// Test if the view has a comparable value. + bool has_value() const { return true; } +}; + +/// @endcond + + +template class rhs_proxy; + +/** Creates an rhs_proxy. + */ +template +inline rhs_proxy +make_proxy(const typename View::value_type& value, const View& view); + +template class view_base; + + +/** Class to represent the right-hand side of + * `*reducer = {min|max}_of(*reducer, value)`. + * + * The only assignment operator for a min/max view class takes a rhs_proxy as + * its operand. This results in the syntactic restriction that the only + * expressions that can be assigned to a min/max view are ones which generate + * an rhs_proxy - that is, expressions of the form `max_of(view, value)` and + * `min_of(view, value)`. + * + * @warning + * The lhs and rhs views in such an assignment must be the same; otherwise, + * the behavior will be undefined. (I.e., `*r1 = min_of(*r1, x)` is legal; + * `*r1 = min_of(*r2, x)` is illegal.) This condition will be checked with a + * runtime assertion when compiled in debug mode. + * + * @tparam View The view class (op_{min|max}[_index]_view) that this proxy + * was created from. + * + * @see view_base + * + * @ingroup ReducersMinMax + */ +template +class rhs_proxy { + typedef typename View::less_type less_type; + typedef typename View::compare_type compare_type; + typedef typename View::value_type value_type; + typedef typename View::content_type content_type; + typedef typename content_type::comp_value_type comp_value_type; + + friend class view_base; + friend rhs_proxy make_proxy( + const typename View::value_type& value, + const View& view); + + typed_indirect_binary_function< + compare_type, comp_value_type, comp_value_type, bool> + m_comp; + const View* m_view; + value_type m_value; + + rhs_proxy& operator=(const rhs_proxy&); // Disable assignment operator + rhs_proxy(); // Disable default constructor + + // Constructor (called from view_base::make_proxy). + rhs_proxy(const View* view, + const value_type& value, + const compare_type* compare) : + m_view(view), m_value(value), m_comp(compare) {} + + // Checks matching view, then return value (called from view_base::assign). + value_type value(const typename View::base* view) const + { + __CILKRTS_ASSERT(view == m_view); + return m_value; + } + +public: + + /** Supports max_of(max_of(view, value), value) and the like. + */ + rhs_proxy calc(const value_type& x) const + { + return rhs_proxy( + m_view, + m_comp( content_type::comp_value(m_value), + content_type::comp_value(x) + ) ? x : m_value, + m_comp.pointer()); + } +}; + + +template +inline rhs_proxy +make_proxy(const typename View::value_type& value, const View& view) +{ + return rhs_proxy(&view, value, view.compare_pointer()); +} + +//@} + +/** Base class for min and max view classes. + * + * This class accumulates the minimum or maximum of a set of values which have + * occurred as arguments to the `calc()` function, as determined by a + * comparator. The accumulated value will be the first `calc()` argument value + * `x` such that `compare(x, y)` is false for every `calc()` argument value + * `y`. + * + * If the comparator is `std::less`, then the accumulated value is the first + * argument value which is not less than any other argument value, i.e., the + * maximum. Similarly, if the comparator is `reverse_predicate`, + * which is equivalent to `std::greater`, then the accumulated value is the + * first argument value which is not greater than any other argument value, + * i.e., the minimum. + * + * @note This class provides the definitions that are required for a class + * that will be used as the parameter of a + * min_max_internal::monoid_base specialization. + * + * @tparam Content A content class that provides the value types and data + * members for the view. + * @tparam Less A "less than" binary predicate that defines the min or + * max function. + * @tparam Compare A binary predicate to be used to compare the values. + * (The same as @a Less for max reducers; its reversal for + * min reducers.) + * + * @see ReducersMinMaxViewContent + * @see op_max_view + * @see op_min_view + * @see op_max_index_view + * @see op_min_index_view + * @see monoid_base + * + * @ingroup ReducersMinMax + */ +template +class view_base : + // comparator_base comes first to ensure that it will get empty base class + // treatment + private comparator_base, + private Content +{ + typedef comparator_base base; + using base::compare; + using Content::value; + using Content::set_value; + using Content::has_value; + using Content::set_is_set; + using Content::comp_value; + typedef Content content_type; + + template friend class rhs_proxy; + template + friend rhs_proxy make_proxy(const typename View::value_type& value, const View& view); + +public: + + /** @name Monoid support. + */ + //@{ + + /** Value type. Required by @ref monoid_with_view. + */ + typedef typename Content::value_type value_type; + + /** The type of the comparator specified by the user, that defines the + * ordering on @a Type. Required by min_max::monoid_base. + */ + typedef Less less_type; + + /** The type of the comparator actually used by the view. Required by + * min_max::monoid_base. (This is the same as the @ref less_type for a + * max reducer, or `reverse_predicate` for a min reducer.) + */ + typedef Compare compare_type; + + /** Reduces two views. Required by @ref monoid_with_view. + */ + void reduce(view_base* other) + { + if ( other->is_set() && + ( !this->is_set() || + compare(this->comp_value(), other->comp_value()) ) ) + { + this->set_value(other->value()); + this->set_is_set(); + } + } + + //@} + + /** Default constructor. Initializes to identity value. + */ + explicit view_base(const compare_type* compare) : + base(compare), Content() {} + + /** Value constructor. + */ + template + view_base(const T1& x1, const compare_type* compare) : + base(compare), Content(x1) {} + + /** Value constructor. + */ + template + view_base(const T1& x1, const T2& x2, const compare_type* compare) : + base(compare), Content(x1, x2) {} + + + /** Move-in constructor. + */ + explicit view_base(move_in_wrapper w, const compare_type* compare) : + base(compare), Content(w.value()) {} + + /** @name Reducer support. + */ + //@{ + + void view_move_in(value_type& v) + { set_value(v); set_is_set();} + void view_move_out(value_type& v) + { v = value(); } + void view_set_value(const value_type& v) + { set_value(v); set_is_set(); } + value_type view_get_value() const + { return value(); } + // view_get_reference() NOT SUPPORTED + + //@} + + /** Sets the contained index data member, without marking the view as set. + * (Meaningless for non-index reducers.) + */ + using Content::set_default_index; + + /** Is the value defined? + */ + using Content::is_set; + + /** Reference to contained value data member. + * @deprecated For legacy reducers only. + */ + using Content::get_reference; + + /** Reference to contained index data member. + * (Meaningless for non-index reducers.) + * @deprecated For legacy reducers only. + */ + using Content::get_index_reference; + +protected: + + /** Updates the min/max value. + */ + void calc(const value_type& x) + { + if (!has_value() || compare(comp_value(), comp_value(x))) set_value(x); + set_is_set(); + } + + /** Assigns the result of a `{min|max}_of(view, value)` expression to the + * view. + * + * @see rhs_proxy + */ + template + void assign(const rhs_proxy& rhs) + { + calc(rhs.value(this)); + } + +}; + + +/** Base class for min and max monoid classes. + * + * The unique characteristic of minimum and maximum reducers is that they + * incorporate a comparator functor that defines what "minimum" or "maximum" + * means. The monoid for a reducer contains the comparator that will be used + * for the reduction. If the comparator is a function or a class with state, + * then each view will have a pointer to the comparator. + * + * This means that the `construct()` functions first construct the monoid + * (possibly with an explicit comparator argument), and then construct the + * view with a pointer to the monoid's comparator. + * + * @tparam View The view class. + * @tparam Align If true, reducers instantiated on this monoid will be + * aligned. By default, library reducers (unlike legacy + * library reducer _wrappers_) are unaligned. + * + * @see view_base + * + * @ingroup ReducersMinMax + */ +template +class monoid_base : public monoid_with_view +{ + typedef typename View::compare_type compare_type; + typedef typename View::less_type less_type; + + const compare_type m_compare; + + const compare_type* compare_pointer() const { return &m_compare; } + +public: + + /** Default constructor uses default comparator. + */ + monoid_base() : m_compare() {} + + /** Constructor. + * + * @param compare The comparator to use. + */ + monoid_base(const compare_type& compare) : m_compare(compare) {} + + /** Creates an identity view. + * + * List view identity constructors take the list allocator as an argument. + * + * @param v The address of the uninitialized memory in which the view + * will be constructed. + */ + void identity(View *v) const { ::new((void*) v) View(compare_pointer()); } + + /** @name construct functions + * + * Min/max monoid `construct()` functions optionally take one or two value + * arguments, a @ref move_in argument, and/or a comparator argument. + */ + //@{ + + template + static void construct(Monoid* monoid, View* view) + { + provisional_guard mg( new((void*) monoid) Monoid ); + mg.confirm_if( new((void*) view) View(monoid->compare_pointer()) ); + } + + template + static void construct(Monoid* monoid, View* view, const T1& x1) + { + provisional_guard mg( new((void*) monoid) Monoid ); + mg.confirm_if( new((void*) view) View(x1, monoid->compare_pointer()) ); + } + + template + static void construct(Monoid* monoid, View* view, + const T1& x1, const T2& x2) + { + provisional_guard mg( new((void*) monoid) Monoid ); + mg.confirm_if( new((void*) view) View(x1, x2, + monoid->compare_pointer()) ); + } + + template + static void construct(Monoid* monoid, View* view, const less_type& compare) + { + provisional_guard mg( new((void*) monoid) Monoid(compare) ); + mg.confirm_if( new((void*) view) View(monoid->compare_pointer()) ); + } + + template + static void construct(Monoid* monoid, View* view, const T1& x1, + const less_type& compare) + { + provisional_guard mg( new((void*) monoid) Monoid(compare) ); + mg.confirm_if( new((void*) view) View(x1, monoid->compare_pointer()) ); + } + + template + static void construct(Monoid* monoid, View* view, + const T1& x1, const T2& x2, const less_type& compare) + { + provisional_guard mg( new((void*) monoid) Monoid(compare) ); + mg.confirm_if( new((void*) view) View(x1, x2, + monoid->compare_pointer()) ); + } + + //@} +}; + +} //namespace min_max_internal + + +/** @defgroup ReducersMinMaxMaxValue Maximum reducers (value only) + * + * These reducers will find the largest value from a set of values. + * + * @ingroup ReducersMinMax + */ +//@{ + +/** The maximum reducer view class. + * + * This is the view class for reducers created with + * `cilk::reducer< cilk::op_max >`. It accumulates the maximum, + * as determined by a comparator, of a set of values which have occurred as + * arguments to the `calc_max()` function. The accumulated value will be the + * first argument `x` such that `compare(x, y)` is false for every argument + * `y`. + * + * If the comparator is `std::less`, then the accumulated value is the first + * argument value which is not less than any other argument value, i.e., the + * maximum. + * + * @note The reducer "dereference" operation (`reducer::operator *()`) + * yields a reference to the view. Thus, for example, the view class's + * `calc_max()` function would be used in an expression like + * `r->calc_max(a)` where `r` is an op_max reducer variable. + * + * @tparam Type The type of the values compared by the reducer. This will + * be the value type of a monoid_with_view that is + * instantiated with this view. + * @tparam Compare A `Strict Weak Ordering` whose argument type is @a Type. It + * defines the "less than" relation used to compute the + * maximum. + * + * @see ReducersMinMax + * @see op_max + */ +template +class op_max_view : public min_max_internal::view_base< + min_max_internal::view_content, + Compare, + Compare> +{ + typedef min_max_internal::view_base< + min_max_internal::view_content, + Compare, + Compare> base; + using base::calc; + using base::assign; + friend class min_max_internal::rhs_proxy; + +public: + + /** @name Constructors. + * + * All op_max_view constructors simply pass their arguments on to the + * @ref view_base base class. + */ + //@{ + + template + op_max_view(const T1& x1) : base(x1) {} + + template + op_max_view(const T1& x1, const T2& x2) : base(x1, x2) {} + + //@} + + /** @name View modifier operations. + */ + //@{ + + /** Maximizes with a value. + * + * If @a x is greater than the current value of the view (as defined by + * the reducer's comparator), or if the view was created without an + * initial value and its value has never been updated (with `calc_max()` + * or `= max_of()`), then the value of the view is set to @a x. + * + * @param x The value to maximize the view's value with. + * + * @return A reference to the view. (Allows chaining + * `view.comp_max(a).comp_max(b)…`.) + */ + op_max_view& calc_max(const Type& x) { calc(x); return *this; } + + /** Assigns the result of a `max_of(view, value)` expression to the view. + * + * @param rhs An rhs_proxy value created by a `max_of(view, value)` + * expression. + * + * @return A reference to the view. + * + * @see min_max_internal::view_base::rhs_proxy + */ + op_max_view& operator=(const min_max_internal::rhs_proxy& rhs) + { assign(rhs); return *this; } + + //@} +}; + + +/** Computes the maximum of the value in an op_max_view and another value. + * + * The result of this computation can only be assigned back to the original + * view or used in another max_of() call. For example, + * + * *reducer = max_of(*reducer, x); + * *reducer = max_of(x, *reducer); + * + * @see min_max_internal::rhs_proxy + */ +template +inline min_max_internal::rhs_proxy< op_max_view > +max_of(const op_max_view& view, const Type& value) +{ + return min_max_internal::make_proxy(value, view); +} + +/// @copydoc max_of(const op_max_view&, const Type&) +template +inline min_max_internal::rhs_proxy< op_max_view > +max_of(const Type& value, const op_max_view& view) +{ + return min_max_internal::make_proxy(value, view); +} + +/** Computes nested maximum. + * + * Compute the maximum of the result of a max_of() call and another value. + * + * The result of this computation can only be assigned back to the original + * view or wrapper, or used in another max_of() call. For example, + * + * *reducer = max_of(x, max_of(y, *reducer)); + * wrapper = max_of(max_of(wrapper, x), y); + * + * @see min_max_internal::rhs_proxy + */ +template +inline min_max_internal::rhs_proxy< op_max_view > +max_of(const min_max_internal::rhs_proxy< op_max_view >& proxy, + const Type& value) +{ + return proxy.calc(value); +} + +/// @copydoc max_of(const min_max_internal::rhs_proxy< op_max_view >&, const Type&) +template +inline min_max_internal::rhs_proxy< op_max_view > +max_of(const Type& value, + const min_max_internal::rhs_proxy< op_max_view >& proxy) +{ + return proxy.calc(value); +} + + +/** Monoid class for maximum reductions. Instantiate the cilk::reducer template + * class with an op_max monoid to create a maximum reducer class. For example, + * to compute the maximum of a set of `int` values: + * + * cilk::reducer< cilk::op_max > r; + * + * @see ReducersMinMax + * @see op_max_view + */ +template , bool Align = false> +class op_max : + public min_max_internal::monoid_base, Align> +{ + typedef min_max_internal::monoid_base, Align> + base; +public: + /// Construct with default comparator. + op_max() {} + /// Construct with specified comparator. + op_max(const Compare& compare) : base(compare) {} +}; + +//@} + + +/** @defgroup ReducersMinMaxMinValue Minimum reducers (value only) + * + * These reducers will find the smallest value from a set of values. + * + * @ingroup ReducersMinMax + */ +//@{ + +/** The minimum reducer view class. + * + * This is the view class for reducers created with + * `cilk::reducer< cilk::op_min >`. It accumulates the minimum, + * as determined by a comparator, of a set of values which have occurred as + * arguments to the `calc_min()` function. The accumulated value will be the + * first argument `x` such that `compare(y, x)` is false for every argument + * `y`. + * + * If the comparator is `std::less`, then the accumulated value is the first + * argument value which no other argument value is less than, i.e., the + * minimum. + * + * @note The reducer "dereference" operation (`reducer::operator *()`) + * yields a reference to the view. Thus, for example, the view class's + * `calc_min()` function would be used in an expression like + * `r->calc_min(a)` where `r` is an op_min reducer variable. + * + * @tparam Type The type of the values compared by the reducer. This will + * be the value type of a monoid_with_view that is + * instantiated with this view. + * @tparam Compare A `Strict Weak Ordering` whose argument type is @a Type. It + * defines the "less than" relation used to compute the + * minimum. + * + * @see ReducersMinMax + * @see op_min + */ +template +class op_min_view : public min_max_internal::view_base< + min_max_internal::view_content, + Compare, + min_max_internal::reverse_predicate > +{ + typedef min_max_internal::view_base< + min_max_internal::view_content, + Compare, + min_max_internal::reverse_predicate > base; + using base::calc; + using base::assign; + friend class min_max_internal::rhs_proxy; + +public: + /** @name Constructors. + * + * All op_min_view constructors simply pass their arguments on to the + * @ref view_base base class. + */ + //@{ + + template + op_min_view(const T1& x1) : base(x1) {} + + template + op_min_view(const T1& x1, const T2& x2) : base(x1, x2) {} + + //@} + + /** @name View modifier operations. + */ + //@{ + + /** Minimizes with a value. + * + * If @a x is less than the current value of the view (as defined by the + * reducer's comparator), or if the view was created without an initial + * value and its value has never been updated (with `calc_min()` or + * `= min_of()`), then the value of the view is set to @a x. + * + * @param x The value to minimize the view's value with. + * + * @return A reference to the view. (Allows chaining + * `view.comp_min(a).comp_min(b)…`.) + */ + op_min_view& calc_min(const Type& x) { calc(x); return *this; } + + /** Assigns the result of a `min_of(view, value)` expression to the view. + * + * @param rhs An rhs_proxy value created by a `min_of(view, value)` + * expression. + * + * @return A reference to the view. + * + * @see min_max_internal::view_base::rhs_proxy + */ + op_min_view& operator=(const min_max_internal::rhs_proxy& rhs) + { assign(rhs); return *this; } +}; + + +/** Computes the minimum of the value in a view and another value. + * + * The result of this computation can only be assigned back to the original + * view or used in another min_of() call. For example, + * + * *reducer = min_of(*reducer, x); + * *reducer = min_of(x, *reducer); + * + * @see min_max_internal::view_base::rhs_proxy + */ +template +inline min_max_internal::rhs_proxy< op_min_view > +min_of(const op_min_view& view, const Type& value) +{ + return min_max_internal::make_proxy(value, view); +} + +/// @copydoc min_of(const op_min_view&, const Type&) +template +inline min_max_internal::rhs_proxy< op_min_view > +min_of(const Type& value, const op_min_view& view) +{ + return min_max_internal::make_proxy(value, view); +} + +/** Computes nested minimum. + * + * Compute the minimum of the result of a min_of() call and another value. + * + * The result of this computation can only be assigned back to the original + * view or wrapper, or used in another min_of() call. For example, + * + * *reducer = min_of(x, min_of(y, *reducer)); + * wrapper = min_of(min_of(wrapper, x), y); + * + * @see min_max_internal::rhs_proxy + */ +template +inline min_max_internal::rhs_proxy< op_min_view > +min_of(const min_max_internal::rhs_proxy< op_min_view >& proxy, + const Type& value) +{ + return proxy.calc(value); +} + +/// @copydoc min_of(const min_max_internal::rhs_proxy< op_min_view >&, const Type&) +template +inline min_max_internal::rhs_proxy< op_min_view > +min_of(const Type& value, + const min_max_internal::rhs_proxy< op_min_view >& proxy) +{ + return proxy.calc(value); +} + + +/** Monoid class for minimum reductions. Instantiate the cilk::reducer template + * class with an op_min monoid to create a minimum reducer class. For example, + * to compute the minimum of a set of `int` values: + * + * cilk::reducer< cilk::op_min > r; + * + * @see ReducersMinMax + * @see op_min_view + */ +template , bool Align = false> +class op_min : public min_max_internal::monoid_base, Align> { + typedef min_max_internal::monoid_base, Align> base; +public: + /// Construct with default comparator. + op_min() {} + /// Construct with specified comparator. + op_min(const Compare& compare) : base(compare) {} +}; + +//@} + + +/** @defgroup ReducersMinMaxMaxIndex Maximum reducers (value and index) + * + * These reducers will find the largest value from a set of values, and its + * index in the set. + * + * @ingroup ReducersMinMax + */ +//@{ + +/** The maximum index reducer view class. + * + * This is the view class for reducers created with + * `cilk::reducer< cilk::op_max_index >`. It accumulates + * the maximum, as determined by a comparator, of a set of values which have + * occurred as arguments to the `calc_max()` function, and records the index + * of the maximum value. The accumulated value will be the first argument `x` + * such that `compare(x, y)` is false for every argument `y`. + * + * If the comparator is `std::less`, then the accumulated value is the first + * argument value which is not less than any other argument value, i.e., the + * maximum. + * + * @note The reducer "dereference" operation (`reducer::operator *()`) + * yields a reference to the view. Thus, for example, the view class's + * `calc_max()` function would be used in an expression like + * `r->calc_max(i, a)`where `r` is an op_max_index reducer + * variable. + * + * @note The word "index" suggests an integer index into an array, but there + * is no restriction on the index type or how it should be used. In + * general, it may be convenient to use it for any kind of key that + * can be used to locate the maximum value in the collection that it + * came from - for example: + * - An index into an array. + * - A key into an STL map. + * - An iterator into any STL container. + * + * @note A max_index reducer is essentially a max reducer whose value type + * is a `std::pair`. This fact is camouflaged in the view + * `calc_max` function, the global `max_of` functions, and the reducer + * value constructor, which can all take an index argument and a value + * argument as an alternative to a single `std::pair` argument. + * However, the reducer `set_value()`, `get_value()`, `move_in()`, and + * `move_out()` functions work only with pairs, not with individual + * value and/or index arguments. + * + * @tparam Index The type of the indices associated with the values. + * @tparam Type The type of the values compared by the reducer. This will + * be the value type of a monoid_with_view that is + * instantiated with this view. + * @tparam Compare Used to compare the values. It must be a binary predicate. + * If it is omitted, then the view computes the conventional + * arithmetic maximum. + * + * @see ReducersMinMax + * @see op_max_index + */ +template +class op_max_index_view : public min_max_internal::view_base< + min_max_internal::index_view_content, + Compare, + Compare> +{ + typedef min_max_internal::view_base< + min_max_internal::index_view_content, + Compare, + Compare> base; + using base::calc; + using base::assign; + typedef std::pair pair_type; + friend class min_max_internal::rhs_proxy; + +public: + /** @name Constructors. + * + * All op_max_index_view constructors simply pass their arguments on to the + * @ref view_base base class, except for the `(index, value [, compare])` + * constructors, which create a `std::pair` containing the index and value. + */ + //@{ + + op_max_index_view() : base() {} + + template + op_max_index_view(const T1& x1) : base(x1) {} + + template + op_max_index_view(const T1& x1, const T2& x2) : base(x1, x2) {} + + template + op_max_index_view(const T1& x1, const T2& x2, const T3& x3) : base(x1, x2, x3) {} + + op_max_index_view(const Index& i, const Type& v) : base(pair_type(i, v)) {} + + op_max_index_view(const Index& i, const Type& v, const typename base::compare_type* c) : + base(pair_type(i, v), c) {} + + //@} + + /** Maximizes with a value and index. + * + * If @a x is greater than the current value of the view (as defined by + * the reducer's comparator), or if the view was created without an + * initial value and its value has never been updated (with `calc_max()` + * or `= max_of()`), then the value of the view is set to @a x, and the + * index is set to @a i.. + * + * @param i The index of the value @a x. + * @param x The value to maximize the view's value with. + * + * @return A reference to the view. (Allows + * `view.comp_max(i, a).comp_max(j, b)…`.) + */ + op_max_index_view& calc_max(const Index& i, const Type& x) + { calc(pair_type(i, x)); return *this; } + + /** Maximizes with an index/value pair. + * + * If @a pair.second is greater than the current value of the view (as + * defined by the reducer's comparator), or if the view was created + * without an initial value and its value has never been updated (with + * `calc_max()` or `= max_of()`), then the value of the view is set to + * @a pair.second, and the index is set to @a pair.first. + * + * @param pair A pair containing a value to maximize the view's value + * with and its associated index. + * + * @return A reference to the view. (Allows + * `view.comp_max(p1).comp_max(p2)…`.) + */ + op_max_index_view& calc_max(const pair_type& pair) + { calc(pair); return *this; } + + /** Assigns the result of a `max_of(view, index, value)` expression to the + * view. + * + * @param rhs An rhs_proxy value created by a `max_of(view, index, value)` + * expression. + * + * @return A reference to the view. + * + * @see min_max_internal::view_base::rhs_proxy + */ + op_max_index_view& operator=(const min_max_internal::rhs_proxy& rhs) + { assign(rhs); return *this; } +}; + + +/** Computes the maximum of the value in a view and another value. + * + * The result of this computation can only be assigned back to the original + * view or used in another max_of() call. For example, + * + * *reducer = max_of(*reducer, i, x); + * *reducer = max_of(i, x, *reducer); + * + * @see min_max_internal::rhs_proxy + */ +template +inline min_max_internal::rhs_proxy< op_max_index_view > +max_of(const op_max_index_view& view, + const Index& index, const Type& value) +{ + return min_max_internal::make_proxy(std::pair(index, value), view); +} + +/// @copydoc max_of(const op_max_index_view&, const Index&, const Type&) +template +inline min_max_internal::rhs_proxy< op_max_index_view > +max_of(const Index& index, const Type& value, + const op_max_index_view& view) +{ + return min_max_internal::make_proxy(std::pair(index, value), view); +} + +/// @copydoc max_of(const op_max_index_view&, const Index&, const Type&) +template +inline min_max_internal::rhs_proxy< op_max_index_view > +max_of(const op_max_index_view& view, + const std::pair& pair) +{ + return min_max_internal::make_proxy(pair, view); +} + +/// @copydoc max_of(const op_max_index_view&, const Index&, const Type&) +template +inline min_max_internal::rhs_proxy< op_max_index_view > +max_of(const std::pair& pair, + const op_max_index_view& view) +{ + return min_max_internal::make_proxy(pair, view); +} + +/** Computes the nested maximum between the value in a view and other values. + * + * Compute the maximum of the result of a max_of() call and another value. + * + * The result of this computation can only be assigned back to the original + * view or used in another max_of() call. For example, + * + * *reducer = max_of(x, max_of(y, *reducer)); + * *reducer = max_of(max_of(*reducer, x), y); + * + * @see min_max_internal::rhs_proxy + */ +template +inline min_max_internal::rhs_proxy< op_max_index_view > +max_of(const min_max_internal::rhs_proxy< op_max_index_view >& proxy, + const Index& index, const Type& value) +{ + return proxy.calc(std::pair(index, value)); +} + +/// @copydoc max_of(const min_max_internal::rhs_proxy< op_max_index_view >&, const Index&, const Type&) +template +inline min_max_internal::rhs_proxy< op_max_index_view > +max_of(const Index& index, const Type& value, + const min_max_internal::rhs_proxy< op_max_index_view >& proxy) +{ + return proxy.calc(std::pair(index, value)); +} + +/// @copydoc max_of(const min_max_internal::rhs_proxy< op_max_index_view >&, const Index&, const Type&) +template +inline min_max_internal::rhs_proxy< op_max_index_view > +max_of(const min_max_internal::rhs_proxy< op_max_index_view >& proxy, + const std::pair& pair) +{ + return proxy.calc(pair); +} + +/// @copydoc max_of(const min_max_internal::rhs_proxy< op_max_index_view >&, const Index&, const Type&) +template +inline min_max_internal::rhs_proxy< op_max_index_view > +max_of(const std::pair& pair, + const min_max_internal::rhs_proxy< op_max_index_view >& proxy) +{ + return proxy.calc(pair); +} + + +/** Monoid class for maximum reductions with index. Instantiate the + * cilk::reducer template class with an op_max_index monoid to create a + * max_index reducer class. For example, to compute the maximum of an array of + * `double` values and the array index of the max value: + * + * cilk::reducer< cilk::op_max_index > r; + * + * @see ReducersMinMax + * @see op_max_index_view + */ +template < typename Index + , typename Type + , typename Compare=std::less + , bool Align = false + > +class op_max_index : public min_max_internal::monoid_base, Align> +{ + typedef min_max_internal::monoid_base< + op_max_index_view, Align> base; +public: + /// Construct with default comparator. + op_max_index() {} + /// Construct with specified comparator. + op_max_index(const Compare& compare) : base(compare) {} +}; + +//@} + + + +/** @defgroup ReducersMinMaxMinIndex Minimum reducers (value and index) + * + * These reducers will find the smallest value from a set of values, and its + * index in the set. + * + * @ingroup ReducersMinMax + */ +//@{ + +/** The minimum index reducer view class. + * + * This is the view class for reducers created with + * `cilk::reducer >`. It accumulates + * the minimum, as determined by a comparator, of a set of values which have + * occurred as arguments to the `calc_min()` function, and records the index + * of the minimum value. The accumulated value will be the first argument `x` + * such that `compare(y, x)` is false for every argument `y`. + * + * If the comparator is `std::less`, then the accumulated value is the first + * argument value which no other argument value is less than, i.e., the + * minimum. + * + * @note The reducer "dereference" operation (`reducer::operator *()`) + * yields a reference to the view. Thus, for example, the view class's + * `calc_min()` function would be + * used in an expression like `r->calc_min(i, a)`where `r` is an + * op_min_index reducer variable. + * + * @note The word "index" suggests an integer index into an array, but there + * is no restriction on the index type or how it should be used. In + * general, it may be convenient to use it for any kind of key that + * can be used to locate the minimum value in the collection that it + * came from - for example: + * - An index into an array. + * - A key into an STL map. + * - An iterator into any STL container. + * + * @note A min_index reducer is essentially a min reducer whose value type + * is a `std::pair`. This fact is camouflaged in the view + * `calc_min` function, the global `min_of` functions, and the reducer + * value constructor, which can all take an index argument and a value + * argument as an alternative to a single `std::pair` argument. + * However, the reducer `set_value()`, `get_value()`, `move_in()`, and + * `move_out()` functions work only with pairs, not with individual + * value and/or index arguments. + * + * @tparam Index The type of the indices associated with the values. + * @tparam Type The type of the values compared by the reducer. This will + * be the value type of a monoid_with_view that is + * instantiated with this view. + * @tparam Compare Used to compare the values. It must be a binary predicate. + * If it is omitted, then the view computes the conventional + * arithmetic minimum. + * + * @see ReducersMinMax + * @see op_min_index + */ +template +class op_min_index_view : public min_max_internal::view_base< + min_max_internal::index_view_content, + Compare, + min_max_internal::reverse_predicate > +{ + typedef min_max_internal::view_base< + min_max_internal::index_view_content, + Compare, + min_max_internal::reverse_predicate > base; + using base::calc; + using base::assign; + typedef std::pair pair_type; + friend class min_max_internal::rhs_proxy; + +public: + /** @name Constructors. + * + * All op_min_index_view constructors simply pass their arguments on to the + * @ref view_base base class, except for the `(index, value [, compare])` + * constructors, which create a `std::pair` containing the index and value. + */ + //@{ + + op_min_index_view() : base() {} + + template + op_min_index_view(const T1& x1) : base(x1) {} + + template + op_min_index_view(const T1& x1, const T2& x2) : base(x1, x2) {} + + template + op_min_index_view(const T1& x1, const T2& x2, const T3& x3) : base(x1, x2, x3) {} + + op_min_index_view(const Index& i, const Type& v) : base(pair_type(i, v)) {} + + op_min_index_view(const Index& i, const Type& v, const typename base::compare_type* c) : + base(pair_type(i, v), c) {} + + //@} + + /** Minimizes with a value and index. + * + * If @a x is greater than the current value of the view (as defined by + * the reducer's comparator), or if the view was created without an + * initial value and its value has never been updated (with `calc_min()` + * or `= min_of()`), then the value of the view is set to @a x, and the + * index is set to @a i.. + * + * @param i The index of the value @a x. + * @param x The value to minimize the view's value with. + * + * @return A reference to the view. (Allows + * `view.comp_min(i, a).comp_min(j, b)…`.) + */ + op_min_index_view& calc_min(const Index& i, const Type& x) + { calc(pair_type(i, x)); return *this; } + + /** Maximizes with an index/value pair. + * + * If @a pair.second is less than the current value of the view (as + * defined by the reducer's comparator), or if the view was created + * without an initial value and its value has never been updated (with + * `calc_min()` or `= min_of()`), then the value of the view is set to + * @a pair.second, and the index is set to @a pair.first. + * + * @param pair A pair containing a value to minimize the view's value + * with and its associated index. + * + * @return A reference to the view. (Allows + * `view.comp_min(p1).comp_min(p2)…`.) + */ + op_min_index_view& calc_min(const pair_type& pair) + { calc(pair); return *this; } + + /** Assigns the result of a `min_of(view, index, value)` expression to the + * view. + * + * @param rhs An rhs_proxy value created by a `min_of(view, index, value)` + * expression. + * + * @return A reference to the view. + * + * @see min_max_internal::view_base::rhs_proxy + */ + op_min_index_view& operator=(const min_max_internal::rhs_proxy& rhs) + { assign(rhs); return *this; } +}; + + +/** Computes the minimum of the value in a view and another value. + * + * The result of this computation can only be assigned back to the original + * view or used in another min_of() call. For example, + * + * *reducer = min_of(*reducer, i, x); + * *reducer = min_of(i, x, *reducer); + * + * @see min_max_internal::min_min_view_base::rhs_proxy + */ +template +inline min_max_internal::rhs_proxy< op_min_index_view > +min_of(const op_min_index_view& view, + const Index& index, const Type& value) +{ + return min_max_internal::make_proxy(std::pair(index, value), view); +} + +/// @copydoc min_of(const op_min_index_view&, const Index&, const Type&) +template +inline min_max_internal::rhs_proxy< op_min_index_view > +min_of(const Index& index, const Type& value, + const op_min_index_view& view) +{ + return min_max_internal::make_proxy(std::pair(index, value), view); +} + +/// @copydoc min_of(const op_min_index_view&, const Index&, const Type&) +template +inline min_max_internal::rhs_proxy< op_min_index_view > +min_of(const op_min_index_view& view, + const std::pair& pair) +{ + return min_max_internal::make_proxy(pair, view); +} + +/// @copydoc min_of(const op_min_index_view&, const Index&, const Type&) +template +inline min_max_internal::rhs_proxy< op_min_index_view > +min_of(const std::pair& pair, + const op_min_index_view& view) +{ + return min_max_internal::make_proxy(pair, view); +} + +/** Computes nested minimum between the value in a view and other values. + * + * Compute the minimum of the result of a min_of() call and another value. + * + * The result of this computation can only be assigned back to the original + * view or used in another min_of() call. For example, + * + * *reducer = min_of(x, min_of(y, *reducer)); + * *reducer = min_of(min_of(*reducer, x), y); + * + * @see min_max_internal::min_min_view_base::rhs_proxy + */ +template +inline min_max_internal::rhs_proxy< op_min_index_view > +min_of(const min_max_internal::rhs_proxy< op_min_index_view >& proxy, + const Index& index, const Type& value) +{ + return proxy.calc(std::pair(index, value)); +} + +/// @copydoc min_of(const min_max_internal::rhs_proxy< op_min_index_view >&, const Index&, const Type&) +template +inline min_max_internal::rhs_proxy< op_min_index_view > +min_of(const Index& index, const Type& value, + const min_max_internal::rhs_proxy< op_min_index_view >& proxy) +{ + return proxy.calc(std::pair(index, value)); +} + +/// @copydoc min_of(const min_max_internal::rhs_proxy< op_min_index_view >&, const Index&, const Type&) +template +inline min_max_internal::rhs_proxy< op_min_index_view > +min_of(const min_max_internal::rhs_proxy< op_min_index_view >& proxy, + const std::pair& pair) +{ + return proxy.calc(pair); +} + +/// @copydoc min_of(const min_max_internal::rhs_proxy< op_min_index_view >&, const Index&, const Type&) +template +inline min_max_internal::rhs_proxy< op_min_index_view > +min_of(const std::pair& pair, + const min_max_internal::rhs_proxy< op_min_index_view >& proxy) +{ + return proxy.calc(pair); +} + + +/** Monoid class for minimum reductions with index. Instantiate the + * cilk::reducer template class with an op_min_index monoid to create a + * min_index reducer class. For example, to compute the minimum of an array of + * `double` values and the array index of the min value: + * + * cilk::reducer< cilk::op_min_index > r; + * + * @see ReducersMinMax + * @see op_min_index_view + */ +template < typename Index + , typename Type + , typename Compare=std::less + , bool Align = false + > +class op_min_index : public min_max_internal::monoid_base, Align> +{ + typedef min_max_internal::monoid_base< + op_min_index_view, Align> base; +public: + /// Construct with default comparator. + op_min_index() {} + /// Construct with specified comparator. + op_min_index(const Compare& compare) : base(compare) {} +}; + +//@} + + +/** Deprecated maximum reducer wrapper class. + * + * reducer_max is the same as @ref reducer<@ref op_max>, except that + * reducer_max is a proxy for the contained view, so that accumulator + * variable update operations can be applied directly to the reducer. For + * example, a value is maximized with a `reducer<%op_max>` with + * `r->calc_max(a)`, but a value can be maximized with a `%reducer_max` with + * `r.calc_max(a)`. + * + * + * @deprecated Users are strongly encouraged to use `reducer` + * reducers rather than the old wrappers like reducer_max. + * The `reducer` reducers show the reducer/monoid/view + * architecture more clearly, are more consistent in their + * implementation, and present a simpler model for new + * user-implemented reducers. + * + * @note Implicit conversions are provided between `%reducer_max` + * and `reducer<%op_max>`. This allows incremental code + * conversion: old code that used `%reducer_max` can pass a + * `%reducer_max` to a converted function that now expects a + * pointer or reference to a `reducer<%op_max>`, and vice + * versa. **But see @ref redminmax_compatibility.** + * + * @tparam Type The value type of the reducer. + * @tparam Compare The "less than" comparator type for the reducer. + * + * @see op_max + * @see op_max_view + * @see reducer + * @see ReducersMinMax + * @ingroup ReducersMinMaxMaxValue + */ +template > +class reducer_max : public reducer< op_max > +{ + __CILKRTS_STATIC_ASSERT( + ::cilk::internal::class_is_empty< + typename ::cilk::internal::binary_functor::type >::value, + "cilk::reducer_max only works with " + "an empty Compare class"); + typedef reducer< op_max > base; +public: + + /// Type of data in a reducer_max. + typedef Type basic_value_type; + + /// The view type for the reducer. + typedef typename base::view_type view_type; + + /// The view type for the reducer. + typedef typename base::view_type View; + + /// The monoid type for the reducer. + typedef typename base::monoid_type monoid_type; + + /// The monoid type for the reducer. + typedef typename base::monoid_type Monoid; + + /// The view's rhs proxy type. + typedef min_max_internal::rhs_proxy rhs_proxy; + + using base::view; + + /** @name Constructors + */ + //@{ + + /// Constructs the wrapper in its identity state (either `!is_set()`, or + /// `value() == identity value`). + reducer_max() : base() {} + + /// Constructs the wrapper with a specified initial value. + explicit reducer_max(const Type& initial_value) : base(initial_value) {} + + /// Constructs the wrapper in its identity state with a specified + /// comparator. + explicit reducer_max(const Compare& comp) : base(comp) {} + + /// Constructs the wrapper with a specified initial value and a specified + /// comparator. + reducer_max(const Type& initial_value, const Compare& comp) + : base(initial_value, comp) {} + + //@} + + /** @name Forwarded functions + * @details Functions that update the contained accumulator variable are + * simply forwarded to the contained @ref op_max_view. */ + //@{ + + /// @copydoc cilk_lib_1_1::min_max_internal::view_content::is_set() const + bool is_set() const { return view().is_set(); } + + /// @copydoc op_max_view::calc_max(const Type&) + reducer_max& calc_max(const Type& x) + { view().calc_max(x); return *this; } + + /// @copydoc op_max_view::operator=(const min_max_internal::rhs_proxy&) + reducer_max& operator=(const rhs_proxy& rhs) + { view() = rhs; return *this; } + + //@} + + /** Allows read-only access to the value within the current view. + * + * @returns A const reference to the value within the current view. + */ + const Type& get_reference() const { return view().get_reference(); } + + /// @name Dereference + /** Dereferencing a wrapper is a no-op. It simply returns the wrapper. + * Combined with the rule that a wrapper forwards view operations to the + * view, this means that view operations can be written the same way on + * reducers and wrappers, which is convenient for incrementally + * converting code using wrappers to code using reducers. That is: + * + * reducer< op_max > r; + * r->calc_max(a); // *r returns the view + * // calc_max is a view member function + * + * reducer_max w; + * w->calc_max(a); // *w returns the wrapper + * // calc_max is a wrapper member function that + * // calls the corresponding view function + */ + //@{ + reducer_max& operator*() { return *this; } + reducer_max const& operator*() const { return *this; } + + reducer_max* operator->() { return this; } + reducer_max const* operator->() const { return this; } + //@} + + /** @name Upcast + * @details In Intel Cilk Plus library 0.9, reducers were always cache-aligned. + * In library 1.0, reducer cache alignment is optional. By default, + * reducers are unaligned (i.e., just naturally aligned), but legacy + * wrappers inherit from cache-aligned reducers for binary compatibility. + * + * This means that a wrapper will automatically be upcast to its aligned + * reducer base class. The following conversion operators provide + * pseudo-upcasts to the corresponding unaligned reducer class. + */ + //@{ + operator reducer< op_max >& () + { + return *reinterpret_cast< reducer< op_max >* >(this); + } + + operator const reducer< op_max >& () const + { + return *reinterpret_cast< const reducer< op_max >* >(this); + } + //@} +}; + + +/// @cond internal +// The legacy definition of max_of(reducer_max, value) has different +// behavior and a different return type than this definition. We add an +// unused third argument to this version of the function to give it a different +// signature, so that they won't end up sharing a single object file entry. +struct max_of_1_0_t {}; +const max_of_1_0_t max_of_1_0 = {}; +/// @endcond + +/** Computes the maximum of the value in a reducer_max and another value. + * + * @deprecated Because reducer_max is deprecated. + * + * The result of this computation can only be assigned back to the original + * reducer or used in another max_of() call. For example, + * + * reducer = max_of(reducer, x); + * reducer = max_of(x, reducer); + * + * @see min_max_internal::rhs_proxy + * + * @ingroup ReducersMinMaxMaxValue + */ +template +inline min_max_internal::rhs_proxy< op_max_view > +max_of(const reducer_max& r, const Type& value, + const max_of_1_0_t& = max_of_1_0) +{ + return min_max_internal::make_proxy(value, r.view()); +} + +/// @copydoc max_of(const reducer_max&, const Type&, const max_of_1_0_t&) +/// @ingroup ReducersMinMaxMaxValue +template +inline min_max_internal::rhs_proxy< op_max_view > +max_of(const Type& value, const reducer_max& r, + const max_of_1_0_t& = max_of_1_0) +{ + return min_max_internal::make_proxy(value, r.view()); +} + + +/** Deprecated minimum reducer wrapper class. + * + * reducer_min is the same as @ref reducer<@ref op_min>, except that + * reducer_min is a proxy for the contained view, so that accumulator + * variable update operations can be applied directly to the reducer. For + * example, a value is minimized with a `reducer<%op_min>` with + * `r->calc_min(a)`, but a value can be minimized with a `%reducer_min` with + * `r.calc_min(a)`. + * + * + * @deprecated Users are strongly encouraged to use `reducer` + * reducers rather than the old wrappers like reducer_min. + * The `reducer` reducers show the reducer/monoid/view + * architecture more clearly, are more consistent in their + * implementation, and present a simpler model for new + * user-implemented reducers. + * + * @note Implicit conversions are provided between `%reducer_min` + * and `reducer<%op_min>`. This allows incremental code + * conversion: old code that used `%reducer_min` can pass a + * `%reducer_min` to a converted function that now expects a + * pointer or reference to a `reducer<%op_min>`, and vice + * versa. **But see @ref redminmax_compatibility.** + * + * @tparam Type The value type of the reducer. + * @tparam Compare The "less than" comparator type for the reducer. + * + * @see op_min + * @see op_min_view + * @see reducer + * @see ReducersMinMax + * @ingroup ReducersMinMaxMinValue + */ +template > +class reducer_min : public reducer< op_min > +{ + __CILKRTS_STATIC_ASSERT( + ::cilk::internal::class_is_empty< + typename ::cilk::internal::binary_functor::type >::value, + "cilk::reducer_min only works with " + "an empty Compare class"); + typedef reducer< op_min > base; +public: + + /// Type of data in a reducer_min. + typedef Type basic_value_type; + + /// The view type for the reducer. + typedef typename base::view_type view_type; + + /// The view type for the reducer. + typedef typename base::view_type View; + + /// The monoid type for the reducer. + typedef typename base::monoid_type monoid_type; + + /// The monoid type for the reducer. + typedef typename base::monoid_type Monoid; + + /// The view's rhs proxy type. + typedef min_max_internal::rhs_proxy rhs_proxy; + + using base::view; + + /** @name Constructors + */ + //@{ + + /// Constructs the wrapper in its identity state (either `!is_set()`, or + /// `value() == identity value`). + reducer_min() : base() {} + + /// Constructs the wrapper with a specified initial value. + explicit reducer_min(const Type& initial_value) : base(initial_value) {} + + /// Constructs the wrapper in its identity state with a specified + /// comparator. + explicit reducer_min(const Compare& comp) : base(comp) {} + + /// Constructs the wrapper with a specified initial value and a specified + /// comparator. + reducer_min(const Type& initial_value, const Compare& comp) + : base(initial_value, comp) {} + + //@} + + /** @name Forwarded functions + * @details Functions that update the contained accumulator variable are + * simply forwarded to the contained @ref op_min_view. */ + //@{ + + /// @copydoc cilk_lib_1_1::min_max_internal::view_content::is_set() const + bool is_set() const { return view().is_set(); } + + /// @copydoc op_min_view::calc_min(const Type&) + reducer_min& calc_min(const Type& x) + { view().calc_min(x); return *this; } + + /// @copydoc op_min_view::operator=(const min_max_internal::rhs_proxy&) + reducer_min& operator=(const rhs_proxy& rhs) + { view() = rhs; return *this; } + + //@} + + /** Allows read-only access to the value within the current view. + * + * @returns A const reference to the value within the current view. + */ + const Type& get_reference() const { return view().get_reference(); } + + /// @name Dereference + /** Dereferencing a wrapper is a no-op. It simply returns the wrapper. + * Combined with the rule that a wrapper forwards view operations to the + * view, this means that view operations can be written the same way on + * reducers and wrappers, which is convenient for incrementally + * converting code using wrappers to code using reducers. That is: + * + * reducer< op_min > r; + * r->calc_min(a); // *r returns the view + * // calc_min is a view member function + * + * reducer_min w; + * w->calc_min(a); // *w returns the wrapper + * // calc_min is a wrapper member function that + * // calls the corresponding view function + */ + //@{ + reducer_min& operator*() { return *this; } + reducer_min const& operator*() const { return *this; } + + reducer_min* operator->() { return this; } + reducer_min const* operator->() const { return this; } + //@} + + /** @name Upcast + * @details In Intel Cilk Plus library 0.9, reducers were always cache-aligned. + * In library 1.0, reducer cache alignment is optional. By default, + * reducers are unaligned (i.e., just naturally aligned), but legacy + * wrappers inherit from cache-aligned reducers for binary compatibility. + * + * This means that a wrapper will automatically be upcast to its aligned + * reducer base class. The following conversion operators provide + * pseudo-upcasts to the corresponding unaligned reducer class. + */ + //@{ + operator reducer< op_min >& () + { + return *reinterpret_cast< reducer< op_min >* >(this); + } + + operator const reducer< op_min >& () const + { + return *reinterpret_cast< const reducer< op_min >* >(this); + } + //@} +}; + + +/** Computes the minimum of a reducer and a value. + * + * @deprecated Because reducer_min is deprecated. + */ +//@{ +// The legacy definition of min_of(reducer_min, value) has different +// behavior and a different return type than this definition. We add an +// unused third argument to this version of the function to give it a different +// signature, so that they won't end up sharing a single object file entry. +struct min_of_1_0_t {}; +const min_of_1_0_t min_of_1_0 = {}; + +template +inline min_max_internal::rhs_proxy< op_min_view > +min_of(const reducer_min& r, const Type& value, + const min_of_1_0_t& = min_of_1_0) +{ + return min_max_internal::make_proxy(value, r.view()); +} + +template +inline min_max_internal::rhs_proxy< op_min_view > +min_of(const Type& value, const reducer_min& r, + const min_of_1_0_t& = min_of_1_0) +{ + return min_max_internal::make_proxy(value, r.view()); +} +//@} + + +/** Deprecated maximum with index reducer wrapper class. + * + * reducer_max_index is the same as @ref reducer<@ref op_max_index>, except + * that reducer_max_index is a proxy for the contained view, so that + * accumulator variable update operations can be applied directly to the + * reducer. For example, a value is maximized with a `reducer<%op_max_index>` + * with `r->calc_max(i, a)`, but a value can be maximized with a + * `%reducer_max` with `r.calc_max(i, aa)`. + * + * + * @deprecated Users are strongly encouraged to use `reducer` + * reducers rather than the old wrappers like reducer_max. + * The `reducer` reducers show the reducer/monoid/view + * architecture more clearly, are more consistent in their + * implementation, and present a simpler model for new + * user-implemented reducers. + * + * @note Implicit conversions are provided between `%reducer_max_index` + * and `reducer<%op_max_index>`. This allows incremental code + * conversion: old code that used `%reducer_max_index` can pass a + * `%reducer_max_index` to a converted function that now expects a + * pointer or reference to a `reducer<%op_max_index>`, and vice + * versa. **But see @ref redminmax_compatibility.** + * + * @tparam Index The index type of the reducer. + * @tparam Type The value type of the reducer. + * @tparam Compare The "less than" comparator type for the reducer. + * + * @see op_max_index + * @see op_max_index_view + * @see reducer + * @see ReducersMinMax + * @ingroup ReducersMinMaxMaxIndex + */ +template < typename Index + , typename Type + , typename Compare = std::less + > +class reducer_max_index : + public reducer< op_max_index > +{ + __CILKRTS_STATIC_ASSERT( + ::cilk::internal::class_is_empty< + typename ::cilk::internal::binary_functor::type >::value, + "cilk::reducer_max_index only works with " + "an empty Compare class"); + typedef reducer< op_max_index > base; +public: + + /// Type of data in a reducer_max_index. + typedef Type basic_value_type; + + /// The view type for the reducer. + typedef typename base::view_type view_type; + + /// The view type for the reducer. + typedef typename base::view_type View; + + /// The monoid type for the reducer. + typedef typename base::monoid_type monoid_type; + + /// The monoid type for the reducer. + typedef typename base::monoid_type Monoid; + + /// The view's rhs proxy type. + typedef min_max_internal::rhs_proxy rhs_proxy; + + using base::view; + + /** @name Constructors + */ + //@{ + + /// Constructs the wrapper in its identity state (`!is_set()`). + reducer_max_index() : base() {} + + /// Construct with a specified initial index and value. + reducer_max_index(const Index& initial_index, + const Type& initial_value) + : base(initial_index, initial_value) {} + + /// Constructs the wrapper with a specified comparator. + explicit reducer_max_index(const Compare& comp) : base(comp) {} + + /// Constructs the wrapper with a specified initial index, value, + /// and comparator. + reducer_max_index(const Index& initial_index, + const Type& initial_value, + const Compare& comp) + : base(initial_index, initial_value, comp) {} + + //@} + + /** @name Set / Get + */ + //@{ + + /// Sets the index and value of this object. + void set_value(const Index& index, const Type& value) + { base::set_value(std::make_pair(index, value)); } + + /// Returns the maximum value. + const Type& get_value() const + { return view().get_reference(); } + + /// Returns the maximum index. + const Index& get_index() const + { return view().get_index_reference(); } + + /// Returns a const reference to value data member in the view. + const Type& get_reference() const + { return view().get_reference(); } + + /// Returns a const reference to index data member in the view. + const Index& get_index_reference() const + { return view().get_index_reference(); } + + //@} + + /** @name Forwarded functions + * @details Functions that update the contained accumulator variable are + * simply forwarded to the contained @ref op_max_view. */ + //@{ + + /// @copydoc cilk_lib_1_1::min_max_internal::view_content::is_set() const + bool is_set() const { return view().is_set(); } + + /// @copydoc op_max_index_view::calc_max(const Index&, const Type&) + reducer_max_index& calc_max(const Index& i, const Type& x) + { view().calc_max(i, x); return *this; } + + /// @copydoc op_max_view::operator=(const min_max_internal::rhs_proxy&) + reducer_max_index& operator=(const rhs_proxy& rhs) + { view() = rhs; return *this; } + + //@} + + /// @name Dereference + /** Dereferencing a wrapper is a no-op. It simply returns the wrapper. + * Combined with the rule that a wrapper forwards view operations to the + * view, this means that view operations can be written the same way on + * reducers and wrappers, which is convenient for incrementally + * converting code using wrappers to code using reducers. That is: + * + * reducer< op_max_index > r; + * r->calc_max(i, a); // *r returns the view + * // calc_max is a view member function + * + * reducer_max_index w; + * w->calc_max(i, a); // *w returns the wrapper + * // calc_max is a wrapper member function that + * // calls the corresponding view function + */ + //@{ + reducer_max_index& operator*() { return *this; } + reducer_max_index const& operator*() const { return *this; } + + reducer_max_index* operator->() { return this; } + reducer_max_index const* operator->() const { return this; } + //@} + + /** @name Upcast + * @details In Intel Cilk Plus library 0.9, reducers were always cache-aligned. + * In library 1.0, reducer cache alignment is optional. By default, + * reducers are unaligned (i.e., just naturally aligned), but legacy + * wrappers inherit from cache-aligned reducers for binary compatibility. + * + * This means that a wrapper will automatically be upcast to its aligned + * reducer base class. The following conversion operators provide + * pseudo-upcasts to the corresponding unaligned reducer class. + */ + //@{ + operator reducer< op_max_index >& () + { + return *reinterpret_cast< reducer< op_max_index >* >(this); + } + + operator const reducer< op_max_index >& () const + { + return *reinterpret_cast< const reducer< op_max_index >* >(this); + } + //@} + +}; + + +/** Deprecated minimum with index reducer wrapper class. + * + * reducer_min_index is the same as @ref reducer<@ref op_min_index>, except + * that reducer_min_index is a proxy for the contained view, so that + * accumulator variable update operations can be applied directly to the + * reducer. For example, a value is minimized with a `reducer<%op_min_index>` + * with `r->calc_min(i, a)`, but a value can be minimized with a + * `%reducer_min` with `r.calc_min(i, aa)`. + * + * + * @deprecated Users are strongly encouraged to use `reducer` + * reducers rather than the old wrappers like reducer_min. + * The `reducer` reducers show the reducer/monoid/view + * architecture more clearly, are more consistent in their + * implementation, and present a simpler model for new + * user-implemented reducers. + * + * @note Implicit conversions are provided between `%reducer_min_index` + * and `reducer<%op_min_index>`. This allows incremental code + * conversion: old code that used `%reducer_min_index` can pass a + * `%reducer_min_index` to a converted function that now expects a + * pointer or reference to a `reducer<%op_min_index>`, and vice + * versa. **But see @ref redminmax_compatibility.** + * + * @tparam Index The index type of the reducer. + * @tparam Type The value type of the reducer. + * @tparam Compare The "less than" comparator type for the reducer. + * + * @see op_min_index + * @see op_min_index_view + * @see reducer + * @see ReducersMinMax + * @ingroup ReducersMinMaxMinIndex + */ +template < typename Index + , typename Type + , typename Compare = std::less + > +class reducer_min_index : + public reducer< op_min_index > +{ + __CILKRTS_STATIC_ASSERT( + ::cilk::internal::class_is_empty< + typename ::cilk::internal::binary_functor::type >::value, + "cilk::reducer_min_index only works with " + "an empty Compare class"); + typedef reducer< op_min_index > base; +public: + + /// Type of data in a reducer_min_index. + typedef Type basic_value_type; + + /// The view type for the reducer. + typedef typename base::view_type view_type; + + /// The view type for the reducer. + typedef typename base::view_type View; + + /// The monoid type for the reducer. + typedef typename base::monoid_type monoid_type; + + /// The monoid type for the reducer. + typedef typename base::monoid_type Monoid; + + /// The view's rhs proxy type. + typedef min_max_internal::rhs_proxy rhs_proxy; + + using base::view; + + /** @name Constructors + */ + //@{ + + /// Constructs the wrapper in its identity state (`!is_set()`). + reducer_min_index() : base() {} + + /// Construct with a specified initial index and value. + reducer_min_index(const Index& initial_index, + const Type& initial_value) + : base(initial_index, initial_value) {} + + /// Constructs the wrapper with a specified comparator. + explicit reducer_min_index(const Compare& comp) : base(comp) {} + + /// Constructs the wrapper with a specified initial index, value, + /// and comparator. + reducer_min_index(const Index& initial_index, + const Type& initial_value, + const Compare& comp) + : base(initial_index, initial_value, comp) {} + + //@} + + /** @name Set / Get + */ + //@{ + + /// Sets the index and value of this object. + void set_value(const Index& index, const Type& value) + { base::set_value(std::make_pair(index, value)); } + + /// Returns the minimum value. + const Type& get_value() const + { return view().get_reference(); } + + /// Returns the minimum index. + const Index& get_index() const + { return view().get_index_reference(); } + + /// Returns a const reference to value data member in the view. + const Type& get_reference() const + { return view().get_reference(); } + + /// Returns a const reference to index data member in the view. + const Index& get_index_reference() const + { return view().get_index_reference(); } + + //@} + + /** @name Forwarded functions + * @details Functions that update the contained accumulator variable are + * simply forwarded to the contained @ref op_min_view. */ + //@{ + + /// @copydoc cilk_lib_1_1::min_max_internal::view_content::is_set() const + bool is_set() const { return view().is_set(); } + + /// @copydoc op_min_index_view::calc_min(const Index&, const Type&) + reducer_min_index& calc_min(const Index& i, const Type& x) + { view().calc_min(i, x); return *this; } + + /// @copydoc op_min_view::operator=(const min_max_internal::rhs_proxy&) + reducer_min_index& operator=(const rhs_proxy& rhs) + { view() = rhs; return *this; } + + //@} + + /// @name Dereference + /** Dereferencing a wrapper is a no-op. It simply returns the wrapper. + * Combined with the rule that a wrapper forwards view operations to the + * view, this means that view operations can be written the same way on + * reducers and wrappers, which is convenient for incrementally + * converting code using wrappers to code using reducers. That is: + * + * reducer< op_min_index > r; + * r->calc_min(i, a); // *r returns the view + * // calc_min is a view member function + * + * reducer_min_index w; + * w->calc_min(i, a); // *w returns the wrapper + * // calc_min is a wrapper member function that + * // calls the corresponding view function + */ + //@{ + reducer_min_index& operator*() { return *this; } + reducer_min_index const& operator*() const { return *this; } + + reducer_min_index* operator->() { return this; } + reducer_min_index const* operator->() const { return this; } + //@} + + /** @name Upcast + * @details In Intel Cilk Plus library 0.9, reducers were always cache-aligned. + * In library 1.0, reducer cache alignment is optional. By default, + * reducers are unaligned (i.e., just naturally aligned), but legacy + * wrappers inherit from cache-aligned reducers for binary compatibility. + * + * This means that a wrapper will automatically be upcast to its aligned + * reducer base class. The following conversion operators provide + * pseudo-upcasts to the corresponding unaligned reducer class. + */ + //@{ + operator reducer< op_min_index >& () + { + return *reinterpret_cast< reducer< op_min_index >* >(this); + } + + operator const reducer< op_min_index >& () const + { + return *reinterpret_cast< const reducer< op_min_index >* >(this); + } + //@} + +}; + + +#ifndef CILK_LIBRARY_0_9_REDUCER_MINMAX +} // namespace cilk_lib_1_1 +using namespace cilk_lib_1_1; +#endif + + +/// @cond internal +/** Metafunction specialization for reducer conversion. + * + * These specializations of the @ref legacy_reducer_downcast template class + * defined in reducer.h causes each `reducer< op_xxxx >` classes to have + * an `operator reducer_xxxx& ()` conversion operator that statically + * downcasts the `reducer` to the corresponding `reducer_xxxx` type. + * (The reverse conversion, from `reducer_xxxx` to `reducer`, is just + * an upcast, which is provided for free by the language.) + */ +template +struct legacy_reducer_downcast< reducer< op_max > > +{ + typedef reducer_max type; +}; + +template +struct legacy_reducer_downcast< reducer< op_min > > +{ + typedef reducer_min type; +}; + +template +struct legacy_reducer_downcast< reducer< op_max_index > > +{ + typedef reducer_max_index type; +}; + +template +struct legacy_reducer_downcast< reducer< op_min_index > > +{ + typedef reducer_min_index type; +}; +/// @endcond + +} // namespace cilk + +#endif // __cplusplus + + +/** @name C language reducer macros + * + * These macros are used to declare and work with numeric minimum and maximum reducers in C + * code. + * + * @see @ref page_reducers_in_c + */ + //@{ + + +#ifdef CILK_C_DEFINE_REDUCERS + +/* Integer min/max constants */ +#include + +/* Wchar_t min/max constants */ +#if defined(_MSC_VER) || defined(__ANDROID__) +# include +#else +# include +#endif + +/* Floating-point min/max constants */ +#include +#ifndef HUGE_VALF + static const unsigned int __huge_valf[] = {0x7f800000}; +# define HUGE_VALF (*((const float *)__huge_valf)) +#endif + +#ifndef HUGE_VALL + static const unsigned int __huge_vall[] = {0, 0, 0x00007f80, 0}; +# define HUGE_VALL (*((const long double *)__huge_vall)) +#endif + +#endif + +/** Declares max reducer type name. + * + * This macro expands into the identifier which is the name of the max reducer + * type for a specified numeric type. + * + * @param tn The @ref reducers_c_type_names "numeric type name" specifying the type of the + * reducer. + * + * @see @ref reducers_c_predefined + */ +#define CILK_C_REDUCER_MAX_TYPE(tn) \ + __CILKRTS_MKIDENT(cilk_c_reducer_max_,tn) + +/** Declares a max reducer object. + * + * This macro expands into a declaration of a max reducer object for a specified numeric + * type. For example: + * + * CILK_C_REDUCER_MAX(my_reducer, double, -DBL_MAX); + * + * @param obj The variable name to be used for the declared reducer object. + * @param tn The @ref reducers_c_type_names "numeric type name" specifying the type of the + * reducer. + * @param v The initial value for the reducer. (A value which can be assigned to the + * numeric type represented by @a tn.) + * + * @see @ref reducers_c_predefined + */ +#define CILK_C_REDUCER_MAX(obj,tn,v) \ + CILK_C_REDUCER_MAX_TYPE(tn) obj = \ + CILK_C_INIT_REDUCER(_Typeof(obj.value), \ + __CILKRTS_MKIDENT(cilk_c_reducer_max_reduce_,tn), \ + __CILKRTS_MKIDENT(cilk_c_reducer_max_identity_,tn), \ + __cilkrts_hyperobject_noop_destroy, v) + +/** Maximizes with a value. + * + * `CILK_C_REDUCER_MAX_CALC(reducer, v)` sets the current view of the + * reducer to the max of its previous value and a specified new value. + * This is equivalent to + * + * REDUCER_VIEW(reducer) = max(REDUCER_VIEW(reducer), v) + * + * @param reducer The reducer whose contained value is to be updated. + * @param v The value that it is to be maximized with. + */ +#define CILK_C_REDUCER_MAX_CALC(reducer, v) do { \ + _Typeof((reducer).value)* view = &(REDUCER_VIEW(reducer)); \ + _Typeof(v) __value = (v); \ + if (*view < __value) { \ + *view = __value; \ + } } while (0) + +/// @cond internal + +/** Declares the max reducer functions for a numeric type. + * + * This macro expands into external function declarations for functions which implement + * the reducer functionality for the max reducer type for a specified numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer type name, + * function names, etc. + */ +#define CILK_C_REDUCER_MAX_DECLARATION(t,tn,id) \ + typedef CILK_C_DECLARE_REDUCER(t) CILK_C_REDUCER_MAX_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_max,tn,l,r); \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_max,tn); + +/** Defines the max reducer functions for a numeric type. + * + * This macro expands into function definitions for functions which implement the + * reducer functionality for the max reducer type for a specified numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer type name, + * function names, etc. + */ +#define CILK_C_REDUCER_MAX_DEFINITION(t,tn,id) \ + typedef CILK_C_DECLARE_REDUCER(t) CILK_C_REDUCER_MAX_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_max,tn,l,r) \ + { if (*(t*)l < *(t*)r) *(t*)l = *(t*)r; } \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_max,tn) \ + { *(t*)v = id; } + +//@{ +/** @def CILK_C_REDUCER_MAX_INSTANCE + * @brief Declare or define implementation functions for a reducer type. + * + * In the runtime source file c_reducers.c, the macro `CILK_C_DEFINE_REDUCERS` will be defined, and + * this macro will generate reducer implementation functions. Everywhere else, `CILK_C_DEFINE_REDUCERS` + * will be undefined, and this macro will expand into external declarations for the functions. + */ +#ifdef CILK_C_DEFINE_REDUCERS +# define CILK_C_REDUCER_MAX_INSTANCE(t,tn,id) \ + CILK_C_REDUCER_MAX_DEFINITION(t,tn,id) +#else +# define CILK_C_REDUCER_MAX_INSTANCE(t,tn,id) \ + CILK_C_REDUCER_MAX_DECLARATION(t,tn,id) +#endif +//@} + +/* Declare or define an instance of the reducer type and its functions for each + * numeric type. + */ +__CILKRTS_BEGIN_EXTERN_C +CILK_C_REDUCER_MAX_INSTANCE(char, char, CHAR_MIN) +CILK_C_REDUCER_MAX_INSTANCE(unsigned char, uchar, 0) +CILK_C_REDUCER_MAX_INSTANCE(signed char, schar, SCHAR_MIN) +CILK_C_REDUCER_MAX_INSTANCE(wchar_t, wchar_t, WCHAR_MIN) +CILK_C_REDUCER_MAX_INSTANCE(short, short, SHRT_MIN) +CILK_C_REDUCER_MAX_INSTANCE(unsigned short, ushort, 0) +CILK_C_REDUCER_MAX_INSTANCE(int, int, INT_MIN) +CILK_C_REDUCER_MAX_INSTANCE(unsigned int, uint, 0) +CILK_C_REDUCER_MAX_INSTANCE(unsigned int, unsigned, 0) // alternate name +CILK_C_REDUCER_MAX_INSTANCE(long, long, LONG_MIN) +CILK_C_REDUCER_MAX_INSTANCE(unsigned long, ulong, 0) +CILK_C_REDUCER_MAX_INSTANCE(long long, longlong, LLONG_MIN) +CILK_C_REDUCER_MAX_INSTANCE(unsigned long long, ulonglong, 0) +CILK_C_REDUCER_MAX_INSTANCE(float, float, -HUGE_VALF) +CILK_C_REDUCER_MAX_INSTANCE(double, double, -HUGE_VAL) +CILK_C_REDUCER_MAX_INSTANCE(long double, longdouble, -HUGE_VALL) +__CILKRTS_END_EXTERN_C + +/// @endcond + +/** Max_index reducer type name. + * + * This macro expands into the identifier which is the name of the max_index reducer + * type for a specified numeric type. + * + * @param tn The @ref reducers_c_type_names "numeric type name" specifying the type of the + * reducer. + * + * @see @ref reducers_c_predefined + */ +#define CILK_C_REDUCER_MAX_INDEX_TYPE(tn) \ + __CILKRTS_MKIDENT(cilk_c_reducer_max_index_,tn) + +/** Declares an op_max_index reducer object. + * + * This macro expands into a declaration of a max_index reducer object for a specified + * numeric type. For example: + * + * CILK_C_REDUCER_MAX_INDEX(my_reducer, double, -DBL_MAX_INDEX); + * + * @param obj The variable name to be used for the declared reducer object. + * @param tn The @ref reducers_c_type_names "numeric type name" specifying the type of the + * reducer. + * @param v The initial value for the reducer. (A value which can be assigned to the + * numeric type represented by @a tn.) + * + * @see @ref reducers_c_predefined + */ +#define CILK_C_REDUCER_MAX_INDEX(obj,tn,v) \ + CILK_C_REDUCER_MAX_INDEX_TYPE(tn) obj = \ + CILK_C_INIT_REDUCER(_Typeof(obj.value), \ + __CILKRTS_MKIDENT(cilk_c_reducer_max_index_reduce_,tn), \ + __CILKRTS_MKIDENT(cilk_c_reducer_max_index_identity_,tn), \ + __cilkrts_hyperobject_noop_destroy, {0, v}) + +/** Maximizes with a value. + * + * `CILK_C_REDUCER_MAX_INDEX_CALC(reducer, i, v)` sets the current view of the + * reducer to the max of its previous value and a specified new value. + * This is equivalent to + * + * REDUCER_VIEW(reducer) = max_index(REDUCER_VIEW(reducer), v) + * + * If the value of the reducer is changed to @a v, then the index of the reducer is + * changed to @a i. + * + * @param reducer The reducer whose contained value and index are to be updated. + * @param i The index associated with the new value. + * @param v The value that it is to be maximized with. + */ +#define CILK_C_REDUCER_MAX_INDEX_CALC(reducer, i, v) do { \ + _Typeof((reducer).value)* view = &(REDUCER_VIEW(reducer)); \ + _Typeof(v) __value = (v); \ + if (view->value < __value) { \ + view->index = (i); \ + view->value = __value; \ + } } while (0) + +/// @cond internal + +/** Declares the max_index view type. + * + * The view of a max_index reducer is a structure containing both the + * maximum value for the reducer and the index that was associated with + * that value in the sequence of input values. + */ +#define CILK_C_REDUCER_MAX_INDEX_VIEW(t,tn) \ + typedef struct { \ + __STDNS ptrdiff_t index; \ + t value; \ + } __CILKRTS_MKIDENT(cilk_c_reducer_max_index_view_,tn) + +/** Declares the max_index reducer functions for a numeric type. + * + * This macro expands into external function declarations for functions which implement + * the reducer functionality for the max_index reducer type for a specified numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer type name, + * function names, etc. + */ +#define CILK_C_REDUCER_MAX_INDEX_DECLARATION(t,tn,id) \ + CILK_C_REDUCER_MAX_INDEX_VIEW(t,tn); \ + typedef CILK_C_DECLARE_REDUCER( \ + __CILKRTS_MKIDENT(cilk_c_reducer_max_index_view_,tn)) \ + CILK_C_REDUCER_MAX_INDEX_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_max_index,tn,l,r); \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_max_index,tn); + +/** Defines the max_index reducer functions for a numeric type. + * + * This macro expands into function definitions for functions which implement the + * reducer functionality for the max_index reducer type for a specified numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer type name, + * function names, etc. + */ +#define CILK_C_REDUCER_MAX_INDEX_DEFINITION(t,tn,id) \ + CILK_C_REDUCER_MAX_INDEX_VIEW(t,tn); \ + typedef CILK_C_DECLARE_REDUCER( \ + __CILKRTS_MKIDENT(cilk_c_reducer_max_index_view_,tn)) \ + CILK_C_REDUCER_MAX_INDEX_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_max_index,tn,l,r) \ + { typedef __CILKRTS_MKIDENT(cilk_c_reducer_max_index_view_,tn) view_t; \ + if (((view_t*)l)->value < ((view_t*)r)->value) \ + *(view_t*)l = *(view_t*)r; } \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_max_index,tn) \ + { typedef __CILKRTS_MKIDENT(cilk_c_reducer_max_index_view_,tn) view_t; \ + ((view_t*)v)->index = 0; ((view_t*)v)->value = id; } + +//@{ +/** @def CILK_C_REDUCER_MAX_INDEX_INSTANCE + * @brief Declare or define implementation functions for a reducer type. + * + * In the runtime source file c_reducers.c, the macro `CILK_C_DEFINE_REDUCERS` will be defined, and + * this macro will generate reducer implementation functions. Everywhere else, `CILK_C_DEFINE_REDUCERS` + * will be undefined, and this macro will expand into external declarations for the functions. + */ +#ifdef CILK_C_DEFINE_REDUCERS +# define CILK_C_REDUCER_MAX_INDEX_INSTANCE(t,tn,id) \ + CILK_C_REDUCER_MAX_INDEX_DEFINITION(t,tn,id) +#else +# define CILK_C_REDUCER_MAX_INDEX_INSTANCE(t,tn,id) \ + CILK_C_REDUCER_MAX_INDEX_DECLARATION(t,tn,id) +#endif +//@} + +/* Declares or defines an instance of the reducer type and its functions for each + * numeric type. + */ +__CILKRTS_BEGIN_EXTERN_C +CILK_C_REDUCER_MAX_INDEX_INSTANCE(char, char, CHAR_MIN) +CILK_C_REDUCER_MAX_INDEX_INSTANCE(unsigned char, uchar, 0) +CILK_C_REDUCER_MAX_INDEX_INSTANCE(signed char, schar, SCHAR_MIN) +CILK_C_REDUCER_MAX_INDEX_INSTANCE(wchar_t, wchar_t, WCHAR_MIN) +CILK_C_REDUCER_MAX_INDEX_INSTANCE(short, short, SHRT_MIN) +CILK_C_REDUCER_MAX_INDEX_INSTANCE(unsigned short, ushort, 0) +CILK_C_REDUCER_MAX_INDEX_INSTANCE(int, int, INT_MIN) +CILK_C_REDUCER_MAX_INDEX_INSTANCE(unsigned int, uint, 0) +CILK_C_REDUCER_MAX_INDEX_INSTANCE(unsigned int, unsigned, 0) // alternate name +CILK_C_REDUCER_MAX_INDEX_INSTANCE(long, long, LONG_MIN) +CILK_C_REDUCER_MAX_INDEX_INSTANCE(unsigned long, ulong, 0) +CILK_C_REDUCER_MAX_INDEX_INSTANCE(long long, longlong, LLONG_MIN) +CILK_C_REDUCER_MAX_INDEX_INSTANCE(unsigned long long, ulonglong, 0) +CILK_C_REDUCER_MAX_INDEX_INSTANCE(float, float, -HUGE_VALF) +CILK_C_REDUCER_MAX_INDEX_INSTANCE(double, double, -HUGE_VAL) +CILK_C_REDUCER_MAX_INDEX_INSTANCE(long double, longdouble, -HUGE_VALL) +__CILKRTS_END_EXTERN_C + +/// @endcond + +/** Declares min reducer type name. + * + * This macro expands into the identifier which is the name of the min reducer + * type for a specified numeric type. + * + * @param tn The @ref reducers_c_type_names "numeric type name" specifying the type of the + * reducer. + * + * @see @ref reducers_c_predefined + */ +#define CILK_C_REDUCER_MIN_TYPE(tn) \ + __CILKRTS_MKIDENT(cilk_c_reducer_min_,tn) + +/** Declares a min reducer object. + * + * This macro expands into a declaration of a min reducer object for a specified numeric + * type. For example: + * + * CILK_C_REDUCER_MIN(my_reducer, double, DBL_MAX); + * + * @param obj The variable name to be used for the declared reducer object. + * @param tn The @ref reducers_c_type_names "numeric type name" specifying the type of the + * reducer. + * @param v The initial value for the reducer. (A value which can be assigned to the + * numeric type represented by @a tn.) + * + * @see @ref reducers_c_predefined + */ +#define CILK_C_REDUCER_MIN(obj,tn,v) \ + CILK_C_REDUCER_MIN_TYPE(tn) obj = \ + CILK_C_INIT_REDUCER(_Typeof(obj.value), \ + __CILKRTS_MKIDENT(cilk_c_reducer_min_reduce_,tn), \ + __CILKRTS_MKIDENT(cilk_c_reducer_min_identity_,tn), \ + __cilkrts_hyperobject_noop_destroy, v) + +/** Minimizes with a value. + * + * `CILK_C_REDUCER_MIN_CALC(reducer, v)` sets the current view of the + * reducer to the min of its previous value and a specified new value. + * This is equivalent to + * + * REDUCER_VIEW(reducer) = min(REDUCER_VIEW(reducer), v) + * + * @param reducer The reducer whose contained value is to be updated. + * @param v The value that it is to be minimized with. + */ +#define CILK_C_REDUCER_MIN_CALC(reducer, v) do { \ + _Typeof((reducer).value)* view = &(REDUCER_VIEW(reducer)); \ + _Typeof(v) __value = (v); \ + if (*view > __value) { \ + *view = __value; \ + } } while (0) + +/// @cond internal + +/** Declares the min reducer functions for a numeric type. + * + * This macro expands into external function declarations for functions which implement + * the reducer functionality for the min reducer type for a specified numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer type name, + * function names, etc. + */ +#define CILK_C_REDUCER_MIN_DECLARATION(t,tn,id) \ + typedef CILK_C_DECLARE_REDUCER(t) CILK_C_REDUCER_MIN_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_min,tn,l,r); \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_min,tn); + +/** Defines the min reducer functions for a numeric type. + * + * This macro expands into function definitions for functions which implement the + * reducer functionality for the min reducer type for a specified numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer type name, + * function names, etc. + */ +#define CILK_C_REDUCER_MIN_DEFINITION(t,tn,id) \ + typedef CILK_C_DECLARE_REDUCER(t) CILK_C_REDUCER_MIN_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_min,tn,l,r) \ + { if (*(t*)l > *(t*)r) *(t*)l = *(t*)r; } \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_min,tn) \ + { *(t*)v = id; } + +//@{ +/** @def CILK_C_REDUCER_MIN_INSTANCE + * @brief Declare or define implementation functions for a reducer type. + * + * In the runtime source file c_reducers.c, the macro `CILK_C_DEFINE_REDUCERS` will be defined, and + * this macro will generate reducer implementation functions. Everywhere else, `CILK_C_DEFINE_REDUCERS` + * will be undefined, and this macro will expand into external declarations for the functions. + */ +#ifdef CILK_C_DEFINE_REDUCERS +# define CILK_C_REDUCER_MIN_INSTANCE(t,tn,id) \ + CILK_C_REDUCER_MIN_DEFINITION(t,tn,id) +#else +# define CILK_C_REDUCER_MIN_INSTANCE(t,tn,id) \ + CILK_C_REDUCER_MIN_DECLARATION(t,tn,id) +#endif +//@} + +/* Declares or defines an instance of the reducer type and its functions for each + * numeric type. + */ +__CILKRTS_BEGIN_EXTERN_C +CILK_C_REDUCER_MIN_INSTANCE(char, char, CHAR_MAX) +CILK_C_REDUCER_MIN_INSTANCE(unsigned char, uchar, CHAR_MAX) +CILK_C_REDUCER_MIN_INSTANCE(signed char, schar, SCHAR_MAX) +CILK_C_REDUCER_MIN_INSTANCE(wchar_t, wchar_t, WCHAR_MAX) +CILK_C_REDUCER_MIN_INSTANCE(short, short, SHRT_MAX) +CILK_C_REDUCER_MIN_INSTANCE(unsigned short, ushort, USHRT_MAX) +CILK_C_REDUCER_MIN_INSTANCE(int, int, INT_MAX) +CILK_C_REDUCER_MIN_INSTANCE(unsigned int, uint, UINT_MAX) +CILK_C_REDUCER_MIN_INSTANCE(unsigned int, unsigned, UINT_MAX) // alternate name +CILK_C_REDUCER_MIN_INSTANCE(long, long, LONG_MAX) +CILK_C_REDUCER_MIN_INSTANCE(unsigned long, ulong, ULONG_MAX) +CILK_C_REDUCER_MIN_INSTANCE(long long, longlong, LLONG_MAX) +CILK_C_REDUCER_MIN_INSTANCE(unsigned long long, ulonglong, ULLONG_MAX) +CILK_C_REDUCER_MIN_INSTANCE(float, float, HUGE_VALF) +CILK_C_REDUCER_MIN_INSTANCE(double, double, HUGE_VAL) +CILK_C_REDUCER_MIN_INSTANCE(long double, longdouble, HUGE_VALL) +__CILKRTS_END_EXTERN_C + +/// @endcond + +/** Declares `min_index` reducer type name. + * + * This macro expands into the identifier which is the name of the min_index reducer + * type for a specified numeric type. + * + * @param tn The @ref reducers_c_type_names "numeric type name" specifying the type of the + * reducer. + * + * @see @ref reducers_c_predefined + */ +#define CILK_C_REDUCER_MIN_INDEX_TYPE(tn) \ + __CILKRTS_MKIDENT(cilk_c_reducer_min_index_,tn) + +/** Declares an op_min_index reducer object. + * + * This macro expands into a declaration of a min_index reducer object for a specified + * numeric type. For example: + * + * CILK_C_REDUCER_MIN_INDEX(my_reducer, double, -DBL_MIN_INDEX); + * + * @param obj The variable name to be used for the declared reducer object. + * @param tn The @ref reducers_c_type_names "numeric type name" specifying the type of the + * reducer. + * @param v The initial value for the reducer. (A value which can be assigned to the + * numeric type represented by @a tn.) + * + * @see @ref reducers_c_predefined + */ +#define CILK_C_REDUCER_MIN_INDEX(obj,tn,v) \ + CILK_C_REDUCER_MIN_INDEX_TYPE(tn) obj = \ + CILK_C_INIT_REDUCER(_Typeof(obj.value), \ + __CILKRTS_MKIDENT(cilk_c_reducer_min_index_reduce_,tn), \ + __CILKRTS_MKIDENT(cilk_c_reducer_min_index_identity_,tn), \ + __cilkrts_hyperobject_noop_destroy, {0, v}) + +/** Minimizes with a value. + * + * `CILK_C_REDUCER_MIN_INDEX_CALC(reducer, i, v)` sets the current view of the + * reducer to the min of its previous value and a specified new value. + * This is equivalent to + * + * REDUCER_VIEW(reducer) = min_index(REDUCER_VIEW(reducer), v) + * + * If the value of the reducer is changed to @a v, then the index of the reducer is + * changed to @a i. + * + * @param reducer The reducer whose contained value and index are to be updated. + * @param i The index associated with the new value. + * @param v The value that it is to be minimized with. + */ +#define CILK_C_REDUCER_MIN_INDEX_CALC(reducer, i, v) do { \ + _Typeof((reducer).value)* view = &(REDUCER_VIEW(reducer)); \ + _Typeof(v) __value = (v); \ + if (view->value > __value) { \ + view->index = (i); \ + view->value = __value; \ + } } while (0) + +/// @cond internal + +/** Declares the min_index view type. + * + * The view of a min_index reducer is a structure containing both the + * minimum value for the reducer and the index that was associated with + * that value in the sequence of input values. + */ +#define CILK_C_REDUCER_MIN_INDEX_VIEW(t,tn) \ + typedef struct { \ + __STDNS ptrdiff_t index; \ + t value; \ + } __CILKRTS_MKIDENT(cilk_c_reducer_min_index_view_,tn) + +/** Declares the min_index reducer functions for a numeric type. + * + * This macro expands into external function declarations for functions which implement + * the reducer functionality for the min_index reducer type for a specified numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer type name, + * function names, etc. + */ +#define CILK_C_REDUCER_MIN_INDEX_DECLARATION(t,tn,id) \ + CILK_C_REDUCER_MIN_INDEX_VIEW(t,tn); \ + typedef CILK_C_DECLARE_REDUCER( \ + __CILKRTS_MKIDENT(cilk_c_reducer_min_index_view_,tn)) \ + CILK_C_REDUCER_MIN_INDEX_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_min_index,tn,l,r); \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_min_index,tn); + +/** Defines the min_index reducer functions for a numeric type. + * + * This macro expands into function definitions for functions which implement the + * reducer functionality for the min_index reducer type for a specified numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer type name, + * function names, etc. + */ +#define CILK_C_REDUCER_MIN_INDEX_DEFINITION(t,tn,id) \ + CILK_C_REDUCER_MIN_INDEX_VIEW(t,tn); \ + typedef CILK_C_DECLARE_REDUCER( \ + __CILKRTS_MKIDENT(cilk_c_reducer_min_index_view_,tn)) \ + CILK_C_REDUCER_MIN_INDEX_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_min_index,tn,l,r) \ + { typedef __CILKRTS_MKIDENT(cilk_c_reducer_min_index_view_,tn) view_t; \ + if (((view_t*)l)->value > ((view_t*)r)->value) \ + *(view_t*)l = *(view_t*)r; } \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_min_index,tn) \ + { typedef __CILKRTS_MKIDENT(cilk_c_reducer_min_index_view_,tn) view_t; \ + ((view_t*)v)->index = 0; ((view_t*)v)->value = id; } + +//@{ +/** @def CILK_C_REDUCER_MIN_INDEX_INSTANCE + * @brief Declares or defines implementation functions for a reducer type. + * + * In the runtime source file c_reducers.c, the macro `CILK_C_DEFINE_REDUCERS` will be defined, and + * this macro will generate reducer implementation functions. Everywhere else, `CILK_C_DEFINE_REDUCERS` + * will be undefined, and this macro will expand into external declarations for the functions. + */ +#ifdef CILK_C_DEFINE_REDUCERS +# define CILK_C_REDUCER_MIN_INDEX_INSTANCE(t,tn,id) \ + CILK_C_REDUCER_MIN_INDEX_DEFINITION(t,tn,id) +#else +# define CILK_C_REDUCER_MIN_INDEX_INSTANCE(t,tn,id) \ + CILK_C_REDUCER_MIN_INDEX_DECLARATION(t,tn,id) +#endif +//@} + +/* Declares or defines an instance of the reducer type and its functions for each + * numeric type. + */ +__CILKRTS_BEGIN_EXTERN_C +CILK_C_REDUCER_MIN_INDEX_INSTANCE(char, char, CHAR_MAX) +CILK_C_REDUCER_MIN_INDEX_INSTANCE(unsigned char, uchar, CHAR_MAX) +CILK_C_REDUCER_MIN_INDEX_INSTANCE(signed char, schar, SCHAR_MAX) +CILK_C_REDUCER_MIN_INDEX_INSTANCE(wchar_t, wchar_t, WCHAR_MAX) +CILK_C_REDUCER_MIN_INDEX_INSTANCE(short, short, SHRT_MAX) +CILK_C_REDUCER_MIN_INDEX_INSTANCE(unsigned short, ushort, USHRT_MAX) +CILK_C_REDUCER_MIN_INDEX_INSTANCE(int, int, INT_MAX) +CILK_C_REDUCER_MIN_INDEX_INSTANCE(unsigned int, uint, UINT_MAX) +CILK_C_REDUCER_MIN_INDEX_INSTANCE(unsigned int, unsigned, UINT_MAX) // alternate name +CILK_C_REDUCER_MIN_INDEX_INSTANCE(long, long, LONG_MAX) +CILK_C_REDUCER_MIN_INDEX_INSTANCE(unsigned long, ulong, ULONG_MAX) +CILK_C_REDUCER_MIN_INDEX_INSTANCE(long long, longlong, LLONG_MAX) +CILK_C_REDUCER_MIN_INDEX_INSTANCE(unsigned long long, ulonglong, ULLONG_MAX) +CILK_C_REDUCER_MIN_INDEX_INSTANCE(float, float, HUGE_VALF) +CILK_C_REDUCER_MIN_INDEX_INSTANCE(double, double, HUGE_VAL) +CILK_C_REDUCER_MIN_INDEX_INSTANCE(long double, longdouble, HUGE_VALL) +__CILKRTS_END_EXTERN_C + +/// @endcond + +//@} + +#endif // defined REDUCER_MIN_MAX_H_INCLUDED diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_opadd.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_opadd.h new file mode 100644 index 0000000000..46d4b6e9b5 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_opadd.h @@ -0,0 +1,701 @@ +/* reducer_opadd.h -*- C++ -*- + * + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + */ + +/** @file reducer_opadd.h + * + * @brief Defines classes for doing parallel addition reductions. + * + * @ingroup ReducersAdd + * + * @see ReducersAdd + */ + +#ifndef REDUCER_OPADD_H_INCLUDED +#define REDUCER_OPADD_H_INCLUDED + +#include + +/** @defgroup ReducersAdd Addition Reducers + * + * Addition reducers allow the computation of the sum of a set of values in + * parallel. + * + * @ingroup Reducers + * + * You should be familiar with @ref pagereducers "Intel(R) Cilk(TM) Plus reducers", + * described in file `reducers.md`, and particularly with @ref reducers_using, + * before trying to use the information in this file. + * + * @section redopadd_usage Usage Example + * + * cilk::reducer< cilk::op_add > r; + * cilk_for (int i = 0; i != N; ++i) { + * *r += a[i]; + * } + * return r.get_value(); + * + * @section redopadd_monoid The Monoid + * + * @subsection redopadd_monoid_values Value Set + * + * The value set of an addition reducer is the set of values of `Type`, which + * is expected to be a builtin numeric type (or something like it, such as + * `std::complex`). + * + * @subsection redopadd_monoid_operator Operator + * + * The operator of an addition reducer is the addition operator, defined by + * the "`+`" binary operator on `Type`. + * + * @subsection redopadd_monoid_identity Identity + * + * The identity value of the reducer is the numeric value "`0`". This is + * expected to be the value of the default constructor `Type()`. + * + * @section redopadd_operations Operations + * + * @subsection redopadd_constructors Constructors + * + * reducer() // identity + * reducer(const Type& value) + * reducer(move_in(Type& variable)) + * + * @subsection redopadd_get_set Set and Get + * + * r.set_value(const Type& value) + * const Type& = r.get_value() const + * r.move_in(Type& variable) + * r.move_out(Type& variable) + * + * @subsection redopadd_initial Initial Values + * + * If an addition reducer is constructed without an explicit initial value, + * then its initial value will be its identity value, as long as `Type` + * satisfies the requirements of @ref redopadd_types. + * + * @subsection redopadd_view_ops View Operations + * + * *r += a + * *r -= a + * ++*r + * --*r + * (*r)++ + * (*r)-- + * *r = *r + a + * *r = *r - a + * *r = *r ± a1 ± a2 … ± an + * + * The post-increment and post-decrement operations do not return a value. (If + * they did, they would expose the value contained in the view, which is + * non-deterministic in the middle of a reduction.) + * + * Note that subtraction operations are allowed on an addition reducer because + * subtraction is equivalent to addition with a negated operand. It is true + * that `(x - y) - z` is not equivalent to `x - (y - z)`, but + * `(x + (-y)) + (-z)` _is_ equivalent to `x + ((-y) + (-z))`. + * + * @section redopadd_floating_point Issues with Floating-Point Types + * + * Because of precision and round-off issues, floating-point addition is not + * really associative. For example, `(1e30 + -1e30) + 1 == 1`, but + * `1e30 + (-1e30 + 1) == 0`. + * + * In many cases, this won't matter, but computations which have been + * carefully ordered to control round-off errors may not deal well with + * being reassociated. In general, you should be sure to understand the + * floating-point behavior of your program before doing any transformation + * that will reassociate its computations. + * + * @section redopadd_types Type and Operator Requirements + * + * `Type` must be `Copy Constructible`, `Default Constructible`, and + * `Assignable`. + * + * The operator "`+=`" must be defined on `Type`, with `x += a` having the + * same meaning as `x = x + a`. In addition, if the code uses the "`-=`", + * pre-increment, post-increment, pre-decrement, or post-decrement operators, + * then the corresponding operators must be defined on `Type`. + * + * The expression `Type()` must be a valid expression which yields the + * identity value (the value of `Type` whose numeric value is zero). + * + * @section redopadd_in_c Addition Reducers in C + * + * The @ref CILK_C_REDUCER_OPADD and @ref CILK_C_REDUCER_OPADD_TYPE macros can + * be used to do addition reductions in C. For example: + * + * CILK_C_REDUCER_OPADD(r, double, 0); + * CILK_C_REGISTER_REDUCER(r); + * cilk_for(int i = 0; i != n; ++i) { + * REDUCER_VIEW(r) += a[i]; + * } + * CILK_C_UNREGISTER_REDUCER(r); + * printf("The sum of the elements of a is %f\n", REDUCER_VIEW(r)); + * + * See @ref reducers_c_predefined. + */ + +#ifdef __cplusplus + +namespace cilk { + +/** The addition reducer view class. + * + * This is the view class for reducers created with + * `cilk::reducer< cilk::op_add >`. It holds the accumulator variable + * for the reduction, and allows only addition and subtraction operations to + * be performed on it. + * + * @note The reducer "dereference" operation (`reducer::operator *()`) + * yields a reference to the view. Thus, for example, the view class's + * `+=` operation would be used in an expression like `*r += a`, where + * `r` is an op_add reducer variable. + * + * @tparam Type The type of the contained accumulator variable. This will + * be the value type of a monoid_with_view that is + * instantiated with this view. + * + * @see ReducersAdd + * @see op_add + * + * @ingroup ReducersAdd + */ +template +class op_add_view : public scalar_view +{ + typedef scalar_view base; + +public: + /** Class to represent the right-hand side of + * `*reducer = *reducer ± value`. + * + * The only assignment operator for the op_add_view class takes an + * rhs_proxy as its operand. This results in the syntactic restriction + * that the only expressions that can be assigned to an op_add_view are + * ones which generate an rhs_proxy - that is, expressions of the form + * `op_add_view ± value ... ± value`. + * + * @warning + * The lhs and rhs views in such an assignment must be the same; + * otherwise, the behavior will be undefined. (I.e., `v1 = v1 + x` is + * legal; `v1 = v2 + x` is illegal.) This condition will be checked with a + * runtime assertion when compiled in debug mode. + * + * @see op_add_view + */ + class rhs_proxy { + friend class op_add_view; + + const op_add_view* m_view; + Type m_value; + + // Constructor is invoked only from op_add_view::operator+() and + // op_add_view::operator-(). + // + rhs_proxy(const op_add_view* view, const Type& value) : + m_view(view), m_value(value) {} + + rhs_proxy& operator=(const rhs_proxy&); // Disable assignment operator + rhs_proxy(); // Disable default constructor + + public: + ///@{ + /** Adds or subtracts an additional rhs value. If `v` is an op_add_view + * and `a1` is a value, then the expression `v + a1` invokes the view's + * `operator+()` to create an rhs_proxy for `(v, a1)`; then + * `v + a1 + a2` invokes the rhs_proxy's `operator+()` to create a new + * rhs_proxy for `(v, a1+a2)`. This allows the right-hand side of an + * assignment to be not just `view ± value`, but + * `view ± value ± value ... ± value`. The effect is that + * + * v = v ± a1 ± a2 ... ± an; + * + * is evaluated as + * + * v = v ± (±a1 ± a2 ... ± an); + */ + rhs_proxy& operator+(const Type& x) { m_value += x; return *this; } + rhs_proxy& operator-(const Type& x) { m_value -= x; return *this; } + ///@} + }; + + + /** Default/identity constructor. This constructor initializes the + * contained value to `Type()`, which is expected to be the identity value + * for addition on `Type`. + */ + op_add_view() : base() {} + + /** Construct with a specified initial value. + */ + explicit op_add_view(const Type& v) : base(v) {} + + /** Reduces the views of two strands. + * + * This function is invoked by the @ref op_add monoid to combine the views + * of two strands when the right strand merges with the left one. It adds + * the value contained in the right-strand view to the value contained in + * the left-strand view, and leaves the value in the right-strand view + * undefined. + * + * @param right A pointer to the right-strand view. (`this` points to + * the left-strand view.) + * + * @note Used only by the @ref op_add monoid to implement the monoid + * reduce operation. + */ + void reduce(op_add_view* right) { this->m_value += right->m_value; } + + /** @name Accumulator variable updates. + * + * These functions support the various syntaxes for incrementing or + * decrementing the accumulator variable contained in the view. + */ + ///@{ + + /** Increments the accumulator variable by @a x. + */ + op_add_view& operator+=(const Type& x) { this->m_value += x; return *this; } + + /** Decrements the accumulator variable by @a x. + */ + op_add_view& operator-=(const Type& x) { this->m_value -= x; return *this; } + + /** Pre-increment. + */ + op_add_view& operator++() { ++this->m_value; return *this; } + + /** Post-increments. + * + * @note Conventionally, post-increment operators return the old value + * of the incremented variable. However, reducer views do not + * expose their contained values, so `view++` does not have a + * return value. + */ + void operator++(int) { this->m_value++; } + + /** Pre-decrements. + */ + op_add_view& operator--() { --this->m_value; return *this; } + + /** Post-decrements. + * + * @note Conventionally, post-decrement operators return the old value + * of the decremented variable. However, reducer views do not + * expose their contained values, so `view--` does not have a + * return value. + */ + void operator--(int) { this->m_value--; } + + /** Creates an object representing `*this + x`. + * + * @see rhs_proxy + */ + rhs_proxy operator+(const Type& x) const { return rhs_proxy(this, x); } + + /** Creates an object representing `*this - x`. + * + * @see rhs_proxy + */ + rhs_proxy operator-(const Type& x) const { return rhs_proxy(this, -x); } + + /** Assigns the result of a `view ± value` expression to the view. Note that + * this is the only assignment operator for this class. + * + * @see rhs_proxy + */ + op_add_view& operator=(const rhs_proxy& rhs) { + __CILKRTS_ASSERT(this == rhs.m_view); + this->m_value += rhs.m_value; + return *this; + } + + ///@} +}; + + +/** Monoid class for addition reductions. Instantiate the cilk::reducer + * template class with an op_add monoid to create an addition reducer class. + * For example, to compute + * the sum of a set of `int` values: + * + * cilk::reducer< cilk::op_add > r; + * + * @tparam Type The reducer value type. + * @tparam Align If `false` (the default), reducers instantiated on this + * monoid will be naturally aligned (the Intel Cilk Plus library 1.0 + * behavior). If `true`, reducers instantiated on this monoid + * will be cache-aligned for binary compatibility with + * reducers in Intel Cilk Plus library version 0.9. + * + * @see ReducersAdd + * @see op_add_view + * + * @ingroup ReducersAdd + */ +template +struct op_add : public monoid_with_view, Align> {}; + +/** **Deprecated** addition reducer wrapper class. + * + * reducer_opadd is the same as @ref reducer<@ref op_add>, except that + * reducer_opadd is a proxy for the contained view, so that accumulator + * variable update operations can be applied directly to the reducer. For + * example, a value is added to a `reducer<%op_add>` with `*r += a`, but a + * value can be added to a `%reducer_opadd` with `r += a`. + * + * @deprecated Users are strongly encouraged to use `reducer` + * reducers rather than the old wrappers like reducer_opadd. + * The `reducer` reducers show the reducer/monoid/view + * architecture more clearly, are more consistent in their + * implementation, and present a simpler model for new + * user-implemented reducers. + * + * @note Implicit conversions are provided between `%reducer_opadd` + * and `reducer<%op_add>`. This allows incremental code + * conversion: old code that used `%reducer_opadd` can pass a + * `%reducer_opadd` to a converted function that now expects a + * pointer or reference to a `reducer<%op_add>`, and vice + * versa. + * + * @tparam Type The value type of the reducer. + * + * @see op_add + * @see reducer + * @see ReducersAdd + * + * @ingroup ReducersAdd + */ +template +class reducer_opadd : public reducer< op_add > +{ + typedef reducer< op_add > base; + using base::view; + + public: + /// The view type for the reducer. + typedef typename base::view_type view_type; + + /// The view's rhs proxy type. + typedef typename view_type::rhs_proxy rhs_proxy; + + /// The view type for the reducer. + typedef view_type View; + + /// The monoid type for the reducer. + typedef typename base::monoid_type Monoid; + + /** @name Constructors + */ + ///@{ + + /** Default (identity) constructor. + * + * Constructs the wrapper with the default initial value of `Type()`. + */ + reducer_opadd() {} + + /** Value constructor. + * + * Constructs the wrapper with a specified initial value. + */ + explicit reducer_opadd(const Type& initial_value) : base(initial_value) {} + + ///@} + + /** @name Forwarded functions + * @details Functions that update the contained accumulator variable are + * simply forwarded to the contained @ref op_add_view. */ + ///@{ + + /// @copydoc op_add_view::operator+=(const Type&) + reducer_opadd& operator+=(const Type& x) { view() += x; return *this; } + + /// @copydoc op_add_view::operator-=(const Type&) + reducer_opadd& operator-=(const Type& x) { view() -= x; return *this; } + + /// @copydoc op_add_view::operator++() + reducer_opadd& operator++() { ++view(); return *this; } + + /// @copydoc op_add_view::operator++(int) + void operator++(int) { view()++; } + + /// @copydoc op_add_view::operator-\-() + reducer_opadd& operator--() { --view(); return *this; } + + /// @copydoc op_add_view::operator-\-(int) + void operator--(int) { view()--; } + + // The legacy definitions of reducer_opadd::operator+() and + // reducer_opadd::operator-() have different behavior and a different + // return type than this definition. The legacy version is defined as a + // member function, so this new version is defined as a free function to + // give it a different signature, so that they won't end up sharing a + // single object file entry. + + /// @copydoc op_add_view::operator+(const Type&) const + friend rhs_proxy operator+(const reducer_opadd& r, const Type& x) + { + return r.view() + x; + } + /// @copydoc op_add_view::operator-(const Type&) const + friend rhs_proxy operator-(const reducer_opadd& r, const Type& x) + { + return r.view() - x; + } + /// @copydoc op_add_view::operator=(const rhs_proxy&) + reducer_opadd& operator=(const rhs_proxy& temp) + { + view() = temp; + return *this; + } + ///@} + + /** @name Dereference + * @details Dereferencing a wrapper is a no-op. It simply returns the + * wrapper. Combined with the rule that the wrapper forwards view + * operations to its contained view, this means that view operations can + * be written the same way on reducers and wrappers, which is convenient + * for incrementally converting old code using wrappers to use reducers + * instead. That is: + * + * reducer< op_add > r; + * *r += a; // *r returns the view + * // operator += is a view member function + * + * reducer_opadd w; + * *w += a; // *w returns the wrapper + * // operator += is a wrapper member function that + * // calls the corresponding view function + */ + ///@{ + reducer_opadd& operator*() { return *this; } + reducer_opadd const& operator*() const { return *this; } + + reducer_opadd* operator->() { return this; } + reducer_opadd const* operator->() const { return this; } + ///@} + + /** @name Upcast + * @details In Intel Cilk Plus library 0.9, reducers were always cache-aligned. + * In library 1.0, reducer cache alignment is optional. By default, + * reducers are unaligned (i.e., just naturally aligned), but legacy + * wrappers inherit from cache-aligned reducers for binary compatibility. + * + * This means that a wrapper will automatically be upcast to its aligned + * reducer base class. The following conversion operators provide + * pseudo-upcasts to the corresponding unaligned reducer class. + */ + ///@{ + operator reducer< op_add >& () + { + return *reinterpret_cast< reducer< op_add >* >(this); + } + operator const reducer< op_add >& () const + { + return *reinterpret_cast< const reducer< op_add >* >(this); + } + ///@} +}; + +/// @cond internal +/** Metafunction specialization for reducer conversion. + * + * This specialization of the @ref legacy_reducer_downcast template class + * defined in reducer.h causes the `reducer< op_add >` class to have an + * `operator reducer_opadd& ()` conversion operator that statically + * downcasts the `reducer` to the corresponding `reducer_opadd` type. + * (The reverse conversion, from `reducer_opadd` to `reducer`, is just + * an upcast, which is provided for free by the language.) + * + * @ingroup ReducersAdd + */ +template +struct legacy_reducer_downcast > > +{ + typedef reducer_opadd type; +}; +/// @endcond + +} // namespace cilk + +#endif // __cplusplus + + +/** @ingroup ReducersAdd + */ +///@{ + +/** @name C Language Reducer Macros + * + * These macros are used to declare and work with numeric op_add reducers in + * C code. + * + * @see @ref page_reducers_in_c + */ + ///@{ + +__CILKRTS_BEGIN_EXTERN_C + +/** Declares opadd reducer type name. + * + * This macro expands into the identifier which is the name of the op_add + * reducer type for a specified numeric type. + * + * @param tn The @ref reducers_c_type_names "numeric type name" specifying + * the type of the reducer. + * + * @see @ref reducers_c_predefined + * @see ReducersAdd + */ +#define CILK_C_REDUCER_OPADD_TYPE(tn) \ + __CILKRTS_MKIDENT(cilk_c_reducer_opadd_,tn) + +/** Declares an op_add reducer object. + * + * This macro expands into a declaration of an op_add reducer object for a + * specified numeric type. For example: + * + * CILK_C_REDUCER_OPADD(my_reducer, double, 0.0); + * + * @param obj The variable name to be used for the declared reducer object. + * @param tn The @ref reducers_c_type_names "numeric type name" specifying + * the type of the reducer. + * @param v The initial value for the reducer. (A value which can be + * assigned to the numeric type represented by @a tn.) + * + * @see @ref reducers_c_predefined + * @see ReducersAdd + */ +#define CILK_C_REDUCER_OPADD(obj,tn,v) \ + CILK_C_REDUCER_OPADD_TYPE(tn) obj = \ + CILK_C_INIT_REDUCER(_Typeof(obj.value), \ + __CILKRTS_MKIDENT(cilk_c_reducer_opadd_reduce_,tn), \ + __CILKRTS_MKIDENT(cilk_c_reducer_opadd_identity_,tn), \ + __cilkrts_hyperobject_noop_destroy, v) + +/// @cond internal + +/** Declares the op_add reducer functions for a numeric type. + * + * This macro expands into external function declarations for functions which + * implement the reducer functionality for the op_add reducer type for a + * specified numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer + * type name, function names, etc. + */ +#define CILK_C_REDUCER_OPADD_DECLARATION(t,tn) \ + typedef CILK_C_DECLARE_REDUCER(t) CILK_C_REDUCER_OPADD_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_opadd,tn,l,r); \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_opadd,tn); + +/** Defines the op_add reducer functions for a numeric type. + * + * This macro expands into function definitions for functions which implement + * the reducer functionality for the op_add reducer type for a specified + * numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer + * type name, function names, etc. + */ +#define CILK_C_REDUCER_OPADD_DEFINITION(t,tn) \ + typedef CILK_C_DECLARE_REDUCER(t) CILK_C_REDUCER_OPADD_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_opadd,tn,l,r) \ + { *(t*)l += *(t*)r; } \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_opadd,tn) \ + { *(t*)v = 0; } + +///@{ +/** @def CILK_C_REDUCER_OPADD_INSTANCE + * @brief Declares or defines implementation functions for a reducer type. + * + * In the runtime source file c_reducers.c, the macro `CILK_C_DEFINE_REDUCERS` + * will be defined, and this macro will generate reducer implementation + * functions. Everywhere else, `CILK_C_DEFINE_REDUCERS` will be undefined, + * and this macro will expand into external declarations for the functions. + */ +#ifdef CILK_C_DEFINE_REDUCERS +# define CILK_C_REDUCER_OPADD_INSTANCE(t,tn) \ + CILK_C_REDUCER_OPADD_DEFINITION(t,tn) +#else +# define CILK_C_REDUCER_OPADD_INSTANCE(t,tn) \ + CILK_C_REDUCER_OPADD_DECLARATION(t,tn) +#endif +///@} + +/* Declares or defines an instance of the reducer type and its functions for each + * numeric type. + */ +CILK_C_REDUCER_OPADD_INSTANCE(char, char) +CILK_C_REDUCER_OPADD_INSTANCE(unsigned char, uchar) +CILK_C_REDUCER_OPADD_INSTANCE(signed char, schar) +CILK_C_REDUCER_OPADD_INSTANCE(wchar_t, wchar_t) +CILK_C_REDUCER_OPADD_INSTANCE(short, short) +CILK_C_REDUCER_OPADD_INSTANCE(unsigned short, ushort) +CILK_C_REDUCER_OPADD_INSTANCE(int, int) +CILK_C_REDUCER_OPADD_INSTANCE(unsigned int, uint) +CILK_C_REDUCER_OPADD_INSTANCE(unsigned int, unsigned) /* alternate name */ +CILK_C_REDUCER_OPADD_INSTANCE(long, long) +CILK_C_REDUCER_OPADD_INSTANCE(unsigned long, ulong) +CILK_C_REDUCER_OPADD_INSTANCE(long long, longlong) +CILK_C_REDUCER_OPADD_INSTANCE(unsigned long long, ulonglong) +CILK_C_REDUCER_OPADD_INSTANCE(float, float) +CILK_C_REDUCER_OPADD_INSTANCE(double, double) +CILK_C_REDUCER_OPADD_INSTANCE(long double, longdouble) + +//@endcond + +__CILKRTS_END_EXTERN_C + +///@} + +///@} + +#endif /* REDUCER_OPADD_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_opand.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_opand.h new file mode 100644 index 0000000000..44d537d4f3 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_opand.h @@ -0,0 +1,614 @@ +/* reducer_opand.h -*- C++ -*- + * + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + */ + +/** @file reducer_opand.h + * + * @brief Defines classes for doing parallel bitwise AND reductions. + * + * @ingroup ReducersAnd + * + * @see ReducersAnd + */ + +#ifndef REDUCER_OPAND_H_INCLUDED +#define REDUCER_OPAND_H_INCLUDED + +#include + +/** @defgroup ReducersAnd Bitwise AND Reducers + * + * Bitwise AND reducers allow the computation of the bitwise AND of a set of + * values in parallel. + * + * @ingroup Reducers + * + * You should be familiar with @ref pagereducers "Intel(R) Cilk(TM) Plus reducers", + * described in file `reducers.md`, and particularly with @ref reducers_using, + * before trying to use the information in this file. + * + * @section redopand_usage Usage Example + * + * cilk::reducer< cilk::op_and > r; + * cilk_for (int i = 0; i != N; ++i) { + * *r &= a[i]; + * } + * unsigned result; + * r.move_out(result); + * + * @section redopand_monoid The Monoid + * + * @subsection redopand_monoid_values Value Set + * + * The value set of a bitwise AND reducer is the set of values of `Type`, + * which is expected to be a builtin integer type which has a representation + * as a sequence of bits (or something like it, such as `bool` or + * `std::bitset`). + * + * @subsection redopand_monoid_operator Operator + * + * The bitwise AND operator is defined by the "`&`" binary operator on `Type`. + * + * @subsection redopand_monoid_identity Identity + * + * The identity value of the reducer is the value whose representation + * contains all 1-bits. This is expected to be the value of the expression + * `~Type()` (i.e., the bitwise negation operator applied to the default value + * of the value type). + * + * @section redopand_operations Operations + * + * @subsection redopand_constructors Constructors + * + * reducer() // identity + * reducer(const Type& value) + * reducer(move_in(Type& variable)) + * + * @subsection redopand_get_set Set and Get + * + * r.set_value(const Type& value) + * const Type& = r.get_value() const + * r.move_in(Type& variable) + * r.move_out(Type& variable) + * + * @subsection redopand_initial Initial Values + * + * If a bitwise AND reducer is constructed without an explicit initial value, + * then its initial value will be its identity value, as long as `Type` + * satisfies the requirements of @ref redopand_types. + * + * @subsection redopand_view_ops View Operations + * + * *r &= a + * *r = *r & a + * *r = *r & a1 & a2 … & an + * + * @section redopand_types Type and Operator Requirements + * + * `Type` must be `Copy Constructible`, `Default Constructible`, and + * `Assignable`. + * + * The operator "`&=`" must be defined on `Type`, with `x &= a` having the + * same meaning as `x = x & a`. + * + * The expression `~ Type()` must be a valid expression which yields the + * identity value (the value of `Type` whose representation consists of all + * 1-bits). + * + * @section redopand_in_c Bitwise AND Reducers in C + * + * The @ref CILK_C_REDUCER_OPAND and @ref CILK_C_REDUCER_OPAND_TYPE macros can + * be used to do bitwise AND reductions in C. For example: + * + * CILK_C_REDUCER_OPAND(r, uint, ~0); + * CILK_C_REGISTER_REDUCER(r); + * cilk_for(int i = 0; i != n; ++i) { + * REDUCER_VIEW(r) &= a[i]; + * } + * CILK_C_UNREGISTER_REDUCER(r); + * printf("The bitwise AND of the elements of a is %x\n", REDUCER_VIEW(r)); + * + * See @ref reducers_c_predefined. + */ + +#ifdef __cplusplus + +namespace cilk { + +/** The bitwise AND reducer view class. + * + * This is the view class for reducers created with + * `cilk::reducer< cilk::op_and >`. It holds the accumulator variable + * for the reduction, and allows only AND operations to be performed on it. + * + * @note The reducer "dereference" operation (`reducer::operator *()`) + * yields a reference to the view. Thus, for example, the view class's + * `&=` operation would be used in an expression like `*r &= a`, where + * `r` is an opmod reducer variable. + * + * @tparam Type The type of the contained accumulator variable. This will + * be the value type of a monoid_with_view that is + * instantiated with this view. + * + * @see ReducersAnd + * @see op_and + * + * @ingroup ReducersAnd + */ +template +class op_and_view : public scalar_view +{ + typedef scalar_view base; + +public: + /** Class to represent the right-hand side of `*reducer = *reducer & value`. + * + * The only assignment operator for the op_and_view class takes an + * rhs_proxy as its operand. This results in the syntactic restriction + * that the only expressions that can be assigned to an op_and_view are + * ones which generate an rhs_proxy - that is, expressions of the form + * `op_and_view & value ... & value`. + * + * @warning + * The lhs and rhs views in such an assignment must be the same; + * otherwise, the behavior will be undefined. (I.e., `v1 = v1 & x` is + * legal; `v1 = v2 & x` is illegal.) This condition will be checked with + * a runtime assertion when compiled in debug mode. + * + * @see op_and_view + */ + class rhs_proxy { + private: + friend class op_and_view; + + const op_and_view* m_view; + Type m_value; + + // Constructor is invoked only from op_and_view::operator&(). + // + rhs_proxy(const op_and_view* view, const Type& value) : m_view(view), m_value(value) {} + + rhs_proxy& operator=(const rhs_proxy&); // Disable assignment operator + rhs_proxy(); // Disable default constructor + + public: + /** Bitwise AND with an additional `rhs` value. If `v` is an op_and_view + * and `a1` is a value, then the expression `v & a1` invokes the + * view's `operator&()` to create an rhs_proxy for `(v, a1)`; then + * `v & a1 & a2` invokes the rhs_proxy's `operator&()` to create a new + * rhs_proxy for `(v, a1&a2)`. This allows the right-hand side of an + * assignment to be not just `view & value`, but + * `view & value & value ... & value`. The effect is that + * + * v = v & a1 & a2 ... & an; + * + * is evaluated as + * + * v = v & (a1 & a2 ... & an); + */ + rhs_proxy& operator&(const Type& x) { m_value &= x; return *this; } + }; + + + /** Default/identity constructor. This constructor initializes the + * contained value to `~ Type()`. + */ + op_and_view() : base(~Type()) {} + + /** Construct with a specified initial value. + */ + explicit op_and_view(const Type& v) : base(v) {} + + + /** Reduces the views of two strands. + * + * This function is invoked by the @ref op_and monoid to combine the views + * of two strands when the right strand merges with the left one. It + * "ANDs" the value contained in the left-strand view with the value + * contained in the right-strand view, and leaves the value in the + * right-strand view undefined. + * + * @param right A pointer to the right-strand view. (`this` points to + * the left-strand view.) + * + * @note Used only by the @ref op_and monoid to implement the monoid + * reduce operation. + */ + void reduce(op_and_view* right) { this->m_value &= right->m_value; } + + /** @name Accumulator variable updates. + * + * These functions support the various syntaxes for "ANDing" the + * accumulator variable contained in the view with some value. + */ + ///@{ + + /** Performs AND between the accumulator variable and @a x. + */ + op_and_view& operator&=(const Type& x) { this->m_value &= x; return *this; } + + /** Creates an object representing `*this & x`. + * + * @see rhs_proxy + */ + rhs_proxy operator&(const Type& x) const { return rhs_proxy(this, x); } + + /** Assigns the result of a `view & value` expression to the view. Note that + * this is the only assignment operator for this class. + * + * @see rhs_proxy + */ + op_and_view& operator=(const rhs_proxy& rhs) { + __CILKRTS_ASSERT(this == rhs.m_view); + this->m_value &= rhs.m_value; + return *this; + } + + ///@} +}; + +/** Monoid class for bitwise AND reductions. Instantiate the cilk::reducer + * template class with an op_and monoid to create a bitwise AND reducer + * class. For example, to compute the bitwise AND of a set of `unsigned long` + * values: + * + * cilk::reducer< cilk::op_and > r; + * + * @tparam Type The reducer value type. + * @tparam Align If `false` (the default), reducers instantiated on this + * monoid will be naturally aligned (the Intel Cilk Plus library 1.0 + * behavior). If `true`, reducers instantiated on this monoid + * will be cache-aligned for binary compatibility with + * reducers in Intel Cilk Plus library version 0.9. + * + * @see ReducersAnd + * @see op_and_view + * + * @ingroup ReducersAnd + */ +template +struct op_and : public monoid_with_view, Align> {}; + +/** Deprecated bitwise AND reducer class. + * + * reducer_opand is the same as @ref reducer<@ref op_and>, except that + * reducer_opand is a proxy for the contained view, so that accumulator + * variable update operations can be applied directly to the reducer. For + * example, a value is "ANDed" with a `reducer<%op_and>` with `*r &= a`, but a + * value can be "ANDed" with a `%reducer_opand` with `r &= a`. + * + * @deprecated Users are strongly encouraged to use `reducer` + * reducers rather than the old wrappers like reducer_opand. + * The `reducer` reducers show the reducer/monoid/view + * architecture more clearly, are more consistent in their + * implementation, and present a simpler model for new + * user-implemented reducers. + * + * @note Implicit conversions are provided between `%reducer_opand` + * and `reducer<%op_and>`. This allows incremental code + * conversion: old code that used `%reducer_opand` can pass a + * `%reducer_opand` to a converted function that now expects a + * pointer or reference to a `reducer<%op_and>`, and vice + * versa. + * + * @tparam Type The value type of the reducer. + * + * @see op_and + * @see reducer + * @see ReducersAnd + * + * @ingroup ReducersAnd + */ +template +class reducer_opand : public reducer< op_and > +{ + typedef reducer< op_and > base; + using base::view; + +public: + /// The view type for the reducer. + typedef typename base::view_type view_type; + + /// The view's rhs proxy type. + typedef typename view_type::rhs_proxy rhs_proxy; + + /// The view type for the reducer. + typedef view_type View; + + /// The monoid type for the reducer. + typedef typename base::monoid_type Monoid; + + /** @name Constructors + */ + ///@{ + + /** Default constructor. + * + * Constructs the wrapper with the default initial value of `Type()` + * (not the identity value). + */ + reducer_opand() : base(Type()) {} + + /** Value constructor. + * + * Constructs the wrapper with a specified initial value. + */ + explicit reducer_opand(const Type& initial_value) : base(initial_value) {} + + ///@} + + /** @name Forwarded functions + * @details Functions that update the contained accumulator variable are + * simply forwarded to the contained @ref op_and_view. */ + ///@{ + + /// @copydoc op_and_view::operator&=(const Type&) + reducer_opand& operator&=(const Type& x) + { + view() &= x; + return *this; + } + + // The legacy definition of reducer_opand::operator&() has different + // behavior and a different return type than this definition. The legacy + // version is defined as a member function, so this new version is defined + // as a free function to give it a different signature, so that they won't + // end up sharing a single object file entry. + + /// @copydoc op_and_view::operator&(const Type&) const + friend rhs_proxy operator&(const reducer_opand& r, const Type& x) + { + return r.view() & x; + } + + /// @copydoc op_and_view::operator=(const rhs_proxy&) + reducer_opand& operator=(const rhs_proxy& temp) + { + view() = temp; + return *this; + } + ///@} + + /** @name Dereference + * @details Dereferencing a wrapper is a no-op. It simply returns the + * wrapper. Combined with the rule that the wrapper forwards view + * operations to its contained view, this means that view operations can + * be written the same way on reducers and wrappers, which is convenient + * for incrementally converting old code using wrappers to use reducers + * instead. That is: + * + * reducer< op_and > r; + * *r &= a; // *r returns the view + * // operator &= is a view member function + * + * reducer_opand w; + * *w &= a; // *w returns the wrapper + * // operator &= is a wrapper member function that + * // calls the corresponding view function + */ + ///@{ + reducer_opand& operator*() { return *this; } + reducer_opand const& operator*() const { return *this; } + + reducer_opand* operator->() { return this; } + reducer_opand const* operator->() const { return this; } + ///@} + + /** @name Upcast + * @details In Intel Cilk Plus library 0.9, reducers were always cache-aligned. + * In library 1.0, reducer cache alignment is optional. By default, + * reducers are unaligned (i.e., just naturally aligned), but legacy + * wrappers inherit from cache-aligned reducers for binary compatibility. + * + * This means that a wrapper will automatically be upcast to its aligned + * reducer base class. The following conversion operators provide + * pseudo-upcasts to the corresponding unaligned reducer class. + */ + ///@{ + operator reducer< op_and >& () + { + return *reinterpret_cast< reducer< op_and >* >(this); + } + operator const reducer< op_and >& () const + { + return *reinterpret_cast< const reducer< op_and >* >(this); + } + ///@} +}; + +/// @cond internal +/** Metafunction specialization for reducer conversion. + * + * This specialization of the @ref legacy_reducer_downcast template class + * defined in reducer.h causes the `reducer< op_and >` class to have an + * `operator reducer_opand& ()` conversion operator that statically + * downcasts the `reducer` to the corresponding `reducer_opand` type. + * (The reverse conversion, from `reducer_opand` to `reducer`, is just + * an upcast, which is provided for free by the language.) + * + * @ingroup ReducersAnd + */ +template +struct legacy_reducer_downcast > > +{ + typedef reducer_opand type; +}; +/// @endcond + +} // namespace cilk + +#endif // __cplusplus + + +/** @ingroup ReducersAdd + */ +///@{ + +/** @name C language reducer macros + * + * These macros are used to declare and work with op_and reducers in C code. + * + * @see @ref page_reducers_in_c + */ + ///@{ + +__CILKRTS_BEGIN_EXTERN_C + +/** Declares `opand` reducer type name. + * + * This macro expands into the identifier which is the name of the op_and + * reducer type for a specified numeric type. + * + * @param tn The @ref reducers_c_type_names "numeric type name" specifying + * the type of the reducer. + * + * @see @ref reducers_c_predefined + * @see ReducersAnd + */ +#define CILK_C_REDUCER_OPAND_TYPE(tn) \ + __CILKRTS_MKIDENT(cilk_c_reducer_opand_,tn) + +/** Declares an op_and reducer object. + * + * This macro expands into a declaration of an op_and reducer object for a + * specified numeric type. For example: + * + * CILK_C_REDUCER_OPAND(my_reducer, ulong, ~0UL); + * + * @param obj The variable name to be used for the declared reducer object. + * @param tn The @ref reducers_c_type_names "numeric type name" specifying + * the type of the reducer. + * @param v The initial value for the reducer. (A value which can be + * assigned to the numeric type represented by @a tn.) + * + * @see @ref reducers_c_predefined + * @see ReducersAnd + */ +#define CILK_C_REDUCER_OPAND(obj,tn,v) \ + CILK_C_REDUCER_OPAND_TYPE(tn) obj = \ + CILK_C_INIT_REDUCER(_Typeof(obj.value), \ + __CILKRTS_MKIDENT(cilk_c_reducer_opand_reduce_,tn), \ + __CILKRTS_MKIDENT(cilk_c_reducer_opand_identity_,tn), \ + __cilkrts_hyperobject_noop_destroy, v) + +/// @cond internal + +/** Declares the op_and reducer functions for a numeric type. + * + * This macro expands into external function declarations for functions which + * implement the reducer functionality for the op_and reducer type for a + * specified numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer + * type name, function names, etc. + */ +#define CILK_C_REDUCER_OPAND_DECLARATION(t,tn) \ + typedef CILK_C_DECLARE_REDUCER(t) CILK_C_REDUCER_OPAND_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_opand,tn,l,r); \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_opand,tn); + +/** Defines the op_and reducer functions for a numeric type. + * + * This macro expands into function definitions for functions which implement + * the reducer functionality for the op_and reducer type for a specified + * numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer + * type name, function names, etc. + */ +#define CILK_C_REDUCER_OPAND_DEFINITION(t,tn) \ + typedef CILK_C_DECLARE_REDUCER(t) CILK_C_REDUCER_OPAND_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_opand,tn,l,r) \ + { *(t*)l &= *(t*)r; } \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_opand,tn) \ + { *(t*)v = ~((t)0); } + +///@{ +/** @def CILK_C_REDUCER_OPAND_INSTANCE + * @brief Declares or defines implementation functions for a reducer type. + * + * In the runtime source file c_reducers.c, the macro `CILK_C_DEFINE_REDUCERS` + * will be defined, and this macro will generate reducer implementation + * functions. Everywhere else, `CILK_C_DEFINE_REDUCERS` will be undefined, and + * this macro will expand into external declarations for the functions. + */ +#ifdef CILK_C_DEFINE_REDUCERS +# define CILK_C_REDUCER_OPAND_INSTANCE(t,tn) \ + CILK_C_REDUCER_OPAND_DEFINITION(t,tn) +#else +# define CILK_C_REDUCER_OPAND_INSTANCE(t,tn) \ + CILK_C_REDUCER_OPAND_DECLARATION(t,tn) +#endif +///@} + +/* Declares or defines an instance of the reducer type and its functions for + * each numeric type. + */ +CILK_C_REDUCER_OPAND_INSTANCE(char, char) +CILK_C_REDUCER_OPAND_INSTANCE(unsigned char, uchar) +CILK_C_REDUCER_OPAND_INSTANCE(signed char, schar) +CILK_C_REDUCER_OPAND_INSTANCE(wchar_t, wchar_t) +CILK_C_REDUCER_OPAND_INSTANCE(short, short) +CILK_C_REDUCER_OPAND_INSTANCE(unsigned short, ushort) +CILK_C_REDUCER_OPAND_INSTANCE(int, int) +CILK_C_REDUCER_OPAND_INSTANCE(unsigned int, uint) +CILK_C_REDUCER_OPAND_INSTANCE(unsigned int, unsigned) /* alternate name */ +CILK_C_REDUCER_OPAND_INSTANCE(long, long) +CILK_C_REDUCER_OPAND_INSTANCE(unsigned long, ulong) +CILK_C_REDUCER_OPAND_INSTANCE(long long, longlong) +CILK_C_REDUCER_OPAND_INSTANCE(unsigned long long, ulonglong) + +//@endcond + +__CILKRTS_END_EXTERN_C + +///@} + +///@} + +#endif /* REDUCER_OPAND_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_opmul.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_opmul.h new file mode 100644 index 0000000000..8a3e2d2a2a --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_opmul.h @@ -0,0 +1,453 @@ +/* reducer_opmul.h -*- C++ -*- + * + * Copyright (C) 2012-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + */ + +/** @file reducer_opmul.h + * + * @brief Defines classes for doing parallel multiplication reductions. + * + * @ingroup ReducersMul + * + * @see ReducersMul + */ + +#ifndef REDUCER_OPMUL_H_INCLUDED +#define REDUCER_OPMUL_H_INCLUDED + +#include + +/** @defgroup ReducersMul Multiplication Reducers + * + * Multiplication reducers allow the computation of the product of a set of + * values in parallel. + * + * @ingroup Reducers + * + * You should be familiar with @ref pagereducers "Intel(R) Cilk(TM) Plus reducers", + * described in file `reducers.md`, and particularly with @ref reducers_using, + * before trying to use the information in this file. + * + * @section redopmul_usage Usage Example + * + * cilk::reducer< cilk::op_mul > r; + * cilk_for (int i = 0; i != N; ++i) { + * *r *= a[i]; + * } + * double product; + * r.move_out(product); + * + * @section redopmul_monoid The Monoid + * + * @subsection redopmul_monoid_values Value Set + * + * The value set of a multiplication reducer is the set of values of `Type`, + * which is expected to be a builtin numeric type (or something like it, such + * as `std::complex`). + * + * @subsection redopmul_monoid_operator Operator + * + * The operator of a multiplication reducer is the multiplication operation, + * defined by the "`*`" binary operator on `Type`. + * + * @subsection redopmul_monoid_identity Identity + * + * The identity value of the reducer is the numeric value "`1`". This is + * expected to be the value of the expression `Type(1)`. + * + * @section redopmul_operations Operations + * + * @subsection redopmul_constructors Constructors + * + * reducer() // identity + * reducer(const Type& value) + * reducer(move_in(Type& variable)) + * + * @subsection redopmul_get_set Set and Get + * + * r.set_value(const Type& value) + * const Type& = r.get_value() const + * r.move_in(Type& variable) + * r.move_out(Type& variable) + * + * @subsection redopmul_initial Initial Values + * + * If a multiplication reducer is constructed without an explicit initial + * value, then its initial value will be its identity value, as long as `Type` + * satisfies the requirements of @ref redopmul_types. + * + * @subsection redopmul_view_ops View Operations + * + * *r *= a + * *r = *r * a + * *r = *r * a1 * a2 … * an + * + * @section redopmul_floating_point Issues with Floating-Point Types + * + * Because of overflow and underflow issues, floating-point multiplication is + * not really associative. For example, `(1e200 * 1e-200) * 1e-200 == 1e-200`, + * but `1e200 * (1e-200 * 1e-200 == 0. + * + * In many cases, this won't matter, but computations which have been + * carefully ordered to control overflow and underflow may not deal well with + * being reassociated. In general, you should be sure to understand the + * floating-point behavior of your program before doing any transformation + * that will reassociate its computations. + * + * @section redopmul_types Type and Operator Requirements + * + * `Type` must be `Copy Constructible`, `Default Constructible`, and + * `Assignable`. + * + * The operator "`*=`" must be defined on `Type`, with `x *= a` having the same + * meaning as `x = x * a`. + * + * The expression `Type(1)` must be a valid expression which yields the + * identity value (the value of `Type` whose numeric value is `1`). + * + * @section redopmul_in_c Multiplication Reducers in C + * + * The @ref CILK_C_REDUCER_OPMUL and @ref CILK_C_REDUCER_OPMUL_TYPE macros can + * be used to do multiplication reductions in C. For example: + * + * CILK_C_REDUCER_OPMUL(r, double, 1); + * CILK_C_REGISTER_REDUCER(r); + * cilk_for(int i = 0; i != n; ++i) { + * REDUCER_VIEW(r) *= a[i]; + * } + * CILK_C_UNREGISTER_REDUCER(r); + * printf("The product of the elements of a is %f\n", REDUCER_VIEW(r)); + * + * See @ref reducers_c_predefined. + */ + +#ifdef __cplusplus + +namespace cilk { + +/** The multiplication reducer view class. + * + * This is the view class for reducers created with + * `cilk::reducer< cilk::op_mul >`. It holds the accumulator variable + * for the reduction, and allows only multiplication operations to be + * performed on it. + * + * @note The reducer "dereference" operation (`reducer::operator *()`) + * yields a reference to the view. Thus, for example, the view class's + * `*=` operation would be used in an expression like `*r *= a`, where + * `r` is an op_mul reducer variable. + * + * @tparam Type The type of the contained accumulator variable. This will + * be the value type of a monoid_with_view that is + * instantiated with this view. + * + * @see ReducersMul + * @see op_mul + * + * @ingroup ReducersMul + */ +template +class op_mul_view : public scalar_view +{ + typedef scalar_view base; + +public: + /** Class to represent the right-hand side of `*reducer = *reducer * value`. + * + * The only assignment operator for the op_mul_view class takes an + * rhs_proxy as its operand. This results in the syntactic restriction + * that the only expressions that can be assigned to an op_mul_view are + * ones which generate an rhs_proxy - that is, expressions of the form + * `op_mul_view * value ... * value`. + * + * @warning + * The lhs and rhs views in such an assignment must be the same; + * otherwise, the behavior will be undefined. (I.e., `v1 = v1 * x` is + * legal; `v1 = v2 * x` is illegal.) This condition will be checked with a + * runtime assertion when compiled in debug mode. + * + * @see op_mul_view + */ + class rhs_proxy { + friend class op_mul_view; + + const op_mul_view* m_view; + Type m_value; + + // Constructor is invoked only from op_mul_view::operator*(). + // + rhs_proxy(const op_mul_view* view, const Type& value) : m_view(view), m_value(value) {} + + rhs_proxy& operator=(const rhs_proxy&); // Disable assignment operator + rhs_proxy(); // Disable default constructor + + public: + /** Multiplies by an additional `rhs` value. If `v` is an op_mul_view and + * `a1` is a value, then the expression `v * a1` invokes the view's + * `operator*()` to create an rhs_proxy for `(v, a1)`; then + * `v * a1 * a2` invokes the rhs_proxy's `operator*()` to create a + * new rhs_proxy for `(v, a1*a2)`. This allows the right-hand side of + * an assignment to be not just `view * value`, but + * `view * value * value ... * value`. The effect is that + * + * v = v * a1 * a2 ... * an; + * + * is evaluated as + * + * v = v * (a1 * a2 ... * an); + */ + rhs_proxy& operator*(const Type& x) { m_value *= x; return *this; } + }; + + + /** Default/identity constructor. This constructor initializes the + * contained value to `Type(1)`, which is expected to be the identity + * value for multiplication on `Type`. + */ + op_mul_view() : base(Type(1)) {} + + /** Construct with a specified initial value. + */ + explicit op_mul_view(const Type& v) : base(v) {} + + /** Reduces two strand views. + * + * This function is invoked by the @ref op_mul monoid to combine the views + * of two strands when the right strand merges with the left one. It + * multiplies the value contained in the left-strand view by the value + * contained in the right-strand view, and leaves the value in the + * right-strand view undefined. + * + * @param right A pointer to the right-strand view. (`this` points to + * the left-strand view.) + * + * @note Used only by the @ref op_mul monoid to implement the monoid + * reduce operation. + */ + void reduce(op_mul_view* right) { this->m_value *= right->m_value; } + + /** @name Accumulator variable updates. + * + * These functions support the various syntaxes for multiplying the + * accumulator variable contained in the view by some value. + */ + ///@{ + + /** Multiplies the accumulator variable by @a x. + */ + op_mul_view& operator*=(const Type& x) { this->m_value *= x; return *this; } + + /** Creates an object representing `*this * x`. + * + * @see rhs_proxy + */ + rhs_proxy operator*(const Type& x) const { return rhs_proxy(this, x); } + + /** Assigns the result of a `view * value` expression to the view. Note that + * this is the only assignment operator for this class. + * + * @see rhs_proxy + */ + op_mul_view& operator=(const rhs_proxy& rhs) { + __CILKRTS_ASSERT(this == rhs.m_view); + this->m_value *= rhs.m_value; + return *this; + } + + ///@} +}; + +/** Monoid class for multiplication reductions. Instantiate the cilk::reducer + * template class with an op_mul monoid to create a multiplication reducer + * class. For example, to compute the product of a set of `double` values: + * + * cilk::reducer< cilk::op_mul > r; + * + * @see ReducersMul + * @see op_mul_view + * + * @ingroup ReducersMul + */ +template +struct op_mul : public monoid_with_view< op_mul_view > {}; + +} // namespace cilk + +#endif // __cplusplus + + +/** @ingroup ReducersAdd + */ +///@{ + +/** @name C language reducer macros + * + * These macros are used to declare and work with numeric op_mul reducers in + * C code. + * + * @see @ref page_reducers_in_c + */ + ///@{ + +__CILKRTS_BEGIN_EXTERN_C + +/** Declares `opmul` reducer type name. + * + * This macro expands into the identifier which is the name of the op_mul + * reducer type for a specified numeric type. + * + * @param tn The @ref reducers_c_type_names "numeric type name" specifying + * the type of the reducer. + * + * @see @ref reducers_c_predefined + * @see ReducersMul + */ +#define CILK_C_REDUCER_OPMUL_TYPE(tn) \ + __CILKRTS_MKIDENT(cilk_c_reducer_opmul_,tn) + +/** Declares an op_mul reducer object. + * + * This macro expands into a declaration of an op_mul reducer object for a + * specified numeric type. For example: + * + * CILK_C_REDUCER_OPMUL(my_reducer, double, 1.0); + * + * @param obj The variable name to be used for the declared reducer object. + * @param tn The @ref reducers_c_type_names "numeric type name" specifying + * the type of the reducer. + * @param v The initial value for the reducer. (A value which can be + * assigned to the numeric type represented by @a tn.) + * + * @see @ref reducers_c_predefined + * @see ReducersMul + */ +#define CILK_C_REDUCER_OPMUL(obj,tn,v) \ + CILK_C_REDUCER_OPMUL_TYPE(tn) obj = \ + CILK_C_INIT_REDUCER(_Typeof(obj.value), \ + __CILKRTS_MKIDENT(cilk_c_reducer_opmul_reduce_,tn), \ + __CILKRTS_MKIDENT(cilk_c_reducer_opmul_identity_,tn), \ + __cilkrts_hyperobject_noop_destroy, v) + +/// @cond internal + +/** Declares the op_mul reducer functions for a numeric type. + * + * This macro expands into external function declarations for functions which + * implement the reducer functionality for the op_mul reducer type for a + * specified numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer + * type name, function names, etc. + */ +#define CILK_C_REDUCER_OPMUL_DECLARATION(t,tn) \ + typedef CILK_C_DECLARE_REDUCER(t) CILK_C_REDUCER_OPMUL_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_opmul,tn,l,r); \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_opmul,tn); + +/** Defines the op_mul reducer functions for a numeric type. + * + * This macro expands into function definitions for functions which implement + * the reducer functionality for the op_mul reducer type for a specified + * numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer + * type name, function names, etc. + */ +#define CILK_C_REDUCER_OPMUL_DEFINITION(t,tn) \ + typedef CILK_C_DECLARE_REDUCER(t) CILK_C_REDUCER_OPMUL_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_opmul,tn,l,r) \ + { *(t*)l *= *(t*)r; } \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_opmul,tn) \ + { *(t*)v = 1; } + +///@{ +/** @def CILK_C_REDUCER_OPMUL_INSTANCE + * @brief Declares or defines implementation functions for a reducer type. + * + * In the runtime source file c_reducers.c, the macro `CILK_C_DEFINE_REDUCERS` + * will be defined, and this macro will generate reducer implementation + * functions. Everywhere else, `CILK_C_DEFINE_REDUCERS` will be undefined, and + * this macro will expand into external declarations for the functions. + */ +#ifdef CILK_C_DEFINE_REDUCERS +# define CILK_C_REDUCER_OPMUL_INSTANCE(t,tn) \ + CILK_C_REDUCER_OPMUL_DEFINITION(t,tn) +#else +# define CILK_C_REDUCER_OPMUL_INSTANCE(t,tn) \ + CILK_C_REDUCER_OPMUL_DECLARATION(t,tn) +#endif +///@} + +/* Declares or defines an instance of the reducer type and its functions for each + * numeric type. + */ +CILK_C_REDUCER_OPMUL_INSTANCE(char, char) +CILK_C_REDUCER_OPMUL_INSTANCE(unsigned char, uchar) +CILK_C_REDUCER_OPMUL_INSTANCE(signed char, schar) +CILK_C_REDUCER_OPMUL_INSTANCE(wchar_t, wchar_t) +CILK_C_REDUCER_OPMUL_INSTANCE(short, short) +CILK_C_REDUCER_OPMUL_INSTANCE(unsigned short, ushort) +CILK_C_REDUCER_OPMUL_INSTANCE(int, int) +CILK_C_REDUCER_OPMUL_INSTANCE(unsigned int, uint) +CILK_C_REDUCER_OPMUL_INSTANCE(unsigned int, unsigned) /* alternate name */ +CILK_C_REDUCER_OPMUL_INSTANCE(long, long) +CILK_C_REDUCER_OPMUL_INSTANCE(unsigned long, ulong) +CILK_C_REDUCER_OPMUL_INSTANCE(long long, longlong) +CILK_C_REDUCER_OPMUL_INSTANCE(unsigned long long, ulonglong) +CILK_C_REDUCER_OPMUL_INSTANCE(float, float) +CILK_C_REDUCER_OPMUL_INSTANCE(double, double) +CILK_C_REDUCER_OPMUL_INSTANCE(long double, longdouble) + +//@endcond + +__CILKRTS_END_EXTERN_C + +///@} + +///@} + +#endif /* REDUCER_OPMUL_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_opor.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_opor.h new file mode 100644 index 0000000000..8d6d520248 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_opor.h @@ -0,0 +1,609 @@ +/* reducer_opor.h -*- C++ -*- + * + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + */ + +/** @file reducer_opor.h + * + * @brief Defines classes for doing parallel bitwise OR reductions. + * + * @ingroup ReducersOr + * + * @see ReducersOr + */ + +#ifndef REDUCER_OPOR_H_INCLUDED +#define REDUCER_OPOR_H_INCLUDED + +#include + +/** @defgroup ReducersOr Bitwise `OR` Reducers + * + * Bitwise `OR` reducers allow the computation of the bitwise `OR` of a set of + * values in parallel. + * + * @ingroup Reducers + * + * You should be familiar with @ref pagereducers "Intel(R) Cilk(TM) Plus reducers", + * described in file `reducers.md`, and particularly with @ref reducers_using, + * before trying to use the information in this file. + * + * @section redopor_usage Usage Example + * + * cilk::reducer< cilk::op_or > r; + * cilk_for (int i = 0; i != N; ++i) { + * *r |= a[i]; + * } + * unsigned result; + * r.move_out(result); + * + * @section redopor_monoid The Monoid + * + * @subsection redopor_monoid_values Value Set + * + * The value set of a bitwise `OR` reducer is the set of values of `Type`, which + * is expected to be a builtin integer type which has a representation as a + * sequence of bits (or something like it, such as `bool` or `std::bitset`). + * + * @subsection redopor_monoid_operator Operator + * + * The operator of a bitwise `OR` reducer is the bitwise OR operator, defined by + * the "`|`" binary operator on `Type`. + * + * @subsection redopor_monoid_identity Identity + * + * The identity value of the reducer is the value whose representation + * contains all 0-bits. This is expected to be the value of the default + * constructor `Type()`. + * + * @section redopor_operations Operations + * + * @subsection redopor_constructors Constructors + * + * reducer() // identity + * reducer(const Type& value) + * reducer(move_in(Type& variable)) + * + * @subsection redopor_get_set Set and Get + * + * r.set_value(const Type& value) + * const Type& = r.get_value() const + * r.move_in(Type& variable) + * r.move_out(Type& variable) + * + * @subsection redopor_initial Initial Values + * + * If a bitwise OR reducer is constructed without an explicit initial value, + * then its initial value will be its identity value, as long as `Type` + * satisfies the requirements of @ref redopor_types. + * + * @subsection redopor_view_ops View Operations + * + * *r |= a + * *r = *r | a + * *r = *r | a1 | a2 … | an + * + * @section redopor_types Type and Operator Requirements + * + * `Type` must be `Copy Constructible`, `Default Constructible`, and + * `Assignable`. + * + * The operator "`|=`" must be defined on `Type`, with `x |= a` having the + * same meaning as `x = x | a`. + * + * The expression `Type()` must be a valid expression which yields the + * identity value (the value of `Type` whose representation consists of all + * 0-bits). + * + * @section redopor_in_c Bitwise OR Reducers in C + * + * The @ref CILK_C_REDUCER_OPOR and @ref CILK_C_REDUCER_OPOR_TYPE macros can + * be used to do bitwise OR reductions in C. For example: + * + * CILK_C_REDUCER_OPOR(r, uint, 0); + * CILK_C_REGISTER_REDUCER(r); + * cilk_for(int i = 0; i != n; ++i) { + * REDUCER_VIEW(r) |= a[i]; + * } + * CILK_C_UNREGISTER_REDUCER(r); + * printf("The bitwise OR of the elements of a is %x\n", REDUCER_VIEW(r)); + * + * See @ref reducers_c_predefined. + */ + +#ifdef __cplusplus + +namespace cilk { + +/** The bitwise OR reducer view class. + * + * This is the view class for reducers created with + * `cilk::reducer< cilk::op_or >`. It holds the accumulator variable for + * the reduction, and allows only `or` operations to be performed on it. + * + * @note The reducer "dereference" operation (`reducer::operator *()`) + * yields a reference to the view. Thus, for example, the view class's + * `|=` operation would be used in an expression like `*r |= a`, where + * `r` is an opmod reducer variable. + * + * @tparam Type The type of the contained accumulator variable. This will + * be the value type of a monoid_with_view that is + * instantiated with this view. + * + * @see ReducersOr + * @see op_or + * + * @ingroup ReducersOr + */ +template +class op_or_view : public scalar_view +{ + typedef scalar_view base; + +public: + /** Class to represent the right-hand side of `*reducer = *reducer | value`. + * + * The only assignment operator for the op_or_view class takes an + * rhs_proxy as its operand. This results in the syntactic restriction + * that the only expressions that can be assigned to an op_or_view are + * ones which generate an rhs_proxy - that is, expressions of the form + * `op_or_view | value ... | value`. + * + * @warning + * The lhs and rhs views in such an assignment must be the same; + * otherwise, the behavior will be undefined. (I.e., `v1 = v1 | x` is + * legal; `v1 = v2 | x` is illegal.) This condition will be checked with + * a runtime assertion when compiled in debug mode. + * + * @see op_or_view + */ + class rhs_proxy { + friend class op_or_view; + + const op_or_view* m_view; + Type m_value; + + // Constructor is invoked only from op_or_view::operator|(). + // + rhs_proxy(const op_or_view* view, const Type& value) : m_view(view), m_value(value) {} + + rhs_proxy& operator=(const rhs_proxy&); // Disable assignment operator + rhs_proxy(); // Disable default constructor + + public: + /** bitwise OR with an additional rhs value. If `v` is an op_or_view + * and `a1` is a value, then the expression `v | a1` invokes the + * view's `operator|()` to create an rhs_proxy for `(v, a1)`; then + * `v | a1 | a2` invokes the rhs_proxy's `operator|()` to create a new + * rhs_proxy for `(v, a1|a2)`. This allows the right-hand side of an + * assignment to be not just `view | value`, but + ( `view | value | value ... | value`. The effect is that + * + * v = v | a1 | a2 ... | an; + * + * is evaluated as + * + * v = v | (a1 | a2 ... | an); + */ + rhs_proxy& operator|(const Type& x) { m_value |= x; return *this; } + }; + + + /** Default/identity constructor. This constructor initializes the + * contained value to `Type()`. + */ + op_or_view() : base() {} + + /** Construct with a specified initial value. + */ + explicit op_or_view(const Type& v) : base(v) {} + + /** Reduces the views of two strands. + * + * This function is invoked by the @ref op_or monoid to combine the views + * of two strands when the right strand merges with the left one. It + * "ORs" the value contained in the left-strand view by the value + * contained in the right-strand view, and leaves the value in the + * right-strand view undefined. + * + * @param right A pointer to the right-strand view. (`this` points to + * the left-strand view.) + * + * @note Used only by the @ref op_or monoid to implement the monoid + * reduce operation. + */ + void reduce(op_or_view* right) { this->m_value |= right->m_value; } + + /** @name Accumulator variable updates. + * + * These functions support the various syntaxes for "ORing" the + * accumulator variable contained in the view with some value. + */ + ///@{ + + /** Perfoms an OR operation between the accumulator variable and @a x. + */ + op_or_view& operator|=(const Type& x) { this->m_value |= x; return *this; } + + /** Creates an object representing `*this | x`. + * + * @see rhs_proxy + */ + rhs_proxy operator|(const Type& x) const { return rhs_proxy(this, x); } + + /** Assigns the result of a `view | value` expression to the view. Note that + * this is the only assignment operator for this class. + * + * @see rhs_proxy + */ + op_or_view& operator=(const rhs_proxy& rhs) { + __CILKRTS_ASSERT(this == rhs.m_view); + this->m_value |= rhs.m_value; + return *this; + } + + ///@} +}; + +/** Monoid class for bitwise OR reductions. Instantiate the cilk::reducer + * template class with an op_or monoid to create a bitwise OR reducer + * class. For example, to compute the bitwise OR of a set of `unsigned long` + * values: + * + * cilk::reducer< cilk::op_or > r; + * + * @tparam Type The reducer value type. + * @tparam Align If `false` (the default), reducers instantiated on this + * monoid will be naturally aligned (the Intel Cilk Plus library 1.0 + * behavior). If `true`, reducers instantiated on this monoid + * will be cache-aligned for binary compatibility with + * reducers in Intel Cilk Plus library version 0.9. + * + * @see ReducersOr + * @see op_or_view + * + * @ingroup ReducersOr + */ +template +struct op_or : public monoid_with_view, Align> {}; + +/** Deprecated bitwise OR reducer class. + * + * reducer_opor is the same as @ref reducer<@ref op_or>, except that + * reducer_opor is a proxy for the contained view, so that accumulator + * variable update operations can be applied directly to the reducer. For + * example, a value is "ORed" with a `reducer<%op_or>` with `*r |= a`, but a + * value can be "ORed" with a `%reducer_opor` with `r |= a`. + * + * @deprecated Users are strongly encouraged to use `reducer` + * reducers rather than the old wrappers like reducer_opor. + * The `reducer` reducers show the reducer/monoid/view + * architecture more clearly, are more consistent in their + * implementation, and present a simpler model for new + * user-implemented reducers. + * + * @note Implicit conversions are provided between `%reducer_opor` + * and `reducer<%op_or>`. This allows incremental code + * conversion: old code that used `%reducer_opor` can pass a + * `%reducer_opor` to a converted function that now expects a + * pointer or reference to a `reducer<%op_or>`, and vice + * versa. + * + * @tparam Type The value type of the reducer. + * + * @see op_or + * @see reducer + * @see ReducersOr + * + * @ingroup ReducersOr + */ +template +class reducer_opor : public reducer< op_or > +{ + typedef reducer< op_or > base; + using base::view; + + public: + /// The view type for the reducer. + typedef typename base::view_type view_type; + + /// The view's rhs proxy type. + typedef typename view_type::rhs_proxy rhs_proxy; + + /// The view type for the reducer. + typedef view_type View; + + /// The monoid type for the reducer. + typedef typename base::monoid_type Monoid; + + /** @name Constructors + */ + ///@{ + + /** Default (identity) constructor. + * + * Constructs the wrapper with the default initial value of `Type()`. + */ + reducer_opor() {} + + /** Value constructor. + * + * Constructs the wrapper with a specified initial value. + */ + explicit reducer_opor(const Type& initial_value) : base(initial_value) {} + + ///@} + + /** @name Forwarded functions + * @details Functions that update the contained accumulator variable are + * simply forwarded to the contained @ref op_and_view. */ + ///@{ + + /// @copydoc op_or_view::operator|=(const Type&) + reducer_opor& operator|=(const Type& x) + { + view() |= x; return *this; + } + + // The legacy definition of reducer_opor::operator|() has different + // behavior and a different return type than this definition. The legacy + // version is defined as a member function, so this new version is defined + // as a free function to give it a different signature, so that they won't + // end up sharing a single object file entry. + + /// @copydoc op_or_view::operator|(const Type&) const + friend rhs_proxy operator|(const reducer_opor& r, const Type& x) + { + return r.view() | x; + } + + /// @copydoc op_and_view::operator=(const rhs_proxy&) + reducer_opor& operator=(const rhs_proxy& temp) + { + view() = temp; return *this; + } + ///@} + + /** @name Dereference + * @details Dereferencing a wrapper is a no-op. It simply returns the + * wrapper. Combined with the rule that the wrapper forwards view + * operations to its contained view, this means that view operations can + * be written the same way on reducers and wrappers, which is convenient + * for incrementally converting old code using wrappers to use reducers + * instead. That is: + * + * reducer< op_and > r; + * *r &= a; // *r returns the view + * // operator &= is a view member function + * + * reducer_opand w; + * *w &= a; // *w returns the wrapper + * // operator &= is a wrapper member function that + * // calls the corresponding view function + */ + ///@{ + reducer_opor& operator*() { return *this; } + reducer_opor const& operator*() const { return *this; } + + reducer_opor* operator->() { return this; } + reducer_opor const* operator->() const { return this; } + ///@} + + /** @name Upcast + * @details In Intel Cilk Plus library 0.9, reducers were always cache-aligned. + * In library 1.0, reducer cache alignment is optional. By default, + * reducers are unaligned (i.e., just naturally aligned), but legacy + * wrappers inherit from cache-aligned reducers for binary compatibility. + * + * This means that a wrapper will automatically be upcast to its aligned + * reducer base class. The following conversion operators provide + * pseudo-upcasts to the corresponding unaligned reducer class. + */ + ///@{ + operator reducer< op_or >& () + { + return *reinterpret_cast< reducer< op_or >* >(this); + } + operator const reducer< op_or >& () const + { + return *reinterpret_cast< const reducer< op_or >* >(this); + } + ///@} + +}; + +/// @cond internal +/** Metafunction specialization for reducer conversion. + * + * This specialization of the @ref legacy_reducer_downcast template class + * defined in reducer.h causes the `reducer< op_or >` class to have an + * `operator reducer_opor& ()` conversion operator that statically + * downcasts the `reducer` to the corresponding `reducer_opor` type. + * (The reverse conversion, from `reducer_opor` to `reducer`, is just + * an upcast, which is provided for free by the language.) + * + * @ingroup ReducersOr + */ +template +struct legacy_reducer_downcast > > +{ + typedef reducer_opor type; +}; +/// @endcond + +} // namespace cilk + +#endif /* __cplusplus */ + + +/** @ingroup ReducersOr + */ +///@{ + +/** @name C language reducer macros + * + * These macros are used to declare and work with op_or reducers in C code. + * + * @see @ref page_reducers_in_c + */ + ///@{ + +__CILKRTS_BEGIN_EXTERN_C + +/** Declares OPOR reducer type name. + * + * This macro expands into the identifier which is the name of the op_or + * reducer type for a specified numeric type. + * + * @param tn The @ref reducers_c_type_names "numeric type name" specifying + * the type of the reducer. + * + * @see @ref reducers_c_predefined + * @see ReducersOr + */ +#define CILK_C_REDUCER_OPOR_TYPE(tn) \ + __CILKRTS_MKIDENT(cilk_c_reducer_opor_,tn) + +/** Declares an op_or reducer object. + * + * This macro expands into a declaration of an op_or reducer object for a + * specified numeric type. For example: + * + * CILK_C_REDUCER_OPOR(my_reducer, ulong, 0); + * + * @param obj The variable name to be used for the declared reducer object. + * @param tn The @ref reducers_c_type_names "numeric type name" specifying + * the type of the reducer. + * @param v The initial value for the reducer. (A value which can be + * assigned to the numeric type represented by @a tn.) + * + * @see @ref reducers_c_predefined + * @see ReducersOr + */ +#define CILK_C_REDUCER_OPOR(obj,tn,v) \ + CILK_C_REDUCER_OPOR_TYPE(tn) obj = \ + CILK_C_INIT_REDUCER(_Typeof(obj.value), \ + __CILKRTS_MKIDENT(cilk_c_reducer_opor_reduce_,tn), \ + __CILKRTS_MKIDENT(cilk_c_reducer_opor_identity_,tn), \ + __cilkrts_hyperobject_noop_destroy, v) + +/// @cond internal + +/** Declares the op_or reducer functions for a numeric type. + * + * This macro expands into external function declarations for functions which + * implement the reducer functionality for the op_or reducer type for a + * specified numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer + * type name, function names, etc. + */ +#define CILK_C_REDUCER_OPOR_DECLARATION(t,tn) \ + typedef CILK_C_DECLARE_REDUCER(t) CILK_C_REDUCER_OPOR_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_opor,tn,l,r); \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_opor,tn); + +/** Defines the op_or reducer functions for a numeric type. + * + * This macro expands into function definitions for functions which implement + * the reducer functionality for the op_or reducer type for a specified + * numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer + * type name, function names, etc. + */ +#define CILK_C_REDUCER_OPOR_DEFINITION(t,tn) \ + typedef CILK_C_DECLARE_REDUCER(t) CILK_C_REDUCER_OPOR_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_opor,tn,l,r) \ + { *(t*)l |= *(t*)r; } \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_opor,tn) \ + { *(t*)v = 0; } + +///@{ +/** @def CILK_C_REDUCER_OPOR_INSTANCE + * @brief Declares or defines implementation functions for a reducer type. + * + * In the runtime source file c_reducers.c, the macro `CILK_C_DEFINE_REDUCERS` + * will be defined, and this macro will generate reducer implementation + * functions. Everywhere else, `CILK_C_DEFINE_REDUCERS` will be undefined, and + * this macro will expand into external declarations for the functions. + */ +#ifdef CILK_C_DEFINE_REDUCERS +# define CILK_C_REDUCER_OPOR_INSTANCE(t,tn) \ + CILK_C_REDUCER_OPOR_DEFINITION(t,tn) +#else +# define CILK_C_REDUCER_OPOR_INSTANCE(t,tn) \ + CILK_C_REDUCER_OPOR_DECLARATION(t,tn) +#endif +///@} + +/* Declare or define an instance of the reducer type and its functions for each + * numeric type. + */ +CILK_C_REDUCER_OPOR_INSTANCE(char, char) +CILK_C_REDUCER_OPOR_INSTANCE(unsigned char, uchar) +CILK_C_REDUCER_OPOR_INSTANCE(signed char, schar) +CILK_C_REDUCER_OPOR_INSTANCE(wchar_t, wchar_t) +CILK_C_REDUCER_OPOR_INSTANCE(short, short) +CILK_C_REDUCER_OPOR_INSTANCE(unsigned short, ushort) +CILK_C_REDUCER_OPOR_INSTANCE(int, int) +CILK_C_REDUCER_OPOR_INSTANCE(unsigned int, uint) +CILK_C_REDUCER_OPOR_INSTANCE(unsigned int, unsigned) /* alternate name */ +CILK_C_REDUCER_OPOR_INSTANCE(long, long) +CILK_C_REDUCER_OPOR_INSTANCE(unsigned long, ulong) +CILK_C_REDUCER_OPOR_INSTANCE(long long, longlong) +CILK_C_REDUCER_OPOR_INSTANCE(unsigned long long, ulonglong) + +//@endcond + +__CILKRTS_END_EXTERN_C + +///@} + +///@} + +#endif /* REDUCER_OPOR_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_opxor.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_opxor.h new file mode 100644 index 0000000000..cb6560f9c5 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_opxor.h @@ -0,0 +1,608 @@ +/* reducer_opxor.h -*- C++ -*- + * + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + */ + +/** @file reducer_opxor.h + * + * @brief Defines classes for doing parallel bitwise or reductions. + * + * @ingroup ReducersXor + * + * @see ReducersXor + */ + +#ifndef REDUCER_OPXOR_H_INCLUDED +#define REDUCER_OPXOR_H_INCLUDED + +#include + +/** @defgroup ReducersXor Bitwise XOR Reducers + * + * Bitwise XOR reducers allow the computation of the bitwise XOR of a set of + * values in parallel. + * + * @ingroup Reducers + * + * You should be familiar with @ref pagereducers "Intel(R) Cilk(TM) Plus reducers", + * described in file `reducers.md`, and particularly with @ref reducers_using, + * before trying to use the information in this file. + * + * @section redopxor_usage Usage Example + * + * cilk::reducer< cilk::op_xor > r; + * cilk_for (int i = 0; i != N; ++i) { + * *r ^= a[i]; + * } + * unsigned result; + * r.move_out(result); + * + * @section redopxor_monoid The Monoid + * + * @subsection redopxor_monoid_values Value Set + * + * The value set of a bitwise XOR reducer is the set of values of `Type`, which + * is expected to be a builtin integer type which has a representation as a + * sequence of bits (or something like it, such as `bool` or `std::bitset`). + * + * @subsection redopxor_monoid_operator Operator + * + * The bitwise XOR operator is defined by the "`^`" binary operator on `Type`. + * + * @subsection redopxor_monoid_identity Identity + * + * The identity value of the reducer is the value whose representation + * contains all 0-bits. This is expected to be the value of the default + * constructor `Type()`. + * + * @section redopxor_operations Operations + * + * @subsection redopxor_constructors Constructors + * + * reducer() // identity + * reducer(const Type& value) + * reducer(move_in(Type& variable)) + * + * @subsection redopxor_get_set Set and Get + * + * r.set_value(const Type& value) + * const Type& = r.get_value() const + * r.move_in(Type& variable) + * r.move_out(Type& variable) + * + * @subsection redopxor_initial Initial Values + * + * If a bitwise XOR reducer is constructed without an explicit initial value, + * then its initial value will be its identity value, as long as `Type` + * satisfies the requirements of @ref redopxor_types. + * + * @subsection redopxor_view_ops View Operations + * + * *r ^= a + * *r = *r ^ a + * *r = *r ^ a1 ^ a2 … ^ an + * + * @section redopxor_types Type and Operator Requirements + * + * `Type` must be `Copy Constructible`, `Default Constructible`, and + * `Assignable`. + * + * The operator "`^=`" must be defined on `Type`, with `x ^= a` having the + * same meaning as `x = x ^ a`. + * + * The expression `Type()` must be a valid expression which yields the + * identity value (the value of `Type` whose representation consists of all + * 0-bits). + * + * @section redopxor_in_c Bitwise XOR Reducers in C + * + * The @ref CILK_C_REDUCER_OPXOR and @ref CILK_C_REDUCER_OPXOR_TYPE macros can + * be used to do bitwise XOR reductions in C. For example: + * + * CILK_C_REDUCER_OPXOR(r, uint, 0); + * CILK_C_REGISTER_REDUCER(r); + * cilk_for(int i = 0; i != n; ++i) { + * REDUCER_VIEW(r) ^= a[i]; + * } + * CILK_C_UNREGISTER_REDUCER(r); + * printf("The bitwise XOR of the elements of a is %x\n", REDUCER_VIEW(r)); + * + * See @ref reducers_c_predefined. + */ + +#ifdef __cplusplus + +namespace cilk { + +/** The bitwise XOR reducer view class. + * + * This is the view class for reducers created with + * `cilk::reducer< cilk::op_xor >`. It holds the accumulator variable + * for the reduction, and allows only `xor` operations to be performed on it. + * + * @note The reducer "dereference" operation (`reducer::operator *()`) + * yields a reference to the view. Thus, for example, the view class's + * `^=` operation would be used in an expression like `*r ^= a`, where + * `r` is an opmod reducer variable. + * + * @tparam Type The type of the contained accumulator variable. This will + * be the value type of a monoid_with_view that is + * instantiated with this view. + * + * @see ReducersXor + * @see op_xor + * + * @ingroup ReducersXor + */ +template +class op_xor_view : public scalar_view +{ + typedef scalar_view base; + +public: + /** Class to represent the right-hand side of `*reducer = *reducer ^ value`. + * + * The only assignment operator for the op_xor_view class takes an + * rhs_proxy as its operand. This results in the syntactic restriction + * that the only expressions that can be assigned to an op_xor_view are + * ones which generate an rhs_proxy - that is, expressions of the form + * `op_xor_view ^ value ... ^ value`. + * + * @warning + * The lhs and rhs views in such an assignment must be the same; + * otherwise, the behavior will be undefined. (I.e., `v1 = v1 ^ x` is + * legal; `v1 = v2 ^ x` is illegal.) This condition will be checked with + * a runtime assertion when compiled in debug mode. + * + * @see op_xor_view + */ + class rhs_proxy { + friend class op_xor_view; + + const op_xor_view* m_view; + Type m_value; + + // Constructor is invoked only from op_xor_view::operator^(). + // + rhs_proxy(const op_xor_view* view, const Type& value) : m_view(view), m_value(value) {} + + rhs_proxy& operator=(const rhs_proxy&); // Disable assignment operator + rhs_proxy(); // Disable default constructor + + public: + /** bitwise XOR with an additional rhs value. If `v` is an op_xor_view + * and `a1` is a value, then the expression `v ^ a1` invokes the + * view's `operator^()` to create an rhs_proxy for `(v, a1)`; then + * `v ^ a1 ^ a2` invokes the rhs_proxy's `operator^()` to create a new + * rhs_proxy for `(v, a1^a2)`. This allows the right-hand side of an + * assignment to be not just `view ^ value`, but + ( `view ^ value ^ value ... ^ value`. The effect is that + * + * v = v ^ a1 ^ a2 ... ^ an; + * + * is evaluated as + * + * v = v ^ (a1 ^ a2 ... ^ an); + */ + rhs_proxy& operator^(const Type& x) { m_value ^= x; return *this; } + }; + + + /** Default/identity constructor. This constructor initializes the + * contained value to `Type()`. + */ + op_xor_view() : base() {} + + /** Construct with a specified initial value. + */ + explicit op_xor_view(const Type& v) : base(v) {} + + /** Reduces the views of two strands. + * + * This function is invoked by the @ref op_xor monoid to combine the views + * of two strands when the right strand merges with the left one. It + * "XORs" the value contained in the left-strand view by the value + * contained in the right-strand view, and leaves the value in the + * right-strand view undefined. + * + * @param right A pointer to the right-strand view. (`this` points to + * the left-strand view.) + * + * @note Used only by the @ref op_xor monoid to implement the monoid + * reduce operation. + */ + void reduce(op_xor_view* right) { this->m_value ^= right->m_value; } + + /** @name Accumulator variable updates. + * + * These functions support the various syntaxes for "XORing" the + * accumulator variable contained in the view with some value. + */ + ///@{ + + /** Performs XOR operation between the accumulator variable and @a x. + */ + op_xor_view& operator^=(const Type& x) { this->m_value ^= x; return *this; } + + /** Creates an object representing `*this ^ x`. + * + * @see rhs_proxy + */ + rhs_proxy operator^(const Type& x) const { return rhs_proxy(this, x); } + + /** Assigns the result of a `view ^ value` expression to the view. Note that + * this is the only assignment operator for this class. + * + * @see rhs_proxy + */ + op_xor_view& operator=(const rhs_proxy& rhs) { + __CILKRTS_ASSERT(this == rhs.m_view); + this->m_value ^= rhs.m_value; + return *this; + } + + ///@} +}; + +/** Monoid class for bitwise XOR reductions. Instantiate the cilk::reducer + * template class with an op_xor monoid to create a bitwise XOR reducer + * class. For example, to compute the bitwise XOR of a set of `unsigned long` + * values: + * + * cilk::reducer< cilk::op_xor > r; + * + * @tparam Type The reducer value type. + * @tparam Align If `false` (the default), reducers instantiated on this + * monoid will be naturally aligned (the Intel Cilk Plus library 1.0 + * behavior). If `true`, reducers instantiated on this monoid + * will be cache-aligned for binary compatibility with + * reducers in Intel Cilk Plus library version 0.9. + * + * @see ReducersXor + * @see op_xor_view + * + * @ingroup ReducersXor + */ +template +struct op_xor : public monoid_with_view, Align> {}; + +/** Deprecated bitwise XOR reducer class. + * + * reducer_opxor is the same as @ref reducer<@ref op_xor>, except that + * reducer_opxor is a proxy for the contained view, so that accumulator + * variable update operations can be applied directly to the reducer. For + * example, a value is "XORed" with a `reducer<%op_xor>` with `*r ^= a`, but a + * value can be "XORed" with a `%reducer_opxor` with `r ^= a`. + * + * @deprecated Users are strongly encouraged to use `reducer` + * reducers rather than the old wrappers like reducer_opand. + * The `reducer` reducers show the reducer/monoid/view + * architecture more clearly, are more consistent in their + * implementation, and present a simpler model for new + * user-implemented reducers. + * + * @note Implicit conversions are provided between `%reducer_opxor` + * and `reducer<%op_xor>`. This allows incremental code + * conversion: old code that used `%reducer_opxor` can pass a + * `%reducer_opxor` to a converted function that now expects a + * pointer or reference to a `reducer<%op_xor>`, and vice + * versa. + * + * @tparam Type The value type of the reducer. + * + * @see op_xor + * @see reducer + * @see ReducersXor + * + * @ingroup ReducersXor + */ +template +class reducer_opxor : public reducer< op_xor > +{ + typedef reducer< op_xor > base; + using base::view; + + public: + /// The view type for the reducer. + typedef typename base::view_type view_type; + + /// The view's rhs proxy type. + typedef typename view_type::rhs_proxy rhs_proxy; + + /// The view type for the reducer. + typedef view_type View; + + /// The monoid type for the reducer. + typedef typename base::monoid_type Monoid; + + /** @name Constructors + */ + ///@{ + + /** Default (identity) constructor. + * + * Constructs the wrapper with the default initial value of `Type()`. + */ + reducer_opxor() {} + + /** Value constructor. + * + * Constructs the wrapper with a specified initial value. + */ + explicit reducer_opxor(const Type& initial_value) : base(initial_value) {} + + ///@} + + /** @name Forwarded functions + * @details Functions that update the contained accumulator variable are + * simply forwarded to the contained @ref op_and_view. */ + ///@{ + + /// @copydoc op_xor_view::operator^=(const Type&) + reducer_opxor& operator^=(const Type& x) + { + view() ^= x; return *this; + } + + // The legacy definition of reducer_opxor::operator^() has different + // behavior and a different return type than this definition. The legacy + // version is defined as a member function, so this new version is defined + // as a free function to give it a different signature, so that they won't + // end up sharing a single object file entry. + + /// @copydoc op_xor_view::operator^(const Type&) const + friend rhs_proxy operator^(const reducer_opxor& r, const Type& x) + { + return r.view() ^ x; + } + + /// @copydoc op_and_view::operator=(const rhs_proxy&) + reducer_opxor& operator=(const rhs_proxy& temp) + { + view() = temp; return *this; + } + ///@} + + /** @name Dereference + * @details Dereferencing a wrapper is a no-op. It simply returns the + * wrapper. Combined with the rule that the wrapper forwards view + * operations to its contained view, this means that view operations can + * be written the same way on reducers and wrappers, which is convenient + * for incrementally converting old code using wrappers to use reducers + * instead. That is: + * + * reducer< op_and > r; + * *r &= a; // *r returns the view + * // operator &= is a view member function + * + * reducer_opand w; + * *w &= a; // *w returns the wrapper + * // operator &= is a wrapper member function that + * // calls the corresponding view function + */ + ///@{ + reducer_opxor& operator*() { return *this; } + reducer_opxor const& operator*() const { return *this; } + + reducer_opxor* operator->() { return this; } + reducer_opxor const* operator->() const { return this; } + ///@} + + /** @name Upcast + * @details In Intel Cilk Plus library 0.9, reducers were always cache-aligned. + * In library 1.0, reducer cache alignment is optional. By default, + * reducers are unaligned (i.e., just naturally aligned), but legacy + * wrappers inherit from cache-aligned reducers for binary compatibility. + * + * This means that a wrapper will automatically be upcast to its aligned + * reducer base class. The following conversion operators provide + * pseudo-upcasts to the corresponding unaligned reducer class. + */ + ///@{ + operator reducer< op_xor >& () + { + return *reinterpret_cast< reducer< op_xor >* >(this); + } + operator const reducer< op_xor >& () const + { + return *reinterpret_cast< const reducer< op_xor >* >(this); + } + ///@} + +}; + +/// @cond internal +/** Metafunction specialization for reducer conversion. + * + * This specialization of the @ref legacy_reducer_downcast template class + * defined in reducer.h causes the `reducer< op_xor >` class to have an + * `operator reducer_opxor& ()` conversion operator that statically + * downcasts the `reducer` to the corresponding `reducer_opxor` type. + * (The reverse conversion, from `reducer_opxor` to `reducer`, is just + * an upcast, which is provided for free by the language.) + * + * @ingroup ReducersXor + */ +template +struct legacy_reducer_downcast > > +{ + typedef reducer_opxor type; +}; +/// @endcond + +} // namespace cilk + +#endif /* __cplusplus */ + + +/** @ingroup ReducersXor + */ +///@{ + +/** @name C language reducer macros + * + * These macros are used to declare and work with op_xor reducers in C code. + * + * @see @ref page_reducers_in_c + */ + ///@{ + +__CILKRTS_BEGIN_EXTERN_C + +/** Declares OPXOR reducer type name. + * + * This macro expands into the identifier which is the name of the op_xor + * reducer type for a specified numeric type. + * + * @param tn The @ref reducers_c_type_names "numeric type name" specifying + * the type of the reducer. + * + * @see @ref reducers_c_predefined + * @see ReducersXor + */ +#define CILK_C_REDUCER_OPXOR_TYPE(tn) \ + __CILKRTS_MKIDENT(cilk_c_reducer_opxor_,tn) + +/** Declares an op_xor reducer object. + * + * This macro expands into a declaration of an op_xor reducer object for a + * specified numeric type. For example: + * + * CILK_C_REDUCER_OPXOR(my_reducer, ulong, 0); + * + * @param obj The variable name to be used for the declared reducer object. + * @param tn The @ref reducers_c_type_names "numeric type name" specifying + * the type of the reducer. + * @param v The initial value for the reducer. (A value which can be + * assigned to the numeric type represented by @a tn.) + * + * @see @ref reducers_c_predefined + * @see ReducersXor + */ +#define CILK_C_REDUCER_OPXOR(obj,tn,v) \ + CILK_C_REDUCER_OPXOR_TYPE(tn) obj = \ + CILK_C_INIT_REDUCER(_Typeof(obj.value), \ + __CILKRTS_MKIDENT(cilk_c_reducer_opxor_reduce_,tn), \ + __CILKRTS_MKIDENT(cilk_c_reducer_opxor_identity_,tn), \ + __cilkrts_hyperobject_noop_destroy, v) + +/// @cond internal + +/** Declares the op_xor reducer functions for a numeric type. + * + * This macro expands into external function declarations for functions which + * implement the reducer functionality for the op_xor reducer type for a + * specified numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer + * type name, function names, etc. + */ +#define CILK_C_REDUCER_OPXOR_DECLARATION(t,tn) \ + typedef CILK_C_DECLARE_REDUCER(t) CILK_C_REDUCER_OPXOR_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_opxor,tn,l,r); \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_opxor,tn); + +/** Defines the op_xor reducer functions for a numeric type. + * + * This macro expands into function definitions for functions which implement + * the reducer functionality for the op_xor reducer type for a specified + * numeric type. + * + * @param t The value type of the reducer. + * @param tn The value "type name" identifier, used to construct the reducer + * type name, function names, etc. + */ +#define CILK_C_REDUCER_OPXOR_DEFINITION(t,tn) \ + typedef CILK_C_DECLARE_REDUCER(t) CILK_C_REDUCER_OPXOR_TYPE(tn); \ + __CILKRTS_DECLARE_REDUCER_REDUCE(cilk_c_reducer_opxor,tn,l,r) \ + { *(t*)l ^= *(t*)r; } \ + __CILKRTS_DECLARE_REDUCER_IDENTITY(cilk_c_reducer_opxor,tn) \ + { *(t*)v = 0; } + +///@{ +/** @def CILK_C_REDUCER_OPXOR_INSTANCE + * @brief Declares or defines implementation functions for a reducer type. + * + * In the runtime source file c_reducers.c, the macro `CILK_C_DEFINE_REDUCERS` + * will be defined, and this macro will generate reducer implementation + * functions. Everywhere else, `CILK_C_DEFINE_REDUCERS` will be undefined, and + * this macro will expand into external declarations for the functions. + */ +#ifdef CILK_C_DEFINE_REDUCERS +# define CILK_C_REDUCER_OPXOR_INSTANCE(t,tn) \ + CILK_C_REDUCER_OPXOR_DEFINITION(t,tn) +#else +# define CILK_C_REDUCER_OPXOR_INSTANCE(t,tn) \ + CILK_C_REDUCER_OPXOR_DECLARATION(t,tn) +#endif +///@} + +/* Declares or defines an instance of the reducer type and its functions for each + * numeric type. + */ +CILK_C_REDUCER_OPXOR_INSTANCE(char, char) +CILK_C_REDUCER_OPXOR_INSTANCE(unsigned char, uchar) +CILK_C_REDUCER_OPXOR_INSTANCE(signed char, schar) +CILK_C_REDUCER_OPXOR_INSTANCE(wchar_t, wchar_t) +CILK_C_REDUCER_OPXOR_INSTANCE(short, short) +CILK_C_REDUCER_OPXOR_INSTANCE(unsigned short, ushort) +CILK_C_REDUCER_OPXOR_INSTANCE(int, int) +CILK_C_REDUCER_OPXOR_INSTANCE(unsigned int, uint) +CILK_C_REDUCER_OPXOR_INSTANCE(unsigned int, unsigned) /* alternate name */ +CILK_C_REDUCER_OPXOR_INSTANCE(long, long) +CILK_C_REDUCER_OPXOR_INSTANCE(unsigned long, ulong) +CILK_C_REDUCER_OPXOR_INSTANCE(long long, longlong) +CILK_C_REDUCER_OPXOR_INSTANCE(unsigned long long, ulonglong) + +//@endcond + +__CILKRTS_END_EXTERN_C + +///@} + +///@} + +#endif /* REDUCER_OPXOR_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_ostream.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_ostream.h new file mode 100644 index 0000000000..793c3c5020 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_ostream.h @@ -0,0 +1,517 @@ +/* reducer_ostream.h -*- C++ -*- + * + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + */ + +/** @file reducer_ostream.h + * + * @brief Defines a class for writing to an ostream in parallel. + * + * @ingroup ReducersOstream + * + * @see @ref ReducersOstream + */ + +#ifndef REDUCER_OSTREAM_H_INCLUDED +#define REDUCER_OSTREAM_H_INCLUDED + +#include +#include +#include + +/** @defgroup ReducersOstream Ostream Reducers + * + * Ostream reducers allow multiple strands to write to an ostream in parallel. + * + * @ingroup Reducers + * + * You should be familiar with @ref pagereducers "Intel(R) Cilk(TM) Plus reducers", + * described in file reducers.md, and particularly with @ref reducers_using, + * before trying to use the information in this file. + * + * @section redostream_usage Usage Example + * + * One of the most common debugging techniques is adding `print` statements + * to the code being debugged. When the code is parallelized, the results can + * be less than satisfactory, as output from multiple strands is mingled in an + * unpredictable way. Like other reducers, an ostream reducer requires minimal + * recoding to guarantee that the output from parallelized computation will be + * ordered the same as though the computation were executed serially. + * + * cilk::reducer r(std::cerr); + * cilk_for (int i = 0; i != data.size(); ++i) { + * *r << "Iteration " << i << ":\n"; + * ... some computation ... + * *r << " Step 1:" << some information; + * ... some more computation ... + * *r << " Step 2:" << some more information; + * ... still more computation ... + * *r << " Step 3:" << still more information; + * } + * + * Output on standard error: + * + * Iteration 1: + * Step 1: ... + * Step 2: ... + * Step 3: ... + * Iteration 2: + * Step 1: ... + * Step 2: ... + * Step 3: ... + * Iteration 3: + * Step 1: ... + * Step 2: ... + * Step 3: ... + * ... + * + * @section redostream_overview Overview + * + * An "ostream reducer" is not really a reducer. It uses the reducer + * technology to coordinate operations on parallel strands to achieve + * the same behavior in a parallel computation that would be seen in a + * serial computation, but it does not have a monoid. It has a "monoid + * class," because that is part of the implementation framework, but it + * does not represent a mathematical monoid: there is no value type, no + * associative operation, and no identity value. The reducer is used for + * its side effect rather than to construct a value. + * + * You might think of an ostream reducer as a relative of a + * @ref ReducersString "string reducer" which uses stream output + * syntax (`stream << value`) instead of string append syntax + * (`string += value`), and which writes its result string to an + * ostream instead of making it available as the reducer value. + * + * Another difference is that "real" reducers protect their contained + * value quite strongly from improper access by the user. Ostream reducers, + * on the other hand, pretty much have to expose the ostream, since normal + * use of an ostream involves accessing its internal state. Furthermore, + * the ostream reducer just coordinates output to an existing ostream - + * there is nothing to keep the user from writing directly to the attached + * stream, with unpredictable results. + * + * @section redostream_operations Operations + * + * In the operation descriptions below, the type name `Ostream` refers to the + * reducer's ostream type, `std::basic_ostream`. + * + * @subsection redostream_constructors Constructors + * + * The only constructor is + * + * reducer(const Ostream& os) + * + * This creates a reducer that is associated with the existing ostream `os`. + * Anything "written to" the reducer will (eventually) be written to `os`. + * + * @subsection redostream_get_set Set and Get + * + * Just as a stream does not have a "value," neither does an ostream + * reducer. Therefore, none of the usual `set_value`, `get_value`, + * `move_in`, or `move_out` functions are available for ostream reducers. + * + * @subsection redostream_initial Initial Values + * + * Ostream reducers do not have default constructors. + * + * @subsection redostream_view_ops View Operations + * + * An ostream reducer view is actually a kind of `std::ostream`. Therefore, + * any operation that can be used on an ostream can be used on an ostream + * reducer view. For example: + * + * reducer r(cout); + * *r << setw(5) << (x=1) << endl; + * + * + * @section redostream_performance Performance Considerations + * + * Ostream reducers work by creating a string stream for each non-leftmost + * view. When two strands are merged, the contents of the string buffer of the + * right view are written to the left view. Since all non-leftmost strands are + * eventually merged, all output is eventually written to the associated + * ostream. + * + * This implementation has two consequences. + * + * First, all output written to an ostream reducer on a stolen strand is kept + * in memory (in a string buffer) until the strand is merged with the leftmost + * strand. This means that some portion of the output written to an ostream + * reducer during a parallel computation - half of the total output, on + * average - will temporarily be held in memory during the computation. + * Obviously, ostream reducers will work better for small and moderate amounts + * of output. + * + * Second, buffered ostream reducer content must be copied at every merge. + * The total amount of copying is potentially proportional to the total amount + * of output multiplied by the number of strands stolen during the computation. + * + * In short, writing to an ostream in a parallel computation with an ostream + * reducer will always be less efficient than writing the same output directly + * to the ostream in a serial computation. The value of the ostream + * reducer is not in the writing of the ostream itself, but in removing the + * race and serialization obstacles that the ostream output would cause in an + * otherwise parallelizable computation. + * + * + * @section redostream_state Stream State + * + * The reducer implementation can correctly order the output that is written + * to an ostream. However, an ostream has additional state that controls its + * behavior, such as its formatting attributes, error state, extensible arrays, * and registered callbacks. If these are modified during the computation, the * reducer implementation cannot guarantee that they will be the same in a + * parallel computation as in a serial computation. In particular: + * + * - In the serial execution, the ostream state in the continuation of a + * spawn will be the same as the state at the end of the spawned function. + * In the parallel execution, if the continuation is stolen, its view will + * contain a newly created ostream with the default initial state. + * - In the serial execution, the ostream state following a sync is the same + * as the state before the sync. In the parallel execution, if the + * continuation is stolen, then the state following the sync will be the + * same as the state at the end of some spawned function. + * + * In short, you must not make any assumptions about the stream state of an + * ostream reducer: + * + * - Following a `cilk_spawn`. + * - Following a `cilk_sync`. + * - At the start of an iteration of a `cilk_for` loop. + * - Following the completion of a `cilk_for` loop. + * + * @section redostream_types Type and Operator Requirements + * + * `std::basic_ostream` must be a valid type. +*/ + +namespace cilk { + +/** @ingroup ReducersOstream */ +//@{ + +/** The ostream reducer view class. + * + * This is the view class for reducers created with + * `cilk::reducer< cilk::op_basic_ostream >`. It holds the + * actual ostream for a parallel strand, and allows only stream output + * operations to be performed on it. + * + * @note The reducer "dereference" operation (`reducer::operator *()`) + * yields a reference to the view. Thus, for example, the view + * class's `<<` operation would be used in an expression like + * `*r << "x = " << x`, where `r` is an ostream reducer. + * + * @tparam Char The ostream element type (not the ostream type). + * @tparam Traits The character traits type. + * + * @see ReducersOstream + * @see op_basic_ostream + */ +template +class op_basic_ostream_view : public std::basic_ostream +{ + typedef std::basic_ostream base; + typedef std::basic_ostream ostream_type; + + // A non-leftmost view is associated with a private string buffer. (The + // leftmost view is associated with the buffer of the reducer's associated + // ostream, so its private buffer is unused.) + // + std::basic_stringbuf m_buffer; + +public: + + /** Value type. Required by @ref monoid_with_view. + */ + typedef ostream_type value_type; + + /** Reduce operation. Required by @ref monoid_with_view. + */ + void reduce(op_basic_ostream_view* other) + { + // Writing an empty buffer results in failure. Testing `sgetc()` is the + // easiest way of checking for an empty buffer. + if (other->m_buffer.sgetc() != Traits::eof()) { + *this << (&other->m_buffer); + } + } + + /** Non-leftmost (identity) view constructor. The view is associated with + * its internal buffer. Required by @ref monoid_base. + */ + op_basic_ostream_view() : base(&m_buffer) {} + + /** Leftmost view constructor. The view is associated with an existing + * ostream. + */ + op_basic_ostream_view(const ostream_type& os) : base(0) + { + base::rdbuf(os.rdbuf()); // Copy stream buffer + base::flags(os.flags()); // Copy formatting flags + base::setstate(os.rdstate()); // Copy error state + } + + /** Sets/gets. + * + * These are all no-ops. + */ + //@{ + + void view_set_value(const value_type&) + { assert("set_value() is not allowed on ostream reducers" && 0); } + const value_type& view_get_value() const + { assert("get_value() is not allowed on ostream reducers" && 0); + return *this; } + typedef value_type const& return_type_for_get_value; + void view_move_in(const value_type&) + { assert("move_in() is not allowed on ostream reducers" && 0); } + void view_move_out(const value_type&) + { assert("move_out() is not allowed on ostream reducers" && 0); } + + //@} +}; + +/** Ostream monoid class. Instantiate the cilk::reducer template class with an + * op_basic_ostream monoid to create an ostream reducer class: + * + * cilk::reducer< cilk::op_basic_string > r; + * + * @tparam Char The stream element type (not the stream type). + * @tparam Traits The character traits type. + * + * @see ReducersOstream + * @see op_basic_ostream_view + * @see reducer_ostream + * @see op_ostream + * @see op_wostream + */ +template, + bool Align = false> +class op_basic_ostream : + public monoid_with_view< op_basic_ostream_view, Align > +{ + typedef monoid_with_view< op_basic_ostream_view, Align > + base; + typedef std::basic_ostream ostream_type; + typedef provisional_guard view_guard; + +public: + + /** View type of the monoid. + */ + typedef typename base::view_type view_type; + + /** @name Construct function. + * + * The only supported ostream reducer constructor takes a reference to + * an existing ostream. + * + * @param os The ostream destination for receive all data written to the + * reducer. + */ + static void construct( + op_basic_ostream* monoid, + view_type* view, + const ostream_type& os) + { + view_guard vg( new((void*) view) view_type(os) ); + vg.confirm_if( new((void*) monoid) op_basic_ostream ); + } +}; + + +/** + * Convenience typedef for narrow ostreams. + */ +typedef op_basic_ostream op_ostream; + +/** + * Convenience typedef for wide ostreams. + */ +typedef op_basic_ostream op_wostream; + +/// @cond internal + +class reducer_ostream; + +/** Metafunction specialization for reducer conversion. + * + * This specialization of the @ref legacy_reducer_downcast template class + * defined in reducer.h causes the `reducer >` class + * to have an `operator reducer_ostream& ()` conversion operator that + * statically downcasts the `reducer >` to + * `reducer_ostream`. (The reverse conversion, from `reducer_ostream` to + * `reducer >`, is just an upcast, which is provided + * for free by the language.) + */ +template +struct legacy_reducer_downcast< + reducer, Align> > > +{ + typedef reducer_ostream type; +}; + +/// @endcond + +/** Deprecated ostream reducer class. + * + * reducer_ostream is the same as @ref cilk::reducer<@ref op_ostream>, except + * that reducer_ostream is a proxy for the contained view, so that ostream + * operations can be applied directly to the reducer. For example, a number is + * written to a `reducer` with `*r << x`, but a number can be + * written to a `reducer_ostream` with `r << x`. + * + * @deprecated Users are strongly encouraged to use `reducer` + * reducers rather than the old wrappers like reducer_ostream. The + * `reducer` reducers show the reducer/monoid/view + * architecture more clearly, are more consistent in their + * implementation, and present a simpler model for new + * user-implemented reducers. + * + * @note Implicit conversions are provided between `%reducer_ostream` + * and `reducer<%op_ostream>`. This allows incremental code + * conversion: old code that used `%reducer_ostream` can pass a + * `%reducer_ostream` to a converted function that now expects a + * pointer or reference to a `reducer<%op_ostream>`, and vice versa. + * + * @tparam Char The stream element type (not the stream type). + * @tparam Traits The character traits type. + * + * @see op_ostream + * @see reducer + * @see ReducersOstream + */ +class reducer_ostream : + public reducer, true> > +{ + typedef reducer, true> > base; + using base::view; +public: + + /// The view type for the reducer. + typedef base::view_type View; + + /// The monoid type for the reducer. + typedef base::monoid_type Monoid; + + /** Constructs an initial `reducer_ostream` from a `std::ostream`. The + * specified stream is used as the eventual destination for all text + * streamed to this hyperobject. + */ + explicit reducer_ostream(const std::ostream &os) : base(os) {} + + /** Returns a modifiable reference to the underlying 'ostream' object. + */ + std::ostream& get_reference() { return view(); } + + /** Writes to the ostream. + */ + template + std::ostream& operator<< (const T &v) + { + return view() << v; + } + + /** + * Calls a manipulator. + * + * @param _Pfn Pointer to the manipulator function. + */ + reducer_ostream& operator<< (std::ostream &(*_Pfn)(std::ostream &)) + { + (*_Pfn)(view()); + return *this; + } + + /** @name Dereference + * @details Dereferencing a wrapper is a no-op. It simply returns the + * wrapper. Combined with the rule that the wrapper forwards view + * operations to its contained view, this means that view operations can + * be written the same way on reducers and wrappers, which is convenient + * for incrementally converting old code using wrappers to use reducers + * instead. That is: + * + * reducer r; + * *r << "a"; // *r returns the view + * // operator<<() is a view member function + * + * reducer_ostream w; + * *w << "a"; // *w returns the wrapper + * // operator<<() is a wrapper member function + * // that calls the corresponding view function + */ + //@{ + reducer_ostream& operator*() { return *this; } + reducer_ostream const& operator*() const { return *this; } + + reducer_ostream* operator->() { return this; } + reducer_ostream const* operator->() const { return this; } + //@} + + /** @name Upcast + * @details In Intel Cilk Plus library 0.9, reducers were always cache-aligned. + * In library 1.0, reducer cache alignment is optional. By default, + * reducers are unaligned (i.e., just naturally aligned), but legacy + * wrappers inherit from cache-aligned reducers for binary compatibility. + * + * This means that a wrapper will automatically be upcast to its aligned + * reducer base class. The following conversion operators provide + * pseudo-upcasts to the corresponding unaligned reducer class. + */ + //@{ + operator reducer& () + { + return *reinterpret_cast< reducer* >(this); + } + operator const reducer& () const + { + return *reinterpret_cast< const reducer* >(this); + } + //@} +}; + +} // namespace cilk + +#endif // REDUCER_OSTREAM_H_INCLUDED diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_string.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_string.h new file mode 100644 index 0000000000..9af65d5534 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cilk/reducer_string.h @@ -0,0 +1,763 @@ +/* reducer_string.h -*- C++ -*- + * + * Copyright (C) 2009-2016, Intel Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ********************************************************************* + * + * PLEASE NOTE: This file is a downstream copy of a file mainitained in + * a repository at cilkplus.org. Changes made to this file that are not + * submitted through the contribution process detailed at + * http://www.cilkplus.org/submit-cilk-contribution will be lost the next + * time that a new version is released. Changes only submitted to the + * GNU compiler collection or posted to the git repository at + * https://bitbucket.org/intelcilkruntime/intel-cilk-runtime.git are + * not tracked. + * + * We welcome your contributions to this open source project. Thank you + * for your assistance in helping us improve Cilk Plus. + */ + +/** @file reducer_string.h + * + * @brief Defines classes for doing parallel string creation by appending. + * + * @ingroup ReducersString + * + * @see ReducersString + */ + +#ifndef REDUCER_STRING_H_INCLUDED +#define REDUCER_STRING_H_INCLUDED + +#include +#include +#include + +/** @defgroup ReducersString String Reducers + * + * String reducers allow the creation of a string by concatenating a set of + * strings or characters in parallel. + * + * @ingroup Reducers + * + * You should be familiar with @ref pagereducers "Intel(R) Cilk(TM) Plus reducers", + * described in file reducers.md, and particularly with @ref reducers_using, + * before trying to use the information in this file. + * + * @section redstring_usage Usage Example + * + * vector data; + * void expensive_string_computation(const Data& x, string& s); + * cilk::reducer r; + * cilk_for (int i = 0; i != data.size(); ++i) { + * string temp; + * expensive_string_computation(data[i], temp); + * *r += temp; + * } + * string result; + * r.move_out(result); + * + * @section redstring_monoid The Monoid + * + * @subsection redstring_monoid_values Value Set + * + * The value set of a string reducer is the set of values of the class + * `std::basic_string`, which we refer to as "the + * reducer's string type". + * + * @subsection redstring_monoid_operator Operator + * + * The operator of a string reducer is the string concatenation operator, + * defined by the "`+`" binary operator on the reducer's string type. + * + * @subsection redstring_monoid_identity Identity + * + * The identity value of a string reducer is the empty string, which is the + * value of the expression + * `std::basic_string([allocator])`. + * + * @section redstring_operations Operations + * + * In the operation descriptions below, the type name `String` refers to the + * reducer's string type, `std::basic_string`. + * + * @subsection redstring_constructors Constructors + * + * Any argument list which is valid for a `std::basic_string` constructor is + * valid for a string reducer constructor. The usual move-in constructor is + * also provided: + * + * reducer(move_in(String& variable)) + * + * @subsection redstring_get_set Set and Get + * + * r.set_value(const String& value) + * const String& = r.get_value() const + * r.move_in(String& variable) + * r.move_out(String& variable) + * + * @subsection redstring_initial Initial Values + * + * A string reducer with no constructor arguments, or with only an allocator + * argument, will initially contain the identity value, an empty string. + * + * @subsection redstring_view_ops View Operations + * + * *r += a + * r->append(a) + * r->append(a, b) + * r->push_back(a) + * + * These operations on string reducer views are the same as the corresponding + * operations on strings. + * + * @section redstring_performance Performance Considerations + * + * String reducers work by creating a string for each view, collecting those + * strings in a list, and then concatenating them into a single result string + * at the end of the computation. This last step takes place in serial code, + * and necessarily takes time proportional to the length of the result string. + * Thus, a parallel string reducer cannot actually speed up the time spent + * directly creating the string. This trivial example would probably be slower + * (because of reducer overhead) than the corresponding serial code: + * + * vector a; + * reducer r; + * cilk_for (int i = 0; i != a.length(); ++i) { + * *r += a[i]; + * } + * string result; + * r.move_out(result); + * + * What a string reducer _can_ do is to allow the _remainder_ of the + * computation to be done in parallel, without having to worry about managing + * the string computation. + * + * The strings for new views are created (by the view identity constructor) + * using the same allocator as the string that was created when the reducer + * was constructed. Note that this allocator is determined when the reducer is + * constructed. The following two examples may have very different behavior: + * + * string a_string; + * + * reducer< op_string reducer1(move_in(a_string)); + * ... parallel computation ... + * reducer1.move_out(a_string); + * + * reducer< op_string reducer2; + * reducer2.move_in(a_string); + * ... parallel computation ... + * reducer2.move_out(a_string); + * + * * `reducer1` will be constructed with the same allocator as `a_string`, + * because the string was specified in the constructor. The `move_in` + * and `move_out` can therefore be done with a `swap` in constant time. + * * `reducer2` will be constructed with a _default_ allocator of type + * `Allocator`, which may not be the same as the allocator of `a_string`. + * Therefore, the `move_in` and `move_out` may have to be done with a copy + * in _O(N)_ time. + * + * (All instances of an allocator type with no internal state (like + * `std::allocator`) are "the same". You only need to worry about the "same + * allocator" issue when you create string reducers with custom allocator + * types.) + * + * @section redstring_types Type and Operator Requirements + * + * `std::basic_string` must be a valid type. +*/ + +namespace cilk { + +/** @ingroup ReducersString */ +//@{ + +/** The string append reducer view class. + * + * This is the view class for reducers created with + * `cilk::reducer< cilk::op_basic_string >`. It holds + * the accumulator variable for the reduction, and allows only append + * operations to be performed on it. + * + * @note The reducer "dereference" operation (`reducer::operator *()`) + * yields a reference to the view. Thus, for example, the view class's + * `append` operation would be used in an expression like + * `r->append(a)`, where `r` is a string append reducer variable. + * + * @tparam Char The string element type (not the string type). + * @tparam Traits The character traits type. + * @tparam Alloc The string allocator type. + * + * @see ReducersString + * @see op_basic_string + */ +template +class op_basic_string_view +{ + typedef std::basic_string string_type; + typedef std::list list_type; + typedef typename string_type::size_type size_type; + + // The view's value is represented by a list of strings and a single + // string. The value is the concatenation of the strings in the list with + // the single string at the end. All string operations apply to the single + // string; reduce operations cause lists of partial strings from multiple + // strands to be combined. + // + mutable string_type m_string; + mutable list_type m_list; + + // Before returning the value of the reducer, concatenate all the strings + // in the list with the single string. + // + void flatten() const + { + if (m_list.empty()) return; + + typename list_type::iterator i; + + size_type len = m_string.size(); + for (i = m_list.begin(); i != m_list.end(); ++i) + len += i->size(); + + string_type result(get_allocator()); + result.reserve(len); + + for (i = m_list.begin(); i != m_list.end(); ++i) + result += *i; + m_list.clear(); + + result += m_string; + result.swap(m_string); + } + +public: + + /** @name Monoid support. + */ + //@{ + + /// Required by @ref monoid_with_view + typedef string_type value_type; + + /// Required by @ref op_string + Alloc get_allocator() const + { + return m_string.get_allocator(); + } + + /** Reduces the views of two strands. + * + * This function is invoked by the @ref op_basic_string monoid to combine + * the views of two strands when the right strand merges with the left + * one. It appends the value contained in the right-strand view to the + * value contained in the left-strand view, and leaves the value in the + * right-strand view undefined. + * + * @param right A pointer to the right-strand view. (`this` points to + * the left-strand view.) + * + * @note Used only by the @ref op_basic_string monoid to implement the + * monoid reduce operation. + */ + void reduce(op_basic_string_view* right) + { + if (!right->m_string.empty() || !right->m_list.empty()) { + // (list, string) + (right_list, right_string) => + // (list + {string} + right_list, right_string) + if (!m_string.empty()) { + // simulate m_list.push_back(std::move(m_string)) + m_list.push_back(string_type(get_allocator())); + m_list.back().swap(m_string); + } + m_list.splice(m_list.end(), right->m_list); + m_string.swap(right->m_string); + } + } + + //@} + + /** @name Passes constructor arguments to the string constructor. + */ + //@{ + + op_basic_string_view() : m_string() {} + + template + op_basic_string_view(const T1& x1) : m_string(x1) {} + + template + op_basic_string_view(const T1& x1, const T2& x2) : m_string(x1, x2) {} + + template + op_basic_string_view(const T1& x1, const T2& x2, const T3& x3) : m_string(x1, x2, x3) {} + + template + op_basic_string_view(const T1& x1, const T2& x2, const T3& x3, const T4& x4) : + m_string(x1, x2, x3, x4) {} + + //@} + + /** Move-in constructor. + */ + explicit op_basic_string_view(move_in_wrapper w) + : m_string(w.value().get_allocator()) + { + m_string.swap(w.value()); + } + + /** @name @ref reducer support. + */ + //@{ + + void view_move_in(string_type& s) + { + m_list.clear(); + if (m_string.get_allocator() == s.get_allocator()) + // Equal allocators. Do a (fast) swap. + m_string.swap(s); + else + // Unequal allocators. Do a (slow) copy. + m_string = s; + s.clear(); + } + + void view_move_out(string_type& s) + { + flatten(); + if (m_string.get_allocator() == s.get_allocator()) + // Equal allocators. Do a (fast) swap. + m_string.swap(s); + else + // Unequal allocators. Do a (slow) copy. + s = m_string; + m_string.clear(); + } + + void view_set_value(const string_type& s) + { m_list.clear(); m_string = s; } + + string_type const& view_get_value() const + { flatten(); return m_string; } + + typedef string_type const& return_type_for_get_value; + + string_type & view_get_reference() + { flatten(); return m_string; } + + string_type const& view_get_reference() const + { flatten(); return m_string; } + + //@} + + /** @name View modifier operations. + * + * @details These simply wrap the corresponding operations on the underlying string. + */ + //@{ + + template + op_basic_string_view& operator +=(const T& x) + { m_string += x; return *this; } + + template + op_basic_string_view& append(const T1& x1) + { m_string.append(x1); return *this; } + + template + op_basic_string_view& append(const T1& x1, const T2& x2) + { m_string.append(x1, x2); return *this; } + + template + op_basic_string_view& append(const T1& x1, const T2& x2, const T3& x3) + { m_string.append(x1, x2, x3); return *this; } + + void push_back(const Char x) { m_string.push_back(x); } + + //@} +}; + + +/** String append monoid class. Instantiate the cilk::reducer template class + * with an op_basic_string monoid to create a string append reducer class. For + * example, to concatenate a collection of standard strings: + * + * cilk::reducer< cilk::op_basic_string > r; + * + * @tparam Char The string element type (not the string type). + * @tparam Traits The character traits type. + * @tparam Alloc The string allocator type. + * @tparam Align If `false` (the default), reducers instantiated on this + * monoid will be naturally aligned (the Intel Cilk Plus library 1.0 + * behavior). If `true`, reducers instantiated on this monoid + * will be cache-aligned for binary compatibility with + * reducers in Intel Cilk Plus library version 0.9. + * + * @see ReducersString + * @see op_basic_string_view + * @see reducer_basic_string + * @see op_string + * @see op_wstring + */ +template, + typename Alloc = std::allocator, + bool Align = false> +class op_basic_string : + public monoid_with_view< op_basic_string_view, Align > +{ + typedef monoid_with_view< op_basic_string_view, Align > + base; + typedef provisional_guard view_guard; + + Alloc m_allocator; + +public: + + /** View type of the monoid. + */ + typedef typename base::view_type view_type; + + /** Constructor. + * + * There is no default constructor for string monoids, because the + * allocator must always be specified. + * + * @param allocator The list allocator to be used when + * identity-constructing new views. + */ + op_basic_string(const Alloc& allocator = Alloc()) : m_allocator(allocator) + {} + + /** Creates an identity view. + * + * String view identity constructors take the string allocator as an + * argument. + * + * @param v The address of the uninitialized memory in which the view + * will be constructed. + */ + void identity(view_type *v) const + { ::new((void*) v) view_type(m_allocator); } + + /** @name Construct functions + * + * A string append reduction monoid must have a copy of the allocator of + * the leftmost view's string, so that it can use it in the `identity` + * operation. This, in turn, requires that string reduction monoids have a + * specialized `construct()` function. + * + * All string reducer monoid `construct()` functions first construct the + * leftmost view, using the arguments that were passed in from the reducer + * constructor. They then call the view's `get_allocator()` function to + * get the string allocator from the string in the leftmost view, and pass + * that to the monoid constructor. + */ + //@{ + + static void construct(op_basic_string* monoid, view_type* view) + { + view_guard vg( new((void*) view) view_type() ); + vg.confirm_if( + new((void*) monoid) op_basic_string(view->get_allocator()) ); + } + + template + static void construct(op_basic_string* monoid, view_type* view, + const T1& x1) + { + view_guard vg( new((void*) view) view_type(x1) ); + vg.confirm_if( + new((void*) monoid) op_basic_string(view->get_allocator()) ); + } + + template + static void construct(op_basic_string* monoid, view_type* view, + const T1& x1, const T2& x2) + { + view_guard vg( new((void*) view) view_type(x1, x2) ); + vg.confirm_if( + new((void*) monoid) op_basic_string(view->get_allocator()) ); + } + + template + static void construct(op_basic_string* monoid, view_type* view, + const T1& x1, const T2& x2, const T3& x3) + { + view_guard vg( new((void*) view) view_type(x1, x2, x3) ); + vg.confirm_if( + new((void*) monoid) op_basic_string(view->get_allocator()) ); + } + + template + static void construct(op_basic_string* monoid, view_type* view, + const T1& x1, const T2& x2, const T3& x3, + const T4& x4) + { + view_guard vg( new((void*) view) view_type(x1, x2, x3, x4) ); + vg.confirm_if( + new((void*) monoid) op_basic_string(view->get_allocator()) ); + } + + //@} +}; + + +/** Convenience typedef for 8-bit strings + */ +typedef op_basic_string op_string; + +/** Convenience typedef for 16-bit strings + */ +typedef op_basic_string op_wstring; + + +/** Deprecated string append reducer class. + * + * reducer_basic_string is the same as @ref reducer<@ref op_basic_string>, + * except that reducer_basic_string is a proxy for the contained view, so that + * accumulator variable update operations can be applied directly to the + * reducer. For example, a value is appended to a `reducer<%op_basic_string>` + * with `r->push_back(a)`, but a value can be appended to a `%reducer_opand` + * with `r.push_back(a)`. + * + * @deprecated Users are strongly encouraged to use `reducer` + * reducers rather than the old wrappers like reducer_basic_string. + * The `reducer` reducers show the reducer/monoid/view + * architecture more clearly, are more consistent in their + * implementation, and present a simpler model for new + * user-implemented reducers. + * + * @note Implicit conversions are provided between `%reducer_basic_string` + * and `reducer<%op_basic_string>`. This allows incremental code + * conversion: old code that used `%reducer_basic_string` can pass a + * `%reducer_basic_string` to a converted function that now expects a + * pointer or reference to a `reducer<%op_basic_string>`, and vice + * versa. + * + * @tparam Char The string element type (not the string type). + * @tparam Traits The character traits type. + * @tparam Alloc The string allocator type. + * + * @see op_basic_string + * @see reducer + * @see ReducersString + */ +template, + typename Alloc = std::allocator > +class reducer_basic_string : + public reducer< op_basic_string > +{ + typedef reducer< op_basic_string > base; + using base::view; +public: + + /// The reducer's string type. + typedef typename base::value_type string_type; + + /// The reducer's primitive component type. + typedef Char basic_value_type; + + /// The string size type. + typedef typename string_type::size_type size_type; + + /// The view type for the reducer. + typedef typename base::view_type View; + + /// The monoid type for the reducer. + typedef typename base::monoid_type Monoid; + + + /** @name Constructors + */ + //@{ + + /** @name Forward constructor calls to the base class. + * + * All basic_string constructor forms are supported. + */ + //@{ + reducer_basic_string() {} + + template + reducer_basic_string(const T1& x1) : + base(x1) {} + + template + reducer_basic_string(const T1& x1, const T2& x2) : + base(x1, x2) {} + + template + reducer_basic_string(const T1& x1, const T2& x2, const T3& x3) : + base(x1, x2, x3) {} + + template + reducer_basic_string(const T1& x1, const T2& x2, const T3& x3, const T4& x4) : + base(x1, x2, x3, x4) {} + //@} + + /** Allows mutable access to the string within the current view. + * + * @warning If this method is called before the parallel calculation is + * complete, the string returned by this method will be a + * partial result. + * + * @returns A mutable reference to the string within the current view. + */ + string_type &get_reference() + { return view().view_get_reference(); } + + /** Allows read-only access to the string within the current view. + * + * @warning If this method is called before the parallel calculation is + * complete, the string returned by this method will be a + * partial result. + * + * @returns A const reference to the string within the current view. + */ + string_type const &get_reference() const + { return view().view_get_reference(); } + + /** @name Appends to the string. + * + * These operations are simply forwarded to the view. + */ + //@{ + void append(const Char *ptr) + { view().append(ptr); } + void append(const Char *ptr, size_type count) + { view().append(ptr, count); } + void append(const string_type &str, size_type offset, size_type count) + { view().append(str, offset, count); } + void append(const string_type &str) + { view().append(str); } + void append(size_type count, Char ch) + { view().append(count, ch); } + + // Appends to the string + reducer_basic_string &operator+=(Char ch) + { view() += ch; return *this; } + reducer_basic_string &operator+=(const Char *ptr) + { view() += ptr; return *this; } + reducer_basic_string &operator+=(const string_type &right) + { view() += right; return *this; } + //@} + + /** @name Dereference + * @details Dereferencing a wrapper is a no-op. It simply returns the + * wrapper. Combined with the rule that the wrapper forwards view + * operations to its contained view, this means that view operations can + * be written the same way on reducers and wrappers, which is convenient + * for incrementally converting old code using wrappers to use reducers + * instead. That is: + * + * reducer r; + * r->push_back(a); // r-> returns the view + * // push_back() is a view member function + * + * reducer_string w; + * w->push_back(a); // *w returns the wrapper + * // push_back() is a wrapper member function + * // that calls the corresponding view function + */ + //@{ + reducer_basic_string& operator*() { return *this; } + reducer_basic_string const& operator*() const { return *this; } + + reducer_basic_string* operator->() { return this; } + reducer_basic_string const* operator->() const { return this; } + //@} + + /** @name Upcast + * @details In Intel Cilk Plus library 0.9, reducers were always cache-aligned. + * In library 1.0, reducer cache alignment is optional. By default, + * reducers are unaligned (i.e., just naturally aligned), but legacy + * wrappers inherit from cache-aligned reducers for binary compatibility. + * + * This means that a wrapper will automatically be upcast to its aligned + * reducer base class. The following conversion operators provide + * pseudo-upcasts to the corresponding unaligned reducer class. + */ + //@{ + operator reducer< op_basic_string >& () + { + return *reinterpret_cast< reducer< + op_basic_string >* + >(this); + } + operator const reducer< op_basic_string >& () const + { + return *reinterpret_cast< const reducer< + op_basic_string >* + >(this); + } + //@} +}; + + +/** Convenience typedef for 8-bit strings + */ +typedef reducer_basic_string reducer_string; + +/** Convenience typedef for 16-bit strings + */ +typedef reducer_basic_string reducer_wstring; + +/// @cond internal + +/// @cond internal +/** Metafunction specialization for reducer conversion. + * + * This specialization of the @ref legacy_reducer_downcast template class + * defined in reducer.h causes the `reducer< op_basic_string >` class to + * have an `operator reducer_basic_string& ()` conversion operator that + * statically downcasts the `reducer` to the corresponding + * `reducer_basic_string` type. (The reverse conversion, from + * `reducer_basic_string` to `reducer`, is just an upcast, + * which is provided for free by the language.) + * + * @ingroup ReducersString + */ +template +struct legacy_reducer_downcast< + reducer > > +{ + typedef reducer_basic_string type; +}; + +/// @endcond + +//@} + +} // namespace cilk + +#endif // REDUCER_STRING_H_INCLUDED diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/clflushoptintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/clflushoptintrin.h new file mode 100644 index 0000000000..1d56b65e31 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/clflushoptintrin.h @@ -0,0 +1,49 @@ +/* Copyright (C) 2013-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#if !defined _X86INTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _CLFLUSHOPTINTRIN_H_INCLUDED +#define _CLFLUSHOPTINTRIN_H_INCLUDED + +#ifndef __CLFLUSHOPT__ +#pragma GCC push_options +#pragma GCC target("clflushopt") +#define __DISABLE_CLFLUSHOPT__ +#endif /* __CLFLUSHOPT__ */ + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_clflushopt (void *__A) +{ + __builtin_ia32_clflushopt (__A); +} + +#ifdef __DISABLE_CLFLUSHOPT__ +#undef __DISABLE_CLFLUSHOPT__ +#pragma GCC pop_options +#endif /* __DISABLE_CLFLUSHOPT__ */ + +#endif /* _CLFLUSHOPTINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/clwbintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/clwbintrin.h new file mode 100644 index 0000000000..37ba9108eb --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/clwbintrin.h @@ -0,0 +1,49 @@ +/* Copyright (C) 2013-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#if !defined _X86INTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _CLWBINTRIN_H_INCLUDED +#define _CLWBINTRIN_H_INCLUDED + +#ifndef __CLWB__ +#pragma GCC push_options +#pragma GCC target("clwb") +#define __DISABLE_CLWB__ +#endif /* __CLWB__ */ + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_clwb (void *__A) +{ + __builtin_ia32_clwb (__A); +} + +#ifdef __DISABLE_CLWB__ +#undef __DISABLE_CLWB__ +#pragma GCC pop_options +#endif /* __DISABLE_CLWB__ */ + +#endif /* _CLWBINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/clzerointrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/clzerointrin.h new file mode 100644 index 0000000000..dc72b1bec9 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/clzerointrin.h @@ -0,0 +1,44 @@ +/* Copyright (C) 2012-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _CLZEROINTRIN_H_INCLUDED +#define _CLZEROINTRIN_H_INCLUDED + +#ifndef __CLZERO__ +#pragma GCC push_options +#pragma GCC target("clzero") +#define __DISABLE_CLZERO__ +#endif /* __CLZERO__ */ + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_clzero (void * __I) +{ + __builtin_ia32_clzero (__I); +} + +#ifdef __DISABLE_CLZERO__ +#undef __DISABLE_CLZERO__ +#pragma GCC pop_options +#endif /* __DISABLE_CLZERO__ */ + +#endif /* _CLZEROINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/cpuid.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cpuid.h new file mode 100644 index 0000000000..b3b0f912c9 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cpuid.h @@ -0,0 +1,273 @@ +/* + * Copyright (C) 2007-2017 Free Software Foundation, Inc. + * + * This file is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 3, or (at your option) any + * later version. + * + * This file is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * Under Section 7 of GPL version 3, you are granted additional + * permissions described in the GCC Runtime Library Exception, version + * 3.1, as published by the Free Software Foundation. + * + * You should have received a copy of the GNU General Public License and + * a copy of the GCC Runtime Library Exception along with this program; + * see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + * . + */ + +/* %ecx */ +#define bit_SSE3 (1 << 0) +#define bit_PCLMUL (1 << 1) +#define bit_LZCNT (1 << 5) +#define bit_SSSE3 (1 << 9) +#define bit_FMA (1 << 12) +#define bit_CMPXCHG16B (1 << 13) +#define bit_SSE4_1 (1 << 19) +#define bit_SSE4_2 (1 << 20) +#define bit_MOVBE (1 << 22) +#define bit_POPCNT (1 << 23) +#define bit_AES (1 << 25) +#define bit_XSAVE (1 << 26) +#define bit_OSXSAVE (1 << 27) +#define bit_AVX (1 << 28) +#define bit_F16C (1 << 29) +#define bit_RDRND (1 << 30) + +/* %edx */ +#define bit_CMPXCHG8B (1 << 8) +#define bit_CMOV (1 << 15) +#define bit_MMX (1 << 23) +#define bit_FXSAVE (1 << 24) +#define bit_SSE (1 << 25) +#define bit_SSE2 (1 << 26) + +/* Extended Features (%eax == 0x80000001) */ +/* %ecx */ +#define bit_LAHF_LM (1 << 0) +#define bit_ABM (1 << 5) +#define bit_SSE4a (1 << 6) +#define bit_PRFCHW (1 << 8) +#define bit_XOP (1 << 11) +#define bit_LWP (1 << 15) +#define bit_FMA4 (1 << 16) +#define bit_TBM (1 << 21) +#define bit_MWAITX (1 << 29) + +/* %edx */ +#define bit_MMXEXT (1 << 22) +#define bit_LM (1 << 29) +#define bit_3DNOWP (1 << 30) +#define bit_3DNOW (1u << 31) + +/* %ebx */ +#define bit_CLZERO (1 << 0) + +/* Extended Features (%eax == 7) */ +/* %ebx */ +#define bit_FSGSBASE (1 << 0) +#define bit_SGX (1 << 2) +#define bit_BMI (1 << 3) +#define bit_HLE (1 << 4) +#define bit_AVX2 (1 << 5) +#define bit_BMI2 (1 << 8) +#define bit_RTM (1 << 11) +#define bit_MPX (1 << 14) +#define bit_AVX512F (1 << 16) +#define bit_AVX512DQ (1 << 17) +#define bit_RDSEED (1 << 18) +#define bit_ADX (1 << 19) +#define bit_AVX512IFMA (1 << 21) +#define bit_CLFLUSHOPT (1 << 23) +#define bit_CLWB (1 << 24) +#define bit_AVX512PF (1 << 26) +#define bit_AVX512ER (1 << 27) +#define bit_AVX512CD (1 << 28) +#define bit_SHA (1 << 29) +#define bit_AVX512BW (1 << 30) +#define bit_AVX512VL (1u << 31) + +/* %ecx */ +#define bit_PREFETCHWT1 (1 << 0) +#define bit_AVX512VBMI (1 << 1) +#define bit_PKU (1 << 3) +#define bit_OSPKE (1 << 4) +#define bit_AVX512VPOPCNTDQ (1 << 14) +#define bit_RDPID (1 << 22) + +/* %edx */ +#define bit_AVX5124VNNIW (1 << 2) +#define bit_AVX5124FMAPS (1 << 3) + +/* XFEATURE_ENABLED_MASK register bits (%eax == 13, %ecx == 0) */ +#define bit_BNDREGS (1 << 3) +#define bit_BNDCSR (1 << 4) + +/* Extended State Enumeration Sub-leaf (%eax == 13, %ecx == 1) */ +#define bit_XSAVEOPT (1 << 0) +#define bit_XSAVEC (1 << 1) +#define bit_XSAVES (1 << 3) + +/* Signatures for different CPU implementations as returned in uses + of cpuid with level 0. */ +#define signature_AMD_ebx 0x68747541 +#define signature_AMD_ecx 0x444d4163 +#define signature_AMD_edx 0x69746e65 + +#define signature_CENTAUR_ebx 0x746e6543 +#define signature_CENTAUR_ecx 0x736c7561 +#define signature_CENTAUR_edx 0x48727561 + +#define signature_CYRIX_ebx 0x69727943 +#define signature_CYRIX_ecx 0x64616574 +#define signature_CYRIX_edx 0x736e4978 + +#define signature_INTEL_ebx 0x756e6547 +#define signature_INTEL_ecx 0x6c65746e +#define signature_INTEL_edx 0x49656e69 + +#define signature_TM1_ebx 0x6e617254 +#define signature_TM1_ecx 0x55504361 +#define signature_TM1_edx 0x74656d73 + +#define signature_TM2_ebx 0x756e6547 +#define signature_TM2_ecx 0x3638784d +#define signature_TM2_edx 0x54656e69 + +#define signature_NSC_ebx 0x646f6547 +#define signature_NSC_ecx 0x43534e20 +#define signature_NSC_edx 0x79622065 + +#define signature_NEXGEN_ebx 0x4778654e +#define signature_NEXGEN_ecx 0x6e657669 +#define signature_NEXGEN_edx 0x72446e65 + +#define signature_RISE_ebx 0x65736952 +#define signature_RISE_ecx 0x65736952 +#define signature_RISE_edx 0x65736952 + +#define signature_SIS_ebx 0x20536953 +#define signature_SIS_ecx 0x20536953 +#define signature_SIS_edx 0x20536953 + +#define signature_UMC_ebx 0x20434d55 +#define signature_UMC_ecx 0x20434d55 +#define signature_UMC_edx 0x20434d55 + +#define signature_VIA_ebx 0x20414956 +#define signature_VIA_ecx 0x20414956 +#define signature_VIA_edx 0x20414956 + +#define signature_VORTEX_ebx 0x74726f56 +#define signature_VORTEX_ecx 0x436f5320 +#define signature_VORTEX_edx 0x36387865 + +#define __cpuid(level, a, b, c, d) \ + __asm__ ("cpuid\n\t" \ + : "=a" (a), "=b" (b), "=c" (c), "=d" (d) \ + : "0" (level)) + +#define __cpuid_count(level, count, a, b, c, d) \ + __asm__ ("cpuid\n\t" \ + : "=a" (a), "=b" (b), "=c" (c), "=d" (d) \ + : "0" (level), "2" (count)) + + +/* Return highest supported input value for cpuid instruction. ext can + be either 0x0 or 0x80000000 to return highest supported value for + basic or extended cpuid information. Function returns 0 if cpuid + is not supported or whatever cpuid returns in eax register. If sig + pointer is non-null, then first four bytes of the signature + (as found in ebx register) are returned in location pointed by sig. */ + +static __inline unsigned int +__get_cpuid_max (unsigned int __ext, unsigned int *__sig) +{ + unsigned int __eax, __ebx, __ecx, __edx; + +#ifndef __x86_64__ + /* See if we can use cpuid. On AMD64 we always can. */ +#if __GNUC__ >= 3 + __asm__ ("pushf{l|d}\n\t" + "pushf{l|d}\n\t" + "pop{l}\t%0\n\t" + "mov{l}\t{%0, %1|%1, %0}\n\t" + "xor{l}\t{%2, %0|%0, %2}\n\t" + "push{l}\t%0\n\t" + "popf{l|d}\n\t" + "pushf{l|d}\n\t" + "pop{l}\t%0\n\t" + "popf{l|d}\n\t" + : "=&r" (__eax), "=&r" (__ebx) + : "i" (0x00200000)); +#else +/* Host GCCs older than 3.0 weren't supporting Intel asm syntax + nor alternatives in i386 code. */ + __asm__ ("pushfl\n\t" + "pushfl\n\t" + "popl\t%0\n\t" + "movl\t%0, %1\n\t" + "xorl\t%2, %0\n\t" + "pushl\t%0\n\t" + "popfl\n\t" + "pushfl\n\t" + "popl\t%0\n\t" + "popfl\n\t" + : "=&r" (__eax), "=&r" (__ebx) + : "i" (0x00200000)); +#endif + + if (!((__eax ^ __ebx) & 0x00200000)) + return 0; +#endif + + /* Host supports cpuid. Return highest supported cpuid input value. */ + __cpuid (__ext, __eax, __ebx, __ecx, __edx); + + if (__sig) + *__sig = __ebx; + + return __eax; +} + +/* Return cpuid data for requested cpuid leaf, as found in returned + eax, ebx, ecx and edx registers. The function checks if cpuid is + supported and returns 1 for valid cpuid information or 0 for + unsupported cpuid leaf. All pointers are required to be non-null. */ + +static __inline int +__get_cpuid (unsigned int __leaf, + unsigned int *__eax, unsigned int *__ebx, + unsigned int *__ecx, unsigned int *__edx) +{ + unsigned int __ext = __leaf & 0x80000000; + unsigned int __maxlevel = __get_cpuid_max (__ext, 0); + + if (__maxlevel == 0 || __maxlevel < __leaf) + return 0; + + __cpuid (__leaf, *__eax, *__ebx, *__ecx, *__edx); + return 1; +} + +/* Same as above, but sub-leaf can be specified. */ + +static __inline int +__get_cpuid_count (unsigned int __leaf, unsigned int __subleaf, + unsigned int *__eax, unsigned int *__ebx, + unsigned int *__ecx, unsigned int *__edx) +{ + unsigned int __ext = __leaf & 0x80000000; + unsigned int __maxlevel = __get_cpuid_max (__ext, 0); + + if (__maxlevel == 0 || __maxlevel < __leaf) + return 0; + + __cpuid_count (__leaf, __subleaf, *__eax, *__ebx, *__ecx, *__edx); + return 1; +} diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/cross-stdarg.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cross-stdarg.h new file mode 100644 index 0000000000..67a2150a9d --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/cross-stdarg.h @@ -0,0 +1,72 @@ +/* Copyright (C) 2002-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef __CROSS_STDARG_H_INCLUDED +#define __CROSS_STDARG_H_INCLUDED + +/* Make sure that for non x64 targets cross builtins are defined. */ +#ifndef __x86_64__ +/* Call abi ms_abi. */ +#define __builtin_ms_va_list __builtin_va_list +#define __builtin_ms_va_copy __builtin_va_copy +#define __builtin_ms_va_start __builtin_va_start +#define __builtin_ms_va_end __builtin_va_end + +/* Call abi sysv_abi. */ +#define __builtin_sysv_va_list __builtin_va_list +#define __builtin_sysv_va_copy __builtin_va_copy +#define __builtin_sysv_va_start __builtin_va_start +#define __builtin_sysv_va_end __builtin_va_end +#endif + +#define __ms_va_copy(__d,__s) __builtin_ms_va_copy(__d,__s) +#define __ms_va_start(__v,__l) __builtin_ms_va_start(__v,__l) +#define __ms_va_arg(__v,__l) __builtin_va_arg(__v,__l) +#define __ms_va_end(__v) __builtin_ms_va_end(__v) + +#define __sysv_va_copy(__d,__s) __builtin_sysv_va_copy(__d,__s) +#define __sysv_va_start(__v,__l) __builtin_sysv_va_start(__v,__l) +#define __sysv_va_arg(__v,__l) __builtin_va_arg(__v,__l) +#define __sysv_va_end(__v) __builtin_sysv_va_end(__v) + +#ifndef __GNUC_SYSV_VA_LIST +#define __GNUC_SYSV_VA_LIST + typedef __builtin_sysv_va_list __gnuc_sysv_va_list; +#endif + +#ifndef _SYSV_VA_LIST_DEFINED +#define _SYSV_VA_LIST_DEFINED + typedef __gnuc_sysv_va_list sysv_va_list; +#endif + +#ifndef __GNUC_MS_VA_LIST +#define __GNUC_MS_VA_LIST + typedef __builtin_ms_va_list __gnuc_ms_va_list; +#endif + +#ifndef _MS_VA_LIST_DEFINED +#define _MS_VA_LIST_DEFINED + typedef __gnuc_ms_va_list ms_va_list; +#endif + +#endif /* __CROSS_STDARG_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/emmintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/emmintrin.h new file mode 100644 index 0000000000..828f4a07a9 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/emmintrin.h @@ -0,0 +1,1565 @@ +/* Copyright (C) 2003-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +/* Implemented from the specification included in the Intel C++ Compiler + User Guide and Reference, version 9.0. */ + +#ifndef _EMMINTRIN_H_INCLUDED +#define _EMMINTRIN_H_INCLUDED + +/* We need definitions from the SSE header files*/ +#include + +#ifndef __SSE2__ +#pragma GCC push_options +#pragma GCC target("sse2") +#define __DISABLE_SSE2__ +#endif /* __SSE2__ */ + +/* SSE2 */ +typedef double __v2df __attribute__ ((__vector_size__ (16))); +typedef long long __v2di __attribute__ ((__vector_size__ (16))); +typedef unsigned long long __v2du __attribute__ ((__vector_size__ (16))); +typedef int __v4si __attribute__ ((__vector_size__ (16))); +typedef unsigned int __v4su __attribute__ ((__vector_size__ (16))); +typedef short __v8hi __attribute__ ((__vector_size__ (16))); +typedef unsigned short __v8hu __attribute__ ((__vector_size__ (16))); +typedef char __v16qi __attribute__ ((__vector_size__ (16))); +typedef unsigned char __v16qu __attribute__ ((__vector_size__ (16))); + +/* The Intel API is flexible enough that we must allow aliasing with other + vector types, and their scalar components. */ +typedef long long __m128i __attribute__ ((__vector_size__ (16), __may_alias__)); +typedef double __m128d __attribute__ ((__vector_size__ (16), __may_alias__)); + +/* Unaligned version of the same types. */ +typedef long long __m128i_u __attribute__ ((__vector_size__ (16), __may_alias__, __aligned__ (1))); +typedef double __m128d_u __attribute__ ((__vector_size__ (16), __may_alias__, __aligned__ (1))); + +/* Create a selector for use with the SHUFPD instruction. */ +#define _MM_SHUFFLE2(fp1,fp0) \ + (((fp1) << 1) | (fp0)) + +/* Create a vector with element 0 as F and the rest zero. */ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set_sd (double __F) +{ + return __extension__ (__m128d){ __F, 0.0 }; +} + +/* Create a vector with both elements equal to F. */ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set1_pd (double __F) +{ + return __extension__ (__m128d){ __F, __F }; +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set_pd1 (double __F) +{ + return _mm_set1_pd (__F); +} + +/* Create a vector with the lower value X and upper value W. */ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set_pd (double __W, double __X) +{ + return __extension__ (__m128d){ __X, __W }; +} + +/* Create a vector with the lower value W and upper value X. */ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_setr_pd (double __W, double __X) +{ + return __extension__ (__m128d){ __W, __X }; +} + +/* Create an undefined vector. */ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_undefined_pd (void) +{ + __m128d __Y = __Y; + return __Y; +} + +/* Create a vector of zeros. */ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_setzero_pd (void) +{ + return __extension__ (__m128d){ 0.0, 0.0 }; +} + +/* Sets the low DPFP value of A from the low value of B. */ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_move_sd (__m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_movsd ((__v2df)__A, (__v2df)__B); +} + +/* Load two DPFP values from P. The address must be 16-byte aligned. */ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_load_pd (double const *__P) +{ + return *(__m128d *)__P; +} + +/* Load two DPFP values from P. The address need not be 16-byte aligned. */ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_loadu_pd (double const *__P) +{ + return *(__m128d_u *)__P; +} + +/* Create a vector with all two elements equal to *P. */ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_load1_pd (double const *__P) +{ + return _mm_set1_pd (*__P); +} + +/* Create a vector with element 0 as *P and the rest zero. */ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_load_sd (double const *__P) +{ + return _mm_set_sd (*__P); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_load_pd1 (double const *__P) +{ + return _mm_load1_pd (__P); +} + +/* Load two DPFP values in reverse order. The address must be aligned. */ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_loadr_pd (double const *__P) +{ + __m128d __tmp = _mm_load_pd (__P); + return __builtin_ia32_shufpd (__tmp, __tmp, _MM_SHUFFLE2 (0,1)); +} + +/* Store two DPFP values. The address must be 16-byte aligned. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_store_pd (double *__P, __m128d __A) +{ + *(__m128d *)__P = __A; +} + +/* Store two DPFP values. The address need not be 16-byte aligned. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_storeu_pd (double *__P, __m128d __A) +{ + *(__m128d_u *)__P = __A; +} + +/* Stores the lower DPFP value. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_store_sd (double *__P, __m128d __A) +{ + *__P = ((__v2df)__A)[0]; +} + +extern __inline double __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsd_f64 (__m128d __A) +{ + return ((__v2df)__A)[0]; +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_storel_pd (double *__P, __m128d __A) +{ + _mm_store_sd (__P, __A); +} + +/* Stores the upper DPFP value. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_storeh_pd (double *__P, __m128d __A) +{ + *__P = ((__v2df)__A)[1]; +} + +/* Store the lower DPFP value across two words. + The address must be 16-byte aligned. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_store1_pd (double *__P, __m128d __A) +{ + _mm_store_pd (__P, __builtin_ia32_shufpd (__A, __A, _MM_SHUFFLE2 (0,0))); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_store_pd1 (double *__P, __m128d __A) +{ + _mm_store1_pd (__P, __A); +} + +/* Store two DPFP values in reverse order. The address must be aligned. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_storer_pd (double *__P, __m128d __A) +{ + _mm_store_pd (__P, __builtin_ia32_shufpd (__A, __A, _MM_SHUFFLE2 (0,1))); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi128_si32 (__m128i __A) +{ + return __builtin_ia32_vec_ext_v4si ((__v4si)__A, 0); +} + +#ifdef __x86_64__ +/* Intel intrinsic. */ +extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi128_si64 (__m128i __A) +{ + return ((__v2di)__A)[0]; +} + +/* Microsoft intrinsic. */ +extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi128_si64x (__m128i __A) +{ + return ((__v2di)__A)[0]; +} +#endif + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_add_pd (__m128d __A, __m128d __B) +{ + return (__m128d) ((__v2df)__A + (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_add_sd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_addsd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sub_pd (__m128d __A, __m128d __B) +{ + return (__m128d) ((__v2df)__A - (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sub_sd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_subsd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mul_pd (__m128d __A, __m128d __B) +{ + return (__m128d) ((__v2df)__A * (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mul_sd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_mulsd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_div_pd (__m128d __A, __m128d __B) +{ + return (__m128d) ((__v2df)__A / (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_div_sd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_divsd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sqrt_pd (__m128d __A) +{ + return (__m128d)__builtin_ia32_sqrtpd ((__v2df)__A); +} + +/* Return pair {sqrt (B[0]), A[1]}. */ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sqrt_sd (__m128d __A, __m128d __B) +{ + __v2df __tmp = __builtin_ia32_movsd ((__v2df)__A, (__v2df)__B); + return (__m128d)__builtin_ia32_sqrtsd ((__v2df)__tmp); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_min_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_minpd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_min_sd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_minsd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_max_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_maxpd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_max_sd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_maxsd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_and_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_andpd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_andnot_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_andnpd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_or_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_orpd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_xor_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_xorpd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpeqpd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmplt_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpltpd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmple_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmplepd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpgtpd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpge_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpgepd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpneq_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpneqpd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpnlt_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpnltpd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpnle_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpnlepd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpngt_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpngtpd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpnge_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpngepd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpord_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpordpd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpunord_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpunordpd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_sd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpeqsd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmplt_sd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpltsd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmple_sd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmplesd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_sd (__m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_movsd ((__v2df) __A, + (__v2df) + __builtin_ia32_cmpltsd ((__v2df) __B, + (__v2df) + __A)); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpge_sd (__m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_movsd ((__v2df) __A, + (__v2df) + __builtin_ia32_cmplesd ((__v2df) __B, + (__v2df) + __A)); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpneq_sd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpneqsd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpnlt_sd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpnltsd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpnle_sd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpnlesd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpngt_sd (__m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_movsd ((__v2df) __A, + (__v2df) + __builtin_ia32_cmpnltsd ((__v2df) __B, + (__v2df) + __A)); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpnge_sd (__m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_movsd ((__v2df) __A, + (__v2df) + __builtin_ia32_cmpnlesd ((__v2df) __B, + (__v2df) + __A)); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpord_sd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpordsd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpunord_sd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_cmpunordsd ((__v2df)__A, (__v2df)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comieq_sd (__m128d __A, __m128d __B) +{ + return __builtin_ia32_comisdeq ((__v2df)__A, (__v2df)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comilt_sd (__m128d __A, __m128d __B) +{ + return __builtin_ia32_comisdlt ((__v2df)__A, (__v2df)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comile_sd (__m128d __A, __m128d __B) +{ + return __builtin_ia32_comisdle ((__v2df)__A, (__v2df)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comigt_sd (__m128d __A, __m128d __B) +{ + return __builtin_ia32_comisdgt ((__v2df)__A, (__v2df)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comige_sd (__m128d __A, __m128d __B) +{ + return __builtin_ia32_comisdge ((__v2df)__A, (__v2df)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comineq_sd (__m128d __A, __m128d __B) +{ + return __builtin_ia32_comisdneq ((__v2df)__A, (__v2df)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_ucomieq_sd (__m128d __A, __m128d __B) +{ + return __builtin_ia32_ucomisdeq ((__v2df)__A, (__v2df)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_ucomilt_sd (__m128d __A, __m128d __B) +{ + return __builtin_ia32_ucomisdlt ((__v2df)__A, (__v2df)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_ucomile_sd (__m128d __A, __m128d __B) +{ + return __builtin_ia32_ucomisdle ((__v2df)__A, (__v2df)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_ucomigt_sd (__m128d __A, __m128d __B) +{ + return __builtin_ia32_ucomisdgt ((__v2df)__A, (__v2df)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_ucomige_sd (__m128d __A, __m128d __B) +{ + return __builtin_ia32_ucomisdge ((__v2df)__A, (__v2df)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_ucomineq_sd (__m128d __A, __m128d __B) +{ + return __builtin_ia32_ucomisdneq ((__v2df)__A, (__v2df)__B); +} + +/* Create a vector of Qi, where i is the element number. */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set_epi64x (long long __q1, long long __q0) +{ + return __extension__ (__m128i)(__v2di){ __q0, __q1 }; +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set_epi64 (__m64 __q1, __m64 __q0) +{ + return _mm_set_epi64x ((long long)__q1, (long long)__q0); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set_epi32 (int __q3, int __q2, int __q1, int __q0) +{ + return __extension__ (__m128i)(__v4si){ __q0, __q1, __q2, __q3 }; +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set_epi16 (short __q7, short __q6, short __q5, short __q4, + short __q3, short __q2, short __q1, short __q0) +{ + return __extension__ (__m128i)(__v8hi){ + __q0, __q1, __q2, __q3, __q4, __q5, __q6, __q7 }; +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set_epi8 (char __q15, char __q14, char __q13, char __q12, + char __q11, char __q10, char __q09, char __q08, + char __q07, char __q06, char __q05, char __q04, + char __q03, char __q02, char __q01, char __q00) +{ + return __extension__ (__m128i)(__v16qi){ + __q00, __q01, __q02, __q03, __q04, __q05, __q06, __q07, + __q08, __q09, __q10, __q11, __q12, __q13, __q14, __q15 + }; +} + +/* Set all of the elements of the vector to A. */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set1_epi64x (long long __A) +{ + return _mm_set_epi64x (__A, __A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set1_epi64 (__m64 __A) +{ + return _mm_set_epi64 (__A, __A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set1_epi32 (int __A) +{ + return _mm_set_epi32 (__A, __A, __A, __A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set1_epi16 (short __A) +{ + return _mm_set_epi16 (__A, __A, __A, __A, __A, __A, __A, __A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set1_epi8 (char __A) +{ + return _mm_set_epi8 (__A, __A, __A, __A, __A, __A, __A, __A, + __A, __A, __A, __A, __A, __A, __A, __A); +} + +/* Create a vector of Qi, where i is the element number. + The parameter order is reversed from the _mm_set_epi* functions. */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_setr_epi64 (__m64 __q0, __m64 __q1) +{ + return _mm_set_epi64 (__q1, __q0); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_setr_epi32 (int __q0, int __q1, int __q2, int __q3) +{ + return _mm_set_epi32 (__q3, __q2, __q1, __q0); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_setr_epi16 (short __q0, short __q1, short __q2, short __q3, + short __q4, short __q5, short __q6, short __q7) +{ + return _mm_set_epi16 (__q7, __q6, __q5, __q4, __q3, __q2, __q1, __q0); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_setr_epi8 (char __q00, char __q01, char __q02, char __q03, + char __q04, char __q05, char __q06, char __q07, + char __q08, char __q09, char __q10, char __q11, + char __q12, char __q13, char __q14, char __q15) +{ + return _mm_set_epi8 (__q15, __q14, __q13, __q12, __q11, __q10, __q09, __q08, + __q07, __q06, __q05, __q04, __q03, __q02, __q01, __q00); +} + +/* Create a vector with element 0 as *P and the rest zero. */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_load_si128 (__m128i const *__P) +{ + return *__P; +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_loadu_si128 (__m128i_u const *__P) +{ + return *__P; +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_loadl_epi64 (__m128i_u const *__P) +{ + return _mm_set_epi64 ((__m64)0LL, *(__m64_u *)__P); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_store_si128 (__m128i *__P, __m128i __B) +{ + *__P = __B; +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_storeu_si128 (__m128i_u *__P, __m128i __B) +{ + *__P = __B; +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_storel_epi64 (__m128i_u *__P, __m128i __B) +{ + *(__m64_u *)__P = (__m64) ((__v2di)__B)[0]; +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movepi64_pi64 (__m128i __B) +{ + return (__m64) ((__v2di)__B)[0]; +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movpi64_epi64 (__m64 __A) +{ + return _mm_set_epi64 ((__m64)0LL, __A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_move_epi64 (__m128i __A) +{ + return (__m128i)__builtin_ia32_movq128 ((__v2di) __A); +} + +/* Create an undefined vector. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_undefined_si128 (void) +{ + __m128i __Y = __Y; + return __Y; +} + +/* Create a vector of zeros. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_setzero_si128 (void) +{ + return __extension__ (__m128i)(__v4si){ 0, 0, 0, 0 }; +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepi32_pd (__m128i __A) +{ + return (__m128d)__builtin_ia32_cvtdq2pd ((__v4si) __A); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepi32_ps (__m128i __A) +{ + return (__m128)__builtin_ia32_cvtdq2ps ((__v4si) __A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtpd_epi32 (__m128d __A) +{ + return (__m128i)__builtin_ia32_cvtpd2dq ((__v2df) __A); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtpd_pi32 (__m128d __A) +{ + return (__m64)__builtin_ia32_cvtpd2pi ((__v2df) __A); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtpd_ps (__m128d __A) +{ + return (__m128)__builtin_ia32_cvtpd2ps ((__v2df) __A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttpd_epi32 (__m128d __A) +{ + return (__m128i)__builtin_ia32_cvttpd2dq ((__v2df) __A); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttpd_pi32 (__m128d __A) +{ + return (__m64)__builtin_ia32_cvttpd2pi ((__v2df) __A); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtpi32_pd (__m64 __A) +{ + return (__m128d)__builtin_ia32_cvtpi2pd ((__v2si) __A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtps_epi32 (__m128 __A) +{ + return (__m128i)__builtin_ia32_cvtps2dq ((__v4sf) __A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttps_epi32 (__m128 __A) +{ + return (__m128i)__builtin_ia32_cvttps2dq ((__v4sf) __A); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtps_pd (__m128 __A) +{ + return (__m128d)__builtin_ia32_cvtps2pd ((__v4sf) __A); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsd_si32 (__m128d __A) +{ + return __builtin_ia32_cvtsd2si ((__v2df) __A); +} + +#ifdef __x86_64__ +/* Intel intrinsic. */ +extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsd_si64 (__m128d __A) +{ + return __builtin_ia32_cvtsd2si64 ((__v2df) __A); +} + +/* Microsoft intrinsic. */ +extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsd_si64x (__m128d __A) +{ + return __builtin_ia32_cvtsd2si64 ((__v2df) __A); +} +#endif + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttsd_si32 (__m128d __A) +{ + return __builtin_ia32_cvttsd2si ((__v2df) __A); +} + +#ifdef __x86_64__ +/* Intel intrinsic. */ +extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttsd_si64 (__m128d __A) +{ + return __builtin_ia32_cvttsd2si64 ((__v2df) __A); +} + +/* Microsoft intrinsic. */ +extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttsd_si64x (__m128d __A) +{ + return __builtin_ia32_cvttsd2si64 ((__v2df) __A); +} +#endif + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsd_ss (__m128 __A, __m128d __B) +{ + return (__m128)__builtin_ia32_cvtsd2ss ((__v4sf) __A, (__v2df) __B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi32_sd (__m128d __A, int __B) +{ + return (__m128d)__builtin_ia32_cvtsi2sd ((__v2df) __A, __B); +} + +#ifdef __x86_64__ +/* Intel intrinsic. */ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi64_sd (__m128d __A, long long __B) +{ + return (__m128d)__builtin_ia32_cvtsi642sd ((__v2df) __A, __B); +} + +/* Microsoft intrinsic. */ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi64x_sd (__m128d __A, long long __B) +{ + return (__m128d)__builtin_ia32_cvtsi642sd ((__v2df) __A, __B); +} +#endif + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtss_sd (__m128d __A, __m128 __B) +{ + return (__m128d)__builtin_ia32_cvtss2sd ((__v2df) __A, (__v4sf)__B); +} + +#ifdef __OPTIMIZE__ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_shuffle_pd(__m128d __A, __m128d __B, const int __mask) +{ + return (__m128d)__builtin_ia32_shufpd ((__v2df)__A, (__v2df)__B, __mask); +} +#else +#define _mm_shuffle_pd(A, B, N) \ + ((__m128d)__builtin_ia32_shufpd ((__v2df)(__m128d)(A), \ + (__v2df)(__m128d)(B), (int)(N))) +#endif + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpackhi_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_unpckhpd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpacklo_pd (__m128d __A, __m128d __B) +{ + return (__m128d)__builtin_ia32_unpcklpd ((__v2df)__A, (__v2df)__B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_loadh_pd (__m128d __A, double const *__B) +{ + return (__m128d)__builtin_ia32_loadhpd ((__v2df)__A, __B); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_loadl_pd (__m128d __A, double const *__B) +{ + return (__m128d)__builtin_ia32_loadlpd ((__v2df)__A, __B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movemask_pd (__m128d __A) +{ + return __builtin_ia32_movmskpd ((__v2df)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_packs_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_packsswb128 ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_packs_epi32 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_packssdw128 ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_packus_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_packuswb128 ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpackhi_epi8 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_punpckhbw128 ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpackhi_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_punpckhwd128 ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpackhi_epi32 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_punpckhdq128 ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpackhi_epi64 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_punpckhqdq128 ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpacklo_epi8 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_punpcklbw128 ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpacklo_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_punpcklwd128 ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpacklo_epi32 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_punpckldq128 ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpacklo_epi64 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_punpcklqdq128 ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_add_epi8 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v16qu)__A + (__v16qu)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_add_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v8hu)__A + (__v8hu)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_add_epi32 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v4su)__A + (__v4su)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_add_epi64 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v2du)__A + (__v2du)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_adds_epi8 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_paddsb128 ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_adds_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_paddsw128 ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_adds_epu8 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_paddusb128 ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_adds_epu16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_paddusw128 ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sub_epi8 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v16qu)__A - (__v16qu)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sub_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v8hu)__A - (__v8hu)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sub_epi32 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v4su)__A - (__v4su)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sub_epi64 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v2du)__A - (__v2du)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_subs_epi8 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_psubsb128 ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_subs_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_psubsw128 ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_subs_epu8 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_psubusb128 ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_subs_epu16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_psubusw128 ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_madd_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_pmaddwd128 ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mulhi_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_pmulhw128 ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mullo_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v8hu)__A * (__v8hu)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mul_su32 (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pmuludq ((__v2si)__A, (__v2si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mul_epu32 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_pmuludq128 ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_slli_epi16 (__m128i __A, int __B) +{ + return (__m128i)__builtin_ia32_psllwi128 ((__v8hi)__A, __B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_slli_epi32 (__m128i __A, int __B) +{ + return (__m128i)__builtin_ia32_pslldi128 ((__v4si)__A, __B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_slli_epi64 (__m128i __A, int __B) +{ + return (__m128i)__builtin_ia32_psllqi128 ((__v2di)__A, __B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srai_epi16 (__m128i __A, int __B) +{ + return (__m128i)__builtin_ia32_psrawi128 ((__v8hi)__A, __B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srai_epi32 (__m128i __A, int __B) +{ + return (__m128i)__builtin_ia32_psradi128 ((__v4si)__A, __B); +} + +#ifdef __OPTIMIZE__ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_bsrli_si128 (__m128i __A, const int __N) +{ + return (__m128i)__builtin_ia32_psrldqi128 (__A, __N * 8); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_bslli_si128 (__m128i __A, const int __N) +{ + return (__m128i)__builtin_ia32_pslldqi128 (__A, __N * 8); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srli_si128 (__m128i __A, const int __N) +{ + return (__m128i)__builtin_ia32_psrldqi128 (__A, __N * 8); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_slli_si128 (__m128i __A, const int __N) +{ + return (__m128i)__builtin_ia32_pslldqi128 (__A, __N * 8); +} +#else +#define _mm_bsrli_si128(A, N) \ + ((__m128i)__builtin_ia32_psrldqi128 ((__m128i)(A), (int)(N) * 8)) +#define _mm_bslli_si128(A, N) \ + ((__m128i)__builtin_ia32_pslldqi128 ((__m128i)(A), (int)(N) * 8)) +#define _mm_srli_si128(A, N) \ + ((__m128i)__builtin_ia32_psrldqi128 ((__m128i)(A), (int)(N) * 8)) +#define _mm_slli_si128(A, N) \ + ((__m128i)__builtin_ia32_pslldqi128 ((__m128i)(A), (int)(N) * 8)) +#endif + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srli_epi16 (__m128i __A, int __B) +{ + return (__m128i)__builtin_ia32_psrlwi128 ((__v8hi)__A, __B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srli_epi32 (__m128i __A, int __B) +{ + return (__m128i)__builtin_ia32_psrldi128 ((__v4si)__A, __B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srli_epi64 (__m128i __A, int __B) +{ + return (__m128i)__builtin_ia32_psrlqi128 ((__v2di)__A, __B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sll_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_psllw128((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sll_epi32 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_pslld128((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sll_epi64 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_psllq128((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sra_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_psraw128 ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sra_epi32 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_psrad128 ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srl_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_psrlw128 ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srl_epi32 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_psrld128 ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srl_epi64 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_psrlq128 ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_and_si128 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v2du)__A & (__v2du)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_andnot_si128 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_pandn128 ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_or_si128 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v2du)__A | (__v2du)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_xor_si128 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v2du)__A ^ (__v2du)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_epi8 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v16qi)__A == (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v8hi)__A == (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_epi32 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v4si)__A == (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmplt_epi8 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v16qi)__A < (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmplt_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v8hi)__A < (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmplt_epi32 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v4si)__A < (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_epi8 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v16qi)__A > (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v8hi)__A > (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_epi32 (__m128i __A, __m128i __B) +{ + return (__m128i) ((__v4si)__A > (__v4si)__B); +} + +#ifdef __OPTIMIZE__ +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_extract_epi16 (__m128i const __A, int const __N) +{ + return (unsigned short) __builtin_ia32_vec_ext_v8hi ((__v8hi)__A, __N); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_insert_epi16 (__m128i const __A, int const __D, int const __N) +{ + return (__m128i) __builtin_ia32_vec_set_v8hi ((__v8hi)__A, __D, __N); +} +#else +#define _mm_extract_epi16(A, N) \ + ((int) (unsigned short) __builtin_ia32_vec_ext_v8hi ((__v8hi)(__m128i)(A), (int)(N))) +#define _mm_insert_epi16(A, D, N) \ + ((__m128i) __builtin_ia32_vec_set_v8hi ((__v8hi)(__m128i)(A), \ + (int)(D), (int)(N))) +#endif + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_max_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_pmaxsw128 ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_max_epu8 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_pmaxub128 ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_min_epi16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_pminsw128 ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_min_epu8 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_pminub128 ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movemask_epi8 (__m128i __A) +{ + return __builtin_ia32_pmovmskb128 ((__v16qi)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mulhi_epu16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_pmulhuw128 ((__v8hi)__A, (__v8hi)__B); +} + +#ifdef __OPTIMIZE__ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_shufflehi_epi16 (__m128i __A, const int __mask) +{ + return (__m128i)__builtin_ia32_pshufhw ((__v8hi)__A, __mask); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_shufflelo_epi16 (__m128i __A, const int __mask) +{ + return (__m128i)__builtin_ia32_pshuflw ((__v8hi)__A, __mask); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_shuffle_epi32 (__m128i __A, const int __mask) +{ + return (__m128i)__builtin_ia32_pshufd ((__v4si)__A, __mask); +} +#else +#define _mm_shufflehi_epi16(A, N) \ + ((__m128i)__builtin_ia32_pshufhw ((__v8hi)(__m128i)(A), (int)(N))) +#define _mm_shufflelo_epi16(A, N) \ + ((__m128i)__builtin_ia32_pshuflw ((__v8hi)(__m128i)(A), (int)(N))) +#define _mm_shuffle_epi32(A, N) \ + ((__m128i)__builtin_ia32_pshufd ((__v4si)(__m128i)(A), (int)(N))) +#endif + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskmoveu_si128 (__m128i __A, __m128i __B, char *__C) +{ + __builtin_ia32_maskmovdqu ((__v16qi)__A, (__v16qi)__B, __C); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_avg_epu8 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_pavgb128 ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_avg_epu16 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_pavgw128 ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sad_epu8 (__m128i __A, __m128i __B) +{ + return (__m128i)__builtin_ia32_psadbw128 ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_stream_si32 (int *__A, int __B) +{ + __builtin_ia32_movnti (__A, __B); +} + +#ifdef __x86_64__ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_stream_si64 (long long int *__A, long long int __B) +{ + __builtin_ia32_movnti64 (__A, __B); +} +#endif + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_stream_si128 (__m128i *__A, __m128i __B) +{ + __builtin_ia32_movntdq ((__v2di *)__A, (__v2di)__B); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_stream_pd (double *__A, __m128d __B) +{ + __builtin_ia32_movntpd (__A, (__v2df)__B); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_clflush (void const *__A) +{ + __builtin_ia32_clflush (__A); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_lfence (void) +{ + __builtin_ia32_lfence (); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mfence (void) +{ + __builtin_ia32_mfence (); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi32_si128 (int __A) +{ + return _mm_set_epi32 (0, 0, 0, __A); +} + +#ifdef __x86_64__ +/* Intel intrinsic. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi64_si128 (long long __A) +{ + return _mm_set_epi64x (0, __A); +} + +/* Microsoft intrinsic. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi64x_si128 (long long __A) +{ + return _mm_set_epi64x (0, __A); +} +#endif + +/* Casts between various SP, DP, INT vector types. Note that these do no + conversion of values, they just change the type. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_castpd_ps(__m128d __A) +{ + return (__m128) __A; +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_castpd_si128(__m128d __A) +{ + return (__m128i) __A; +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_castps_pd(__m128 __A) +{ + return (__m128d) __A; +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_castps_si128(__m128 __A) +{ + return (__m128i) __A; +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_castsi128_ps(__m128i __A) +{ + return (__m128) __A; +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_castsi128_pd(__m128i __A) +{ + return (__m128d) __A; +} + +#ifdef __DISABLE_SSE2__ +#undef __DISABLE_SSE2__ +#pragma GCC pop_options +#endif /* __DISABLE_SSE2__ */ + +#endif /* _EMMINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/f16cintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/f16cintrin.h new file mode 100644 index 0000000000..488e97d674 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/f16cintrin.h @@ -0,0 +1,98 @@ +/* Copyright (C) 2011-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#if !defined _X86INTRIN_H_INCLUDED && !defined _IMMINTRIN_H_INCLUDED +# error "Never use directly; include or instead." +#endif + +#ifndef _F16CINTRIN_H_INCLUDED +#define _F16CINTRIN_H_INCLUDED + +#ifndef __F16C__ +#pragma GCC push_options +#pragma GCC target("f16c") +#define __DISABLE_F16C__ +#endif /* __F16C__ */ + +extern __inline float __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_cvtsh_ss (unsigned short __S) +{ + __v8hi __H = __extension__ (__v8hi){ (short) __S, 0, 0, 0, 0, 0, 0, 0 }; + __v4sf __A = __builtin_ia32_vcvtph2ps (__H); + return __builtin_ia32_vec_ext_v4sf (__A, 0); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtph_ps (__m128i __A) +{ + return (__m128) __builtin_ia32_vcvtph2ps ((__v8hi) __A); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtph_ps (__m128i __A) +{ + return (__m256) __builtin_ia32_vcvtph2ps256 ((__v8hi) __A); +} + +#ifdef __OPTIMIZE__ +extern __inline unsigned short __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_cvtss_sh (float __F, const int __I) +{ + __v4sf __A = __extension__ (__v4sf){ __F, 0, 0, 0 }; + __v8hi __H = __builtin_ia32_vcvtps2ph (__A, __I); + return (unsigned short) __builtin_ia32_vec_ext_v8hi (__H, 0); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtps_ph (__m128 __A, const int __I) +{ + return (__m128i) __builtin_ia32_vcvtps2ph ((__v4sf) __A, __I); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_cvtps_ph (__m256 __A, const int __I) +{ + return (__m128i) __builtin_ia32_vcvtps2ph256 ((__v8sf) __A, __I); +} +#else +#define _cvtss_sh(__F, __I) \ + (__extension__ \ + ({ \ + __v4sf __A = __extension__ (__v4sf){ __F, 0, 0, 0 }; \ + __v8hi __H = __builtin_ia32_vcvtps2ph (__A, __I); \ + (unsigned short) __builtin_ia32_vec_ext_v8hi (__H, 0); \ + })) + +#define _mm_cvtps_ph(A, I) \ + ((__m128i) __builtin_ia32_vcvtps2ph ((__v4sf)(__m128) A, (int) (I))) + +#define _mm256_cvtps_ph(A, I) \ + ((__m128i) __builtin_ia32_vcvtps2ph256 ((__v8sf)(__m256) A, (int) (I))) +#endif /* __OPTIMIZE */ + +#ifdef __DISABLE_F16C__ +#undef __DISABLE_F16C__ +#pragma GCC pop_options +#endif /* __DISABLE_F16C__ */ + +#endif /* _F16CINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/float.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/float.h new file mode 100644 index 0000000000..658017fd63 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/float.h @@ -0,0 +1,506 @@ +/* Copyright (C) 2002-2017 Free Software Foundation, Inc. + +This file is part of GCC. + +GCC is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3, or (at your option) +any later version. + +GCC is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +Under Section 7 of GPL version 3, you are granted additional +permissions described in the GCC Runtime Library Exception, version +3.1, as published by the Free Software Foundation. + +You should have received a copy of the GNU General Public License and +a copy of the GCC Runtime Library Exception along with this program; +see the files COPYING3 and COPYING.RUNTIME respectively. If not, see +. */ + +/* + * ISO C Standard: 5.2.4.2.2 Characteristics of floating types + */ + +#ifndef _FLOAT_H___ +#define _FLOAT_H___ + +/* Radix of exponent representation, b. */ +#undef FLT_RADIX +#define FLT_RADIX __FLT_RADIX__ + +/* Number of base-FLT_RADIX digits in the significand, p. */ +#undef FLT_MANT_DIG +#undef DBL_MANT_DIG +#undef LDBL_MANT_DIG +#define FLT_MANT_DIG __FLT_MANT_DIG__ +#define DBL_MANT_DIG __DBL_MANT_DIG__ +#define LDBL_MANT_DIG __LDBL_MANT_DIG__ + +/* Number of decimal digits, q, such that any floating-point number with q + decimal digits can be rounded into a floating-point number with p radix b + digits and back again without change to the q decimal digits, + + p * log10(b) if b is a power of 10 + floor((p - 1) * log10(b)) otherwise +*/ +#undef FLT_DIG +#undef DBL_DIG +#undef LDBL_DIG +#define FLT_DIG __FLT_DIG__ +#define DBL_DIG __DBL_DIG__ +#define LDBL_DIG __LDBL_DIG__ + +/* Minimum int x such that FLT_RADIX**(x-1) is a normalized float, emin */ +#undef FLT_MIN_EXP +#undef DBL_MIN_EXP +#undef LDBL_MIN_EXP +#define FLT_MIN_EXP __FLT_MIN_EXP__ +#define DBL_MIN_EXP __DBL_MIN_EXP__ +#define LDBL_MIN_EXP __LDBL_MIN_EXP__ + +/* Minimum negative integer such that 10 raised to that power is in the + range of normalized floating-point numbers, + + ceil(log10(b) * (emin - 1)) +*/ +#undef FLT_MIN_10_EXP +#undef DBL_MIN_10_EXP +#undef LDBL_MIN_10_EXP +#define FLT_MIN_10_EXP __FLT_MIN_10_EXP__ +#define DBL_MIN_10_EXP __DBL_MIN_10_EXP__ +#define LDBL_MIN_10_EXP __LDBL_MIN_10_EXP__ + +/* Maximum int x such that FLT_RADIX**(x-1) is a representable float, emax. */ +#undef FLT_MAX_EXP +#undef DBL_MAX_EXP +#undef LDBL_MAX_EXP +#define FLT_MAX_EXP __FLT_MAX_EXP__ +#define DBL_MAX_EXP __DBL_MAX_EXP__ +#define LDBL_MAX_EXP __LDBL_MAX_EXP__ + +/* Maximum integer such that 10 raised to that power is in the range of + representable finite floating-point numbers, + + floor(log10((1 - b**-p) * b**emax)) +*/ +#undef FLT_MAX_10_EXP +#undef DBL_MAX_10_EXP +#undef LDBL_MAX_10_EXP +#define FLT_MAX_10_EXP __FLT_MAX_10_EXP__ +#define DBL_MAX_10_EXP __DBL_MAX_10_EXP__ +#define LDBL_MAX_10_EXP __LDBL_MAX_10_EXP__ + +/* Maximum representable finite floating-point number, + + (1 - b**-p) * b**emax +*/ +#undef FLT_MAX +#undef DBL_MAX +#undef LDBL_MAX +#define FLT_MAX __FLT_MAX__ +#define DBL_MAX __DBL_MAX__ +#define LDBL_MAX __LDBL_MAX__ + +/* The difference between 1 and the least value greater than 1 that is + representable in the given floating point type, b**1-p. */ +#undef FLT_EPSILON +#undef DBL_EPSILON +#undef LDBL_EPSILON +#define FLT_EPSILON __FLT_EPSILON__ +#define DBL_EPSILON __DBL_EPSILON__ +#define LDBL_EPSILON __LDBL_EPSILON__ + +/* Minimum normalized positive floating-point number, b**(emin - 1). */ +#undef FLT_MIN +#undef DBL_MIN +#undef LDBL_MIN +#define FLT_MIN __FLT_MIN__ +#define DBL_MIN __DBL_MIN__ +#define LDBL_MIN __LDBL_MIN__ + +/* Addition rounds to 0: zero, 1: nearest, 2: +inf, 3: -inf, -1: unknown. */ +/* ??? This is supposed to change with calls to fesetround in . */ +#undef FLT_ROUNDS +#define FLT_ROUNDS 1 + +#if (defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) \ + || (defined (__cplusplus) && __cplusplus >= 201103L) +/* The floating-point expression evaluation method. The precise + definitions of these values are generalised to include support for + the interchange and extended types defined in ISO/IEC TS 18661-3. + Prior to this (for C99/C11) the definitions were: + + -1 indeterminate + 0 evaluate all operations and constants just to the range and + precision of the type + 1 evaluate operations and constants of type float and double + to the range and precision of the double type, evaluate + long double operations and constants to the range and + precision of the long double type + 2 evaluate all operations and constants to the range and + precision of the long double type + + The TS 18661-3 definitions are: + + -1 indeterminate + 0 evaluate all operations and constants, whose semantic type has + at most the range and precision of float, to the range and + precision of float; evaluate all other operations and constants + to the range and precision of the semantic type. + 1 evaluate all operations and constants, whose semantic type has + at most the range and precision of double, to the range and + precision of double; evaluate all other operations and constants + to the range and precision of the semantic type. + 2 evaluate all operations and constants, whose semantic type has + at most the range and precision of long double, to the range and + precision of long double; evaluate all other operations and + constants to the range and precision of the semantic type. + N where _FloatN is a supported interchange floating type + evaluate all operations and constants, whose semantic type has + at most the range and precision of the _FloatN type, to the + range and precision of the _FloatN type; evaluate all other + operations and constants to the range and precision of the + semantic type. + N + 1, where _FloatNx is a supported extended floating type + evaluate operations and constants, whose semantic type has at + most the range and precision of the _FloatNx type, to the range + and precision of the _FloatNx type; evaluate all other + operations and constants to the range and precision of the + semantic type. + + The compiler predefines two macros: + + __FLT_EVAL_METHOD__ + Which, depending on the value given for + -fpermitted-flt-eval-methods, may be limited to only those values + for FLT_EVAL_METHOD defined in C99/C11. + + __FLT_EVAL_METHOD_TS_18661_3__ + Which always permits the values for FLT_EVAL_METHOD defined in + ISO/IEC TS 18661-3. + + Here we want to use __FLT_EVAL_METHOD__, unless + __STDC_WANT_IEC_60559_TYPES_EXT__ is defined, in which case the user + is specifically asking for the ISO/IEC TS 18661-3 types, so we use + __FLT_EVAL_METHOD_TS_18661_3__. + + ??? This ought to change with the setting of the fp control word; + the value provided by the compiler assumes the widest setting. */ +#undef FLT_EVAL_METHOD +#ifdef __STDC_WANT_IEC_60559_TYPES_EXT__ +#define FLT_EVAL_METHOD __FLT_EVAL_METHOD_TS_18661_3__ +#else +#define FLT_EVAL_METHOD __FLT_EVAL_METHOD__ +#endif + +/* Number of decimal digits, n, such that any floating-point number in the + widest supported floating type with pmax radix b digits can be rounded + to a floating-point number with n decimal digits and back again without + change to the value, + + pmax * log10(b) if b is a power of 10 + ceil(1 + pmax * log10(b)) otherwise +*/ +#undef DECIMAL_DIG +#define DECIMAL_DIG __DECIMAL_DIG__ + +#endif /* C99 */ + +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +/* Versions of DECIMAL_DIG for each floating-point type. */ +#undef FLT_DECIMAL_DIG +#undef DBL_DECIMAL_DIG +#undef LDBL_DECIMAL_DIG +#define FLT_DECIMAL_DIG __FLT_DECIMAL_DIG__ +#define DBL_DECIMAL_DIG __DBL_DECIMAL_DIG__ +#define LDBL_DECIMAL_DIG __LDBL_DECIMAL_DIG__ + +/* Whether types support subnormal numbers. */ +#undef FLT_HAS_SUBNORM +#undef DBL_HAS_SUBNORM +#undef LDBL_HAS_SUBNORM +#define FLT_HAS_SUBNORM __FLT_HAS_DENORM__ +#define DBL_HAS_SUBNORM __DBL_HAS_DENORM__ +#define LDBL_HAS_SUBNORM __LDBL_HAS_DENORM__ + +/* Minimum positive values, including subnormals. */ +#undef FLT_TRUE_MIN +#undef DBL_TRUE_MIN +#undef LDBL_TRUE_MIN +#define FLT_TRUE_MIN __FLT_DENORM_MIN__ +#define DBL_TRUE_MIN __DBL_DENORM_MIN__ +#define LDBL_TRUE_MIN __LDBL_DENORM_MIN__ + +#endif /* C11 */ + +#ifdef __STDC_WANT_IEC_60559_BFP_EXT__ +/* Number of decimal digits for which conversions between decimal + character strings and binary formats, in both directions, are + correctly rounded. */ +#define CR_DECIMAL_DIG __UINTMAX_MAX__ +#endif + +#ifdef __STDC_WANT_IEC_60559_TYPES_EXT__ +/* Constants for _FloatN and _FloatNx types from TS 18661-3. See + comments above for their semantics. */ + +#ifdef __FLT16_MANT_DIG__ +#undef FLT16_MANT_DIG +#define FLT16_MANT_DIG __FLT16_MANT_DIG__ +#undef FLT16_DIG +#define FLT16_DIG __FLT16_DIG__ +#undef FLT16_MIN_EXP +#define FLT16_MIN_EXP __FLT16_MIN_EXP__ +#undef FLT16_MIN_10_EXP +#define FLT16_MIN_10_EXP __FLT16_MIN_10_EXP__ +#undef FLT16_MAX_EXP +#define FLT16_MAX_EXP __FLT16_MAX_EXP__ +#undef FLT16_MAX_10_EXP +#define FLT16_MAX_10_EXP __FLT16_MAX_10_EXP__ +#undef FLT16_MAX +#define FLT16_MAX __FLT16_MAX__ +#undef FLT16_EPSILON +#define FLT16_EPSILON __FLT16_EPSILON__ +#undef FLT16_MIN +#define FLT16_MIN __FLT16_MIN__ +#undef FLT16_DECIMAL_DIG +#define FLT16_DECIMAL_DIG __FLT16_DECIMAL_DIG__ +#undef FLT16_TRUE_MIN +#define FLT16_TRUE_MIN __FLT16_DENORM_MIN__ +#endif /* __FLT16_MANT_DIG__. */ + +#ifdef __FLT32_MANT_DIG__ +#undef FLT32_MANT_DIG +#define FLT32_MANT_DIG __FLT32_MANT_DIG__ +#undef FLT32_DIG +#define FLT32_DIG __FLT32_DIG__ +#undef FLT32_MIN_EXP +#define FLT32_MIN_EXP __FLT32_MIN_EXP__ +#undef FLT32_MIN_10_EXP +#define FLT32_MIN_10_EXP __FLT32_MIN_10_EXP__ +#undef FLT32_MAX_EXP +#define FLT32_MAX_EXP __FLT32_MAX_EXP__ +#undef FLT32_MAX_10_EXP +#define FLT32_MAX_10_EXP __FLT32_MAX_10_EXP__ +#undef FLT32_MAX +#define FLT32_MAX __FLT32_MAX__ +#undef FLT32_EPSILON +#define FLT32_EPSILON __FLT32_EPSILON__ +#undef FLT32_MIN +#define FLT32_MIN __FLT32_MIN__ +#undef FLT32_DECIMAL_DIG +#define FLT32_DECIMAL_DIG __FLT32_DECIMAL_DIG__ +#undef FLT32_TRUE_MIN +#define FLT32_TRUE_MIN __FLT32_DENORM_MIN__ +#endif /* __FLT32_MANT_DIG__. */ + +#ifdef __FLT64_MANT_DIG__ +#undef FLT64_MANT_DIG +#define FLT64_MANT_DIG __FLT64_MANT_DIG__ +#undef FLT64_DIG +#define FLT64_DIG __FLT64_DIG__ +#undef FLT64_MIN_EXP +#define FLT64_MIN_EXP __FLT64_MIN_EXP__ +#undef FLT64_MIN_10_EXP +#define FLT64_MIN_10_EXP __FLT64_MIN_10_EXP__ +#undef FLT64_MAX_EXP +#define FLT64_MAX_EXP __FLT64_MAX_EXP__ +#undef FLT64_MAX_10_EXP +#define FLT64_MAX_10_EXP __FLT64_MAX_10_EXP__ +#undef FLT64_MAX +#define FLT64_MAX __FLT64_MAX__ +#undef FLT64_EPSILON +#define FLT64_EPSILON __FLT64_EPSILON__ +#undef FLT64_MIN +#define FLT64_MIN __FLT64_MIN__ +#undef FLT64_DECIMAL_DIG +#define FLT64_DECIMAL_DIG __FLT64_DECIMAL_DIG__ +#undef FLT64_TRUE_MIN +#define FLT64_TRUE_MIN __FLT64_DENORM_MIN__ +#endif /* __FLT64_MANT_DIG__. */ + +#ifdef __FLT128_MANT_DIG__ +#undef FLT128_MANT_DIG +#define FLT128_MANT_DIG __FLT128_MANT_DIG__ +#undef FLT128_DIG +#define FLT128_DIG __FLT128_DIG__ +#undef FLT128_MIN_EXP +#define FLT128_MIN_EXP __FLT128_MIN_EXP__ +#undef FLT128_MIN_10_EXP +#define FLT128_MIN_10_EXP __FLT128_MIN_10_EXP__ +#undef FLT128_MAX_EXP +#define FLT128_MAX_EXP __FLT128_MAX_EXP__ +#undef FLT128_MAX_10_EXP +#define FLT128_MAX_10_EXP __FLT128_MAX_10_EXP__ +#undef FLT128_MAX +#define FLT128_MAX __FLT128_MAX__ +#undef FLT128_EPSILON +#define FLT128_EPSILON __FLT128_EPSILON__ +#undef FLT128_MIN +#define FLT128_MIN __FLT128_MIN__ +#undef FLT128_DECIMAL_DIG +#define FLT128_DECIMAL_DIG __FLT128_DECIMAL_DIG__ +#undef FLT128_TRUE_MIN +#define FLT128_TRUE_MIN __FLT128_DENORM_MIN__ +#endif /* __FLT128_MANT_DIG__. */ + +#ifdef __FLT32X_MANT_DIG__ +#undef FLT32X_MANT_DIG +#define FLT32X_MANT_DIG __FLT32X_MANT_DIG__ +#undef FLT32X_DIG +#define FLT32X_DIG __FLT32X_DIG__ +#undef FLT32X_MIN_EXP +#define FLT32X_MIN_EXP __FLT32X_MIN_EXP__ +#undef FLT32X_MIN_10_EXP +#define FLT32X_MIN_10_EXP __FLT32X_MIN_10_EXP__ +#undef FLT32X_MAX_EXP +#define FLT32X_MAX_EXP __FLT32X_MAX_EXP__ +#undef FLT32X_MAX_10_EXP +#define FLT32X_MAX_10_EXP __FLT32X_MAX_10_EXP__ +#undef FLT32X_MAX +#define FLT32X_MAX __FLT32X_MAX__ +#undef FLT32X_EPSILON +#define FLT32X_EPSILON __FLT32X_EPSILON__ +#undef FLT32X_MIN +#define FLT32X_MIN __FLT32X_MIN__ +#undef FLT32X_DECIMAL_DIG +#define FLT32X_DECIMAL_DIG __FLT32X_DECIMAL_DIG__ +#undef FLT32X_TRUE_MIN +#define FLT32X_TRUE_MIN __FLT32X_DENORM_MIN__ +#endif /* __FLT32X_MANT_DIG__. */ + +#ifdef __FLT64X_MANT_DIG__ +#undef FLT64X_MANT_DIG +#define FLT64X_MANT_DIG __FLT64X_MANT_DIG__ +#undef FLT64X_DIG +#define FLT64X_DIG __FLT64X_DIG__ +#undef FLT64X_MIN_EXP +#define FLT64X_MIN_EXP __FLT64X_MIN_EXP__ +#undef FLT64X_MIN_10_EXP +#define FLT64X_MIN_10_EXP __FLT64X_MIN_10_EXP__ +#undef FLT64X_MAX_EXP +#define FLT64X_MAX_EXP __FLT64X_MAX_EXP__ +#undef FLT64X_MAX_10_EXP +#define FLT64X_MAX_10_EXP __FLT64X_MAX_10_EXP__ +#undef FLT64X_MAX +#define FLT64X_MAX __FLT64X_MAX__ +#undef FLT64X_EPSILON +#define FLT64X_EPSILON __FLT64X_EPSILON__ +#undef FLT64X_MIN +#define FLT64X_MIN __FLT64X_MIN__ +#undef FLT64X_DECIMAL_DIG +#define FLT64X_DECIMAL_DIG __FLT64X_DECIMAL_DIG__ +#undef FLT64X_TRUE_MIN +#define FLT64X_TRUE_MIN __FLT64X_DENORM_MIN__ +#endif /* __FLT64X_MANT_DIG__. */ + +#ifdef __FLT128X_MANT_DIG__ +#undef FLT128X_MANT_DIG +#define FLT128X_MANT_DIG __FLT128X_MANT_DIG__ +#undef FLT128X_DIG +#define FLT128X_DIG __FLT128X_DIG__ +#undef FLT128X_MIN_EXP +#define FLT128X_MIN_EXP __FLT128X_MIN_EXP__ +#undef FLT128X_MIN_10_EXP +#define FLT128X_MIN_10_EXP __FLT128X_MIN_10_EXP__ +#undef FLT128X_MAX_EXP +#define FLT128X_MAX_EXP __FLT128X_MAX_EXP__ +#undef FLT128X_MAX_10_EXP +#define FLT128X_MAX_10_EXP __FLT128X_MAX_10_EXP__ +#undef FLT128X_MAX +#define FLT128X_MAX __FLT128X_MAX__ +#undef FLT128X_EPSILON +#define FLT128X_EPSILON __FLT128X_EPSILON__ +#undef FLT128X_MIN +#define FLT128X_MIN __FLT128X_MIN__ +#undef FLT128X_DECIMAL_DIG +#define FLT128X_DECIMAL_DIG __FLT128X_DECIMAL_DIG__ +#undef FLT128X_TRUE_MIN +#define FLT128X_TRUE_MIN __FLT128X_DENORM_MIN__ +#endif /* __FLT128X_MANT_DIG__. */ + +#endif /* __STDC_WANT_IEC_60559_TYPES_EXT__. */ + +#ifdef __STDC_WANT_DEC_FP__ +/* Draft Technical Report 24732, extension for decimal floating-point + arithmetic: Characteristic of decimal floating types . */ + +/* Number of base-FLT_RADIX digits in the significand, p. */ +#undef DEC32_MANT_DIG +#undef DEC64_MANT_DIG +#undef DEC128_MANT_DIG +#define DEC32_MANT_DIG __DEC32_MANT_DIG__ +#define DEC64_MANT_DIG __DEC64_MANT_DIG__ +#define DEC128_MANT_DIG __DEC128_MANT_DIG__ + +/* Minimum exponent. */ +#undef DEC32_MIN_EXP +#undef DEC64_MIN_EXP +#undef DEC128_MIN_EXP +#define DEC32_MIN_EXP __DEC32_MIN_EXP__ +#define DEC64_MIN_EXP __DEC64_MIN_EXP__ +#define DEC128_MIN_EXP __DEC128_MIN_EXP__ + +/* Maximum exponent. */ +#undef DEC32_MAX_EXP +#undef DEC64_MAX_EXP +#undef DEC128_MAX_EXP +#define DEC32_MAX_EXP __DEC32_MAX_EXP__ +#define DEC64_MAX_EXP __DEC64_MAX_EXP__ +#define DEC128_MAX_EXP __DEC128_MAX_EXP__ + +/* Maximum representable finite decimal floating-point number + (there are 6, 15, and 33 9s after the decimal points respectively). */ +#undef DEC32_MAX +#undef DEC64_MAX +#undef DEC128_MAX +#define DEC32_MAX __DEC32_MAX__ +#define DEC64_MAX __DEC64_MAX__ +#define DEC128_MAX __DEC128_MAX__ + +/* The difference between 1 and the least value greater than 1 that is + representable in the given floating point type. */ +#undef DEC32_EPSILON +#undef DEC64_EPSILON +#undef DEC128_EPSILON +#define DEC32_EPSILON __DEC32_EPSILON__ +#define DEC64_EPSILON __DEC64_EPSILON__ +#define DEC128_EPSILON __DEC128_EPSILON__ + +/* Minimum normalized positive floating-point number. */ +#undef DEC32_MIN +#undef DEC64_MIN +#undef DEC128_MIN +#define DEC32_MIN __DEC32_MIN__ +#define DEC64_MIN __DEC64_MIN__ +#define DEC128_MIN __DEC128_MIN__ + +/* Minimum subnormal positive floating-point number. */ +#undef DEC32_SUBNORMAL_MIN +#undef DEC64_SUBNORMAL_MIN +#undef DEC128_SUBNORMAL_MIN +#define DEC32_SUBNORMAL_MIN __DEC32_SUBNORMAL_MIN__ +#define DEC64_SUBNORMAL_MIN __DEC64_SUBNORMAL_MIN__ +#define DEC128_SUBNORMAL_MIN __DEC128_SUBNORMAL_MIN__ + +/* The floating-point expression evaluation method. + -1 indeterminate + 0 evaluate all operations and constants just to the range and + precision of the type + 1 evaluate operations and constants of type _Decimal32 + and _Decimal64 to the range and precision of the _Decimal64 + type, evaluate _Decimal128 operations and constants to the + range and precision of the _Decimal128 type; + 2 evaluate all operations and constants to the range and + precision of the _Decimal128 type. */ + +#undef DEC_EVAL_METHOD +#define DEC_EVAL_METHOD __DEC_EVAL_METHOD__ + +#endif /* __STDC_WANT_DEC_FP__ */ + +#endif /* _FLOAT_H___ */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/fma4intrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/fma4intrin.h new file mode 100644 index 0000000000..7230c9c9ae --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/fma4intrin.h @@ -0,0 +1,241 @@ +/* Copyright (C) 2007-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _X86INTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _FMA4INTRIN_H_INCLUDED +#define _FMA4INTRIN_H_INCLUDED + +/* We need definitions from the SSE4A, SSE3, SSE2 and SSE header files. */ +#include + +#ifndef __FMA4__ +#pragma GCC push_options +#pragma GCC target("fma4") +#define __DISABLE_FMA4__ +#endif /* __FMA4__ */ + +/* 128b Floating point multiply/add type instructions. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_macc_ps (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddps ((__v4sf)__A, (__v4sf)__B, (__v4sf)__C); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_macc_pd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddpd ((__v2df)__A, (__v2df)__B, (__v2df)__C); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_macc_ss (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddss ((__v4sf)__A, (__v4sf)__B, (__v4sf)__C); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_macc_sd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddsd ((__v2df)__A, (__v2df)__B, (__v2df)__C); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_msub_ps (__m128 __A, __m128 __B, __m128 __C) + +{ + return (__m128) __builtin_ia32_vfmaddps ((__v4sf)__A, (__v4sf)__B, -(__v4sf)__C); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_msub_pd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddpd ((__v2df)__A, (__v2df)__B, -(__v2df)__C); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_msub_ss (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddss ((__v4sf)__A, (__v4sf)__B, -(__v4sf)__C); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_msub_sd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddsd ((__v2df)__A, (__v2df)__B, -(__v2df)__C); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_nmacc_ps (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddps (-(__v4sf)__A, (__v4sf)__B, (__v4sf)__C); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_nmacc_pd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddpd (-(__v2df)__A, (__v2df)__B, (__v2df)__C); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_nmacc_ss (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddss (-(__v4sf)__A, (__v4sf)__B, (__v4sf)__C); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_nmacc_sd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddsd (-(__v2df)__A, (__v2df)__B, (__v2df)__C); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_nmsub_ps (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddps (-(__v4sf)__A, (__v4sf)__B, -(__v4sf)__C); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_nmsub_pd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddpd (-(__v2df)__A, (__v2df)__B, -(__v2df)__C); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_nmsub_ss (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddss (-(__v4sf)__A, (__v4sf)__B, -(__v4sf)__C); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_nmsub_sd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddsd (-(__v2df)__A, (__v2df)__B, -(__v2df)__C); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maddsub_ps (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddsubps ((__v4sf)__A, (__v4sf)__B, (__v4sf)__C); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maddsub_pd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddsubpd ((__v2df)__A, (__v2df)__B, (__v2df)__C); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_msubadd_ps (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddsubps ((__v4sf)__A, (__v4sf)__B, -(__v4sf)__C); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_msubadd_pd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddsubpd ((__v2df)__A, (__v2df)__B, -(__v2df)__C); +} + +/* 256b Floating point multiply/add type instructions. */ +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_macc_ps (__m256 __A, __m256 __B, __m256 __C) +{ + return (__m256) __builtin_ia32_vfmaddps256 ((__v8sf)__A, (__v8sf)__B, (__v8sf)__C); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_macc_pd (__m256d __A, __m256d __B, __m256d __C) +{ + return (__m256d) __builtin_ia32_vfmaddpd256 ((__v4df)__A, (__v4df)__B, (__v4df)__C); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_msub_ps (__m256 __A, __m256 __B, __m256 __C) + +{ + return (__m256) __builtin_ia32_vfmaddps256 ((__v8sf)__A, (__v8sf)__B, -(__v8sf)__C); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_msub_pd (__m256d __A, __m256d __B, __m256d __C) +{ + return (__m256d) __builtin_ia32_vfmaddpd256 ((__v4df)__A, (__v4df)__B, -(__v4df)__C); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_nmacc_ps (__m256 __A, __m256 __B, __m256 __C) +{ + return (__m256) __builtin_ia32_vfmaddps256 (-(__v8sf)__A, (__v8sf)__B, (__v8sf)__C); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_nmacc_pd (__m256d __A, __m256d __B, __m256d __C) +{ + return (__m256d) __builtin_ia32_vfmaddpd256 (-(__v4df)__A, (__v4df)__B, (__v4df)__C); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_nmsub_ps (__m256 __A, __m256 __B, __m256 __C) +{ + return (__m256) __builtin_ia32_vfmaddps256 (-(__v8sf)__A, (__v8sf)__B, -(__v8sf)__C); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_nmsub_pd (__m256d __A, __m256d __B, __m256d __C) +{ + return (__m256d) __builtin_ia32_vfmaddpd256 (-(__v4df)__A, (__v4df)__B, -(__v4df)__C); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maddsub_ps (__m256 __A, __m256 __B, __m256 __C) +{ + return (__m256) __builtin_ia32_vfmaddsubps256 ((__v8sf)__A, (__v8sf)__B, (__v8sf)__C); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_maddsub_pd (__m256d __A, __m256d __B, __m256d __C) +{ + return (__m256d) __builtin_ia32_vfmaddsubpd256 ((__v4df)__A, (__v4df)__B, (__v4df)__C); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_msubadd_ps (__m256 __A, __m256 __B, __m256 __C) +{ + return (__m256) __builtin_ia32_vfmaddsubps256 ((__v8sf)__A, (__v8sf)__B, -(__v8sf)__C); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_msubadd_pd (__m256d __A, __m256d __B, __m256d __C) +{ + return (__m256d) __builtin_ia32_vfmaddsubpd256 ((__v4df)__A, (__v4df)__B, -(__v4df)__C); +} + +#ifdef __DISABLE_FMA4__ +#undef __DISABLE_FMA4__ +#pragma GCC pop_options +#endif /* __DISABLE_FMA4__ */ + +#endif diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/fmaintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/fmaintrin.h new file mode 100644 index 0000000000..2b1531e433 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/fmaintrin.h @@ -0,0 +1,302 @@ +/* Copyright (C) 2011-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _FMAINTRIN_H_INCLUDED +#define _FMAINTRIN_H_INCLUDED + +#ifndef __FMA__ +#pragma GCC push_options +#pragma GCC target("fma") +#define __DISABLE_FMA__ +#endif /* __FMA__ */ + +extern __inline __m128d +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fmadd_pd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d)__builtin_ia32_vfmaddpd ((__v2df)__A, (__v2df)__B, + (__v2df)__C); +} + +extern __inline __m256d +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_fmadd_pd (__m256d __A, __m256d __B, __m256d __C) +{ + return (__m256d)__builtin_ia32_vfmaddpd256 ((__v4df)__A, (__v4df)__B, + (__v4df)__C); +} + +extern __inline __m128 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fmadd_ps (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128)__builtin_ia32_vfmaddps ((__v4sf)__A, (__v4sf)__B, + (__v4sf)__C); +} + +extern __inline __m256 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_fmadd_ps (__m256 __A, __m256 __B, __m256 __C) +{ + return (__m256)__builtin_ia32_vfmaddps256 ((__v8sf)__A, (__v8sf)__B, + (__v8sf)__C); +} + +extern __inline __m128d +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fmadd_sd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d) __builtin_ia32_vfmaddsd3 ((__v2df)__A, (__v2df)__B, + (__v2df)__C); +} + +extern __inline __m128 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fmadd_ss (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128) __builtin_ia32_vfmaddss3 ((__v4sf)__A, (__v4sf)__B, + (__v4sf)__C); +} + +extern __inline __m128d +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fmsub_pd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d)__builtin_ia32_vfmaddpd ((__v2df)__A, (__v2df)__B, + -(__v2df)__C); +} + +extern __inline __m256d +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_fmsub_pd (__m256d __A, __m256d __B, __m256d __C) +{ + return (__m256d)__builtin_ia32_vfmaddpd256 ((__v4df)__A, (__v4df)__B, + -(__v4df)__C); +} + +extern __inline __m128 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fmsub_ps (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128)__builtin_ia32_vfmaddps ((__v4sf)__A, (__v4sf)__B, + -(__v4sf)__C); +} + +extern __inline __m256 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_fmsub_ps (__m256 __A, __m256 __B, __m256 __C) +{ + return (__m256)__builtin_ia32_vfmaddps256 ((__v8sf)__A, (__v8sf)__B, + -(__v8sf)__C); +} + +extern __inline __m128d +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fmsub_sd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d)__builtin_ia32_vfmaddsd3 ((__v2df)__A, (__v2df)__B, + -(__v2df)__C); +} + +extern __inline __m128 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fmsub_ss (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128)__builtin_ia32_vfmaddss3 ((__v4sf)__A, (__v4sf)__B, + -(__v4sf)__C); +} + +extern __inline __m128d +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fnmadd_pd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d)__builtin_ia32_vfmaddpd (-(__v2df)__A, (__v2df)__B, + (__v2df)__C); +} + +extern __inline __m256d +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_fnmadd_pd (__m256d __A, __m256d __B, __m256d __C) +{ + return (__m256d)__builtin_ia32_vfmaddpd256 (-(__v4df)__A, (__v4df)__B, + (__v4df)__C); +} + +extern __inline __m128 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fnmadd_ps (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128)__builtin_ia32_vfmaddps (-(__v4sf)__A, (__v4sf)__B, + (__v4sf)__C); +} + +extern __inline __m256 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_fnmadd_ps (__m256 __A, __m256 __B, __m256 __C) +{ + return (__m256)__builtin_ia32_vfmaddps256 (-(__v8sf)__A, (__v8sf)__B, + (__v8sf)__C); +} + +extern __inline __m128d +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fnmadd_sd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d)__builtin_ia32_vfmaddsd3 ((__v2df)__A, -(__v2df)__B, + (__v2df)__C); +} + +extern __inline __m128 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fnmadd_ss (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128)__builtin_ia32_vfmaddss3 ((__v4sf)__A, -(__v4sf)__B, + (__v4sf)__C); +} + +extern __inline __m128d +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fnmsub_pd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d)__builtin_ia32_vfmaddpd (-(__v2df)__A, (__v2df)__B, + -(__v2df)__C); +} + +extern __inline __m256d +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_fnmsub_pd (__m256d __A, __m256d __B, __m256d __C) +{ + return (__m256d)__builtin_ia32_vfmaddpd256 (-(__v4df)__A, (__v4df)__B, + -(__v4df)__C); +} + +extern __inline __m128 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fnmsub_ps (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128)__builtin_ia32_vfmaddps (-(__v4sf)__A, (__v4sf)__B, + -(__v4sf)__C); +} + +extern __inline __m256 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_fnmsub_ps (__m256 __A, __m256 __B, __m256 __C) +{ + return (__m256)__builtin_ia32_vfmaddps256 (-(__v8sf)__A, (__v8sf)__B, + -(__v8sf)__C); +} + +extern __inline __m128d +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fnmsub_sd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d)__builtin_ia32_vfmaddsd3 ((__v2df)__A, -(__v2df)__B, + -(__v2df)__C); +} + +extern __inline __m128 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fnmsub_ss (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128)__builtin_ia32_vfmaddss3 ((__v4sf)__A, -(__v4sf)__B, + -(__v4sf)__C); +} + +extern __inline __m128d +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fmaddsub_pd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d)__builtin_ia32_vfmaddsubpd ((__v2df)__A, (__v2df)__B, + (__v2df)__C); +} + +extern __inline __m256d +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_fmaddsub_pd (__m256d __A, __m256d __B, __m256d __C) +{ + return (__m256d)__builtin_ia32_vfmaddsubpd256 ((__v4df)__A, + (__v4df)__B, + (__v4df)__C); +} + +extern __inline __m128 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fmaddsub_ps (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128)__builtin_ia32_vfmaddsubps ((__v4sf)__A, (__v4sf)__B, + (__v4sf)__C); +} + +extern __inline __m256 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_fmaddsub_ps (__m256 __A, __m256 __B, __m256 __C) +{ + return (__m256)__builtin_ia32_vfmaddsubps256 ((__v8sf)__A, + (__v8sf)__B, + (__v8sf)__C); +} + +extern __inline __m128d +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fmsubadd_pd (__m128d __A, __m128d __B, __m128d __C) +{ + return (__m128d)__builtin_ia32_vfmaddsubpd ((__v2df)__A, (__v2df)__B, + -(__v2df)__C); +} + +extern __inline __m256d +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_fmsubadd_pd (__m256d __A, __m256d __B, __m256d __C) +{ + return (__m256d)__builtin_ia32_vfmaddsubpd256 ((__v4df)__A, + (__v4df)__B, + -(__v4df)__C); +} + +extern __inline __m128 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_fmsubadd_ps (__m128 __A, __m128 __B, __m128 __C) +{ + return (__m128)__builtin_ia32_vfmaddsubps ((__v4sf)__A, (__v4sf)__B, + -(__v4sf)__C); +} + +extern __inline __m256 +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_fmsubadd_ps (__m256 __A, __m256 __B, __m256 __C) +{ + return (__m256)__builtin_ia32_vfmaddsubps256 ((__v8sf)__A, + (__v8sf)__B, + -(__v8sf)__C); +} + +#ifdef __DISABLE_FMA__ +#undef __DISABLE_FMA__ +#pragma GCC pop_options +#endif /* __DISABLE_FMA__ */ + +#endif diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/fxsrintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/fxsrintrin.h new file mode 100644 index 0000000000..d04cf9c12f --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/fxsrintrin.h @@ -0,0 +1,73 @@ +/* Copyright (C) 2012-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +/* #if !defined _X86INTRIN_H_INCLUDED && !defined _IMMINTRIN_H_INCLUDED */ +/* # error "Never use directly; include instead." */ +/* #endif */ + +#ifndef _FXSRINTRIN_H_INCLUDED +#define _FXSRINTRIN_H_INCLUDED + +#ifndef __FXSR__ +#pragma GCC push_options +#pragma GCC target("fxsr") +#define __DISABLE_FXSR__ +#endif /* __FXSR__ */ + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_fxsave (void *__P) +{ + __builtin_ia32_fxsave (__P); +} + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_fxrstor (void *__P) +{ + __builtin_ia32_fxrstor (__P); +} + +#ifdef __x86_64__ +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_fxsave64 (void *__P) +{ + __builtin_ia32_fxsave64 (__P); +} + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_fxrstor64 (void *__P) +{ + __builtin_ia32_fxrstor64 (__P); +} +#endif + +#ifdef __DISABLE_FXSR__ +#undef __DISABLE_FXSR__ +#pragma GCC pop_options +#endif /* __DISABLE_FXSR__ */ + + +#endif /* _FXSRINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/ia32intrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/ia32intrin.h new file mode 100644 index 0000000000..5f954fce85 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/ia32intrin.h @@ -0,0 +1,313 @@ +/* Copyright (C) 2009-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _X86INTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +/* 32bit bsf */ +extern __inline int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__bsfd (int __X) +{ + return __builtin_ctz (__X); +} + +/* 32bit bsr */ +extern __inline int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__bsrd (int __X) +{ + return __builtin_ia32_bsrsi (__X); +} + +/* 32bit bswap */ +extern __inline int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__bswapd (int __X) +{ + return __builtin_bswap32 (__X); +} + +#ifndef __iamcu__ + +#ifndef __SSE4_2__ +#pragma GCC push_options +#pragma GCC target("sse4.2") +#define __DISABLE_SSE4_2__ +#endif /* __SSE4_2__ */ + +/* 32bit accumulate CRC32 (polynomial 0x11EDC6F41) value. */ +extern __inline unsigned int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__crc32b (unsigned int __C, unsigned char __V) +{ + return __builtin_ia32_crc32qi (__C, __V); +} + +extern __inline unsigned int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__crc32w (unsigned int __C, unsigned short __V) +{ + return __builtin_ia32_crc32hi (__C, __V); +} + +extern __inline unsigned int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__crc32d (unsigned int __C, unsigned int __V) +{ + return __builtin_ia32_crc32si (__C, __V); +} + +#ifdef __DISABLE_SSE4_2__ +#undef __DISABLE_SSE4_2__ +#pragma GCC pop_options +#endif /* __DISABLE_SSE4_2__ */ + +#endif /* __iamcu__ */ + +/* 32bit popcnt */ +extern __inline int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__popcntd (unsigned int __X) +{ + return __builtin_popcount (__X); +} + +#ifndef __iamcu__ + +/* rdpmc */ +extern __inline unsigned long long +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__rdpmc (int __S) +{ + return __builtin_ia32_rdpmc (__S); +} + +#endif /* __iamcu__ */ + +/* rdtsc */ +extern __inline unsigned long long +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__rdtsc (void) +{ + return __builtin_ia32_rdtsc (); +} + +#ifndef __iamcu__ + +/* rdtscp */ +extern __inline unsigned long long +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__rdtscp (unsigned int *__A) +{ + return __builtin_ia32_rdtscp (__A); +} + +#endif /* __iamcu__ */ + +/* 8bit rol */ +extern __inline unsigned char +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__rolb (unsigned char __X, int __C) +{ + return __builtin_ia32_rolqi (__X, __C); +} + +/* 16bit rol */ +extern __inline unsigned short +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__rolw (unsigned short __X, int __C) +{ + return __builtin_ia32_rolhi (__X, __C); +} + +/* 32bit rol */ +extern __inline unsigned int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__rold (unsigned int __X, int __C) +{ + return (__X << __C) | (__X >> (32 - __C)); +} + +/* 8bit ror */ +extern __inline unsigned char +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__rorb (unsigned char __X, int __C) +{ + return __builtin_ia32_rorqi (__X, __C); +} + +/* 16bit ror */ +extern __inline unsigned short +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__rorw (unsigned short __X, int __C) +{ + return __builtin_ia32_rorhi (__X, __C); +} + +/* 32bit ror */ +extern __inline unsigned int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__rord (unsigned int __X, int __C) +{ + return (__X >> __C) | (__X << (32 - __C)); +} + +/* Pause */ +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__pause (void) +{ + __builtin_ia32_pause (); +} + +#ifdef __x86_64__ +/* 64bit bsf */ +extern __inline int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__bsfq (long long __X) +{ + return __builtin_ctzll (__X); +} + +/* 64bit bsr */ +extern __inline int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__bsrq (long long __X) +{ + return __builtin_ia32_bsrdi (__X); +} + +/* 64bit bswap */ +extern __inline long long +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__bswapq (long long __X) +{ + return __builtin_bswap64 (__X); +} + +#ifndef __SSE4_2__ +#pragma GCC push_options +#pragma GCC target("sse4.2") +#define __DISABLE_SSE4_2__ +#endif /* __SSE4_2__ */ + +/* 64bit accumulate CRC32 (polynomial 0x11EDC6F41) value. */ +extern __inline unsigned long long +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__crc32q (unsigned long long __C, unsigned long long __V) +{ + return __builtin_ia32_crc32di (__C, __V); +} + +#ifdef __DISABLE_SSE4_2__ +#undef __DISABLE_SSE4_2__ +#pragma GCC pop_options +#endif /* __DISABLE_SSE4_2__ */ + +/* 64bit popcnt */ +extern __inline long long +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__popcntq (unsigned long long __X) +{ + return __builtin_popcountll (__X); +} + +/* 64bit rol */ +extern __inline unsigned long long +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__rolq (unsigned long long __X, int __C) +{ + return (__X << __C) | (__X >> (64 - __C)); +} + +/* 64bit ror */ +extern __inline unsigned long long +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__rorq (unsigned long long __X, int __C) +{ + return (__X >> __C) | (__X << (64 - __C)); +} + +/* Read flags register */ +extern __inline unsigned long long +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__readeflags (void) +{ + return __builtin_ia32_readeflags_u64 (); +} + +/* Write flags register */ +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__writeeflags (unsigned long long __X) +{ + __builtin_ia32_writeeflags_u64 (__X); +} + +#define _bswap64(a) __bswapq(a) +#define _popcnt64(a) __popcntq(a) +#else + +/* Read flags register */ +extern __inline unsigned int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__readeflags (void) +{ + return __builtin_ia32_readeflags_u32 (); +} + +/* Write flags register */ +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__writeeflags (unsigned int __X) +{ + __builtin_ia32_writeeflags_u32 (__X); +} + +#endif + +/* On LP64 systems, longs are 64-bit. Use the appropriate rotate + * function. */ +#ifdef __LP64__ +#define _lrotl(a,b) __rolq((a), (b)) +#define _lrotr(a,b) __rorq((a), (b)) +#else +#define _lrotl(a,b) __rold((a), (b)) +#define _lrotr(a,b) __rord((a), (b)) +#endif + +#define _bit_scan_forward(a) __bsfd(a) +#define _bit_scan_reverse(a) __bsrd(a) +#define _bswap(a) __bswapd(a) +#define _popcnt32(a) __popcntd(a) +#ifndef __iamcu__ +#define _rdpmc(a) __rdpmc(a) +#define _rdtscp(a) __rdtscp(a) +#endif /* __iamcu__ */ +#define _rdtsc() __rdtsc() +#define _rotwl(a,b) __rolw((a), (b)) +#define _rotwr(a,b) __rorw((a), (b)) +#define _rotl(a,b) __rold((a), (b)) +#define _rotr(a,b) __rord((a), (b)) diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/immintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/immintrin.h new file mode 100644 index 0000000000..b52f58efa4 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/immintrin.h @@ -0,0 +1,217 @@ +/* Copyright (C) 2008-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +#define _IMMINTRIN_H_INCLUDED + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#ifndef __RDRND__ +#pragma GCC push_options +#pragma GCC target("rdrnd") +#define __DISABLE_RDRND__ +#endif /* __RDRND__ */ +extern __inline int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_rdrand16_step (unsigned short *__P) +{ + return __builtin_ia32_rdrand16_step (__P); +} + +extern __inline int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_rdrand32_step (unsigned int *__P) +{ + return __builtin_ia32_rdrand32_step (__P); +} +#ifdef __DISABLE_RDRND__ +#undef __DISABLE_RDRND__ +#pragma GCC pop_options +#endif /* __DISABLE_RDRND__ */ + +#ifndef __RDPID__ +#pragma GCC push_options +#pragma GCC target("rdpid") +#define __DISABLE_RDPID__ +#endif /* __RDPID__ */ +extern __inline unsigned int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_rdpid_u32 (void) +{ + return __builtin_ia32_rdpid (); +} +#ifdef __DISABLE_RDPID__ +#undef __DISABLE_RDPID__ +#pragma GCC pop_options +#endif /* __DISABLE_RDPID__ */ + +#ifdef __x86_64__ + +#ifndef __FSGSBASE__ +#pragma GCC push_options +#pragma GCC target("fsgsbase") +#define __DISABLE_FSGSBASE__ +#endif /* __FSGSBASE__ */ +extern __inline unsigned int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_readfsbase_u32 (void) +{ + return __builtin_ia32_rdfsbase32 (); +} + +extern __inline unsigned long long +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_readfsbase_u64 (void) +{ + return __builtin_ia32_rdfsbase64 (); +} + +extern __inline unsigned int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_readgsbase_u32 (void) +{ + return __builtin_ia32_rdgsbase32 (); +} + +extern __inline unsigned long long +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_readgsbase_u64 (void) +{ + return __builtin_ia32_rdgsbase64 (); +} + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_writefsbase_u32 (unsigned int __B) +{ + __builtin_ia32_wrfsbase32 (__B); +} + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_writefsbase_u64 (unsigned long long __B) +{ + __builtin_ia32_wrfsbase64 (__B); +} + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_writegsbase_u32 (unsigned int __B) +{ + __builtin_ia32_wrgsbase32 (__B); +} + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_writegsbase_u64 (unsigned long long __B) +{ + __builtin_ia32_wrgsbase64 (__B); +} +#ifdef __DISABLE_FSGSBASE__ +#undef __DISABLE_FSGSBASE__ +#pragma GCC pop_options +#endif /* __DISABLE_FSGSBASE__ */ + +#ifndef __RDRND__ +#pragma GCC push_options +#pragma GCC target("rdrnd") +#define __DISABLE_RDRND__ +#endif /* __RDRND__ */ +extern __inline int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_rdrand64_step (unsigned long long *__P) +{ + return __builtin_ia32_rdrand64_step (__P); +} +#ifdef __DISABLE_RDRND__ +#undef __DISABLE_RDRND__ +#pragma GCC pop_options +#endif /* __DISABLE_RDRND__ */ + +#endif /* __x86_64__ */ + +#endif /* _IMMINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/iso646.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/iso646.h new file mode 100644 index 0000000000..5a6b50335e --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/iso646.h @@ -0,0 +1,45 @@ +/* Copyright (C) 1997-2017 Free Software Foundation, Inc. + +This file is part of GCC. + +GCC is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3, or (at your option) +any later version. + +GCC is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +Under Section 7 of GPL version 3, you are granted additional +permissions described in the GCC Runtime Library Exception, version +3.1, as published by the Free Software Foundation. + +You should have received a copy of the GNU General Public License and +a copy of the GCC Runtime Library Exception along with this program; +see the files COPYING3 and COPYING.RUNTIME respectively. If not, see +. */ + +/* + * ISO C Standard: 7.9 Alternative spellings + */ + +#ifndef _ISO646_H +#define _ISO646_H + +#ifndef __cplusplus +#define and && +#define and_eq &= +#define bitand & +#define bitor | +#define compl ~ +#define not ! +#define not_eq != +#define or || +#define or_eq |= +#define xor ^ +#define xor_eq ^= +#endif + +#endif diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/limits.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/limits.h new file mode 100644 index 0000000000..743481d784 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/limits.h @@ -0,0 +1,197 @@ +/* Copyright (C) 1992-2017 Free Software Foundation, Inc. + +This file is part of GCC. + +GCC is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 3, or (at your option) any later +version. + +GCC is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +Under Section 7 of GPL version 3, you are granted additional +permissions described in the GCC Runtime Library Exception, version +3.1, as published by the Free Software Foundation. + +You should have received a copy of the GNU General Public License and +a copy of the GCC Runtime Library Exception along with this program; +see the files COPYING3 and COPYING.RUNTIME respectively. If not, see +. */ + +/* This administrivia gets added to the beginning of limits.h + if the system has its own version of limits.h. */ + +/* We use _GCC_LIMITS_H_ because we want this not to match + any macros that the system's limits.h uses for its own purposes. */ +#ifndef _GCC_LIMITS_H_ /* Terminated in limity.h. */ +#define _GCC_LIMITS_H_ + +#ifndef _LIBC_LIMITS_H_ +/* Use "..." so that we find syslimits.h only in this same directory. */ +#include "syslimits.h" +#endif +/* Copyright (C) 1991-2017 Free Software Foundation, Inc. + +This file is part of GCC. + +GCC is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 3, or (at your option) any later +version. + +GCC is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +Under Section 7 of GPL version 3, you are granted additional +permissions described in the GCC Runtime Library Exception, version +3.1, as published by the Free Software Foundation. + +You should have received a copy of the GNU General Public License and +a copy of the GCC Runtime Library Exception along with this program; +see the files COPYING3 and COPYING.RUNTIME respectively. If not, see +. */ + +#ifndef _LIMITS_H___ +#define _LIMITS_H___ + +/* Number of bits in a `char'. */ +#undef CHAR_BIT +#define CHAR_BIT __CHAR_BIT__ + +/* Maximum length of a multibyte character. */ +#ifndef MB_LEN_MAX +#define MB_LEN_MAX 1 +#endif + +/* Minimum and maximum values a `signed char' can hold. */ +#undef SCHAR_MIN +#define SCHAR_MIN (-SCHAR_MAX - 1) +#undef SCHAR_MAX +#define SCHAR_MAX __SCHAR_MAX__ + +/* Maximum value an `unsigned char' can hold. (Minimum is 0). */ +#undef UCHAR_MAX +#if __SCHAR_MAX__ == __INT_MAX__ +# define UCHAR_MAX (SCHAR_MAX * 2U + 1U) +#else +# define UCHAR_MAX (SCHAR_MAX * 2 + 1) +#endif + +/* Minimum and maximum values a `char' can hold. */ +#ifdef __CHAR_UNSIGNED__ +# undef CHAR_MIN +# if __SCHAR_MAX__ == __INT_MAX__ +# define CHAR_MIN 0U +# else +# define CHAR_MIN 0 +# endif +# undef CHAR_MAX +# define CHAR_MAX UCHAR_MAX +#else +# undef CHAR_MIN +# define CHAR_MIN SCHAR_MIN +# undef CHAR_MAX +# define CHAR_MAX SCHAR_MAX +#endif + +/* Minimum and maximum values a `signed short int' can hold. */ +#undef SHRT_MIN +#define SHRT_MIN (-SHRT_MAX - 1) +#undef SHRT_MAX +#define SHRT_MAX __SHRT_MAX__ + +/* Maximum value an `unsigned short int' can hold. (Minimum is 0). */ +#undef USHRT_MAX +#if __SHRT_MAX__ == __INT_MAX__ +# define USHRT_MAX (SHRT_MAX * 2U + 1U) +#else +# define USHRT_MAX (SHRT_MAX * 2 + 1) +#endif + +/* Minimum and maximum values a `signed int' can hold. */ +#undef INT_MIN +#define INT_MIN (-INT_MAX - 1) +#undef INT_MAX +#define INT_MAX __INT_MAX__ + +/* Maximum value an `unsigned int' can hold. (Minimum is 0). */ +#undef UINT_MAX +#define UINT_MAX (INT_MAX * 2U + 1U) + +/* Minimum and maximum values a `signed long int' can hold. + (Same as `int'). */ +#undef LONG_MIN +#define LONG_MIN (-LONG_MAX - 1L) +#undef LONG_MAX +#define LONG_MAX __LONG_MAX__ + +/* Maximum value an `unsigned long int' can hold. (Minimum is 0). */ +#undef ULONG_MAX +#define ULONG_MAX (LONG_MAX * 2UL + 1UL) + +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L +/* Minimum and maximum values a `signed long long int' can hold. */ +# undef LLONG_MIN +# define LLONG_MIN (-LLONG_MAX - 1LL) +# undef LLONG_MAX +# define LLONG_MAX __LONG_LONG_MAX__ + +/* Maximum value an `unsigned long long int' can hold. (Minimum is 0). */ +# undef ULLONG_MAX +# define ULLONG_MAX (LLONG_MAX * 2ULL + 1ULL) +#endif + +#if defined (__GNU_LIBRARY__) ? defined (__USE_GNU) : !defined (__STRICT_ANSI__) +/* Minimum and maximum values a `signed long long int' can hold. */ +# undef LONG_LONG_MIN +# define LONG_LONG_MIN (-LONG_LONG_MAX - 1LL) +# undef LONG_LONG_MAX +# define LONG_LONG_MAX __LONG_LONG_MAX__ + +/* Maximum value an `unsigned long long int' can hold. (Minimum is 0). */ +# undef ULONG_LONG_MAX +# define ULONG_LONG_MAX (LONG_LONG_MAX * 2ULL + 1ULL) +#endif + +#ifdef __STDC_WANT_IEC_60559_BFP_EXT__ +/* TS 18661-1 widths of integer types. */ +# undef CHAR_WIDTH +# define CHAR_WIDTH __SCHAR_WIDTH__ +# undef SCHAR_WIDTH +# define SCHAR_WIDTH __SCHAR_WIDTH__ +# undef UCHAR_WIDTH +# define UCHAR_WIDTH __SCHAR_WIDTH__ +# undef SHRT_WIDTH +# define SHRT_WIDTH __SHRT_WIDTH__ +# undef USHRT_WIDTH +# define USHRT_WIDTH __SHRT_WIDTH__ +# undef INT_WIDTH +# define INT_WIDTH __INT_WIDTH__ +# undef UINT_WIDTH +# define UINT_WIDTH __INT_WIDTH__ +# undef LONG_WIDTH +# define LONG_WIDTH __LONG_WIDTH__ +# undef ULONG_WIDTH +# define ULONG_WIDTH __LONG_WIDTH__ +# undef LLONG_WIDTH +# define LLONG_WIDTH __LONG_LONG_WIDTH__ +# undef ULLONG_WIDTH +# define ULLONG_WIDTH __LONG_LONG_WIDTH__ +#endif + +#endif /* _LIMITS_H___ */ +/* This administrivia gets added to the end of limits.h + if the system has its own version of limits.h. */ + +#else /* not _GCC_LIMITS_H_ */ + +#ifdef _GCC_NEXT_LIMITS_H +#include_next /* recurse down to the real one */ +#endif + +#endif /* not _GCC_LIMITS_H_ */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/lwpintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/lwpintrin.h new file mode 100644 index 0000000000..5c095aba18 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/lwpintrin.h @@ -0,0 +1,107 @@ +/* Copyright (C) 2007-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _X86INTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _LWPINTRIN_H_INCLUDED +#define _LWPINTRIN_H_INCLUDED + +#ifndef __LWP__ +#pragma GCC push_options +#pragma GCC target("lwp") +#define __DISABLE_LWP__ +#endif /* __LWP__ */ + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__llwpcb (void *__pcbAddress) +{ + __builtin_ia32_llwpcb (__pcbAddress); +} + +extern __inline void * __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__slwpcb (void) +{ + return __builtin_ia32_slwpcb (); +} + +#ifdef __OPTIMIZE__ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__lwpval32 (unsigned int __data2, unsigned int __data1, unsigned int __flags) +{ + __builtin_ia32_lwpval32 (__data2, __data1, __flags); +} + +#ifdef __x86_64__ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__lwpval64 (unsigned long long __data2, unsigned int __data1, + unsigned int __flags) +{ + __builtin_ia32_lwpval64 (__data2, __data1, __flags); +} +#endif +#else +#define __lwpval32(D2, D1, F) \ + (__builtin_ia32_lwpval32 ((unsigned int) (D2), (unsigned int) (D1), \ + (unsigned int) (F))) +#ifdef __x86_64__ +#define __lwpval64(D2, D1, F) \ + (__builtin_ia32_lwpval64 ((unsigned long long) (D2), (unsigned int) (D1), \ + (unsigned int) (F))) +#endif +#endif + + +#ifdef __OPTIMIZE__ +extern __inline unsigned char __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__lwpins32 (unsigned int __data2, unsigned int __data1, unsigned int __flags) +{ + return __builtin_ia32_lwpins32 (__data2, __data1, __flags); +} + +#ifdef __x86_64__ +extern __inline unsigned char __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__lwpins64 (unsigned long long __data2, unsigned int __data1, + unsigned int __flags) +{ + return __builtin_ia32_lwpins64 (__data2, __data1, __flags); +} +#endif +#else +#define __lwpins32(D2, D1, F) \ + (__builtin_ia32_lwpins32 ((unsigned int) (D2), (unsigned int) (D1), \ + (unsigned int) (F))) +#ifdef __x86_64__ +#define __lwpins64(D2, D1, F) \ + (__builtin_ia32_lwpins64 ((unsigned long long) (D2), (unsigned int) (D1), \ + (unsigned int) (F))) +#endif +#endif + +#ifdef __DISABLE_LWP__ +#undef __DISABLE_LWP__ +#pragma GCC pop_options +#endif /* __DISABLE_LWP__ */ + +#endif /* _LWPINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/lzcntintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/lzcntintrin.h new file mode 100644 index 0000000000..401656cdab --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/lzcntintrin.h @@ -0,0 +1,75 @@ +/* Copyright (C) 2009-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#if !defined _X86INTRIN_H_INCLUDED && !defined _IMMINTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + + +#ifndef _LZCNTINTRIN_H_INCLUDED +#define _LZCNTINTRIN_H_INCLUDED + +#ifndef __LZCNT__ +#pragma GCC push_options +#pragma GCC target("lzcnt") +#define __DISABLE_LZCNT__ +#endif /* __LZCNT__ */ + +extern __inline unsigned short __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__lzcnt16 (unsigned short __X) +{ + return __builtin_ia32_lzcnt_u16 (__X); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__lzcnt32 (unsigned int __X) +{ + return __builtin_ia32_lzcnt_u32 (__X); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_lzcnt_u32 (unsigned int __X) +{ + return __builtin_ia32_lzcnt_u32 (__X); +} + +#ifdef __x86_64__ +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__lzcnt64 (unsigned long long __X) +{ + return __builtin_ia32_lzcnt_u64 (__X); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_lzcnt_u64 (unsigned long long __X) +{ + return __builtin_ia32_lzcnt_u64 (__X); +} +#endif + +#ifdef __DISABLE_LZCNT__ +#undef __DISABLE_LZCNT__ +#pragma GCC pop_options +#endif /* __DISABLE_LZCNT__ */ + +#endif /* _LZCNTINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/mm3dnow.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/mm3dnow.h new file mode 100644 index 0000000000..2d5c538493 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/mm3dnow.h @@ -0,0 +1,233 @@ +/* Copyright (C) 2004-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +/* Implemented from the mm3dnow.h (of supposedly AMD origin) included with + MSVC 7.1. */ + +#ifndef _MM3DNOW_H_INCLUDED +#define _MM3DNOW_H_INCLUDED + +#include +#include + +#if defined __x86_64__ && !defined __SSE__ || !defined __3dNOW__ +#pragma GCC push_options +#ifdef __x86_64__ +#pragma GCC target("sse,3dnow") +#else +#pragma GCC target("3dnow") +#endif +#define __DISABLE_3dNOW__ +#endif /* __3dNOW__ */ + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_femms (void) +{ + __builtin_ia32_femms(); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pavgusb (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pavgusb ((__v8qi)__A, (__v8qi)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pf2id (__m64 __A) +{ + return (__m64)__builtin_ia32_pf2id ((__v2sf)__A); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfacc (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pfacc ((__v2sf)__A, (__v2sf)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfadd (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pfadd ((__v2sf)__A, (__v2sf)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfcmpeq (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pfcmpeq ((__v2sf)__A, (__v2sf)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfcmpge (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pfcmpge ((__v2sf)__A, (__v2sf)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfcmpgt (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pfcmpgt ((__v2sf)__A, (__v2sf)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfmax (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pfmax ((__v2sf)__A, (__v2sf)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfmin (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pfmin ((__v2sf)__A, (__v2sf)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfmul (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pfmul ((__v2sf)__A, (__v2sf)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfrcp (__m64 __A) +{ + return (__m64)__builtin_ia32_pfrcp ((__v2sf)__A); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfrcpit1 (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pfrcpit1 ((__v2sf)__A, (__v2sf)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfrcpit2 (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pfrcpit2 ((__v2sf)__A, (__v2sf)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfrsqrt (__m64 __A) +{ + return (__m64)__builtin_ia32_pfrsqrt ((__v2sf)__A); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfrsqit1 (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pfrsqit1 ((__v2sf)__A, (__v2sf)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfsub (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pfsub ((__v2sf)__A, (__v2sf)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfsubr (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pfsubr ((__v2sf)__A, (__v2sf)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pi2fd (__m64 __A) +{ + return (__m64)__builtin_ia32_pi2fd ((__v2si)__A); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pmulhrw (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pmulhrw ((__v4hi)__A, (__v4hi)__B); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_prefetch (void *__P) +{ + __builtin_prefetch (__P, 0, 3 /* _MM_HINT_T0 */); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_from_float (float __A) +{ + return __extension__ (__m64)(__v2sf){ __A, 0.0f }; +} + +extern __inline float __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_to_float (__m64 __A) +{ + union { __v2sf v; float a[2]; } __tmp; + __tmp.v = (__v2sf)__A; + return __tmp.a[0]; +} + +#ifdef __DISABLE_3dNOW__ +#undef __DISABLE_3dNOW__ +#pragma GCC pop_options +#endif /* __DISABLE_3dNOW__ */ + +#if defined __x86_64__ && !defined __SSE__ || !defined __3dNOW_A__ +#pragma GCC push_options +#ifdef __x86_64__ +#pragma GCC target("sse,3dnowa") +#else +#pragma GCC target("3dnowa") +#endif +#define __DISABLE_3dNOW_A__ +#endif /* __3dNOW_A__ */ + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pf2iw (__m64 __A) +{ + return (__m64)__builtin_ia32_pf2iw ((__v2sf)__A); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfnacc (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pfnacc ((__v2sf)__A, (__v2sf)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pfpnacc (__m64 __A, __m64 __B) +{ + return (__m64)__builtin_ia32_pfpnacc ((__v2sf)__A, (__v2sf)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pi2fw (__m64 __A) +{ + return (__m64)__builtin_ia32_pi2fw ((__v2si)__A); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pswapd (__m64 __A) +{ + return (__m64)__builtin_ia32_pswapdsf ((__v2sf)__A); +} + +#ifdef __DISABLE_3dNOW_A__ +#undef __DISABLE_3dNOW_A__ +#pragma GCC pop_options +#endif /* __DISABLE_3dNOW_A__ */ + +#endif /* _MM3DNOW_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/mm_malloc.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/mm_malloc.h new file mode 100644 index 0000000000..73d9e887b4 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/mm_malloc.h @@ -0,0 +1,57 @@ +/* Copyright (C) 2004-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _MM_MALLOC_H_INCLUDED +#define _MM_MALLOC_H_INCLUDED + +#include + +/* We can't depend on since the prototype of posix_memalign + may not be visible. */ +#ifndef __cplusplus +extern int posix_memalign (void **, size_t, size_t); +#else +extern "C" int posix_memalign (void **, size_t, size_t) throw (); +#endif + +static __inline void * +_mm_malloc (size_t __size, size_t __alignment) +{ + void *__ptr; + if (__alignment == 1) + return malloc (__size); + if (__alignment == 2 || (sizeof (void *) == 8 && __alignment == 4)) + __alignment = sizeof (void *); + if (posix_memalign (&__ptr, __alignment, __size) == 0) + return __ptr; + else + return NULL; +} + +static __inline void +_mm_free (void *__ptr) +{ + free (__ptr); +} + +#endif /* _MM_MALLOC_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/mmintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/mmintrin.h new file mode 100644 index 0000000000..2cb73e34e9 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/mmintrin.h @@ -0,0 +1,949 @@ +/* Copyright (C) 2002-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +/* Implemented from the specification included in the Intel C++ Compiler + User Guide and Reference, version 9.0. */ + +#ifndef _MMINTRIN_H_INCLUDED +#define _MMINTRIN_H_INCLUDED + +#if defined __x86_64__ && !defined __SSE__ || !defined __MMX__ +#pragma GCC push_options +#ifdef __x86_64__ +#pragma GCC target("sse,mmx") +#else +#pragma GCC target("mmx") +#endif +#define __DISABLE_MMX__ +#endif /* __MMX__ */ + +/* The Intel API is flexible enough that we must allow aliasing with other + vector types, and their scalar components. */ +typedef int __m64 __attribute__ ((__vector_size__ (8), __may_alias__)); + +/* Unaligned version of the same type */ +typedef int __m64_u __attribute__ ((__vector_size__ (8), __may_alias__, __aligned__ (1))); + +/* Internal data types for implementing the intrinsics. */ +typedef int __v2si __attribute__ ((__vector_size__ (8))); +typedef short __v4hi __attribute__ ((__vector_size__ (8))); +typedef char __v8qi __attribute__ ((__vector_size__ (8))); +typedef long long __v1di __attribute__ ((__vector_size__ (8))); +typedef float __v2sf __attribute__ ((__vector_size__ (8))); + +/* Empty the multimedia state. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_empty (void) +{ + __builtin_ia32_emms (); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_empty (void) +{ + _mm_empty (); +} + +/* Convert I to a __m64 object. The integer is zero-extended to 64-bits. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi32_si64 (int __i) +{ + return (__m64) __builtin_ia32_vec_init_v2si (__i, 0); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_from_int (int __i) +{ + return _mm_cvtsi32_si64 (__i); +} + +#ifdef __x86_64__ +/* Convert I to a __m64 object. */ + +/* Intel intrinsic. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_from_int64 (long long __i) +{ + return (__m64) __i; +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi64_m64 (long long __i) +{ + return (__m64) __i; +} + +/* Microsoft intrinsic. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi64x_si64 (long long __i) +{ + return (__m64) __i; +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set_pi64x (long long __i) +{ + return (__m64) __i; +} +#endif + +/* Convert the lower 32 bits of the __m64 object into an integer. */ +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi64_si32 (__m64 __i) +{ + return __builtin_ia32_vec_ext_v2si ((__v2si)__i, 0); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_to_int (__m64 __i) +{ + return _mm_cvtsi64_si32 (__i); +} + +#ifdef __x86_64__ +/* Convert the __m64 object to a 64bit integer. */ + +/* Intel intrinsic. */ +extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_to_int64 (__m64 __i) +{ + return (long long)__i; +} + +extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtm64_si64 (__m64 __i) +{ + return (long long)__i; +} + +/* Microsoft intrinsic. */ +extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi64_si64x (__m64 __i) +{ + return (long long)__i; +} +#endif + +/* Pack the four 16-bit values from M1 into the lower four 8-bit values of + the result, and the four 16-bit values from M2 into the upper four 8-bit + values of the result, all with signed saturation. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_packs_pi16 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_packsswb ((__v4hi)__m1, (__v4hi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_packsswb (__m64 __m1, __m64 __m2) +{ + return _mm_packs_pi16 (__m1, __m2); +} + +/* Pack the two 32-bit values from M1 in to the lower two 16-bit values of + the result, and the two 32-bit values from M2 into the upper two 16-bit + values of the result, all with signed saturation. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_packs_pi32 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_packssdw ((__v2si)__m1, (__v2si)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_packssdw (__m64 __m1, __m64 __m2) +{ + return _mm_packs_pi32 (__m1, __m2); +} + +/* Pack the four 16-bit values from M1 into the lower four 8-bit values of + the result, and the four 16-bit values from M2 into the upper four 8-bit + values of the result, all with unsigned saturation. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_packs_pu16 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_packuswb ((__v4hi)__m1, (__v4hi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_packuswb (__m64 __m1, __m64 __m2) +{ + return _mm_packs_pu16 (__m1, __m2); +} + +/* Interleave the four 8-bit values from the high half of M1 with the four + 8-bit values from the high half of M2. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpackhi_pi8 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_punpckhbw ((__v8qi)__m1, (__v8qi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_punpckhbw (__m64 __m1, __m64 __m2) +{ + return _mm_unpackhi_pi8 (__m1, __m2); +} + +/* Interleave the two 16-bit values from the high half of M1 with the two + 16-bit values from the high half of M2. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpackhi_pi16 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_punpckhwd ((__v4hi)__m1, (__v4hi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_punpckhwd (__m64 __m1, __m64 __m2) +{ + return _mm_unpackhi_pi16 (__m1, __m2); +} + +/* Interleave the 32-bit value from the high half of M1 with the 32-bit + value from the high half of M2. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpackhi_pi32 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_punpckhdq ((__v2si)__m1, (__v2si)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_punpckhdq (__m64 __m1, __m64 __m2) +{ + return _mm_unpackhi_pi32 (__m1, __m2); +} + +/* Interleave the four 8-bit values from the low half of M1 with the four + 8-bit values from the low half of M2. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpacklo_pi8 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_punpcklbw ((__v8qi)__m1, (__v8qi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_punpcklbw (__m64 __m1, __m64 __m2) +{ + return _mm_unpacklo_pi8 (__m1, __m2); +} + +/* Interleave the two 16-bit values from the low half of M1 with the two + 16-bit values from the low half of M2. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpacklo_pi16 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_punpcklwd ((__v4hi)__m1, (__v4hi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_punpcklwd (__m64 __m1, __m64 __m2) +{ + return _mm_unpacklo_pi16 (__m1, __m2); +} + +/* Interleave the 32-bit value from the low half of M1 with the 32-bit + value from the low half of M2. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpacklo_pi32 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_punpckldq ((__v2si)__m1, (__v2si)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_punpckldq (__m64 __m1, __m64 __m2) +{ + return _mm_unpacklo_pi32 (__m1, __m2); +} + +/* Add the 8-bit values in M1 to the 8-bit values in M2. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_add_pi8 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_paddb ((__v8qi)__m1, (__v8qi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_paddb (__m64 __m1, __m64 __m2) +{ + return _mm_add_pi8 (__m1, __m2); +} + +/* Add the 16-bit values in M1 to the 16-bit values in M2. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_add_pi16 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_paddw ((__v4hi)__m1, (__v4hi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_paddw (__m64 __m1, __m64 __m2) +{ + return _mm_add_pi16 (__m1, __m2); +} + +/* Add the 32-bit values in M1 to the 32-bit values in M2. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_add_pi32 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_paddd ((__v2si)__m1, (__v2si)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_paddd (__m64 __m1, __m64 __m2) +{ + return _mm_add_pi32 (__m1, __m2); +} + +/* Add the 64-bit values in M1 to the 64-bit values in M2. */ +#ifndef __SSE2__ +#pragma GCC push_options +#pragma GCC target("sse2,mmx") +#define __DISABLE_SSE2__ +#endif /* __SSE2__ */ + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_add_si64 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_paddq ((__v1di)__m1, (__v1di)__m2); +} +#ifdef __DISABLE_SSE2__ +#undef __DISABLE_SSE2__ +#pragma GCC pop_options +#endif /* __DISABLE_SSE2__ */ + +/* Add the 8-bit values in M1 to the 8-bit values in M2 using signed + saturated arithmetic. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_adds_pi8 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_paddsb ((__v8qi)__m1, (__v8qi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_paddsb (__m64 __m1, __m64 __m2) +{ + return _mm_adds_pi8 (__m1, __m2); +} + +/* Add the 16-bit values in M1 to the 16-bit values in M2 using signed + saturated arithmetic. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_adds_pi16 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_paddsw ((__v4hi)__m1, (__v4hi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_paddsw (__m64 __m1, __m64 __m2) +{ + return _mm_adds_pi16 (__m1, __m2); +} + +/* Add the 8-bit values in M1 to the 8-bit values in M2 using unsigned + saturated arithmetic. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_adds_pu8 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_paddusb ((__v8qi)__m1, (__v8qi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_paddusb (__m64 __m1, __m64 __m2) +{ + return _mm_adds_pu8 (__m1, __m2); +} + +/* Add the 16-bit values in M1 to the 16-bit values in M2 using unsigned + saturated arithmetic. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_adds_pu16 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_paddusw ((__v4hi)__m1, (__v4hi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_paddusw (__m64 __m1, __m64 __m2) +{ + return _mm_adds_pu16 (__m1, __m2); +} + +/* Subtract the 8-bit values in M2 from the 8-bit values in M1. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sub_pi8 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_psubb ((__v8qi)__m1, (__v8qi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psubb (__m64 __m1, __m64 __m2) +{ + return _mm_sub_pi8 (__m1, __m2); +} + +/* Subtract the 16-bit values in M2 from the 16-bit values in M1. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sub_pi16 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_psubw ((__v4hi)__m1, (__v4hi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psubw (__m64 __m1, __m64 __m2) +{ + return _mm_sub_pi16 (__m1, __m2); +} + +/* Subtract the 32-bit values in M2 from the 32-bit values in M1. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sub_pi32 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_psubd ((__v2si)__m1, (__v2si)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psubd (__m64 __m1, __m64 __m2) +{ + return _mm_sub_pi32 (__m1, __m2); +} + +/* Add the 64-bit values in M1 to the 64-bit values in M2. */ +#ifndef __SSE2__ +#pragma GCC push_options +#pragma GCC target("sse2,mmx") +#define __DISABLE_SSE2__ +#endif /* __SSE2__ */ + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sub_si64 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_psubq ((__v1di)__m1, (__v1di)__m2); +} +#ifdef __DISABLE_SSE2__ +#undef __DISABLE_SSE2__ +#pragma GCC pop_options +#endif /* __DISABLE_SSE2__ */ + +/* Subtract the 8-bit values in M2 from the 8-bit values in M1 using signed + saturating arithmetic. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_subs_pi8 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_psubsb ((__v8qi)__m1, (__v8qi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psubsb (__m64 __m1, __m64 __m2) +{ + return _mm_subs_pi8 (__m1, __m2); +} + +/* Subtract the 16-bit values in M2 from the 16-bit values in M1 using + signed saturating arithmetic. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_subs_pi16 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_psubsw ((__v4hi)__m1, (__v4hi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psubsw (__m64 __m1, __m64 __m2) +{ + return _mm_subs_pi16 (__m1, __m2); +} + +/* Subtract the 8-bit values in M2 from the 8-bit values in M1 using + unsigned saturating arithmetic. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_subs_pu8 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_psubusb ((__v8qi)__m1, (__v8qi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psubusb (__m64 __m1, __m64 __m2) +{ + return _mm_subs_pu8 (__m1, __m2); +} + +/* Subtract the 16-bit values in M2 from the 16-bit values in M1 using + unsigned saturating arithmetic. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_subs_pu16 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_psubusw ((__v4hi)__m1, (__v4hi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psubusw (__m64 __m1, __m64 __m2) +{ + return _mm_subs_pu16 (__m1, __m2); +} + +/* Multiply four 16-bit values in M1 by four 16-bit values in M2 producing + four 32-bit intermediate results, which are then summed by pairs to + produce two 32-bit results. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_madd_pi16 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_pmaddwd ((__v4hi)__m1, (__v4hi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pmaddwd (__m64 __m1, __m64 __m2) +{ + return _mm_madd_pi16 (__m1, __m2); +} + +/* Multiply four signed 16-bit values in M1 by four signed 16-bit values in + M2 and produce the high 16 bits of the 32-bit results. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mulhi_pi16 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_pmulhw ((__v4hi)__m1, (__v4hi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pmulhw (__m64 __m1, __m64 __m2) +{ + return _mm_mulhi_pi16 (__m1, __m2); +} + +/* Multiply four 16-bit values in M1 by four 16-bit values in M2 and produce + the low 16 bits of the results. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mullo_pi16 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_pmullw ((__v4hi)__m1, (__v4hi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pmullw (__m64 __m1, __m64 __m2) +{ + return _mm_mullo_pi16 (__m1, __m2); +} + +/* Shift four 16-bit values in M left by COUNT. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sll_pi16 (__m64 __m, __m64 __count) +{ + return (__m64) __builtin_ia32_psllw ((__v4hi)__m, (__v4hi)__count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psllw (__m64 __m, __m64 __count) +{ + return _mm_sll_pi16 (__m, __count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_slli_pi16 (__m64 __m, int __count) +{ + return (__m64) __builtin_ia32_psllwi ((__v4hi)__m, __count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psllwi (__m64 __m, int __count) +{ + return _mm_slli_pi16 (__m, __count); +} + +/* Shift two 32-bit values in M left by COUNT. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sll_pi32 (__m64 __m, __m64 __count) +{ + return (__m64) __builtin_ia32_pslld ((__v2si)__m, (__v2si)__count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pslld (__m64 __m, __m64 __count) +{ + return _mm_sll_pi32 (__m, __count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_slli_pi32 (__m64 __m, int __count) +{ + return (__m64) __builtin_ia32_pslldi ((__v2si)__m, __count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pslldi (__m64 __m, int __count) +{ + return _mm_slli_pi32 (__m, __count); +} + +/* Shift the 64-bit value in M left by COUNT. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sll_si64 (__m64 __m, __m64 __count) +{ + return (__m64) __builtin_ia32_psllq ((__v1di)__m, (__v1di)__count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psllq (__m64 __m, __m64 __count) +{ + return _mm_sll_si64 (__m, __count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_slli_si64 (__m64 __m, int __count) +{ + return (__m64) __builtin_ia32_psllqi ((__v1di)__m, __count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psllqi (__m64 __m, int __count) +{ + return _mm_slli_si64 (__m, __count); +} + +/* Shift four 16-bit values in M right by COUNT; shift in the sign bit. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sra_pi16 (__m64 __m, __m64 __count) +{ + return (__m64) __builtin_ia32_psraw ((__v4hi)__m, (__v4hi)__count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psraw (__m64 __m, __m64 __count) +{ + return _mm_sra_pi16 (__m, __count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srai_pi16 (__m64 __m, int __count) +{ + return (__m64) __builtin_ia32_psrawi ((__v4hi)__m, __count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psrawi (__m64 __m, int __count) +{ + return _mm_srai_pi16 (__m, __count); +} + +/* Shift two 32-bit values in M right by COUNT; shift in the sign bit. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sra_pi32 (__m64 __m, __m64 __count) +{ + return (__m64) __builtin_ia32_psrad ((__v2si)__m, (__v2si)__count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psrad (__m64 __m, __m64 __count) +{ + return _mm_sra_pi32 (__m, __count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srai_pi32 (__m64 __m, int __count) +{ + return (__m64) __builtin_ia32_psradi ((__v2si)__m, __count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psradi (__m64 __m, int __count) +{ + return _mm_srai_pi32 (__m, __count); +} + +/* Shift four 16-bit values in M right by COUNT; shift in zeros. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srl_pi16 (__m64 __m, __m64 __count) +{ + return (__m64) __builtin_ia32_psrlw ((__v4hi)__m, (__v4hi)__count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psrlw (__m64 __m, __m64 __count) +{ + return _mm_srl_pi16 (__m, __count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srli_pi16 (__m64 __m, int __count) +{ + return (__m64) __builtin_ia32_psrlwi ((__v4hi)__m, __count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psrlwi (__m64 __m, int __count) +{ + return _mm_srli_pi16 (__m, __count); +} + +/* Shift two 32-bit values in M right by COUNT; shift in zeros. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srl_pi32 (__m64 __m, __m64 __count) +{ + return (__m64) __builtin_ia32_psrld ((__v2si)__m, (__v2si)__count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psrld (__m64 __m, __m64 __count) +{ + return _mm_srl_pi32 (__m, __count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srli_pi32 (__m64 __m, int __count) +{ + return (__m64) __builtin_ia32_psrldi ((__v2si)__m, __count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psrldi (__m64 __m, int __count) +{ + return _mm_srli_pi32 (__m, __count); +} + +/* Shift the 64-bit value in M left by COUNT; shift in zeros. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srl_si64 (__m64 __m, __m64 __count) +{ + return (__m64) __builtin_ia32_psrlq ((__v1di)__m, (__v1di)__count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psrlq (__m64 __m, __m64 __count) +{ + return _mm_srl_si64 (__m, __count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_srli_si64 (__m64 __m, int __count) +{ + return (__m64) __builtin_ia32_psrlqi ((__v1di)__m, __count); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psrlqi (__m64 __m, int __count) +{ + return _mm_srli_si64 (__m, __count); +} + +/* Bit-wise AND the 64-bit values in M1 and M2. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_and_si64 (__m64 __m1, __m64 __m2) +{ + return __builtin_ia32_pand (__m1, __m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pand (__m64 __m1, __m64 __m2) +{ + return _mm_and_si64 (__m1, __m2); +} + +/* Bit-wise complement the 64-bit value in M1 and bit-wise AND it with the + 64-bit value in M2. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_andnot_si64 (__m64 __m1, __m64 __m2) +{ + return __builtin_ia32_pandn (__m1, __m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pandn (__m64 __m1, __m64 __m2) +{ + return _mm_andnot_si64 (__m1, __m2); +} + +/* Bit-wise inclusive OR the 64-bit values in M1 and M2. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_or_si64 (__m64 __m1, __m64 __m2) +{ + return __builtin_ia32_por (__m1, __m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_por (__m64 __m1, __m64 __m2) +{ + return _mm_or_si64 (__m1, __m2); +} + +/* Bit-wise exclusive OR the 64-bit values in M1 and M2. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_xor_si64 (__m64 __m1, __m64 __m2) +{ + return __builtin_ia32_pxor (__m1, __m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pxor (__m64 __m1, __m64 __m2) +{ + return _mm_xor_si64 (__m1, __m2); +} + +/* Compare eight 8-bit values. The result of the comparison is 0xFF if the + test is true and zero if false. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_pi8 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_pcmpeqb ((__v8qi)__m1, (__v8qi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pcmpeqb (__m64 __m1, __m64 __m2) +{ + return _mm_cmpeq_pi8 (__m1, __m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_pi8 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_pcmpgtb ((__v8qi)__m1, (__v8qi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pcmpgtb (__m64 __m1, __m64 __m2) +{ + return _mm_cmpgt_pi8 (__m1, __m2); +} + +/* Compare four 16-bit values. The result of the comparison is 0xFFFF if + the test is true and zero if false. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_pi16 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_pcmpeqw ((__v4hi)__m1, (__v4hi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pcmpeqw (__m64 __m1, __m64 __m2) +{ + return _mm_cmpeq_pi16 (__m1, __m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_pi16 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_pcmpgtw ((__v4hi)__m1, (__v4hi)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pcmpgtw (__m64 __m1, __m64 __m2) +{ + return _mm_cmpgt_pi16 (__m1, __m2); +} + +/* Compare two 32-bit values. The result of the comparison is 0xFFFFFFFF if + the test is true and zero if false. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_pi32 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_pcmpeqd ((__v2si)__m1, (__v2si)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pcmpeqd (__m64 __m1, __m64 __m2) +{ + return _mm_cmpeq_pi32 (__m1, __m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_pi32 (__m64 __m1, __m64 __m2) +{ + return (__m64) __builtin_ia32_pcmpgtd ((__v2si)__m1, (__v2si)__m2); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pcmpgtd (__m64 __m1, __m64 __m2) +{ + return _mm_cmpgt_pi32 (__m1, __m2); +} + +/* Creates a 64-bit zero. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_setzero_si64 (void) +{ + return (__m64)0LL; +} + +/* Creates a vector of two 32-bit values; I0 is least significant. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set_pi32 (int __i1, int __i0) +{ + return (__m64) __builtin_ia32_vec_init_v2si (__i0, __i1); +} + +/* Creates a vector of four 16-bit values; W0 is least significant. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set_pi16 (short __w3, short __w2, short __w1, short __w0) +{ + return (__m64) __builtin_ia32_vec_init_v4hi (__w0, __w1, __w2, __w3); +} + +/* Creates a vector of eight 8-bit values; B0 is least significant. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set_pi8 (char __b7, char __b6, char __b5, char __b4, + char __b3, char __b2, char __b1, char __b0) +{ + return (__m64) __builtin_ia32_vec_init_v8qi (__b0, __b1, __b2, __b3, + __b4, __b5, __b6, __b7); +} + +/* Similar, but with the arguments in reverse order. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_setr_pi32 (int __i0, int __i1) +{ + return _mm_set_pi32 (__i1, __i0); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_setr_pi16 (short __w0, short __w1, short __w2, short __w3) +{ + return _mm_set_pi16 (__w3, __w2, __w1, __w0); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_setr_pi8 (char __b0, char __b1, char __b2, char __b3, + char __b4, char __b5, char __b6, char __b7) +{ + return _mm_set_pi8 (__b7, __b6, __b5, __b4, __b3, __b2, __b1, __b0); +} + +/* Creates a vector of two 32-bit values, both elements containing I. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set1_pi32 (int __i) +{ + return _mm_set_pi32 (__i, __i); +} + +/* Creates a vector of four 16-bit values, all elements containing W. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set1_pi16 (short __w) +{ + return _mm_set_pi16 (__w, __w, __w, __w); +} + +/* Creates a vector of eight 8-bit values, all elements containing B. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set1_pi8 (char __b) +{ + return _mm_set_pi8 (__b, __b, __b, __b, __b, __b, __b, __b); +} +#ifdef __DISABLE_MMX__ +#undef __DISABLE_MMX__ +#pragma GCC pop_options +#endif /* __DISABLE_MMX__ */ + +#endif /* _MMINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/mwaitxintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/mwaitxintrin.h new file mode 100644 index 0000000000..aaa4c29e6f --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/mwaitxintrin.h @@ -0,0 +1,50 @@ +/* Copyright (C) 2012-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _MWAITXINTRIN_H_INCLUDED +#define _MWAITXINTRIN_H_INCLUDED + +#ifndef __MWAITX__ +#pragma GCC push_options +#pragma GCC target("mwaitx") +#define __DISABLE_MWAITX__ +#endif /* __MWAITX__ */ + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_monitorx (void const * __P, unsigned int __E, unsigned int __H) +{ + __builtin_ia32_monitorx (__P, __E, __H); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mwaitx (unsigned int __E, unsigned int __H, unsigned int __C) +{ + __builtin_ia32_mwaitx (__E, __H, __C); +} + +#ifdef __DISABLE_MWAITX__ +#undef __DISABLE_MWAITX__ +#pragma GCC pop_options +#endif /* __DISABLE_MWAITX__ */ + +#endif /* _MWAITXINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/nmmintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/nmmintrin.h new file mode 100644 index 0000000000..6ea57d326d --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/nmmintrin.h @@ -0,0 +1,33 @@ +/* Copyright (C) 2007-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +/* Implemented from the specification included in the Intel C++ Compiler + User Guide and Reference, version 10.0. */ + +#ifndef _NMMINTRIN_H_INCLUDED +#define _NMMINTRIN_H_INCLUDED + +/* We just include SSE4.1 header file. */ +#include + +#endif /* _NMMINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/omp.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/omp.h new file mode 100644 index 0000000000..04f170b690 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/omp.h @@ -0,0 +1,165 @@ +/* Copyright (C) 2005-2017 Free Software Foundation, Inc. + Contributed by Richard Henderson . + + This file is part of the GNU Offloading and Multi Processing Library + (libgomp). + + Libgomp is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _OMP_H +#define _OMP_H 1 + +#ifndef _LIBGOMP_OMP_LOCK_DEFINED +#define _LIBGOMP_OMP_LOCK_DEFINED 1 +/* These two structures get edited by the libgomp build process to + reflect the shape of the two types. Their internals are private + to the library. */ + +typedef struct +{ + unsigned char _x[4] + __attribute__((__aligned__(4))); +} omp_lock_t; + +typedef struct +{ + unsigned char _x[8 + sizeof (void *)] + __attribute__((__aligned__(sizeof (void *)))); +} omp_nest_lock_t; +#endif + +typedef enum omp_sched_t +{ + omp_sched_static = 1, + omp_sched_dynamic = 2, + omp_sched_guided = 3, + omp_sched_auto = 4 +} omp_sched_t; + +typedef enum omp_proc_bind_t +{ + omp_proc_bind_false = 0, + omp_proc_bind_true = 1, + omp_proc_bind_master = 2, + omp_proc_bind_close = 3, + omp_proc_bind_spread = 4 +} omp_proc_bind_t; + +typedef enum omp_lock_hint_t +{ + omp_lock_hint_none = 0, + omp_lock_hint_uncontended = 1, + omp_lock_hint_contended = 2, + omp_lock_hint_nonspeculative = 4, + omp_lock_hint_speculative = 8, +} omp_lock_hint_t; + +#ifdef __cplusplus +extern "C" { +# define __GOMP_NOTHROW throw () +#else +# define __GOMP_NOTHROW __attribute__((__nothrow__)) +#endif + +extern void omp_set_num_threads (int) __GOMP_NOTHROW; +extern int omp_get_num_threads (void) __GOMP_NOTHROW; +extern int omp_get_max_threads (void) __GOMP_NOTHROW; +extern int omp_get_thread_num (void) __GOMP_NOTHROW; +extern int omp_get_num_procs (void) __GOMP_NOTHROW; + +extern int omp_in_parallel (void) __GOMP_NOTHROW; + +extern void omp_set_dynamic (int) __GOMP_NOTHROW; +extern int omp_get_dynamic (void) __GOMP_NOTHROW; + +extern void omp_set_nested (int) __GOMP_NOTHROW; +extern int omp_get_nested (void) __GOMP_NOTHROW; + +extern void omp_init_lock (omp_lock_t *) __GOMP_NOTHROW; +extern void omp_init_lock_with_hint (omp_lock_t *, omp_lock_hint_t) + __GOMP_NOTHROW; +extern void omp_destroy_lock (omp_lock_t *) __GOMP_NOTHROW; +extern void omp_set_lock (omp_lock_t *) __GOMP_NOTHROW; +extern void omp_unset_lock (omp_lock_t *) __GOMP_NOTHROW; +extern int omp_test_lock (omp_lock_t *) __GOMP_NOTHROW; + +extern void omp_init_nest_lock (omp_nest_lock_t *) __GOMP_NOTHROW; +extern void omp_init_nest_lock_with_hint (omp_nest_lock_t *, omp_lock_hint_t) + __GOMP_NOTHROW; +extern void omp_destroy_nest_lock (omp_nest_lock_t *) __GOMP_NOTHROW; +extern void omp_set_nest_lock (omp_nest_lock_t *) __GOMP_NOTHROW; +extern void omp_unset_nest_lock (omp_nest_lock_t *) __GOMP_NOTHROW; +extern int omp_test_nest_lock (omp_nest_lock_t *) __GOMP_NOTHROW; + +extern double omp_get_wtime (void) __GOMP_NOTHROW; +extern double omp_get_wtick (void) __GOMP_NOTHROW; + +extern void omp_set_schedule (omp_sched_t, int) __GOMP_NOTHROW; +extern void omp_get_schedule (omp_sched_t *, int *) __GOMP_NOTHROW; +extern int omp_get_thread_limit (void) __GOMP_NOTHROW; +extern void omp_set_max_active_levels (int) __GOMP_NOTHROW; +extern int omp_get_max_active_levels (void) __GOMP_NOTHROW; +extern int omp_get_level (void) __GOMP_NOTHROW; +extern int omp_get_ancestor_thread_num (int) __GOMP_NOTHROW; +extern int omp_get_team_size (int) __GOMP_NOTHROW; +extern int omp_get_active_level (void) __GOMP_NOTHROW; + +extern int omp_in_final (void) __GOMP_NOTHROW; + +extern int omp_get_cancellation (void) __GOMP_NOTHROW; +extern omp_proc_bind_t omp_get_proc_bind (void) __GOMP_NOTHROW; +extern int omp_get_num_places (void) __GOMP_NOTHROW; +extern int omp_get_place_num_procs (int) __GOMP_NOTHROW; +extern void omp_get_place_proc_ids (int, int *) __GOMP_NOTHROW; +extern int omp_get_place_num (void) __GOMP_NOTHROW; +extern int omp_get_partition_num_places (void) __GOMP_NOTHROW; +extern void omp_get_partition_place_nums (int *) __GOMP_NOTHROW; + +extern void omp_set_default_device (int) __GOMP_NOTHROW; +extern int omp_get_default_device (void) __GOMP_NOTHROW; +extern int omp_get_num_devices (void) __GOMP_NOTHROW; +extern int omp_get_num_teams (void) __GOMP_NOTHROW; +extern int omp_get_team_num (void) __GOMP_NOTHROW; + +extern int omp_is_initial_device (void) __GOMP_NOTHROW; +extern int omp_get_initial_device (void) __GOMP_NOTHROW; +extern int omp_get_max_task_priority (void) __GOMP_NOTHROW; + +extern void *omp_target_alloc (__SIZE_TYPE__, int) __GOMP_NOTHROW; +extern void omp_target_free (void *, int) __GOMP_NOTHROW; +extern int omp_target_is_present (void *, int) __GOMP_NOTHROW; +extern int omp_target_memcpy (void *, void *, __SIZE_TYPE__, __SIZE_TYPE__, + __SIZE_TYPE__, int, int) __GOMP_NOTHROW; +extern int omp_target_memcpy_rect (void *, void *, __SIZE_TYPE__, int, + const __SIZE_TYPE__ *, + const __SIZE_TYPE__ *, + const __SIZE_TYPE__ *, + const __SIZE_TYPE__ *, + const __SIZE_TYPE__ *, int, int) + __GOMP_NOTHROW; +extern int omp_target_associate_ptr (void *, void *, __SIZE_TYPE__, + __SIZE_TYPE__, int) __GOMP_NOTHROW; +extern int omp_target_disassociate_ptr (void *, int) __GOMP_NOTHROW; + +#ifdef __cplusplus +} +#endif + +#endif /* _OMP_H */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/openacc.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/openacc.h new file mode 100644 index 0000000000..53d0c3955c --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/openacc.h @@ -0,0 +1,131 @@ +/* OpenACC Runtime Library User-facing Declarations + + Copyright (C) 2013-2017 Free Software Foundation, Inc. + + Contributed by Mentor Embedded. + + This file is part of the GNU Offloading and Multi Processing Library + (libgomp). + + Libgomp is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU General Public License for + more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _OPENACC_H +#define _OPENACC_H 1 + +/* The OpenACC standard is silent on whether or not including + might or must not include other header files. We chose to include + some. */ +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#if __cplusplus >= 201103 +# define __GOACC_NOTHROW noexcept +#elif __cplusplus +# define __GOACC_NOTHROW throw () +#else /* Not C++ */ +# define __GOACC_NOTHROW __attribute__ ((__nothrow__)) +#endif + +/* Types */ +typedef enum acc_device_t { + /* Keep in sync with include/gomp-constants.h. */ + acc_device_none = 0, + acc_device_default = 1, + acc_device_host = 2, + /* acc_device_host_nonshm = 3 removed. */ + acc_device_not_host = 4, + acc_device_nvidia = 5, + _ACC_device_hwm, + /* Ensure enumeration is layout compatible with int. */ + _ACC_highest = __INT_MAX__, + _ACC_neg = -1 +} acc_device_t; + +typedef enum acc_async_t { + /* Keep in sync with include/gomp-constants.h. */ + acc_async_noval = -1, + acc_async_sync = -2 +} acc_async_t; + +int acc_get_num_devices (acc_device_t) __GOACC_NOTHROW; +void acc_set_device_type (acc_device_t) __GOACC_NOTHROW; +acc_device_t acc_get_device_type (void) __GOACC_NOTHROW; +void acc_set_device_num (int, acc_device_t) __GOACC_NOTHROW; +int acc_get_device_num (acc_device_t) __GOACC_NOTHROW; +int acc_async_test (int) __GOACC_NOTHROW; +int acc_async_test_all (void) __GOACC_NOTHROW; +void acc_wait (int) __GOACC_NOTHROW; +void acc_wait_async (int, int) __GOACC_NOTHROW; +void acc_wait_all (void) __GOACC_NOTHROW; +void acc_wait_all_async (int) __GOACC_NOTHROW; +void acc_init (acc_device_t) __GOACC_NOTHROW; +void acc_shutdown (acc_device_t) __GOACC_NOTHROW; +#ifdef __cplusplus +int acc_on_device (int __arg) __GOACC_NOTHROW; +#else +int acc_on_device (acc_device_t __arg) __GOACC_NOTHROW; +#endif +void *acc_malloc (size_t) __GOACC_NOTHROW; +void acc_free (void *) __GOACC_NOTHROW; +/* Some of these would be more correct with const qualifiers, but + the standard specifies otherwise. */ +void *acc_copyin (void *, size_t) __GOACC_NOTHROW; +void *acc_present_or_copyin (void *, size_t) __GOACC_NOTHROW; +void *acc_create (void *, size_t) __GOACC_NOTHROW; +void *acc_present_or_create (void *, size_t) __GOACC_NOTHROW; +void acc_copyout (void *, size_t) __GOACC_NOTHROW; +void acc_delete (void *, size_t) __GOACC_NOTHROW; +void acc_update_device (void *, size_t) __GOACC_NOTHROW; +void acc_update_self (void *, size_t) __GOACC_NOTHROW; +void acc_map_data (void *, void *, size_t) __GOACC_NOTHROW; +void acc_unmap_data (void *) __GOACC_NOTHROW; +void *acc_deviceptr (void *) __GOACC_NOTHROW; +void *acc_hostptr (void *) __GOACC_NOTHROW; +int acc_is_present (void *, size_t) __GOACC_NOTHROW; +void acc_memcpy_to_device (void *, void *, size_t) __GOACC_NOTHROW; +void acc_memcpy_from_device (void *, void *, size_t) __GOACC_NOTHROW; + +/* Old names. OpenACC does not specify whether these can or must + not be macros, inlines or aliases for the new names. */ +#define acc_pcreate acc_present_or_create +#define acc_pcopyin acc_present_or_copyin + +/* CUDA-specific routines. */ +void *acc_get_current_cuda_device (void) __GOACC_NOTHROW; +void *acc_get_current_cuda_context (void) __GOACC_NOTHROW; +void *acc_get_cuda_stream (int) __GOACC_NOTHROW; +int acc_set_cuda_stream (int, void *) __GOACC_NOTHROW; + +#ifdef __cplusplus +} + +/* Forwarding function with correctly typed arg. */ + +#pragma acc routine seq +inline int acc_on_device (acc_device_t __arg) __GOACC_NOTHROW +{ + return acc_on_device ((int) __arg); +} +#endif + +#endif /* _OPENACC_H */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/pkuintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/pkuintrin.h new file mode 100644 index 0000000000..0f4886ca82 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/pkuintrin.h @@ -0,0 +1,56 @@ +/* Copyright (C) 2015-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#if !defined _X86INTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _PKUINTRIN_H_INCLUDED +#define _PKUINTRIN_H_INCLUDED + +#ifndef __PKU__ +#pragma GCC push_options +#pragma GCC target("pku") +#define __DISABLE_PKU__ +#endif /* __PKU__ */ + +extern __inline unsigned int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_rdpkru_u32 (void) +{ + return __builtin_ia32_rdpkru (); +} + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_wrpkru (unsigned int __key) +{ + __builtin_ia32_wrpkru (__key); +} + +#ifdef __DISABLE_PKU__ +#undef __DISABLE_PKU__ +#pragma GCC pop_options +#endif /* __DISABLE_PKU__ */ + +#endif /* _PKUINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/pmmintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/pmmintrin.h new file mode 100644 index 0000000000..39cc1c6761 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/pmmintrin.h @@ -0,0 +1,132 @@ +/* Copyright (C) 2003-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +/* Implemented from the specification included in the Intel C++ Compiler + User Guide and Reference, version 9.0. */ + +#ifndef _PMMINTRIN_H_INCLUDED +#define _PMMINTRIN_H_INCLUDED + +/* We need definitions from the SSE2 and SSE header files*/ +#include + +#ifndef __SSE3__ +#pragma GCC push_options +#pragma GCC target("sse3") +#define __DISABLE_SSE3__ +#endif /* __SSE3__ */ + +/* Additional bits in the MXCSR. */ +#define _MM_DENORMALS_ZERO_MASK 0x0040 +#define _MM_DENORMALS_ZERO_ON 0x0040 +#define _MM_DENORMALS_ZERO_OFF 0x0000 + +#define _MM_SET_DENORMALS_ZERO_MODE(mode) \ + _mm_setcsr ((_mm_getcsr () & ~_MM_DENORMALS_ZERO_MASK) | (mode)) +#define _MM_GET_DENORMALS_ZERO_MODE() \ + (_mm_getcsr() & _MM_DENORMALS_ZERO_MASK) + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_addsub_ps (__m128 __X, __m128 __Y) +{ + return (__m128) __builtin_ia32_addsubps ((__v4sf)__X, (__v4sf)__Y); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hadd_ps (__m128 __X, __m128 __Y) +{ + return (__m128) __builtin_ia32_haddps ((__v4sf)__X, (__v4sf)__Y); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hsub_ps (__m128 __X, __m128 __Y) +{ + return (__m128) __builtin_ia32_hsubps ((__v4sf)__X, (__v4sf)__Y); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movehdup_ps (__m128 __X) +{ + return (__m128) __builtin_ia32_movshdup ((__v4sf)__X); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_moveldup_ps (__m128 __X) +{ + return (__m128) __builtin_ia32_movsldup ((__v4sf)__X); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_addsub_pd (__m128d __X, __m128d __Y) +{ + return (__m128d) __builtin_ia32_addsubpd ((__v2df)__X, (__v2df)__Y); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hadd_pd (__m128d __X, __m128d __Y) +{ + return (__m128d) __builtin_ia32_haddpd ((__v2df)__X, (__v2df)__Y); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hsub_pd (__m128d __X, __m128d __Y) +{ + return (__m128d) __builtin_ia32_hsubpd ((__v2df)__X, (__v2df)__Y); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_loaddup_pd (double const *__P) +{ + return _mm_load1_pd (__P); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movedup_pd (__m128d __X) +{ + return _mm_shuffle_pd (__X, __X, _MM_SHUFFLE2 (0,0)); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_lddqu_si128 (__m128i const *__P) +{ + return (__m128i) __builtin_ia32_lddqu ((char const *)__P); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_monitor (void const * __P, unsigned int __E, unsigned int __H) +{ + __builtin_ia32_monitor (__P, __E, __H); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mwait (unsigned int __E, unsigned int __H) +{ + __builtin_ia32_mwait (__E, __H); +} + +#ifdef __DISABLE_SSE3__ +#undef __DISABLE_SSE3__ +#pragma GCC pop_options +#endif /* __DISABLE_SSE3__ */ + +#endif /* _PMMINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/popcntintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/popcntintrin.h new file mode 100644 index 0000000000..733fd75553 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/popcntintrin.h @@ -0,0 +1,53 @@ +/* Copyright (C) 2009-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _POPCNTINTRIN_H_INCLUDED +#define _POPCNTINTRIN_H_INCLUDED + +#ifndef __POPCNT__ +#pragma GCC push_options +#pragma GCC target("popcnt") +#define __DISABLE_POPCNT__ +#endif /* __POPCNT__ */ + +/* Calculate a number of bits set to 1. */ +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_popcnt_u32 (unsigned int __X) +{ + return __builtin_popcount (__X); +} + +#ifdef __x86_64__ +extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_popcnt_u64 (unsigned long long __X) +{ + return __builtin_popcountll (__X); +} +#endif + +#ifdef __DISABLE_POPCNT__ +#undef __DISABLE_POPCNT__ +#pragma GCC pop_options +#endif /* __DISABLE_POPCNT__ */ + +#endif /* _POPCNTINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/prfchwintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/prfchwintrin.h new file mode 100644 index 0000000000..11a06c8951 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/prfchwintrin.h @@ -0,0 +1,37 @@ +/* Copyright (C) 2012-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#if !defined _X86INTRIN_H_INCLUDED && !defined _MM3DNOW_H_INCLUDED +# error "Never use directly; include or instead." +#endif + +#ifndef _PRFCHWINTRIN_H_INCLUDED +#define _PRFCHWINTRIN_H_INCLUDED + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_prefetchw (void *__P) +{ + __builtin_prefetch (__P, 1, 3 /* _MM_HINT_T0 */); +} + +#endif /* _PRFCHWINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/rdseedintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/rdseedintrin.h new file mode 100644 index 0000000000..2b731df7d8 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/rdseedintrin.h @@ -0,0 +1,66 @@ +/* Copyright (C) 2012-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#if !defined _X86INTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _RDSEEDINTRIN_H_INCLUDED +#define _RDSEEDINTRIN_H_INCLUDED + +#ifndef __RDSEED__ +#pragma GCC push_options +#pragma GCC target("rdseed") +#define __DISABLE_RDSEED__ +#endif /* __RDSEED__ */ + + +extern __inline int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_rdseed16_step (unsigned short *__p) +{ + return __builtin_ia32_rdseed_hi_step (__p); +} + +extern __inline int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_rdseed32_step (unsigned int *__p) +{ + return __builtin_ia32_rdseed_si_step (__p); +} + +#ifdef __x86_64__ +extern __inline int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_rdseed64_step (unsigned long long *__p) +{ + return __builtin_ia32_rdseed_di_step (__p); +} +#endif + +#ifdef __DISABLE_RDSEED__ +#undef __DISABLE_RDSEED__ +#pragma GCC pop_options +#endif /* __DISABLE_RDSEED__ */ + +#endif /* _RDSEEDINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/rtmintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/rtmintrin.h new file mode 100644 index 0000000000..ef2601eed9 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/rtmintrin.h @@ -0,0 +1,84 @@ +/* Copyright (C) 2012-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _RTMINTRIN_H_INCLUDED +#define _RTMINTRIN_H_INCLUDED + +#ifndef __RTM__ +#pragma GCC push_options +#pragma GCC target("rtm") +#define __DISABLE_RTM__ +#endif /* __RTM__ */ + +#define _XBEGIN_STARTED (~0u) +#define _XABORT_EXPLICIT (1 << 0) +#define _XABORT_RETRY (1 << 1) +#define _XABORT_CONFLICT (1 << 2) +#define _XABORT_CAPACITY (1 << 3) +#define _XABORT_DEBUG (1 << 4) +#define _XABORT_NESTED (1 << 5) +#define _XABORT_CODE(x) (((x) >> 24) & 0xFF) + +/* Start an RTM code region. Return _XBEGIN_STARTED on success and the + abort condition otherwise. */ +extern __inline unsigned int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_xbegin (void) +{ + return __builtin_ia32_xbegin (); +} + +/* Specify the end of an RTM code region. If it corresponds to the + outermost transaction, then attempts the transaction commit. If the + commit fails, then control is transferred to the outermost transaction + fallback handler. */ +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_xend (void) +{ + __builtin_ia32_xend (); +} + +/* Force an RTM abort condition. The control is transferred to the + outermost transaction fallback handler with the abort condition IMM. */ +#ifdef __OPTIMIZE__ +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_xabort (const unsigned int __imm) +{ + __builtin_ia32_xabort (__imm); +} +#else +#define _xabort(N) __builtin_ia32_xabort (N) +#endif /* __OPTIMIZE__ */ + +#ifdef __DISABLE_RTM__ +#undef __DISABLE_RTM__ +#pragma GCC pop_options +#endif /* __DISABLE_RTM__ */ + +#endif /* _RTMINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/sanitizer/asan_interface.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/sanitizer/asan_interface.h new file mode 100644 index 0000000000..448a0bcba0 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/sanitizer/asan_interface.h @@ -0,0 +1,149 @@ +//===-- sanitizer/asan_interface.h ------------------------------*- C++ -*-===// +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of AddressSanitizer. +// +// Public interface header. +//===----------------------------------------------------------------------===// +#ifndef SANITIZER_ASAN_INTERFACE_H +#define SANITIZER_ASAN_INTERFACE_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + // Marks memory region [addr, addr+size) as unaddressable. + // This memory must be previously allocated by the user program. Accessing + // addresses in this region from instrumented code is forbidden until + // this region is unpoisoned. This function is not guaranteed to poison + // the whole region - it may poison only subregion of [addr, addr+size) due + // to ASan alignment restrictions. + // Method is NOT thread-safe in the sense that no two threads can + // (un)poison memory in the same memory region simultaneously. + void __asan_poison_memory_region(void const volatile *addr, size_t size); + // Marks memory region [addr, addr+size) as addressable. + // This memory must be previously allocated by the user program. Accessing + // addresses in this region is allowed until this region is poisoned again. + // This function may unpoison a superregion of [addr, addr+size) due to + // ASan alignment restrictions. + // Method is NOT thread-safe in the sense that no two threads can + // (un)poison memory in the same memory region simultaneously. + void __asan_unpoison_memory_region(void const volatile *addr, size_t size); + +// User code should use macros instead of functions. +#if __has_feature(address_sanitizer) || defined(__SANITIZE_ADDRESS__) +#define ASAN_POISON_MEMORY_REGION(addr, size) \ + __asan_poison_memory_region((addr), (size)) +#define ASAN_UNPOISON_MEMORY_REGION(addr, size) \ + __asan_unpoison_memory_region((addr), (size)) +#else +#define ASAN_POISON_MEMORY_REGION(addr, size) \ + ((void)(addr), (void)(size)) +#define ASAN_UNPOISON_MEMORY_REGION(addr, size) \ + ((void)(addr), (void)(size)) +#endif + + // Returns 1 if addr is poisoned (i.e. 1-byte read/write access to this + // address will result in error report from AddressSanitizer). + // Otherwise returns 0. + int __asan_address_is_poisoned(void const volatile *addr); + + // If at least one byte in [beg, beg+size) is poisoned, return the address + // of the first such byte. Otherwise return 0. + void *__asan_region_is_poisoned(void *beg, size_t size); + + // Print the description of addr (useful when debugging in gdb). + void __asan_describe_address(void *addr); + + // Useful for calling from a debugger to get information about an ASan error. + // Returns 1 if an error has been (or is being) reported, otherwise returns 0. + int __asan_report_present(); + + // Useful for calling from a debugger to get information about an ASan error. + // If an error has been (or is being) reported, the following functions return + // the pc, bp, sp, address, access type (0 = read, 1 = write), access size and + // bug description (e.g. "heap-use-after-free"). Otherwise they return 0. + void *__asan_get_report_pc(); + void *__asan_get_report_bp(); + void *__asan_get_report_sp(); + void *__asan_get_report_address(); + int __asan_get_report_access_type(); + size_t __asan_get_report_access_size(); + const char *__asan_get_report_description(); + + // Useful for calling from the debugger to get information about a pointer. + // Returns the category of the given pointer as a constant string. + // Possible return values are "global", "stack", "stack-fake", "heap", + // "heap-invalid", "shadow-low", "shadow-gap", "shadow-high", "unknown". + // If global or stack, tries to also return the variable name, address and + // size. If heap, tries to return the chunk address and size. 'name' should + // point to an allocated buffer of size 'name_size'. + const char *__asan_locate_address(void *addr, char *name, size_t name_size, + void **region_address, size_t *region_size); + + // Useful for calling from the debugger to get the allocation stack trace + // and thread ID for a heap address. Stores up to 'size' frames into 'trace', + // returns the number of stored frames or 0 on error. + size_t __asan_get_alloc_stack(void *addr, void **trace, size_t size, + int *thread_id); + + // Useful for calling from the debugger to get the free stack trace + // and thread ID for a heap address. Stores up to 'size' frames into 'trace', + // returns the number of stored frames or 0 on error. + size_t __asan_get_free_stack(void *addr, void **trace, size_t size, + int *thread_id); + + // Useful for calling from the debugger to get the current shadow memory + // mapping. + void __asan_get_shadow_mapping(size_t *shadow_scale, size_t *shadow_offset); + + // This is an internal function that is called to report an error. + // However it is still a part of the interface because users may want to + // set a breakpoint on this function in a debugger. + void __asan_report_error(void *pc, void *bp, void *sp, + void *addr, int is_write, size_t access_size); + + // Deprecated. Call __sanitizer_set_death_callback instead. + void __asan_set_death_callback(void (*callback)(void)); + + void __asan_set_error_report_callback(void (*callback)(const char*)); + + // User may provide function that would be called right when ASan detects + // an error. This can be used to notice cases when ASan detects an error, but + // the program crashes before ASan report is printed. + void __asan_on_error(); + + // Prints accumulated stats to stderr. Used for debugging. + void __asan_print_accumulated_stats(); + + // This function may be optionally provided by user and should return + // a string containing ASan runtime options. See asan_flags.h for details. + const char* __asan_default_options(); + + // The following 2 functions facilitate garbage collection in presence of + // asan's fake stack. + + // Returns an opaque handler to be used later in __asan_addr_is_in_fake_stack. + // Returns NULL if the current thread does not have a fake stack. + void *__asan_get_current_fake_stack(); + + // If fake_stack is non-NULL and addr belongs to a fake frame in + // fake_stack, returns the address on real stack that corresponds to + // the fake frame and sets beg/end to the boundaries of this fake frame. + // Otherwise returns NULL and does not touch beg/end. + // If beg/end are NULL, they are not touched. + // This function may be called from a thread other than the owner of + // fake_stack, but the owner thread need to be alive. + void *__asan_addr_is_in_fake_stack(void *fake_stack, void *addr, void **beg, + void **end); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // SANITIZER_ASAN_INTERFACE_H diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/sanitizer/common_interface_defs.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/sanitizer/common_interface_defs.h new file mode 100644 index 0000000000..fd38c55a23 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/sanitizer/common_interface_defs.h @@ -0,0 +1,187 @@ +//===-- sanitizer/common_interface_defs.h -----------------------*- C++ -*-===// +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Common part of the public sanitizer interface. +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_COMMON_INTERFACE_DEFS_H +#define SANITIZER_COMMON_INTERFACE_DEFS_H + +#include +#include + +// GCC does not understand __has_feature. +#if !defined(__has_feature) +# define __has_feature(x) 0 +#endif + +#ifdef __cplusplus +extern "C" { +#endif + // Arguments for __sanitizer_sandbox_on_notify() below. + typedef struct { + // Enable sandbox support in sanitizer coverage. + int coverage_sandboxed; + // File descriptor to write coverage data to. If -1 is passed, a file will + // be pre-opened by __sanitizer_sandobx_on_notify(). This field has no + // effect if coverage_sandboxed == 0. + intptr_t coverage_fd; + // If non-zero, split the coverage data into well-formed blocks. This is + // useful when coverage_fd is a socket descriptor. Each block will contain + // a header, allowing data from multiple processes to be sent over the same + // socket. + unsigned int coverage_max_block_size; + } __sanitizer_sandbox_arguments; + + // Tell the tools to write their reports to "path." instead of stderr. + void __sanitizer_set_report_path(const char *path); + // Tell the tools to write their reports to the provided file descriptor + // (casted to void *). + void __sanitizer_set_report_fd(void *fd); + + // Notify the tools that the sandbox is going to be turned on. The reserved + // parameter will be used in the future to hold a structure with functions + // that the tools may call to bypass the sandbox. + void __sanitizer_sandbox_on_notify(__sanitizer_sandbox_arguments *args); + + // This function is called by the tool when it has just finished reporting + // an error. 'error_summary' is a one-line string that summarizes + // the error message. This function can be overridden by the client. + void __sanitizer_report_error_summary(const char *error_summary); + + // Some of the sanitizers (e.g. asan/tsan) may miss bugs that happen + // in unaligned loads/stores. In order to find such bugs reliably one needs + // to replace plain unaligned loads/stores with these calls. + uint16_t __sanitizer_unaligned_load16(const void *p); + uint32_t __sanitizer_unaligned_load32(const void *p); + uint64_t __sanitizer_unaligned_load64(const void *p); + void __sanitizer_unaligned_store16(void *p, uint16_t x); + void __sanitizer_unaligned_store32(void *p, uint32_t x); + void __sanitizer_unaligned_store64(void *p, uint64_t x); + + // Annotate the current state of a contiguous container, such as + // std::vector, std::string or similar. + // A contiguous container is a container that keeps all of its elements + // in a contiguous region of memory. The container owns the region of memory + // [beg, end); the memory [beg, mid) is used to store the current elements + // and the memory [mid, end) is reserved for future elements; + // beg <= mid <= end. For example, in "std::vector<> v" + // beg = &v[0]; + // end = beg + v.capacity() * sizeof(v[0]); + // mid = beg + v.size() * sizeof(v[0]); + // + // This annotation tells the Sanitizer tool about the current state of the + // container so that the tool can report errors when memory from [mid, end) + // is accessed. Insert this annotation into methods like push_back/pop_back. + // Supply the old and the new values of mid (old_mid/new_mid). + // In the initial state mid == end and so should be the final + // state when the container is destroyed or when it reallocates the storage. + // + // Use with caution and don't use for anything other than vector-like classes. + // + // For AddressSanitizer, 'beg' should be 8-aligned and 'end' should + // be either 8-aligned or it should point to the end of a separate heap-, + // stack-, or global- allocated buffer. I.e. the following will not work: + // int64_t x[2]; // 16 bytes, 8-aligned. + // char *beg = (char *)&x[0]; + // char *end = beg + 12; // Not 8 aligned, not the end of the buffer. + // This however will work fine: + // int32_t x[3]; // 12 bytes, but 8-aligned under AddressSanitizer. + // char *beg = (char*)&x[0]; + // char *end = beg + 12; // Not 8-aligned, but is the end of the buffer. + void __sanitizer_annotate_contiguous_container(const void *beg, + const void *end, + const void *old_mid, + const void *new_mid); + // Returns true if the contiguous container [beg, end) is properly poisoned + // (e.g. with __sanitizer_annotate_contiguous_container), i.e. if + // - [beg, mid) is addressable, + // - [mid, end) is unaddressable. + // Full verification requires O(end-beg) time; this function tries to avoid + // such complexity by touching only parts of the container around beg/mid/end. + int __sanitizer_verify_contiguous_container(const void *beg, const void *mid, + const void *end); + + // Similar to __sanitizer_verify_contiguous_container but returns the address + // of the first improperly poisoned byte otherwise. Returns null if the area + // is poisoned properly. + const void *__sanitizer_contiguous_container_find_bad_address( + const void *beg, const void *mid, const void *end); + + // Print the stack trace leading to this call. Useful for debugging user code. + void __sanitizer_print_stack_trace(); + + // Symbolizes the supplied 'pc' using the format string 'fmt'. + // Outputs at most 'out_buf_size' bytes into 'out_buf'. + // The format syntax is described in + // lib/sanitizer_common/sanitizer_stacktrace_printer.h. + void __sanitizer_symbolize_pc(void *pc, const char *fmt, char *out_buf, + size_t out_buf_size); + // Same as __sanitizer_symbolize_pc, but for data section (i.e. globals). + void __sanitizer_symbolize_global(void *data_ptr, const char *fmt, + char *out_buf, size_t out_buf_size); + + // Sets the callback to be called right before death on error. + // Passing 0 will unset the callback. + void __sanitizer_set_death_callback(void (*callback)(void)); + + // Interceptor hooks. + // Whenever a libc function interceptor is called it checks if the + // corresponding weak hook is defined, and it so -- calls it. + // The primary use case is data-flow-guided fuzzing, where the fuzzer needs + // to know what is being passed to libc functions, e.g. memcmp. + // FIXME: implement more hooks. + void __sanitizer_weak_hook_memcmp(void *called_pc, const void *s1, + const void *s2, size_t n, int result); + void __sanitizer_weak_hook_strncmp(void *called_pc, const char *s1, + const char *s2, size_t n, int result); + void __sanitizer_weak_hook_strncasecmp(void *called_pc, const char *s1, + const char *s2, size_t n, int result); + void __sanitizer_weak_hook_strcmp(void *called_pc, const char *s1, + const char *s2, int result); + void __sanitizer_weak_hook_strcasecmp(void *called_pc, const char *s1, + const char *s2, int result); + void __sanitizer_weak_hook_strstr(void *called_pc, const char *s1, + const char *s2, char *result); + void __sanitizer_weak_hook_strcasestr(void *called_pc, const char *s1, + const char *s2, char *result); + void __sanitizer_weak_hook_memmem(void *called_pc, + const void *s1, size_t len1, + const void *s2, size_t len2, void *result); + + // Prints stack traces for all live heap allocations ordered by total + // allocation size until `top_percent` of total live heap is shown. + // `top_percent` should be between 1 and 100. + // Experimental feature currently available only with asan on Linux/x86_64. + void __sanitizer_print_memory_profile(size_t top_percent); + + // Fiber annotation interface. + // Before switching to a different stack, one must call + // __sanitizer_start_switch_fiber with a pointer to the bottom of the + // destination stack and its size. When code starts running on the new stack, + // it must call __sanitizer_finish_switch_fiber to finalize the switch. + // The start_switch function takes a void** to store the current fake stack if + // there is one (it is needed when detect_stack_use_after_return is enabled). + // When restoring a stack, this pointer must be given to the finish_switch + // function. In most cases, this void* can be stored on the stack just before + // switching. When leaving a fiber definitely, null must be passed as first + // argument to the start_switch function so that the fake stack is destroyed. + // If you do not want support for stack use-after-return detection, you can + // always pass null to these two functions. + // Note that the fake stack mechanism is disabled during fiber switch, so if a + // signal callback runs during the switch, it will not benefit from the stack + // use-after-return detection. + void __sanitizer_start_switch_fiber(void **fake_stack_save, + const void *bottom, size_t size); + void __sanitizer_finish_switch_fiber(void *fake_stack_save, + const void **bottom_old, + size_t *size_old); +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // SANITIZER_COMMON_INTERFACE_DEFS_H diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/sanitizer/lsan_interface.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/sanitizer/lsan_interface.h new file mode 100644 index 0000000000..bdbe39084f --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/sanitizer/lsan_interface.h @@ -0,0 +1,82 @@ +//===-- sanitizer/lsan_interface.h ------------------------------*- C++ -*-===// +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of LeakSanitizer. +// +// Public interface header. +//===----------------------------------------------------------------------===// +#ifndef SANITIZER_LSAN_INTERFACE_H +#define SANITIZER_LSAN_INTERFACE_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + // Allocations made between calls to __lsan_disable() and __lsan_enable() will + // be treated as non-leaks. Disable/enable pairs may be nested. + void __lsan_disable(); + void __lsan_enable(); + + // The heap object into which p points will be treated as a non-leak. + void __lsan_ignore_object(const void *p); + + // Memory regions registered through this interface will be treated as sources + // of live pointers during leak checking. Useful if you store pointers in + // mapped memory. + // Points of note: + // - __lsan_unregister_root_region() must be called with the same pointer and + // size that have earlier been passed to __lsan_register_root_region() + // - LSan will skip any inaccessible memory when scanning a root region. E.g., + // if you map memory within a larger region that you have mprotect'ed, you can + // register the entire large region. + // - the implementation is not optimized for performance. This interface is + // intended to be used for a small number of relatively static regions. + void __lsan_register_root_region(const void *p, size_t size); + void __lsan_unregister_root_region(const void *p, size_t size); + + // Check for leaks now. This function behaves identically to the default + // end-of-process leak check. In particular, it will terminate the process if + // leaks are found and the exitcode runtime flag is non-zero. + // Subsequent calls to this function will have no effect and end-of-process + // leak check will not run. Effectively, end-of-process leak check is moved to + // the time of first invocation of this function. + // By calling this function early during process shutdown, you can instruct + // LSan to ignore shutdown-only leaks which happen later on. + void __lsan_do_leak_check(); + + // Check for leaks now. Returns zero if no leaks have been found or if leak + // detection is disabled, non-zero otherwise. + // This function may be called repeatedly, e.g. to periodically check a + // long-running process. It prints a leak report if appropriate, but does not + // terminate the process. It does not affect the behavior of + // __lsan_do_leak_check() or the end-of-process leak check, and is not + // affected by them. + int __lsan_do_recoverable_leak_check(); + + // The user may optionally provide this function to disallow leak checking + // for the program it is linked into (if the return value is non-zero). This + // function must be defined as returning a constant value; any behavior beyond + // that is unsupported. + int __lsan_is_turned_off(); + + // This function may be optionally provided by the user and should return + // a string containing LSan suppressions. + const char *__lsan_default_suppressions(); +#ifdef __cplusplus +} // extern "C" + +namespace __lsan { +class ScopedDisabler { + public: + ScopedDisabler() { __lsan_disable(); } + ~ScopedDisabler() { __lsan_enable(); } +}; +} // namespace __lsan +#endif + +#endif // SANITIZER_LSAN_INTERFACE_H diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/sgxintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/sgxintrin.h new file mode 100644 index 0000000000..4248076971 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/sgxintrin.h @@ -0,0 +1,177 @@ +#ifndef _SGXINTRIN_H_INCLUDED +#define _SGXINTRIN_H_INCLUDED + +#ifndef __SGX__ +#pragma GCC push_options +#pragma GCC target("sgx") +#define __DISABLE_SGX__ +#endif /* __SGX__ */ + +#define __encls_bc(leaf, b, c, retval) \ + __asm__ __volatile__ ("encls\n\t" \ + : "=a" (retval) \ + : "a" (leaf), "b" (b), "c" (c) \ + : "cc") + +#define __encls_bcd(leaf, b, c, d, retval) \ + __asm__ __volatile__("encls\n\t" \ + : "=a" (retval) \ + : "a" (leaf), "b" (b), "c" (c), "d" (d) \ + : "cc") + +#define __encls_c(leaf, c, retval) \ + __asm__ __volatile__("encls\n\t" \ + : "=a" (retval) \ + : "a" (leaf), "c" (c) \ + : "cc") + +#define __encls_edbgrd(leaf, b, c, retval) \ + __asm__ __volatile__("encls\n\t" \ + : "=a" (retval), "=b" (b) \ + : "a" (leaf), "c" (c)) + +#define __encls_generic(leaf, b, c, d, retval) \ + __asm__ __volatile__("encls\n\t" \ + : "=a" (retval), "=b" (b), "=c" (c), "=d" (d)\ + : "a" (leaf), "b" (b), "c" (c), "d" (d) \ + : "cc") + +#define __enclu_bc(leaf, b, c, retval) \ + __asm__ __volatile__("enclu\n\t" \ + : "=a" (retval) \ + : "a" (leaf), "b" (b), "c" (c) \ + : "cc") + +#define __enclu_bcd(leaf, b, c, d, retval) \ + __asm__ __volatile__("enclu\n\t" \ + : "=a" (retval) \ + : "a" (leaf), "b" (b), "c" (c), "d" (d) \ + : "cc") + +#define __enclu_eenter(leaf, b, c, retval) \ + __asm__ __volatile__("enclu\n\t" \ + : "=a" (retval), "=c" (c) \ + : "a" (leaf), "b" (b), "c" (c) \ + : "cc") + +#define __enclu_eexit(leaf, b, c, retval) \ + __asm__ __volatile__("enclu\n\t" \ + : "=a" (retval), "=c" (c) \ + : "a" (leaf), "b" (b) \ + : "cc") + +#define __enclu_generic(leaf, b, c, d, retval) \ + __asm__ __volatile__("enclu\n\t" \ + : "=a" (retval), "=b" (b), "=c" (c), "=d" (d)\ + : "a" (leaf), "b" (b), "c" (c), "d" (d) \ + : "cc") + + +extern __inline int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_encls_u32 (const int __L, size_t __D[]) +{ + enum __encls_type + { + __SGX_ECREATE = 0x00, + __SGX_EADD = 0x01, + __SGX_EINIT = 0x02, + __SGX_EREMOVE = 0x03, + __SGX_EDBGRD = 0x04, + __SGX_EDBGWR = 0x05, + __SGX_EEXTEND = 0x06, + __SGX_ELDB = 0x07, + __SGX_ELDU = 0x08, + __SGX_EBLOCK = 0x09, + __SGX_EPA = 0x0A, + __SGX_EWB = 0x0B, + __SGX_ETRACK = 0x0C, + __SGX_EAUG = 0x0D, + __SGX_EMODPR = 0x0E, + __SGX_EMODT = 0x0F + }; + enum __encls_type __T = (enum __encls_type)__L; + int __R = 0; + if (!__builtin_constant_p (__T)) + __encls_generic (__L, __D[0], __D[1], __D[2], __R); + else switch (__T) + { + case __SGX_ECREATE: + case __SGX_EADD: + case __SGX_EDBGWR: + case __SGX_EEXTEND: + case __SGX_EPA: + case __SGX_EMODPR: + case __SGX_EMODT: + case __SGX_EAUG: + __encls_bc (__L, __D[0], __D[1], __R); + break; + case __SGX_EINIT: + case __SGX_ELDB: + case __SGX_ELDU: + case __SGX_EWB: + __encls_bcd (__L, __D[0], __D[1], __D[2], __R); + break; + case __SGX_EREMOVE: + case __SGX_EBLOCK: + case __SGX_ETRACK: + __encls_c (__L, __D[1], __R); + break; + case __SGX_EDBGRD: + __encls_edbgrd (__L, __D[0], __D[1], __R); + break; + default: + return -1; + } + return __R; +} + +extern __inline int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_enclu_u32 (const int __L, size_t __D[]) +{ + enum __enclu_type + { + __SGX_EREPORT = 0x00, + __SGX_EGETKEY = 0x01, + __SGX_EENTER = 0x02, + __SGX_ERESUME = 0x03, + __SGX_EEXIT = 0x04, + __SGX_EACCEPT = 0x05, + __SGX_EMODPE = 0x06, + __SGX_EACCEPTCOPY = 0x07 + }; + enum __enclu_type __T = (enum __enclu_type) __L; + int __R = 0; + if (!__builtin_constant_p (__T)) + __enclu_generic (__L, __D[0], __D[1], __D[2], __R); + else switch (__T) + { + case __SGX_EREPORT: + case __SGX_EACCEPTCOPY: + __enclu_bcd (__L, __D[0], __D[1], __D[2], __R); + break; + case __SGX_EGETKEY: + case __SGX_ERESUME: + case __SGX_EACCEPT: + case __SGX_EMODPE: + __enclu_bc (__L, __D[0], __D[1], __R); + break; + case __SGX_EENTER: + __enclu_eenter (__L, __D[0], __D[1], __R); + break; + case __SGX_EEXIT: + __enclu_eexit (__L, __D[0], __D[1], __R); + break; + default: + return -1; + } + return __R; +} + +#ifdef __DISABLE_SGX__ +#undef __DISABLE_SGX__ +#pragma GCC pop_options +#endif /* __DISABLE_SGX__ */ + +#endif /* _SGXINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/shaintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/shaintrin.h new file mode 100644 index 0000000000..dfcf7be321 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/shaintrin.h @@ -0,0 +1,98 @@ +/* Copyright (C) 2013-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +#error "Never use directly; include instead." +#endif + +#ifndef _SHAINTRIN_H_INCLUDED +#define _SHAINTRIN_H_INCLUDED + +#ifndef __SHA__ +#pragma GCC push_options +#pragma GCC target("sha") +#define __DISABLE_SHA__ +#endif /* __SHA__ */ + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sha1msg1_epu32 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_sha1msg1 ((__v4si) __A, (__v4si) __B); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sha1msg2_epu32 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_sha1msg2 ((__v4si) __A, (__v4si) __B); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sha1nexte_epu32 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_sha1nexte ((__v4si) __A, (__v4si) __B); +} + +#ifdef __OPTIMIZE__ +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sha1rnds4_epu32 (__m128i __A, __m128i __B, const int __I) +{ + return (__m128i) __builtin_ia32_sha1rnds4 ((__v4si) __A, (__v4si) __B, __I); +} +#else +#define _mm_sha1rnds4_epu32(A, B, I) \ + ((__m128i) __builtin_ia32_sha1rnds4 ((__v4si)(__m128i)A, \ + (__v4si)(__m128i)B, (int)I)) +#endif + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sha256msg1_epu32 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_sha256msg1 ((__v4si) __A, (__v4si) __B); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sha256msg2_epu32 (__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_sha256msg2 ((__v4si) __A, (__v4si) __B); +} + +extern __inline __m128i +__attribute__ ((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sha256rnds2_epu32 (__m128i __A, __m128i __B, __m128i __C) +{ + return (__m128i) __builtin_ia32_sha256rnds2 ((__v4si) __A, (__v4si) __B, + (__v4si) __C); +} + +#ifdef __DISABLE_SHA__ +#undef __DISABLE_SHA__ +#pragma GCC pop_options +#endif /* __DISABLE_SHA__ */ + +#endif /* _SHAINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/smmintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/smmintrin.h new file mode 100644 index 0000000000..f4a273f8cb --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/smmintrin.h @@ -0,0 +1,862 @@ +/* Copyright (C) 2007-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +/* Implemented from the specification included in the Intel C++ Compiler + User Guide and Reference, version 10.0. */ + +#ifndef _SMMINTRIN_H_INCLUDED +#define _SMMINTRIN_H_INCLUDED + +/* We need definitions from the SSSE3, SSE3, SSE2 and SSE header + files. */ +#include + +#ifndef __SSE4_1__ +#pragma GCC push_options +#pragma GCC target("sse4.1") +#define __DISABLE_SSE4_1__ +#endif /* __SSE4_1__ */ + +/* Rounding mode macros. */ +#define _MM_FROUND_TO_NEAREST_INT 0x00 +#define _MM_FROUND_TO_NEG_INF 0x01 +#define _MM_FROUND_TO_POS_INF 0x02 +#define _MM_FROUND_TO_ZERO 0x03 +#define _MM_FROUND_CUR_DIRECTION 0x04 + +#define _MM_FROUND_RAISE_EXC 0x00 +#define _MM_FROUND_NO_EXC 0x08 + +#define _MM_FROUND_NINT \ + (_MM_FROUND_TO_NEAREST_INT | _MM_FROUND_RAISE_EXC) +#define _MM_FROUND_FLOOR \ + (_MM_FROUND_TO_NEG_INF | _MM_FROUND_RAISE_EXC) +#define _MM_FROUND_CEIL \ + (_MM_FROUND_TO_POS_INF | _MM_FROUND_RAISE_EXC) +#define _MM_FROUND_TRUNC \ + (_MM_FROUND_TO_ZERO | _MM_FROUND_RAISE_EXC) +#define _MM_FROUND_RINT \ + (_MM_FROUND_CUR_DIRECTION | _MM_FROUND_RAISE_EXC) +#define _MM_FROUND_NEARBYINT \ + (_MM_FROUND_CUR_DIRECTION | _MM_FROUND_NO_EXC) + +/* Test Instruction */ +/* Packed integer 128-bit bitwise comparison. Return 1 if + (__V & __M) == 0. */ +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_testz_si128 (__m128i __M, __m128i __V) +{ + return __builtin_ia32_ptestz128 ((__v2di)__M, (__v2di)__V); +} + +/* Packed integer 128-bit bitwise comparison. Return 1 if + (__V & ~__M) == 0. */ +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_testc_si128 (__m128i __M, __m128i __V) +{ + return __builtin_ia32_ptestc128 ((__v2di)__M, (__v2di)__V); +} + +/* Packed integer 128-bit bitwise comparison. Return 1 if + (__V & __M) != 0 && (__V & ~__M) != 0. */ +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_testnzc_si128 (__m128i __M, __m128i __V) +{ + return __builtin_ia32_ptestnzc128 ((__v2di)__M, (__v2di)__V); +} + +/* Macros for packed integer 128-bit comparison intrinsics. */ +#define _mm_test_all_zeros(M, V) _mm_testz_si128 ((M), (V)) + +#define _mm_test_all_ones(V) \ + _mm_testc_si128 ((V), _mm_cmpeq_epi32 ((V), (V))) + +#define _mm_test_mix_ones_zeros(M, V) _mm_testnzc_si128 ((M), (V)) + +/* Packed/scalar double precision floating point rounding. */ + +#ifdef __OPTIMIZE__ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_round_pd (__m128d __V, const int __M) +{ + return (__m128d) __builtin_ia32_roundpd ((__v2df)__V, __M); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_round_sd(__m128d __D, __m128d __V, const int __M) +{ + return (__m128d) __builtin_ia32_roundsd ((__v2df)__D, + (__v2df)__V, + __M); +} +#else +#define _mm_round_pd(V, M) \ + ((__m128d) __builtin_ia32_roundpd ((__v2df)(__m128d)(V), (int)(M))) + +#define _mm_round_sd(D, V, M) \ + ((__m128d) __builtin_ia32_roundsd ((__v2df)(__m128d)(D), \ + (__v2df)(__m128d)(V), (int)(M))) +#endif + +/* Packed/scalar single precision floating point rounding. */ + +#ifdef __OPTIMIZE__ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_round_ps (__m128 __V, const int __M) +{ + return (__m128) __builtin_ia32_roundps ((__v4sf)__V, __M); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_round_ss (__m128 __D, __m128 __V, const int __M) +{ + return (__m128) __builtin_ia32_roundss ((__v4sf)__D, + (__v4sf)__V, + __M); +} +#else +#define _mm_round_ps(V, M) \ + ((__m128) __builtin_ia32_roundps ((__v4sf)(__m128)(V), (int)(M))) + +#define _mm_round_ss(D, V, M) \ + ((__m128) __builtin_ia32_roundss ((__v4sf)(__m128)(D), \ + (__v4sf)(__m128)(V), (int)(M))) +#endif + +/* Macros for ceil/floor intrinsics. */ +#define _mm_ceil_pd(V) _mm_round_pd ((V), _MM_FROUND_CEIL) +#define _mm_ceil_sd(D, V) _mm_round_sd ((D), (V), _MM_FROUND_CEIL) + +#define _mm_floor_pd(V) _mm_round_pd((V), _MM_FROUND_FLOOR) +#define _mm_floor_sd(D, V) _mm_round_sd ((D), (V), _MM_FROUND_FLOOR) + +#define _mm_ceil_ps(V) _mm_round_ps ((V), _MM_FROUND_CEIL) +#define _mm_ceil_ss(D, V) _mm_round_ss ((D), (V), _MM_FROUND_CEIL) + +#define _mm_floor_ps(V) _mm_round_ps ((V), _MM_FROUND_FLOOR) +#define _mm_floor_ss(D, V) _mm_round_ss ((D), (V), _MM_FROUND_FLOOR) + +/* SSE4.1 */ + +/* Integer blend instructions - select data from 2 sources using + constant/variable mask. */ + +#ifdef __OPTIMIZE__ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_blend_epi16 (__m128i __X, __m128i __Y, const int __M) +{ + return (__m128i) __builtin_ia32_pblendw128 ((__v8hi)__X, + (__v8hi)__Y, + __M); +} +#else +#define _mm_blend_epi16(X, Y, M) \ + ((__m128i) __builtin_ia32_pblendw128 ((__v8hi)(__m128i)(X), \ + (__v8hi)(__m128i)(Y), (int)(M))) +#endif + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_blendv_epi8 (__m128i __X, __m128i __Y, __m128i __M) +{ + return (__m128i) __builtin_ia32_pblendvb128 ((__v16qi)__X, + (__v16qi)__Y, + (__v16qi)__M); +} + +/* Single precision floating point blend instructions - select data + from 2 sources using constant/variable mask. */ + +#ifdef __OPTIMIZE__ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_blend_ps (__m128 __X, __m128 __Y, const int __M) +{ + return (__m128) __builtin_ia32_blendps ((__v4sf)__X, + (__v4sf)__Y, + __M); +} +#else +#define _mm_blend_ps(X, Y, M) \ + ((__m128) __builtin_ia32_blendps ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (int)(M))) +#endif + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_blendv_ps (__m128 __X, __m128 __Y, __m128 __M) +{ + return (__m128) __builtin_ia32_blendvps ((__v4sf)__X, + (__v4sf)__Y, + (__v4sf)__M); +} + +/* Double precision floating point blend instructions - select data + from 2 sources using constant/variable mask. */ + +#ifdef __OPTIMIZE__ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_blend_pd (__m128d __X, __m128d __Y, const int __M) +{ + return (__m128d) __builtin_ia32_blendpd ((__v2df)__X, + (__v2df)__Y, + __M); +} +#else +#define _mm_blend_pd(X, Y, M) \ + ((__m128d) __builtin_ia32_blendpd ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (int)(M))) +#endif + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_blendv_pd (__m128d __X, __m128d __Y, __m128d __M) +{ + return (__m128d) __builtin_ia32_blendvpd ((__v2df)__X, + (__v2df)__Y, + (__v2df)__M); +} + +/* Dot product instructions with mask-defined summing and zeroing parts + of result. */ + +#ifdef __OPTIMIZE__ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_dp_ps (__m128 __X, __m128 __Y, const int __M) +{ + return (__m128) __builtin_ia32_dpps ((__v4sf)__X, + (__v4sf)__Y, + __M); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_dp_pd (__m128d __X, __m128d __Y, const int __M) +{ + return (__m128d) __builtin_ia32_dppd ((__v2df)__X, + (__v2df)__Y, + __M); +} +#else +#define _mm_dp_ps(X, Y, M) \ + ((__m128) __builtin_ia32_dpps ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), (int)(M))) + +#define _mm_dp_pd(X, Y, M) \ + ((__m128d) __builtin_ia32_dppd ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), (int)(M))) +#endif + +/* Packed integer 64-bit comparison, zeroing or filling with ones + corresponding parts of result. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_epi64 (__m128i __X, __m128i __Y) +{ + return (__m128i) ((__v2di)__X == (__v2di)__Y); +} + +/* Min/max packed integer instructions. */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_min_epi8 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_pminsb128 ((__v16qi)__X, (__v16qi)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_max_epi8 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_pmaxsb128 ((__v16qi)__X, (__v16qi)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_min_epu16 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_pminuw128 ((__v8hi)__X, (__v8hi)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_max_epu16 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_pmaxuw128 ((__v8hi)__X, (__v8hi)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_min_epi32 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_pminsd128 ((__v4si)__X, (__v4si)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_max_epi32 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_pmaxsd128 ((__v4si)__X, (__v4si)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_min_epu32 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_pminud128 ((__v4si)__X, (__v4si)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_max_epu32 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_pmaxud128 ((__v4si)__X, (__v4si)__Y); +} + +/* Packed integer 32-bit multiplication with truncation of upper + halves of results. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mullo_epi32 (__m128i __X, __m128i __Y) +{ + return (__m128i) ((__v4su)__X * (__v4su)__Y); +} + +/* Packed integer 32-bit multiplication of 2 pairs of operands + with two 64-bit results. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mul_epi32 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_pmuldq128 ((__v4si)__X, (__v4si)__Y); +} + +/* Insert single precision float into packed single precision array + element selected by index N. The bits [7-6] of N define S + index, the bits [5-4] define D index, and bits [3-0] define + zeroing mask for D. */ + +#ifdef __OPTIMIZE__ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_insert_ps (__m128 __D, __m128 __S, const int __N) +{ + return (__m128) __builtin_ia32_insertps128 ((__v4sf)__D, + (__v4sf)__S, + __N); +} +#else +#define _mm_insert_ps(D, S, N) \ + ((__m128) __builtin_ia32_insertps128 ((__v4sf)(__m128)(D), \ + (__v4sf)(__m128)(S), (int)(N))) +#endif + +/* Helper macro to create the N value for _mm_insert_ps. */ +#define _MM_MK_INSERTPS_NDX(S, D, M) (((S) << 6) | ((D) << 4) | (M)) + +/* Extract binary representation of single precision float from packed + single precision array element of X selected by index N. */ + +#ifdef __OPTIMIZE__ +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_extract_ps (__m128 __X, const int __N) +{ + union { int i; float f; } __tmp; + __tmp.f = __builtin_ia32_vec_ext_v4sf ((__v4sf)__X, __N); + return __tmp.i; +} +#else +#define _mm_extract_ps(X, N) \ + (__extension__ \ + ({ \ + union { int i; float f; } __tmp; \ + __tmp.f = __builtin_ia32_vec_ext_v4sf ((__v4sf)(__m128)(X), (int)(N)); \ + __tmp.i; \ + })) +#endif + +/* Extract binary representation of single precision float into + D from packed single precision array element of S selected + by index N. */ +#define _MM_EXTRACT_FLOAT(D, S, N) \ + { (D) = __builtin_ia32_vec_ext_v4sf ((__v4sf)(S), (N)); } + +/* Extract specified single precision float element into the lower + part of __m128. */ +#define _MM_PICK_OUT_PS(X, N) \ + _mm_insert_ps (_mm_setzero_ps (), (X), \ + _MM_MK_INSERTPS_NDX ((N), 0, 0x0e)) + +/* Insert integer, S, into packed integer array element of D + selected by index N. */ + +#ifdef __OPTIMIZE__ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_insert_epi8 (__m128i __D, int __S, const int __N) +{ + return (__m128i) __builtin_ia32_vec_set_v16qi ((__v16qi)__D, + __S, __N); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_insert_epi32 (__m128i __D, int __S, const int __N) +{ + return (__m128i) __builtin_ia32_vec_set_v4si ((__v4si)__D, + __S, __N); +} + +#ifdef __x86_64__ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_insert_epi64 (__m128i __D, long long __S, const int __N) +{ + return (__m128i) __builtin_ia32_vec_set_v2di ((__v2di)__D, + __S, __N); +} +#endif +#else +#define _mm_insert_epi8(D, S, N) \ + ((__m128i) __builtin_ia32_vec_set_v16qi ((__v16qi)(__m128i)(D), \ + (int)(S), (int)(N))) + +#define _mm_insert_epi32(D, S, N) \ + ((__m128i) __builtin_ia32_vec_set_v4si ((__v4si)(__m128i)(D), \ + (int)(S), (int)(N))) + +#ifdef __x86_64__ +#define _mm_insert_epi64(D, S, N) \ + ((__m128i) __builtin_ia32_vec_set_v2di ((__v2di)(__m128i)(D), \ + (long long)(S), (int)(N))) +#endif +#endif + +/* Extract integer from packed integer array element of X selected by + index N. */ + +#ifdef __OPTIMIZE__ +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_extract_epi8 (__m128i __X, const int __N) +{ + return (unsigned char) __builtin_ia32_vec_ext_v16qi ((__v16qi)__X, __N); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_extract_epi32 (__m128i __X, const int __N) +{ + return __builtin_ia32_vec_ext_v4si ((__v4si)__X, __N); +} + +#ifdef __x86_64__ +extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_extract_epi64 (__m128i __X, const int __N) +{ + return __builtin_ia32_vec_ext_v2di ((__v2di)__X, __N); +} +#endif +#else +#define _mm_extract_epi8(X, N) \ + ((int) (unsigned char) __builtin_ia32_vec_ext_v16qi ((__v16qi)(__m128i)(X), (int)(N))) +#define _mm_extract_epi32(X, N) \ + ((int) __builtin_ia32_vec_ext_v4si ((__v4si)(__m128i)(X), (int)(N))) + +#ifdef __x86_64__ +#define _mm_extract_epi64(X, N) \ + ((long long) __builtin_ia32_vec_ext_v2di ((__v2di)(__m128i)(X), (int)(N))) +#endif +#endif + +/* Return horizontal packed word minimum and its index in bits [15:0] + and bits [18:16] respectively. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_minpos_epu16 (__m128i __X) +{ + return (__m128i) __builtin_ia32_phminposuw128 ((__v8hi)__X); +} + +/* Packed integer sign-extension. */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepi8_epi32 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pmovsxbd128 ((__v16qi)__X); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepi16_epi32 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pmovsxwd128 ((__v8hi)__X); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepi8_epi64 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pmovsxbq128 ((__v16qi)__X); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepi32_epi64 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pmovsxdq128 ((__v4si)__X); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepi16_epi64 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pmovsxwq128 ((__v8hi)__X); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepi8_epi16 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pmovsxbw128 ((__v16qi)__X); +} + +/* Packed integer zero-extension. */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepu8_epi32 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pmovzxbd128 ((__v16qi)__X); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepu16_epi32 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pmovzxwd128 ((__v8hi)__X); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepu8_epi64 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pmovzxbq128 ((__v16qi)__X); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepu32_epi64 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pmovzxdq128 ((__v4si)__X); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepu16_epi64 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pmovzxwq128 ((__v8hi)__X); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtepu8_epi16 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pmovzxbw128 ((__v16qi)__X); +} + +/* Pack 8 double words from 2 operands into 8 words of result with + unsigned saturation. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_packus_epi32 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_packusdw128 ((__v4si)__X, (__v4si)__Y); +} + +/* Sum absolute 8-bit integer difference of adjacent groups of 4 + byte integers in the first 2 operands. Starting offsets within + operands are determined by the 3rd mask operand. */ + +#ifdef __OPTIMIZE__ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mpsadbw_epu8 (__m128i __X, __m128i __Y, const int __M) +{ + return (__m128i) __builtin_ia32_mpsadbw128 ((__v16qi)__X, + (__v16qi)__Y, __M); +} +#else +#define _mm_mpsadbw_epu8(X, Y, M) \ + ((__m128i) __builtin_ia32_mpsadbw128 ((__v16qi)(__m128i)(X), \ + (__v16qi)(__m128i)(Y), (int)(M))) +#endif + +/* Load double quadword using non-temporal aligned hint. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_stream_load_si128 (__m128i *__X) +{ + return (__m128i) __builtin_ia32_movntdqa ((__v2di *) __X); +} + +#ifndef __SSE4_2__ +#pragma GCC push_options +#pragma GCC target("sse4.2") +#define __DISABLE_SSE4_2__ +#endif /* __SSE4_2__ */ + +/* These macros specify the source data format. */ +#define _SIDD_UBYTE_OPS 0x00 +#define _SIDD_UWORD_OPS 0x01 +#define _SIDD_SBYTE_OPS 0x02 +#define _SIDD_SWORD_OPS 0x03 + +/* These macros specify the comparison operation. */ +#define _SIDD_CMP_EQUAL_ANY 0x00 +#define _SIDD_CMP_RANGES 0x04 +#define _SIDD_CMP_EQUAL_EACH 0x08 +#define _SIDD_CMP_EQUAL_ORDERED 0x0c + +/* These macros specify the polarity. */ +#define _SIDD_POSITIVE_POLARITY 0x00 +#define _SIDD_NEGATIVE_POLARITY 0x10 +#define _SIDD_MASKED_POSITIVE_POLARITY 0x20 +#define _SIDD_MASKED_NEGATIVE_POLARITY 0x30 + +/* These macros specify the output selection in _mm_cmpXstri (). */ +#define _SIDD_LEAST_SIGNIFICANT 0x00 +#define _SIDD_MOST_SIGNIFICANT 0x40 + +/* These macros specify the output selection in _mm_cmpXstrm (). */ +#define _SIDD_BIT_MASK 0x00 +#define _SIDD_UNIT_MASK 0x40 + +/* Intrinsics for text/string processing. */ + +#ifdef __OPTIMIZE__ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpistrm (__m128i __X, __m128i __Y, const int __M) +{ + return (__m128i) __builtin_ia32_pcmpistrm128 ((__v16qi)__X, + (__v16qi)__Y, + __M); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpistri (__m128i __X, __m128i __Y, const int __M) +{ + return __builtin_ia32_pcmpistri128 ((__v16qi)__X, + (__v16qi)__Y, + __M); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpestrm (__m128i __X, int __LX, __m128i __Y, int __LY, const int __M) +{ + return (__m128i) __builtin_ia32_pcmpestrm128 ((__v16qi)__X, __LX, + (__v16qi)__Y, __LY, + __M); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpestri (__m128i __X, int __LX, __m128i __Y, int __LY, const int __M) +{ + return __builtin_ia32_pcmpestri128 ((__v16qi)__X, __LX, + (__v16qi)__Y, __LY, + __M); +} +#else +#define _mm_cmpistrm(X, Y, M) \ + ((__m128i) __builtin_ia32_pcmpistrm128 ((__v16qi)(__m128i)(X), \ + (__v16qi)(__m128i)(Y), (int)(M))) +#define _mm_cmpistri(X, Y, M) \ + ((int) __builtin_ia32_pcmpistri128 ((__v16qi)(__m128i)(X), \ + (__v16qi)(__m128i)(Y), (int)(M))) + +#define _mm_cmpestrm(X, LX, Y, LY, M) \ + ((__m128i) __builtin_ia32_pcmpestrm128 ((__v16qi)(__m128i)(X), \ + (int)(LX), (__v16qi)(__m128i)(Y), \ + (int)(LY), (int)(M))) +#define _mm_cmpestri(X, LX, Y, LY, M) \ + ((int) __builtin_ia32_pcmpestri128 ((__v16qi)(__m128i)(X), (int)(LX), \ + (__v16qi)(__m128i)(Y), (int)(LY), \ + (int)(M))) +#endif + +/* Intrinsics for text/string processing and reading values of + EFlags. */ + +#ifdef __OPTIMIZE__ +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpistra (__m128i __X, __m128i __Y, const int __M) +{ + return __builtin_ia32_pcmpistria128 ((__v16qi)__X, + (__v16qi)__Y, + __M); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpistrc (__m128i __X, __m128i __Y, const int __M) +{ + return __builtin_ia32_pcmpistric128 ((__v16qi)__X, + (__v16qi)__Y, + __M); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpistro (__m128i __X, __m128i __Y, const int __M) +{ + return __builtin_ia32_pcmpistrio128 ((__v16qi)__X, + (__v16qi)__Y, + __M); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpistrs (__m128i __X, __m128i __Y, const int __M) +{ + return __builtin_ia32_pcmpistris128 ((__v16qi)__X, + (__v16qi)__Y, + __M); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpistrz (__m128i __X, __m128i __Y, const int __M) +{ + return __builtin_ia32_pcmpistriz128 ((__v16qi)__X, + (__v16qi)__Y, + __M); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpestra (__m128i __X, int __LX, __m128i __Y, int __LY, const int __M) +{ + return __builtin_ia32_pcmpestria128 ((__v16qi)__X, __LX, + (__v16qi)__Y, __LY, + __M); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpestrc (__m128i __X, int __LX, __m128i __Y, int __LY, const int __M) +{ + return __builtin_ia32_pcmpestric128 ((__v16qi)__X, __LX, + (__v16qi)__Y, __LY, + __M); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpestro (__m128i __X, int __LX, __m128i __Y, int __LY, const int __M) +{ + return __builtin_ia32_pcmpestrio128 ((__v16qi)__X, __LX, + (__v16qi)__Y, __LY, + __M); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpestrs (__m128i __X, int __LX, __m128i __Y, int __LY, const int __M) +{ + return __builtin_ia32_pcmpestris128 ((__v16qi)__X, __LX, + (__v16qi)__Y, __LY, + __M); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpestrz (__m128i __X, int __LX, __m128i __Y, int __LY, const int __M) +{ + return __builtin_ia32_pcmpestriz128 ((__v16qi)__X, __LX, + (__v16qi)__Y, __LY, + __M); +} +#else +#define _mm_cmpistra(X, Y, M) \ + ((int) __builtin_ia32_pcmpistria128 ((__v16qi)(__m128i)(X), \ + (__v16qi)(__m128i)(Y), (int)(M))) +#define _mm_cmpistrc(X, Y, M) \ + ((int) __builtin_ia32_pcmpistric128 ((__v16qi)(__m128i)(X), \ + (__v16qi)(__m128i)(Y), (int)(M))) +#define _mm_cmpistro(X, Y, M) \ + ((int) __builtin_ia32_pcmpistrio128 ((__v16qi)(__m128i)(X), \ + (__v16qi)(__m128i)(Y), (int)(M))) +#define _mm_cmpistrs(X, Y, M) \ + ((int) __builtin_ia32_pcmpistris128 ((__v16qi)(__m128i)(X), \ + (__v16qi)(__m128i)(Y), (int)(M))) +#define _mm_cmpistrz(X, Y, M) \ + ((int) __builtin_ia32_pcmpistriz128 ((__v16qi)(__m128i)(X), \ + (__v16qi)(__m128i)(Y), (int)(M))) + +#define _mm_cmpestra(X, LX, Y, LY, M) \ + ((int) __builtin_ia32_pcmpestria128 ((__v16qi)(__m128i)(X), (int)(LX), \ + (__v16qi)(__m128i)(Y), (int)(LY), \ + (int)(M))) +#define _mm_cmpestrc(X, LX, Y, LY, M) \ + ((int) __builtin_ia32_pcmpestric128 ((__v16qi)(__m128i)(X), (int)(LX), \ + (__v16qi)(__m128i)(Y), (int)(LY), \ + (int)(M))) +#define _mm_cmpestro(X, LX, Y, LY, M) \ + ((int) __builtin_ia32_pcmpestrio128 ((__v16qi)(__m128i)(X), (int)(LX), \ + (__v16qi)(__m128i)(Y), (int)(LY), \ + (int)(M))) +#define _mm_cmpestrs(X, LX, Y, LY, M) \ + ((int) __builtin_ia32_pcmpestris128 ((__v16qi)(__m128i)(X), (int)(LX), \ + (__v16qi)(__m128i)(Y), (int)(LY), \ + (int)(M))) +#define _mm_cmpestrz(X, LX, Y, LY, M) \ + ((int) __builtin_ia32_pcmpestriz128 ((__v16qi)(__m128i)(X), (int)(LX), \ + (__v16qi)(__m128i)(Y), (int)(LY), \ + (int)(M))) +#endif + +/* Packed integer 64-bit comparison, zeroing or filling with ones + corresponding parts of result. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_epi64 (__m128i __X, __m128i __Y) +{ + return (__m128i) ((__v2di)__X > (__v2di)__Y); +} + +#ifdef __DISABLE_SSE4_2__ +#undef __DISABLE_SSE4_2__ +#pragma GCC pop_options +#endif /* __DISABLE_SSE4_2__ */ + +#ifdef __DISABLE_SSE4_1__ +#undef __DISABLE_SSE4_1__ +#pragma GCC pop_options +#endif /* __DISABLE_SSE4_1__ */ + +#include + +#ifndef __SSE4_1__ +#pragma GCC push_options +#pragma GCC target("sse4.1") +#define __DISABLE_SSE4_1__ +#endif /* __SSE4_1__ */ + +#ifndef __SSE4_2__ +#pragma GCC push_options +#pragma GCC target("sse4.2") +#define __DISABLE_SSE4_2__ +#endif /* __SSE4_1__ */ + +/* Accumulate CRC32 (polynomial 0x11EDC6F41) value. */ +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_crc32_u8 (unsigned int __C, unsigned char __V) +{ + return __builtin_ia32_crc32qi (__C, __V); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_crc32_u16 (unsigned int __C, unsigned short __V) +{ + return __builtin_ia32_crc32hi (__C, __V); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_crc32_u32 (unsigned int __C, unsigned int __V) +{ + return __builtin_ia32_crc32si (__C, __V); +} + +#ifdef __x86_64__ +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_crc32_u64 (unsigned long long __C, unsigned long long __V) +{ + return __builtin_ia32_crc32di (__C, __V); +} +#endif + +#ifdef __DISABLE_SSE4_2__ +#undef __DISABLE_SSE4_2__ +#pragma GCC pop_options +#endif /* __DISABLE_SSE4_2__ */ + +#ifdef __DISABLE_SSE4_1__ +#undef __DISABLE_SSE4_1__ +#pragma GCC pop_options +#endif /* __DISABLE_SSE4_1__ */ + +#endif /* _SMMINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/stdalign.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/stdalign.h new file mode 100644 index 0000000000..243d40e54e --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/stdalign.h @@ -0,0 +1,39 @@ +/* Copyright (C) 2011-2017 Free Software Foundation, Inc. + +This file is part of GCC. + +GCC is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3, or (at your option) +any later version. + +GCC is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +Under Section 7 of GPL version 3, you are granted additional +permissions described in the GCC Runtime Library Exception, version +3.1, as published by the Free Software Foundation. + +You should have received a copy of the GNU General Public License and +a copy of the GCC Runtime Library Exception along with this program; +see the files COPYING3 and COPYING.RUNTIME respectively. If not, see +. */ + +/* ISO C1X: 7.15 Alignment . */ + +#ifndef _STDALIGN_H +#define _STDALIGN_H + +#ifndef __cplusplus + +#define alignas _Alignas +#define alignof _Alignof + +#define __alignas_is_defined 1 +#define __alignof_is_defined 1 + +#endif + +#endif /* stdalign.h */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/stdarg.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/stdarg.h new file mode 100644 index 0000000000..aa248be5da --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/stdarg.h @@ -0,0 +1,127 @@ +/* Copyright (C) 1989-2017 Free Software Foundation, Inc. + +This file is part of GCC. + +GCC is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3, or (at your option) +any later version. + +GCC is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +Under Section 7 of GPL version 3, you are granted additional +permissions described in the GCC Runtime Library Exception, version +3.1, as published by the Free Software Foundation. + +You should have received a copy of the GNU General Public License and +a copy of the GCC Runtime Library Exception along with this program; +see the files COPYING3 and COPYING.RUNTIME respectively. If not, see +. */ + +/* + * ISO C Standard: 7.15 Variable arguments + */ + +#ifndef _STDARG_H +#ifndef _ANSI_STDARG_H_ +#ifndef __need___va_list +#define _STDARG_H +#define _ANSI_STDARG_H_ +#endif /* not __need___va_list */ +#undef __need___va_list + +/* Define __gnuc_va_list. */ + +#ifndef __GNUC_VA_LIST +#define __GNUC_VA_LIST +typedef __builtin_va_list __gnuc_va_list; +#endif + +/* Define the standard macros for the user, + if this invocation was from the user program. */ +#ifdef _STDARG_H + +#define va_start(v,l) __builtin_va_start(v,l) +#define va_end(v) __builtin_va_end(v) +#define va_arg(v,l) __builtin_va_arg(v,l) +#if !defined(__STRICT_ANSI__) || __STDC_VERSION__ + 0 >= 199900L \ + || __cplusplus + 0 >= 201103L +#define va_copy(d,s) __builtin_va_copy(d,s) +#endif +#define __va_copy(d,s) __builtin_va_copy(d,s) + +/* Define va_list, if desired, from __gnuc_va_list. */ +/* We deliberately do not define va_list when called from + stdio.h, because ANSI C says that stdio.h is not supposed to define + va_list. stdio.h needs to have access to that data type, + but must not use that name. It should use the name __gnuc_va_list, + which is safe because it is reserved for the implementation. */ + +#ifdef _BSD_VA_LIST +#undef _BSD_VA_LIST +#endif + +#if defined(__svr4__) || (defined(_SCO_DS) && !defined(__VA_LIST)) +/* SVR4.2 uses _VA_LIST for an internal alias for va_list, + so we must avoid testing it and setting it here. + SVR4 uses _VA_LIST as a flag in stdarg.h, but we should + have no conflict with that. */ +#ifndef _VA_LIST_ +#define _VA_LIST_ +#ifdef __i860__ +#ifndef _VA_LIST +#define _VA_LIST va_list +#endif +#endif /* __i860__ */ +typedef __gnuc_va_list va_list; +#ifdef _SCO_DS +#define __VA_LIST +#endif +#endif /* _VA_LIST_ */ +#else /* not __svr4__ || _SCO_DS */ + +/* The macro _VA_LIST_ is the same thing used by this file in Ultrix. + But on BSD NET2 we must not test or define or undef it. + (Note that the comments in NET 2's ansi.h + are incorrect for _VA_LIST_--see stdio.h!) */ +#if !defined (_VA_LIST_) || defined (__BSD_NET2__) || defined (____386BSD____) || defined (__bsdi__) || defined (__sequent__) || defined (__FreeBSD__) || defined(WINNT) +/* The macro _VA_LIST_DEFINED is used in Windows NT 3.5 */ +#ifndef _VA_LIST_DEFINED +/* The macro _VA_LIST is used in SCO Unix 3.2. */ +#ifndef _VA_LIST +/* The macro _VA_LIST_T_H is used in the Bull dpx2 */ +#ifndef _VA_LIST_T_H +/* The macro __va_list__ is used by BeOS. */ +#ifndef __va_list__ +typedef __gnuc_va_list va_list; +#endif /* not __va_list__ */ +#endif /* not _VA_LIST_T_H */ +#endif /* not _VA_LIST */ +#endif /* not _VA_LIST_DEFINED */ +#if !(defined (__BSD_NET2__) || defined (____386BSD____) || defined (__bsdi__) || defined (__sequent__) || defined (__FreeBSD__)) +#define _VA_LIST_ +#endif +#ifndef _VA_LIST +#define _VA_LIST +#endif +#ifndef _VA_LIST_DEFINED +#define _VA_LIST_DEFINED +#endif +#ifndef _VA_LIST_T_H +#define _VA_LIST_T_H +#endif +#ifndef __va_list__ +#define __va_list__ +#endif + +#endif /* not _VA_LIST_, except on certain systems */ + +#endif /* not __svr4__ */ + +#endif /* _STDARG_H */ + +#endif /* not _ANSI_STDARG_H_ */ +#endif /* not _STDARG_H */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/stdatomic.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/stdatomic.h new file mode 100644 index 0000000000..fb29a142fc --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/stdatomic.h @@ -0,0 +1,243 @@ +/* Copyright (C) 2013-2017 Free Software Foundation, Inc. + +This file is part of GCC. + +GCC is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3, or (at your option) +any later version. + +GCC is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +Under Section 7 of GPL version 3, you are granted additional +permissions described in the GCC Runtime Library Exception, version +3.1, as published by the Free Software Foundation. + +You should have received a copy of the GNU General Public License and +a copy of the GCC Runtime Library Exception along with this program; +see the files COPYING3 and COPYING.RUNTIME respectively. If not, see +. */ + +/* ISO C11 Standard: 7.17 Atomics . */ + +#ifndef _STDATOMIC_H +#define _STDATOMIC_H + +typedef enum + { + memory_order_relaxed = __ATOMIC_RELAXED, + memory_order_consume = __ATOMIC_CONSUME, + memory_order_acquire = __ATOMIC_ACQUIRE, + memory_order_release = __ATOMIC_RELEASE, + memory_order_acq_rel = __ATOMIC_ACQ_REL, + memory_order_seq_cst = __ATOMIC_SEQ_CST + } memory_order; + + +typedef _Atomic _Bool atomic_bool; +typedef _Atomic char atomic_char; +typedef _Atomic signed char atomic_schar; +typedef _Atomic unsigned char atomic_uchar; +typedef _Atomic short atomic_short; +typedef _Atomic unsigned short atomic_ushort; +typedef _Atomic int atomic_int; +typedef _Atomic unsigned int atomic_uint; +typedef _Atomic long atomic_long; +typedef _Atomic unsigned long atomic_ulong; +typedef _Atomic long long atomic_llong; +typedef _Atomic unsigned long long atomic_ullong; +typedef _Atomic __CHAR16_TYPE__ atomic_char16_t; +typedef _Atomic __CHAR32_TYPE__ atomic_char32_t; +typedef _Atomic __WCHAR_TYPE__ atomic_wchar_t; +typedef _Atomic __INT_LEAST8_TYPE__ atomic_int_least8_t; +typedef _Atomic __UINT_LEAST8_TYPE__ atomic_uint_least8_t; +typedef _Atomic __INT_LEAST16_TYPE__ atomic_int_least16_t; +typedef _Atomic __UINT_LEAST16_TYPE__ atomic_uint_least16_t; +typedef _Atomic __INT_LEAST32_TYPE__ atomic_int_least32_t; +typedef _Atomic __UINT_LEAST32_TYPE__ atomic_uint_least32_t; +typedef _Atomic __INT_LEAST64_TYPE__ atomic_int_least64_t; +typedef _Atomic __UINT_LEAST64_TYPE__ atomic_uint_least64_t; +typedef _Atomic __INT_FAST8_TYPE__ atomic_int_fast8_t; +typedef _Atomic __UINT_FAST8_TYPE__ atomic_uint_fast8_t; +typedef _Atomic __INT_FAST16_TYPE__ atomic_int_fast16_t; +typedef _Atomic __UINT_FAST16_TYPE__ atomic_uint_fast16_t; +typedef _Atomic __INT_FAST32_TYPE__ atomic_int_fast32_t; +typedef _Atomic __UINT_FAST32_TYPE__ atomic_uint_fast32_t; +typedef _Atomic __INT_FAST64_TYPE__ atomic_int_fast64_t; +typedef _Atomic __UINT_FAST64_TYPE__ atomic_uint_fast64_t; +typedef _Atomic __INTPTR_TYPE__ atomic_intptr_t; +typedef _Atomic __UINTPTR_TYPE__ atomic_uintptr_t; +typedef _Atomic __SIZE_TYPE__ atomic_size_t; +typedef _Atomic __PTRDIFF_TYPE__ atomic_ptrdiff_t; +typedef _Atomic __INTMAX_TYPE__ atomic_intmax_t; +typedef _Atomic __UINTMAX_TYPE__ atomic_uintmax_t; + + +#define ATOMIC_VAR_INIT(VALUE) (VALUE) + +/* Initialize an atomic object pointed to by PTR with VAL. */ +#define atomic_init(PTR, VAL) \ + atomic_store_explicit (PTR, VAL, __ATOMIC_RELAXED) + +#define kill_dependency(Y) \ + __extension__ \ + ({ \ + __auto_type __kill_dependency_tmp = (Y); \ + __kill_dependency_tmp; \ + }) + +extern void atomic_thread_fence (memory_order); +#define atomic_thread_fence(MO) __atomic_thread_fence (MO) +extern void atomic_signal_fence (memory_order); +#define atomic_signal_fence(MO) __atomic_signal_fence (MO) +#define atomic_is_lock_free(OBJ) __atomic_is_lock_free (sizeof (*(OBJ)), (OBJ)) + +#define ATOMIC_BOOL_LOCK_FREE __GCC_ATOMIC_BOOL_LOCK_FREE +#define ATOMIC_CHAR_LOCK_FREE __GCC_ATOMIC_CHAR_LOCK_FREE +#define ATOMIC_CHAR16_T_LOCK_FREE __GCC_ATOMIC_CHAR16_T_LOCK_FREE +#define ATOMIC_CHAR32_T_LOCK_FREE __GCC_ATOMIC_CHAR32_T_LOCK_FREE +#define ATOMIC_WCHAR_T_LOCK_FREE __GCC_ATOMIC_WCHAR_T_LOCK_FREE +#define ATOMIC_SHORT_LOCK_FREE __GCC_ATOMIC_SHORT_LOCK_FREE +#define ATOMIC_INT_LOCK_FREE __GCC_ATOMIC_INT_LOCK_FREE +#define ATOMIC_LONG_LOCK_FREE __GCC_ATOMIC_LONG_LOCK_FREE +#define ATOMIC_LLONG_LOCK_FREE __GCC_ATOMIC_LLONG_LOCK_FREE +#define ATOMIC_POINTER_LOCK_FREE __GCC_ATOMIC_POINTER_LOCK_FREE + + +/* Note that these macros require __typeof__ and __auto_type to remove + _Atomic qualifiers (and const qualifiers, if those are valid on + macro operands). + + Also note that the header file uses the generic form of __atomic + builtins, which requires the address to be taken of the value + parameter, and then we pass that value on. This allows the macros + to work for any type, and the compiler is smart enough to convert + these to lock-free _N variants if possible, and throw away the + temps. */ + +#define atomic_store_explicit(PTR, VAL, MO) \ + __extension__ \ + ({ \ + __auto_type __atomic_store_ptr = (PTR); \ + __typeof__ (*__atomic_store_ptr) __atomic_store_tmp = (VAL); \ + __atomic_store (__atomic_store_ptr, &__atomic_store_tmp, (MO)); \ + }) + +#define atomic_store(PTR, VAL) \ + atomic_store_explicit (PTR, VAL, __ATOMIC_SEQ_CST) + + +#define atomic_load_explicit(PTR, MO) \ + __extension__ \ + ({ \ + __auto_type __atomic_load_ptr = (PTR); \ + __typeof__ (*__atomic_load_ptr) __atomic_load_tmp; \ + __atomic_load (__atomic_load_ptr, &__atomic_load_tmp, (MO)); \ + __atomic_load_tmp; \ + }) + +#define atomic_load(PTR) atomic_load_explicit (PTR, __ATOMIC_SEQ_CST) + + +#define atomic_exchange_explicit(PTR, VAL, MO) \ + __extension__ \ + ({ \ + __auto_type __atomic_exchange_ptr = (PTR); \ + __typeof__ (*__atomic_exchange_ptr) __atomic_exchange_val = (VAL); \ + __typeof__ (*__atomic_exchange_ptr) __atomic_exchange_tmp; \ + __atomic_exchange (__atomic_exchange_ptr, &__atomic_exchange_val, \ + &__atomic_exchange_tmp, (MO)); \ + __atomic_exchange_tmp; \ + }) + +#define atomic_exchange(PTR, VAL) \ + atomic_exchange_explicit (PTR, VAL, __ATOMIC_SEQ_CST) + + +#define atomic_compare_exchange_strong_explicit(PTR, VAL, DES, SUC, FAIL) \ + __extension__ \ + ({ \ + __auto_type __atomic_compare_exchange_ptr = (PTR); \ + __typeof__ (*__atomic_compare_exchange_ptr) __atomic_compare_exchange_tmp \ + = (DES); \ + __atomic_compare_exchange (__atomic_compare_exchange_ptr, (VAL), \ + &__atomic_compare_exchange_tmp, 0, \ + (SUC), (FAIL)); \ + }) + +#define atomic_compare_exchange_strong(PTR, VAL, DES) \ + atomic_compare_exchange_strong_explicit (PTR, VAL, DES, __ATOMIC_SEQ_CST, \ + __ATOMIC_SEQ_CST) + +#define atomic_compare_exchange_weak_explicit(PTR, VAL, DES, SUC, FAIL) \ + __extension__ \ + ({ \ + __auto_type __atomic_compare_exchange_ptr = (PTR); \ + __typeof__ (*__atomic_compare_exchange_ptr) __atomic_compare_exchange_tmp \ + = (DES); \ + __atomic_compare_exchange (__atomic_compare_exchange_ptr, (VAL), \ + &__atomic_compare_exchange_tmp, 1, \ + (SUC), (FAIL)); \ + }) + +#define atomic_compare_exchange_weak(PTR, VAL, DES) \ + atomic_compare_exchange_weak_explicit (PTR, VAL, DES, __ATOMIC_SEQ_CST, \ + __ATOMIC_SEQ_CST) + + + +#define atomic_fetch_add(PTR, VAL) __atomic_fetch_add ((PTR), (VAL), \ + __ATOMIC_SEQ_CST) +#define atomic_fetch_add_explicit(PTR, VAL, MO) \ + __atomic_fetch_add ((PTR), (VAL), (MO)) + +#define atomic_fetch_sub(PTR, VAL) __atomic_fetch_sub ((PTR), (VAL), \ + __ATOMIC_SEQ_CST) +#define atomic_fetch_sub_explicit(PTR, VAL, MO) \ + __atomic_fetch_sub ((PTR), (VAL), (MO)) + +#define atomic_fetch_or(PTR, VAL) __atomic_fetch_or ((PTR), (VAL), \ + __ATOMIC_SEQ_CST) +#define atomic_fetch_or_explicit(PTR, VAL, MO) \ + __atomic_fetch_or ((PTR), (VAL), (MO)) + +#define atomic_fetch_xor(PTR, VAL) __atomic_fetch_xor ((PTR), (VAL), \ + __ATOMIC_SEQ_CST) +#define atomic_fetch_xor_explicit(PTR, VAL, MO) \ + __atomic_fetch_xor ((PTR), (VAL), (MO)) + +#define atomic_fetch_and(PTR, VAL) __atomic_fetch_and ((PTR), (VAL), \ + __ATOMIC_SEQ_CST) +#define atomic_fetch_and_explicit(PTR, VAL, MO) \ + __atomic_fetch_and ((PTR), (VAL), (MO)) + + +typedef _Atomic struct +{ +#if __GCC_ATOMIC_TEST_AND_SET_TRUEVAL == 1 + _Bool __val; +#else + unsigned char __val; +#endif +} atomic_flag; + +#define ATOMIC_FLAG_INIT { 0 } + + +extern _Bool atomic_flag_test_and_set (volatile atomic_flag *); +#define atomic_flag_test_and_set(PTR) \ + __atomic_test_and_set ((PTR), __ATOMIC_SEQ_CST) +extern _Bool atomic_flag_test_and_set_explicit (volatile atomic_flag *, + memory_order); +#define atomic_flag_test_and_set_explicit(PTR, MO) \ + __atomic_test_and_set ((PTR), (MO)) + +extern void atomic_flag_clear (volatile atomic_flag *); +#define atomic_flag_clear(PTR) __atomic_clear ((PTR), __ATOMIC_SEQ_CST) +extern void atomic_flag_clear_explicit (volatile atomic_flag *, memory_order); +#define atomic_flag_clear_explicit(PTR, MO) __atomic_clear ((PTR), (MO)) + +#endif /* _STDATOMIC_H */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/stdbool.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/stdbool.h new file mode 100644 index 0000000000..a69fc3a444 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/stdbool.h @@ -0,0 +1,54 @@ +/* Copyright (C) 1998-2017 Free Software Foundation, Inc. + +This file is part of GCC. + +GCC is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3, or (at your option) +any later version. + +GCC is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +Under Section 7 of GPL version 3, you are granted additional +permissions described in the GCC Runtime Library Exception, version +3.1, as published by the Free Software Foundation. + +You should have received a copy of the GNU General Public License and +a copy of the GCC Runtime Library Exception along with this program; +see the files COPYING3 and COPYING.RUNTIME respectively. If not, see +. */ + +/* + * ISO C Standard: 7.16 Boolean type and values + */ + +#ifndef _STDBOOL_H +#define _STDBOOL_H + +#ifndef __cplusplus + +#define bool _Bool +#define true 1 +#define false 0 + +#else /* __cplusplus */ + +/* Supporting _Bool in C++ is a GCC extension. */ +#define _Bool bool + +#if __cplusplus < 201103L +/* Defining these macros in C++98 is a GCC extension. */ +#define bool bool +#define false false +#define true true +#endif + +#endif /* __cplusplus */ + +/* Signal that all the definitions are present. */ +#define __bool_true_false_are_defined 1 + +#endif /* stdbool.h */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/stddef.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/stddef.h new file mode 100644 index 0000000000..872f451cac --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/stddef.h @@ -0,0 +1,451 @@ +/* Copyright (C) 1989-2017 Free Software Foundation, Inc. + +This file is part of GCC. + +GCC is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3, or (at your option) +any later version. + +GCC is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +Under Section 7 of GPL version 3, you are granted additional +permissions described in the GCC Runtime Library Exception, version +3.1, as published by the Free Software Foundation. + +You should have received a copy of the GNU General Public License and +a copy of the GCC Runtime Library Exception along with this program; +see the files COPYING3 and COPYING.RUNTIME respectively. If not, see +. */ + +/* + * ISO C Standard: 7.17 Common definitions + */ +#if (!defined(_STDDEF_H) && !defined(_STDDEF_H_) && !defined(_ANSI_STDDEF_H) \ + && !defined(__STDDEF_H__)) \ + || defined(__need_wchar_t) || defined(__need_size_t) \ + || defined(__need_ptrdiff_t) || defined(__need_NULL) \ + || defined(__need_wint_t) + +/* Any one of these symbols __need_* means that GNU libc + wants us just to define one data type. So don't define + the symbols that indicate this file's entire job has been done. */ +#if (!defined(__need_wchar_t) && !defined(__need_size_t) \ + && !defined(__need_ptrdiff_t) && !defined(__need_NULL) \ + && !defined(__need_wint_t)) +#define _STDDEF_H +#define _STDDEF_H_ +/* snaroff@next.com says the NeXT needs this. */ +#define _ANSI_STDDEF_H +#endif + +#ifndef __sys_stdtypes_h +/* This avoids lossage on SunOS but only if stdtypes.h comes first. + There's no way to win with the other order! Sun lossage. */ + +/* On 4.3bsd-net2, make sure ansi.h is included, so we have + one less case to deal with in the following. */ +#if defined (__BSD_NET2__) || defined (____386BSD____) || (defined (__FreeBSD__) && (__FreeBSD__ < 5)) || defined(__NetBSD__) +#include +#endif +/* On FreeBSD 5, machine/ansi.h does not exist anymore... */ +#if defined (__FreeBSD__) && (__FreeBSD__ >= 5) +#include +#endif + +/* In 4.3bsd-net2, machine/ansi.h defines these symbols, which are + defined if the corresponding type is *not* defined. + FreeBSD-2.1 defines _MACHINE_ANSI_H_ instead of _ANSI_H_. + NetBSD defines _I386_ANSI_H_ and _X86_64_ANSI_H_ instead of _ANSI_H_ */ +#if defined(_ANSI_H_) || defined(_MACHINE_ANSI_H_) || defined(_X86_64_ANSI_H_) || defined(_I386_ANSI_H_) +#if !defined(_SIZE_T_) && !defined(_BSD_SIZE_T_) +#define _SIZE_T +#endif +#if !defined(_PTRDIFF_T_) && !defined(_BSD_PTRDIFF_T_) +#define _PTRDIFF_T +#endif +/* On BSD/386 1.1, at least, machine/ansi.h defines _BSD_WCHAR_T_ + instead of _WCHAR_T_. */ +#if !defined(_WCHAR_T_) && !defined(_BSD_WCHAR_T_) +#ifndef _BSD_WCHAR_T_ +#define _WCHAR_T +#endif +#endif +/* Undef _FOO_T_ if we are supposed to define foo_t. */ +#if defined (__need_ptrdiff_t) || defined (_STDDEF_H_) +#undef _PTRDIFF_T_ +#undef _BSD_PTRDIFF_T_ +#endif +#if defined (__need_size_t) || defined (_STDDEF_H_) +#undef _SIZE_T_ +#undef _BSD_SIZE_T_ +#endif +#if defined (__need_wchar_t) || defined (_STDDEF_H_) +#undef _WCHAR_T_ +#undef _BSD_WCHAR_T_ +#endif +#endif /* defined(_ANSI_H_) || defined(_MACHINE_ANSI_H_) || defined(_X86_64_ANSI_H_) || defined(_I386_ANSI_H_) */ + +/* Sequent's header files use _PTRDIFF_T_ in some conflicting way. + Just ignore it. */ +#if defined (__sequent__) && defined (_PTRDIFF_T_) +#undef _PTRDIFF_T_ +#endif + +/* On VxWorks, may have defined macros like + _TYPE_size_t which will typedef size_t. fixincludes patched the + vxTypesBase.h so that this macro is only defined if _GCC_SIZE_T is + not defined, and so that defining this macro defines _GCC_SIZE_T. + If we find that the macros are still defined at this point, we must + invoke them so that the type is defined as expected. */ +#if defined (_TYPE_ptrdiff_t) && (defined (__need_ptrdiff_t) || defined (_STDDEF_H_)) +_TYPE_ptrdiff_t; +#undef _TYPE_ptrdiff_t +#endif +#if defined (_TYPE_size_t) && (defined (__need_size_t) || defined (_STDDEF_H_)) +_TYPE_size_t; +#undef _TYPE_size_t +#endif +#if defined (_TYPE_wchar_t) && (defined (__need_wchar_t) || defined (_STDDEF_H_)) +_TYPE_wchar_t; +#undef _TYPE_wchar_t +#endif + +/* In case nobody has defined these types, but we aren't running under + GCC 2.00, make sure that __PTRDIFF_TYPE__, __SIZE_TYPE__, and + __WCHAR_TYPE__ have reasonable values. This can happen if the + parts of GCC is compiled by an older compiler, that actually + include gstddef.h, such as collect2. */ + +/* Signed type of difference of two pointers. */ + +/* Define this type if we are doing the whole job, + or if we want this type in particular. */ +#if defined (_STDDEF_H) || defined (__need_ptrdiff_t) +#ifndef _PTRDIFF_T /* in case has defined it. */ +#ifndef _T_PTRDIFF_ +#ifndef _T_PTRDIFF +#ifndef __PTRDIFF_T +#ifndef _PTRDIFF_T_ +#ifndef _BSD_PTRDIFF_T_ +#ifndef ___int_ptrdiff_t_h +#ifndef _GCC_PTRDIFF_T +#ifndef _PTRDIFF_T_DECLARED /* DragonFly */ +#define _PTRDIFF_T +#define _T_PTRDIFF_ +#define _T_PTRDIFF +#define __PTRDIFF_T +#define _PTRDIFF_T_ +#define _BSD_PTRDIFF_T_ +#define ___int_ptrdiff_t_h +#define _GCC_PTRDIFF_T +#define _PTRDIFF_T_DECLARED +#ifndef __PTRDIFF_TYPE__ +#define __PTRDIFF_TYPE__ long int +#endif +typedef __PTRDIFF_TYPE__ ptrdiff_t; +#endif /* _PTRDIFF_T_DECLARED */ +#endif /* _GCC_PTRDIFF_T */ +#endif /* ___int_ptrdiff_t_h */ +#endif /* _BSD_PTRDIFF_T_ */ +#endif /* _PTRDIFF_T_ */ +#endif /* __PTRDIFF_T */ +#endif /* _T_PTRDIFF */ +#endif /* _T_PTRDIFF_ */ +#endif /* _PTRDIFF_T */ + +/* If this symbol has done its job, get rid of it. */ +#undef __need_ptrdiff_t + +#endif /* _STDDEF_H or __need_ptrdiff_t. */ + +/* Unsigned type of `sizeof' something. */ + +/* Define this type if we are doing the whole job, + or if we want this type in particular. */ +#if defined (_STDDEF_H) || defined (__need_size_t) +#ifndef __size_t__ /* BeOS */ +#ifndef __SIZE_T__ /* Cray Unicos/Mk */ +#ifndef _SIZE_T /* in case has defined it. */ +#ifndef _SYS_SIZE_T_H +#ifndef _T_SIZE_ +#ifndef _T_SIZE +#ifndef __SIZE_T +#ifndef _SIZE_T_ +#ifndef _BSD_SIZE_T_ +#ifndef _SIZE_T_DEFINED_ +#ifndef _SIZE_T_DEFINED +#ifndef _BSD_SIZE_T_DEFINED_ /* Darwin */ +#ifndef _SIZE_T_DECLARED /* FreeBSD 5 */ +#ifndef ___int_size_t_h +#ifndef _GCC_SIZE_T +#ifndef _SIZET_ +#ifndef __size_t +#define __size_t__ /* BeOS */ +#define __SIZE_T__ /* Cray Unicos/Mk */ +#define _SIZE_T +#define _SYS_SIZE_T_H +#define _T_SIZE_ +#define _T_SIZE +#define __SIZE_T +#define _SIZE_T_ +#define _BSD_SIZE_T_ +#define _SIZE_T_DEFINED_ +#define _SIZE_T_DEFINED +#define _BSD_SIZE_T_DEFINED_ /* Darwin */ +#define _SIZE_T_DECLARED /* FreeBSD 5 */ +#define ___int_size_t_h +#define _GCC_SIZE_T +#define _SIZET_ +#if (defined (__FreeBSD__) && (__FreeBSD__ >= 5)) \ + || defined(__DragonFly__) \ + || defined(__FreeBSD_kernel__) +/* __size_t is a typedef on FreeBSD 5, must not trash it. */ +#elif defined (__VMS__) +/* __size_t is also a typedef on VMS. */ +#else +#define __size_t +#endif +#ifndef __SIZE_TYPE__ +#define __SIZE_TYPE__ long unsigned int +#endif +#if !(defined (__GNUG__) && defined (size_t)) +typedef __SIZE_TYPE__ size_t; +#ifdef __BEOS__ +typedef long ssize_t; +#endif /* __BEOS__ */ +#endif /* !(defined (__GNUG__) && defined (size_t)) */ +#endif /* __size_t */ +#endif /* _SIZET_ */ +#endif /* _GCC_SIZE_T */ +#endif /* ___int_size_t_h */ +#endif /* _SIZE_T_DECLARED */ +#endif /* _BSD_SIZE_T_DEFINED_ */ +#endif /* _SIZE_T_DEFINED */ +#endif /* _SIZE_T_DEFINED_ */ +#endif /* _BSD_SIZE_T_ */ +#endif /* _SIZE_T_ */ +#endif /* __SIZE_T */ +#endif /* _T_SIZE */ +#endif /* _T_SIZE_ */ +#endif /* _SYS_SIZE_T_H */ +#endif /* _SIZE_T */ +#endif /* __SIZE_T__ */ +#endif /* __size_t__ */ +#undef __need_size_t +#endif /* _STDDEF_H or __need_size_t. */ + + +/* Wide character type. + Locale-writers should change this as necessary to + be big enough to hold unique values not between 0 and 127, + and not (wchar_t) -1, for each defined multibyte character. */ + +/* Define this type if we are doing the whole job, + or if we want this type in particular. */ +#if defined (_STDDEF_H) || defined (__need_wchar_t) +#ifndef __wchar_t__ /* BeOS */ +#ifndef __WCHAR_T__ /* Cray Unicos/Mk */ +#ifndef _WCHAR_T +#ifndef _T_WCHAR_ +#ifndef _T_WCHAR +#ifndef __WCHAR_T +#ifndef _WCHAR_T_ +#ifndef _BSD_WCHAR_T_ +#ifndef _BSD_WCHAR_T_DEFINED_ /* Darwin */ +#ifndef _BSD_RUNE_T_DEFINED_ /* Darwin */ +#ifndef _WCHAR_T_DECLARED /* FreeBSD 5 */ +#ifndef _WCHAR_T_DEFINED_ +#ifndef _WCHAR_T_DEFINED +#ifndef _WCHAR_T_H +#ifndef ___int_wchar_t_h +#ifndef __INT_WCHAR_T_H +#ifndef _GCC_WCHAR_T +#define __wchar_t__ /* BeOS */ +#define __WCHAR_T__ /* Cray Unicos/Mk */ +#define _WCHAR_T +#define _T_WCHAR_ +#define _T_WCHAR +#define __WCHAR_T +#define _WCHAR_T_ +#define _BSD_WCHAR_T_ +#define _WCHAR_T_DEFINED_ +#define _WCHAR_T_DEFINED +#define _WCHAR_T_H +#define ___int_wchar_t_h +#define __INT_WCHAR_T_H +#define _GCC_WCHAR_T +#define _WCHAR_T_DECLARED + +/* On BSD/386 1.1, at least, machine/ansi.h defines _BSD_WCHAR_T_ + instead of _WCHAR_T_, and _BSD_RUNE_T_ (which, unlike the other + symbols in the _FOO_T_ family, stays defined even after its + corresponding type is defined). If we define wchar_t, then we + must undef _WCHAR_T_; for BSD/386 1.1 (and perhaps others), if + we undef _WCHAR_T_, then we must also define rune_t, since + headers like runetype.h assume that if machine/ansi.h is included, + and _BSD_WCHAR_T_ is not defined, then rune_t is available. + machine/ansi.h says, "Note that _WCHAR_T_ and _RUNE_T_ must be of + the same type." */ +#ifdef _BSD_WCHAR_T_ +#undef _BSD_WCHAR_T_ +#ifdef _BSD_RUNE_T_ +#if !defined (_ANSI_SOURCE) && !defined (_POSIX_SOURCE) +typedef _BSD_RUNE_T_ rune_t; +#define _BSD_WCHAR_T_DEFINED_ +#define _BSD_RUNE_T_DEFINED_ /* Darwin */ +#if defined (__FreeBSD__) && (__FreeBSD__ < 5) +/* Why is this file so hard to maintain properly? In contrast to + the comment above regarding BSD/386 1.1, on FreeBSD for as long + as the symbol has existed, _BSD_RUNE_T_ must not stay defined or + redundant typedefs will occur when stdlib.h is included after this file. */ +#undef _BSD_RUNE_T_ +#endif +#endif +#endif +#endif +/* FreeBSD 5 can't be handled well using "traditional" logic above + since it no longer defines _BSD_RUNE_T_ yet still desires to export + rune_t in some cases... */ +#if defined (__FreeBSD__) && (__FreeBSD__ >= 5) +#if !defined (_ANSI_SOURCE) && !defined (_POSIX_SOURCE) +#if __BSD_VISIBLE +#ifndef _RUNE_T_DECLARED +typedef __rune_t rune_t; +#define _RUNE_T_DECLARED +#endif +#endif +#endif +#endif + +#ifndef __WCHAR_TYPE__ +#define __WCHAR_TYPE__ int +#endif +#ifndef __cplusplus +typedef __WCHAR_TYPE__ wchar_t; +#endif +#endif +#endif +#endif +#endif +#endif +#endif +#endif /* _WCHAR_T_DECLARED */ +#endif /* _BSD_RUNE_T_DEFINED_ */ +#endif +#endif +#endif +#endif +#endif +#endif +#endif +#endif /* __WCHAR_T__ */ +#endif /* __wchar_t__ */ +#undef __need_wchar_t +#endif /* _STDDEF_H or __need_wchar_t. */ + +#if defined (__need_wint_t) +#ifndef _WINT_T +#define _WINT_T + +#ifndef __WINT_TYPE__ +#define __WINT_TYPE__ unsigned int +#endif +typedef __WINT_TYPE__ wint_t; +#endif +#undef __need_wint_t +#endif + +/* In 4.3bsd-net2, leave these undefined to indicate that size_t, etc. + are already defined. */ +/* BSD/OS 3.1 and FreeBSD [23].x require the MACHINE_ANSI_H check here. */ +/* NetBSD 5 requires the I386_ANSI_H and X86_64_ANSI_H checks here. */ +#if defined(_ANSI_H_) || defined(_MACHINE_ANSI_H_) || defined(_X86_64_ANSI_H_) || defined(_I386_ANSI_H_) +/* The references to _GCC_PTRDIFF_T_, _GCC_SIZE_T_, and _GCC_WCHAR_T_ + are probably typos and should be removed before 2.8 is released. */ +#ifdef _GCC_PTRDIFF_T_ +#undef _PTRDIFF_T_ +#undef _BSD_PTRDIFF_T_ +#endif +#ifdef _GCC_SIZE_T_ +#undef _SIZE_T_ +#undef _BSD_SIZE_T_ +#endif +#ifdef _GCC_WCHAR_T_ +#undef _WCHAR_T_ +#undef _BSD_WCHAR_T_ +#endif +/* The following ones are the real ones. */ +#ifdef _GCC_PTRDIFF_T +#undef _PTRDIFF_T_ +#undef _BSD_PTRDIFF_T_ +#endif +#ifdef _GCC_SIZE_T +#undef _SIZE_T_ +#undef _BSD_SIZE_T_ +#endif +#ifdef _GCC_WCHAR_T +#undef _WCHAR_T_ +#undef _BSD_WCHAR_T_ +#endif +#endif /* _ANSI_H_ || _MACHINE_ANSI_H_ || _X86_64_ANSI_H_ || _I386_ANSI_H_ */ + +#endif /* __sys_stdtypes_h */ + +/* A null pointer constant. */ + +#if defined (_STDDEF_H) || defined (__need_NULL) +#undef NULL /* in case has defined it. */ +#ifdef __GNUG__ +#define NULL __null +#else /* G++ */ +#ifndef __cplusplus +#define NULL ((void *)0) +#else /* C++ */ +#define NULL 0 +#endif /* C++ */ +#endif /* G++ */ +#endif /* NULL not defined and or need NULL. */ +#undef __need_NULL + +#ifdef _STDDEF_H + +/* Offset of member MEMBER in a struct of type TYPE. */ +#define offsetof(TYPE, MEMBER) __builtin_offsetof (TYPE, MEMBER) + +#if (defined (__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \ + || (defined(__cplusplus) && __cplusplus >= 201103L) +#ifndef _GCC_MAX_ALIGN_T +#define _GCC_MAX_ALIGN_T +/* Type whose alignment is supported in every context and is at least + as great as that of any standard type not using alignment + specifiers. */ +typedef struct { + long long __max_align_ll __attribute__((__aligned__(__alignof__(long long)))); + long double __max_align_ld __attribute__((__aligned__(__alignof__(long double)))); + /* _Float128 is defined as a basic type, so max_align_t must be + sufficiently aligned for it. This code must work in C++, so we + use __float128 here; that is only available on some + architectures, but only on i386 is extra alignment needed for + __float128. */ +#ifdef __i386__ + __float128 __max_align_f128 __attribute__((__aligned__(__alignof(__float128)))); +#endif +} max_align_t; +#endif +#endif /* C11 or C++11. */ + +#if defined(__cplusplus) && __cplusplus >= 201103L +#ifndef _GXX_NULLPTR_T +#define _GXX_NULLPTR_T + typedef decltype(nullptr) nullptr_t; +#endif +#endif /* C++11. */ + +#endif /* _STDDEF_H was defined this time */ + +#endif /* !_STDDEF_H && !_STDDEF_H_ && !_ANSI_STDDEF_H && !__STDDEF_H__ + || __need_XXX was not defined before */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/stdfix.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/stdfix.h new file mode 100644 index 0000000000..9de1b72dca --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/stdfix.h @@ -0,0 +1,204 @@ +/* Copyright (C) 2007-2017 Free Software Foundation, Inc. + +This file is part of GCC. + +GCC is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3, or (at your option) +any later version. + +GCC is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +Under Section 7 of GPL version 3, you are granted additional +permissions described in the GCC Runtime Library Exception, version +3.1, as published by the Free Software Foundation. + +You should have received a copy of the GNU General Public License and +a copy of the GCC Runtime Library Exception along with this program; +see the files COPYING3 and COPYING.RUNTIME respectively. If not, see +. */ + +/* ISO/IEC JTC1 SC22 WG14 N1169 + * Date: 2006-04-04 + * ISO/IEC TR 18037 + * Programming languages - C - Extensions to support embedded processors + */ + +#ifndef _STDFIX_H +#define _STDFIX_H + +/* 7.18a.1 Introduction. */ + +#undef fract +#undef accum +#undef sat +#define fract _Fract +#define accum _Accum +#define sat _Sat + +/* 7.18a.3 Precision macros. */ + +#undef SFRACT_FBIT +#undef SFRACT_MIN +#undef SFRACT_MAX +#undef SFRACT_EPSILON +#define SFRACT_FBIT __SFRACT_FBIT__ +#define SFRACT_MIN __SFRACT_MIN__ +#define SFRACT_MAX __SFRACT_MAX__ +#define SFRACT_EPSILON __SFRACT_EPSILON__ + +#undef USFRACT_FBIT +#undef USFRACT_MIN +#undef USFRACT_MAX +#undef USFRACT_EPSILON +#define USFRACT_FBIT __USFRACT_FBIT__ +#define USFRACT_MIN __USFRACT_MIN__ /* GCC extension. */ +#define USFRACT_MAX __USFRACT_MAX__ +#define USFRACT_EPSILON __USFRACT_EPSILON__ + +#undef FRACT_FBIT +#undef FRACT_MIN +#undef FRACT_MAX +#undef FRACT_EPSILON +#define FRACT_FBIT __FRACT_FBIT__ +#define FRACT_MIN __FRACT_MIN__ +#define FRACT_MAX __FRACT_MAX__ +#define FRACT_EPSILON __FRACT_EPSILON__ + +#undef UFRACT_FBIT +#undef UFRACT_MIN +#undef UFRACT_MAX +#undef UFRACT_EPSILON +#define UFRACT_FBIT __UFRACT_FBIT__ +#define UFRACT_MIN __UFRACT_MIN__ /* GCC extension. */ +#define UFRACT_MAX __UFRACT_MAX__ +#define UFRACT_EPSILON __UFRACT_EPSILON__ + +#undef LFRACT_FBIT +#undef LFRACT_MIN +#undef LFRACT_MAX +#undef LFRACT_EPSILON +#define LFRACT_FBIT __LFRACT_FBIT__ +#define LFRACT_MIN __LFRACT_MIN__ +#define LFRACT_MAX __LFRACT_MAX__ +#define LFRACT_EPSILON __LFRACT_EPSILON__ + +#undef ULFRACT_FBIT +#undef ULFRACT_MIN +#undef ULFRACT_MAX +#undef ULFRACT_EPSILON +#define ULFRACT_FBIT __ULFRACT_FBIT__ +#define ULFRACT_MIN __ULFRACT_MIN__ /* GCC extension. */ +#define ULFRACT_MAX __ULFRACT_MAX__ +#define ULFRACT_EPSILON __ULFRACT_EPSILON__ + +#undef LLFRACT_FBIT +#undef LLFRACT_MIN +#undef LLFRACT_MAX +#undef LLFRACT_EPSILON +#define LLFRACT_FBIT __LLFRACT_FBIT__ /* GCC extension. */ +#define LLFRACT_MIN __LLFRACT_MIN__ /* GCC extension. */ +#define LLFRACT_MAX __LLFRACT_MAX__ /* GCC extension. */ +#define LLFRACT_EPSILON __LLFRACT_EPSILON__ /* GCC extension. */ + +#undef ULLFRACT_FBIT +#undef ULLFRACT_MIN +#undef ULLFRACT_MAX +#undef ULLFRACT_EPSILON +#define ULLFRACT_FBIT __ULLFRACT_FBIT__ /* GCC extension. */ +#define ULLFRACT_MIN __ULLFRACT_MIN__ /* GCC extension. */ +#define ULLFRACT_MAX __ULLFRACT_MAX__ /* GCC extension. */ +#define ULLFRACT_EPSILON __ULLFRACT_EPSILON__ /* GCC extension. */ + +#undef SACCUM_FBIT +#undef SACCUM_IBIT +#undef SACCUM_MIN +#undef SACCUM_MAX +#undef SACCUM_EPSILON +#define SACCUM_FBIT __SACCUM_FBIT__ +#define SACCUM_IBIT __SACCUM_IBIT__ +#define SACCUM_MIN __SACCUM_MIN__ +#define SACCUM_MAX __SACCUM_MAX__ +#define SACCUM_EPSILON __SACCUM_EPSILON__ + +#undef USACCUM_FBIT +#undef USACCUM_IBIT +#undef USACCUM_MIN +#undef USACCUM_MAX +#undef USACCUM_EPSILON +#define USACCUM_FBIT __USACCUM_FBIT__ +#define USACCUM_IBIT __USACCUM_IBIT__ +#define USACCUM_MIN __USACCUM_MIN__ /* GCC extension. */ +#define USACCUM_MAX __USACCUM_MAX__ +#define USACCUM_EPSILON __USACCUM_EPSILON__ + +#undef ACCUM_FBIT +#undef ACCUM_IBIT +#undef ACCUM_MIN +#undef ACCUM_MAX +#undef ACCUM_EPSILON +#define ACCUM_FBIT __ACCUM_FBIT__ +#define ACCUM_IBIT __ACCUM_IBIT__ +#define ACCUM_MIN __ACCUM_MIN__ +#define ACCUM_MAX __ACCUM_MAX__ +#define ACCUM_EPSILON __ACCUM_EPSILON__ + +#undef UACCUM_FBIT +#undef UACCUM_IBIT +#undef UACCUM_MIN +#undef UACCUM_MAX +#undef UACCUM_EPSILON +#define UACCUM_FBIT __UACCUM_FBIT__ +#define UACCUM_IBIT __UACCUM_IBIT__ +#define UACCUM_MIN __UACCUM_MIN__ /* GCC extension. */ +#define UACCUM_MAX __UACCUM_MAX__ +#define UACCUM_EPSILON __UACCUM_EPSILON__ + +#undef LACCUM_FBIT +#undef LACCUM_IBIT +#undef LACCUM_MIN +#undef LACCUM_MAX +#undef LACCUM_EPSILON +#define LACCUM_FBIT __LACCUM_FBIT__ +#define LACCUM_IBIT __LACCUM_IBIT__ +#define LACCUM_MIN __LACCUM_MIN__ +#define LACCUM_MAX __LACCUM_MAX__ +#define LACCUM_EPSILON __LACCUM_EPSILON__ + +#undef ULACCUM_FBIT +#undef ULACCUM_IBIT +#undef ULACCUM_MIN +#undef ULACCUM_MAX +#undef ULACCUM_EPSILON +#define ULACCUM_FBIT __ULACCUM_FBIT__ +#define ULACCUM_IBIT __ULACCUM_IBIT__ +#define ULACCUM_MIN __ULACCUM_MIN__ /* GCC extension. */ +#define ULACCUM_MAX __ULACCUM_MAX__ +#define ULACCUM_EPSILON __ULACCUM_EPSILON__ + +#undef LLACCUM_FBIT +#undef LLACCUM_IBIT +#undef LLACCUM_MIN +#undef LLACCUM_MAX +#undef LLACCUM_EPSILON +#define LLACCUM_FBIT __LLACCUM_FBIT__ /* GCC extension. */ +#define LLACCUM_IBIT __LLACCUM_IBIT__ /* GCC extension. */ +#define LLACCUM_MIN __LLACCUM_MIN__ /* GCC extension. */ +#define LLACCUM_MAX __LLACCUM_MAX__ /* GCC extension. */ +#define LLACCUM_EPSILON __LLACCUM_EPSILON__ /* GCC extension. */ + +#undef ULLACCUM_FBIT +#undef ULLACCUM_IBIT +#undef ULLACCUM_MIN +#undef ULLACCUM_MAX +#undef ULLACCUM_EPSILON +#define ULLACCUM_FBIT __ULLACCUM_FBIT__ /* GCC extension. */ +#define ULLACCUM_IBIT __ULLACCUM_IBIT__ /* GCC extension. */ +#define ULLACCUM_MIN __ULLACCUM_MIN__ /* GCC extension. */ +#define ULLACCUM_MAX __ULLACCUM_MAX__ /* GCC extension. */ +#define ULLACCUM_EPSILON __ULLACCUM_EPSILON__ /* GCC extension. */ + +#endif /* _STDFIX_H */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/stdint-gcc.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/stdint-gcc.h new file mode 100644 index 0000000000..0ee7e35234 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/stdint-gcc.h @@ -0,0 +1,364 @@ +/* Copyright (C) 2008-2017 Free Software Foundation, Inc. + +This file is part of GCC. + +GCC is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3, or (at your option) +any later version. + +GCC is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +Under Section 7 of GPL version 3, you are granted additional +permissions described in the GCC Runtime Library Exception, version +3.1, as published by the Free Software Foundation. + +You should have received a copy of the GNU General Public License and +a copy of the GCC Runtime Library Exception along with this program; +see the files COPYING3 and COPYING.RUNTIME respectively. If not, see +. */ + +/* + * ISO C Standard: 7.18 Integer types + */ + +#ifndef _GCC_STDINT_H +#define _GCC_STDINT_H + +/* 7.8.1.1 Exact-width integer types */ + +#ifdef __INT8_TYPE__ +typedef __INT8_TYPE__ int8_t; +#endif +#ifdef __INT16_TYPE__ +typedef __INT16_TYPE__ int16_t; +#endif +#ifdef __INT32_TYPE__ +typedef __INT32_TYPE__ int32_t; +#endif +#ifdef __INT64_TYPE__ +typedef __INT64_TYPE__ int64_t; +#endif +#ifdef __UINT8_TYPE__ +typedef __UINT8_TYPE__ uint8_t; +#endif +#ifdef __UINT16_TYPE__ +typedef __UINT16_TYPE__ uint16_t; +#endif +#ifdef __UINT32_TYPE__ +typedef __UINT32_TYPE__ uint32_t; +#endif +#ifdef __UINT64_TYPE__ +typedef __UINT64_TYPE__ uint64_t; +#endif + +/* 7.8.1.2 Minimum-width integer types */ + +typedef __INT_LEAST8_TYPE__ int_least8_t; +typedef __INT_LEAST16_TYPE__ int_least16_t; +typedef __INT_LEAST32_TYPE__ int_least32_t; +typedef __INT_LEAST64_TYPE__ int_least64_t; +typedef __UINT_LEAST8_TYPE__ uint_least8_t; +typedef __UINT_LEAST16_TYPE__ uint_least16_t; +typedef __UINT_LEAST32_TYPE__ uint_least32_t; +typedef __UINT_LEAST64_TYPE__ uint_least64_t; + +/* 7.8.1.3 Fastest minimum-width integer types */ + +typedef __INT_FAST8_TYPE__ int_fast8_t; +typedef __INT_FAST16_TYPE__ int_fast16_t; +typedef __INT_FAST32_TYPE__ int_fast32_t; +typedef __INT_FAST64_TYPE__ int_fast64_t; +typedef __UINT_FAST8_TYPE__ uint_fast8_t; +typedef __UINT_FAST16_TYPE__ uint_fast16_t; +typedef __UINT_FAST32_TYPE__ uint_fast32_t; +typedef __UINT_FAST64_TYPE__ uint_fast64_t; + +/* 7.8.1.4 Integer types capable of holding object pointers */ + +#ifdef __INTPTR_TYPE__ +typedef __INTPTR_TYPE__ intptr_t; +#endif +#ifdef __UINTPTR_TYPE__ +typedef __UINTPTR_TYPE__ uintptr_t; +#endif + +/* 7.8.1.5 Greatest-width integer types */ + +typedef __INTMAX_TYPE__ intmax_t; +typedef __UINTMAX_TYPE__ uintmax_t; + +#if (!defined __cplusplus || __cplusplus >= 201103L \ + || defined __STDC_LIMIT_MACROS) + +/* 7.18.2 Limits of specified-width integer types */ + +#ifdef __INT8_MAX__ +# undef INT8_MAX +# define INT8_MAX __INT8_MAX__ +# undef INT8_MIN +# define INT8_MIN (-INT8_MAX - 1) +#endif +#ifdef __UINT8_MAX__ +# undef UINT8_MAX +# define UINT8_MAX __UINT8_MAX__ +#endif +#ifdef __INT16_MAX__ +# undef INT16_MAX +# define INT16_MAX __INT16_MAX__ +# undef INT16_MIN +# define INT16_MIN (-INT16_MAX - 1) +#endif +#ifdef __UINT16_MAX__ +# undef UINT16_MAX +# define UINT16_MAX __UINT16_MAX__ +#endif +#ifdef __INT32_MAX__ +# undef INT32_MAX +# define INT32_MAX __INT32_MAX__ +# undef INT32_MIN +# define INT32_MIN (-INT32_MAX - 1) +#endif +#ifdef __UINT32_MAX__ +# undef UINT32_MAX +# define UINT32_MAX __UINT32_MAX__ +#endif +#ifdef __INT64_MAX__ +# undef INT64_MAX +# define INT64_MAX __INT64_MAX__ +# undef INT64_MIN +# define INT64_MIN (-INT64_MAX - 1) +#endif +#ifdef __UINT64_MAX__ +# undef UINT64_MAX +# define UINT64_MAX __UINT64_MAX__ +#endif + +#undef INT_LEAST8_MAX +#define INT_LEAST8_MAX __INT_LEAST8_MAX__ +#undef INT_LEAST8_MIN +#define INT_LEAST8_MIN (-INT_LEAST8_MAX - 1) +#undef UINT_LEAST8_MAX +#define UINT_LEAST8_MAX __UINT_LEAST8_MAX__ +#undef INT_LEAST16_MAX +#define INT_LEAST16_MAX __INT_LEAST16_MAX__ +#undef INT_LEAST16_MIN +#define INT_LEAST16_MIN (-INT_LEAST16_MAX - 1) +#undef UINT_LEAST16_MAX +#define UINT_LEAST16_MAX __UINT_LEAST16_MAX__ +#undef INT_LEAST32_MAX +#define INT_LEAST32_MAX __INT_LEAST32_MAX__ +#undef INT_LEAST32_MIN +#define INT_LEAST32_MIN (-INT_LEAST32_MAX - 1) +#undef UINT_LEAST32_MAX +#define UINT_LEAST32_MAX __UINT_LEAST32_MAX__ +#undef INT_LEAST64_MAX +#define INT_LEAST64_MAX __INT_LEAST64_MAX__ +#undef INT_LEAST64_MIN +#define INT_LEAST64_MIN (-INT_LEAST64_MAX - 1) +#undef UINT_LEAST64_MAX +#define UINT_LEAST64_MAX __UINT_LEAST64_MAX__ + +#undef INT_FAST8_MAX +#define INT_FAST8_MAX __INT_FAST8_MAX__ +#undef INT_FAST8_MIN +#define INT_FAST8_MIN (-INT_FAST8_MAX - 1) +#undef UINT_FAST8_MAX +#define UINT_FAST8_MAX __UINT_FAST8_MAX__ +#undef INT_FAST16_MAX +#define INT_FAST16_MAX __INT_FAST16_MAX__ +#undef INT_FAST16_MIN +#define INT_FAST16_MIN (-INT_FAST16_MAX - 1) +#undef UINT_FAST16_MAX +#define UINT_FAST16_MAX __UINT_FAST16_MAX__ +#undef INT_FAST32_MAX +#define INT_FAST32_MAX __INT_FAST32_MAX__ +#undef INT_FAST32_MIN +#define INT_FAST32_MIN (-INT_FAST32_MAX - 1) +#undef UINT_FAST32_MAX +#define UINT_FAST32_MAX __UINT_FAST32_MAX__ +#undef INT_FAST64_MAX +#define INT_FAST64_MAX __INT_FAST64_MAX__ +#undef INT_FAST64_MIN +#define INT_FAST64_MIN (-INT_FAST64_MAX - 1) +#undef UINT_FAST64_MAX +#define UINT_FAST64_MAX __UINT_FAST64_MAX__ + +#ifdef __INTPTR_MAX__ +# undef INTPTR_MAX +# define INTPTR_MAX __INTPTR_MAX__ +# undef INTPTR_MIN +# define INTPTR_MIN (-INTPTR_MAX - 1) +#endif +#ifdef __UINTPTR_MAX__ +# undef UINTPTR_MAX +# define UINTPTR_MAX __UINTPTR_MAX__ +#endif + +#undef INTMAX_MAX +#define INTMAX_MAX __INTMAX_MAX__ +#undef INTMAX_MIN +#define INTMAX_MIN (-INTMAX_MAX - 1) +#undef UINTMAX_MAX +#define UINTMAX_MAX __UINTMAX_MAX__ + +/* 7.18.3 Limits of other integer types */ + +#undef PTRDIFF_MAX +#define PTRDIFF_MAX __PTRDIFF_MAX__ +#undef PTRDIFF_MIN +#define PTRDIFF_MIN (-PTRDIFF_MAX - 1) + +#undef SIG_ATOMIC_MAX +#define SIG_ATOMIC_MAX __SIG_ATOMIC_MAX__ +#undef SIG_ATOMIC_MIN +#define SIG_ATOMIC_MIN __SIG_ATOMIC_MIN__ + +#undef SIZE_MAX +#define SIZE_MAX __SIZE_MAX__ + +#undef WCHAR_MAX +#define WCHAR_MAX __WCHAR_MAX__ +#undef WCHAR_MIN +#define WCHAR_MIN __WCHAR_MIN__ + +#undef WINT_MAX +#define WINT_MAX __WINT_MAX__ +#undef WINT_MIN +#define WINT_MIN __WINT_MIN__ + +#endif /* (!defined __cplusplus || __cplusplus >= 201103L + || defined __STDC_LIMIT_MACROS) */ + +#if (!defined __cplusplus || __cplusplus >= 201103L \ + || defined __STDC_CONSTANT_MACROS) + +#undef INT8_C +#define INT8_C(c) __INT8_C(c) +#undef INT16_C +#define INT16_C(c) __INT16_C(c) +#undef INT32_C +#define INT32_C(c) __INT32_C(c) +#undef INT64_C +#define INT64_C(c) __INT64_C(c) +#undef UINT8_C +#define UINT8_C(c) __UINT8_C(c) +#undef UINT16_C +#define UINT16_C(c) __UINT16_C(c) +#undef UINT32_C +#define UINT32_C(c) __UINT32_C(c) +#undef UINT64_C +#define UINT64_C(c) __UINT64_C(c) +#undef INTMAX_C +#define INTMAX_C(c) __INTMAX_C(c) +#undef UINTMAX_C +#define UINTMAX_C(c) __UINTMAX_C(c) + +#endif /* (!defined __cplusplus || __cplusplus >= 201103L + || defined __STDC_CONSTANT_MACROS) */ + +#ifdef __STDC_WANT_IEC_60559_BFP_EXT__ +/* TS 18661-1 widths of integer types. */ + +#ifdef __INT8_TYPE__ +# undef INT8_WIDTH +# define INT8_WIDTH 8 +#endif +#ifdef __UINT8_TYPE__ +# undef UINT8_WIDTH +# define UINT8_WIDTH 8 +#endif +#ifdef __INT16_TYPE__ +# undef INT16_WIDTH +# define INT16_WIDTH 16 +#endif +#ifdef __UINT16_TYPE__ +# undef UINT16_WIDTH +# define UINT16_WIDTH 16 +#endif +#ifdef __INT32_TYPE__ +# undef INT32_WIDTH +# define INT32_WIDTH 32 +#endif +#ifdef __UINT32_TYPE__ +# undef UINT32_WIDTH +# define UINT32_WIDTH 32 +#endif +#ifdef __INT64_TYPE__ +# undef INT64_WIDTH +# define INT64_WIDTH 64 +#endif +#ifdef __UINT64_TYPE__ +# undef UINT64_WIDTH +# define UINT64_WIDTH 64 +#endif + +#undef INT_LEAST8_WIDTH +#define INT_LEAST8_WIDTH __INT_LEAST8_WIDTH__ +#undef UINT_LEAST8_WIDTH +#define UINT_LEAST8_WIDTH __INT_LEAST8_WIDTH__ +#undef INT_LEAST16_WIDTH +#define INT_LEAST16_WIDTH __INT_LEAST16_WIDTH__ +#undef UINT_LEAST16_WIDTH +#define UINT_LEAST16_WIDTH __INT_LEAST16_WIDTH__ +#undef INT_LEAST32_WIDTH +#define INT_LEAST32_WIDTH __INT_LEAST32_WIDTH__ +#undef UINT_LEAST32_WIDTH +#define UINT_LEAST32_WIDTH __INT_LEAST32_WIDTH__ +#undef INT_LEAST64_WIDTH +#define INT_LEAST64_WIDTH __INT_LEAST64_WIDTH__ +#undef UINT_LEAST64_WIDTH +#define UINT_LEAST64_WIDTH __INT_LEAST64_WIDTH__ + +#undef INT_FAST8_WIDTH +#define INT_FAST8_WIDTH __INT_FAST8_WIDTH__ +#undef UINT_FAST8_WIDTH +#define UINT_FAST8_WIDTH __INT_FAST8_WIDTH__ +#undef INT_FAST16_WIDTH +#define INT_FAST16_WIDTH __INT_FAST16_WIDTH__ +#undef UINT_FAST16_WIDTH +#define UINT_FAST16_WIDTH __INT_FAST16_WIDTH__ +#undef INT_FAST32_WIDTH +#define INT_FAST32_WIDTH __INT_FAST32_WIDTH__ +#undef UINT_FAST32_WIDTH +#define UINT_FAST32_WIDTH __INT_FAST32_WIDTH__ +#undef INT_FAST64_WIDTH +#define INT_FAST64_WIDTH __INT_FAST64_WIDTH__ +#undef UINT_FAST64_WIDTH +#define UINT_FAST64_WIDTH __INT_FAST64_WIDTH__ + +#ifdef __INTPTR_TYPE__ +# undef INTPTR_WIDTH +# define INTPTR_WIDTH __INTPTR_WIDTH__ +#endif +#ifdef __UINTPTR_TYPE__ +# undef UINTPTR_WIDTH +# define UINTPTR_WIDTH __INTPTR_WIDTH__ +#endif + +#undef INTMAX_WIDTH +#define INTMAX_WIDTH __INTMAX_WIDTH__ +#undef UINTMAX_WIDTH +#define UINTMAX_WIDTH __INTMAX_WIDTH__ + +#undef PTRDIFF_WIDTH +#define PTRDIFF_WIDTH __PTRDIFF_WIDTH__ + +#undef SIG_ATOMIC_WIDTH +#define SIG_ATOMIC_WIDTH __SIG_ATOMIC_WIDTH__ + +#undef SIZE_WIDTH +#define SIZE_WIDTH __SIZE_WIDTH__ + +#undef WCHAR_WIDTH +#define WCHAR_WIDTH __WCHAR_WIDTH__ + +#undef WINT_WIDTH +#define WINT_WIDTH __WINT_WIDTH__ + +#endif + +#endif /* _GCC_STDINT_H */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/stdint.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/stdint.h new file mode 100644 index 0000000000..83b6f70c83 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/stdint.h @@ -0,0 +1,14 @@ +#ifndef _GCC_WRAP_STDINT_H +#if __STDC_HOSTED__ +# if defined __cplusplus && __cplusplus >= 201103L +# undef __STDC_LIMIT_MACROS +# define __STDC_LIMIT_MACROS +# undef __STDC_CONSTANT_MACROS +# define __STDC_CONSTANT_MACROS +# endif +# include_next +#else +# include "stdint-gcc.h" +#endif +#define _GCC_WRAP_STDINT_H +#endif diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/stdnoreturn.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/stdnoreturn.h new file mode 100644 index 0000000000..739c2e31fc --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/stdnoreturn.h @@ -0,0 +1,35 @@ +/* Copyright (C) 2011-2017 Free Software Foundation, Inc. + +This file is part of GCC. + +GCC is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3, or (at your option) +any later version. + +GCC is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +Under Section 7 of GPL version 3, you are granted additional +permissions described in the GCC Runtime Library Exception, version +3.1, as published by the Free Software Foundation. + +You should have received a copy of the GNU General Public License and +a copy of the GCC Runtime Library Exception along with this program; +see the files COPYING3 and COPYING.RUNTIME respectively. If not, see +. */ + +/* ISO C1X: 7.23 _Noreturn . */ + +#ifndef _STDNORETURN_H +#define _STDNORETURN_H + +#ifndef __cplusplus + +#define noreturn _Noreturn + +#endif + +#endif /* stdnoreturn.h */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/syslimits.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/syslimits.h new file mode 100644 index 0000000000..a3628025e8 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/syslimits.h @@ -0,0 +1,8 @@ +/* syslimits.h stands for the system's own limits.h file. + If we can use it ok unmodified, then we install this text. + If fixincludes fixes it, then the fixed version is installed + instead of this text. */ + +#define _GCC_NEXT_LIMITS_H /* tell gcc's limits.h to recurse */ +#include_next +#undef _GCC_NEXT_LIMITS_H diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/tbmintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/tbmintrin.h new file mode 100644 index 0000000000..dfdf163f9c --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/tbmintrin.h @@ -0,0 +1,180 @@ +/* Copyright (C) 2010-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _X86INTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _TBMINTRIN_H_INCLUDED +#define _TBMINTRIN_H_INCLUDED + +#ifndef __TBM__ +#pragma GCC push_options +#pragma GCC target("tbm") +#define __DISABLE_TBM__ +#endif /* __TBM__ */ + +#ifdef __OPTIMIZE__ +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__bextri_u32 (unsigned int __X, const unsigned int __I) +{ + return __builtin_ia32_bextri_u32 (__X, __I); +} +#else +#define __bextri_u32(X, I) \ + ((unsigned int)__builtin_ia32_bextri_u32 ((unsigned int)(X), \ + (unsigned int)(I))) +#endif /*__OPTIMIZE__ */ + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blcfill_u32 (unsigned int __X) +{ + return __X & (__X + 1); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blci_u32 (unsigned int __X) +{ + return __X | ~(__X + 1); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blcic_u32 (unsigned int __X) +{ + return ~__X & (__X + 1); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blcmsk_u32 (unsigned int __X) +{ + return __X ^ (__X + 1); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blcs_u32 (unsigned int __X) +{ + return __X | (__X + 1); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blsfill_u32 (unsigned int __X) +{ + return __X | (__X - 1); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blsic_u32 (unsigned int __X) +{ + return ~__X | (__X - 1); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__t1mskc_u32 (unsigned int __X) +{ + return ~__X | (__X + 1); +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__tzmsk_u32 (unsigned int __X) +{ + return ~__X & (__X - 1); +} + + + +#ifdef __x86_64__ +#ifdef __OPTIMIZE__ +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__bextri_u64 (unsigned long long __X, const unsigned int __I) +{ + return __builtin_ia32_bextri_u64 (__X, __I); +} +#else +#define __bextri_u64(X, I) \ + ((unsigned long long)__builtin_ia32_bextri_u64 ((unsigned long long)(X), \ + (unsigned long long)(I))) +#endif /*__OPTIMIZE__ */ + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blcfill_u64 (unsigned long long __X) +{ + return __X & (__X + 1); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blci_u64 (unsigned long long __X) +{ + return __X | ~(__X + 1); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blcic_u64 (unsigned long long __X) +{ + return ~__X & (__X + 1); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blcmsk_u64 (unsigned long long __X) +{ + return __X ^ (__X + 1); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blcs_u64 (unsigned long long __X) +{ + return __X | (__X + 1); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blsfill_u64 (unsigned long long __X) +{ + return __X | (__X - 1); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__blsic_u64 (unsigned long long __X) +{ + return ~__X | (__X - 1); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__t1mskc_u64 (unsigned long long __X) +{ + return ~__X | (__X + 1); +} + +extern __inline unsigned long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +__tzmsk_u64 (unsigned long long __X) +{ + return ~__X & (__X - 1); +} + + +#endif /* __x86_64__ */ + +#ifdef __DISABLE_TBM__ +#undef __DISABLE_TBM__ +#pragma GCC pop_options +#endif /* __DISABLE_TBM__ */ + +#endif /* _TBMINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/tmmintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/tmmintrin.h new file mode 100644 index 0000000000..93b016d291 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/tmmintrin.h @@ -0,0 +1,249 @@ +/* Copyright (C) 2006-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +/* Implemented from the specification included in the Intel C++ Compiler + User Guide and Reference, version 9.1. */ + +#ifndef _TMMINTRIN_H_INCLUDED +#define _TMMINTRIN_H_INCLUDED + +/* We need definitions from the SSE3, SSE2 and SSE header files*/ +#include + +#ifndef __SSSE3__ +#pragma GCC push_options +#pragma GCC target("ssse3") +#define __DISABLE_SSSE3__ +#endif /* __SSSE3__ */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hadd_epi16 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_phaddw128 ((__v8hi)__X, (__v8hi)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hadd_epi32 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_phaddd128 ((__v4si)__X, (__v4si)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hadds_epi16 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_phaddsw128 ((__v8hi)__X, (__v8hi)__Y); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hadd_pi16 (__m64 __X, __m64 __Y) +{ + return (__m64) __builtin_ia32_phaddw ((__v4hi)__X, (__v4hi)__Y); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hadd_pi32 (__m64 __X, __m64 __Y) +{ + return (__m64) __builtin_ia32_phaddd ((__v2si)__X, (__v2si)__Y); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hadds_pi16 (__m64 __X, __m64 __Y) +{ + return (__m64) __builtin_ia32_phaddsw ((__v4hi)__X, (__v4hi)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hsub_epi16 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_phsubw128 ((__v8hi)__X, (__v8hi)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hsub_epi32 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_phsubd128 ((__v4si)__X, (__v4si)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hsubs_epi16 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_phsubsw128 ((__v8hi)__X, (__v8hi)__Y); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hsub_pi16 (__m64 __X, __m64 __Y) +{ + return (__m64) __builtin_ia32_phsubw ((__v4hi)__X, (__v4hi)__Y); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hsub_pi32 (__m64 __X, __m64 __Y) +{ + return (__m64) __builtin_ia32_phsubd ((__v2si)__X, (__v2si)__Y); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hsubs_pi16 (__m64 __X, __m64 __Y) +{ + return (__m64) __builtin_ia32_phsubsw ((__v4hi)__X, (__v4hi)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maddubs_epi16 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_pmaddubsw128 ((__v16qi)__X, (__v16qi)__Y); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maddubs_pi16 (__m64 __X, __m64 __Y) +{ + return (__m64) __builtin_ia32_pmaddubsw ((__v8qi)__X, (__v8qi)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mulhrs_epi16 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_pmulhrsw128 ((__v8hi)__X, (__v8hi)__Y); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mulhrs_pi16 (__m64 __X, __m64 __Y) +{ + return (__m64) __builtin_ia32_pmulhrsw ((__v4hi)__X, (__v4hi)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_shuffle_epi8 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_pshufb128 ((__v16qi)__X, (__v16qi)__Y); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_shuffle_pi8 (__m64 __X, __m64 __Y) +{ + return (__m64) __builtin_ia32_pshufb ((__v8qi)__X, (__v8qi)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sign_epi8 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_psignb128 ((__v16qi)__X, (__v16qi)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sign_epi16 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_psignw128 ((__v8hi)__X, (__v8hi)__Y); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sign_epi32 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_psignd128 ((__v4si)__X, (__v4si)__Y); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sign_pi8 (__m64 __X, __m64 __Y) +{ + return (__m64) __builtin_ia32_psignb ((__v8qi)__X, (__v8qi)__Y); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sign_pi16 (__m64 __X, __m64 __Y) +{ + return (__m64) __builtin_ia32_psignw ((__v4hi)__X, (__v4hi)__Y); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sign_pi32 (__m64 __X, __m64 __Y) +{ + return (__m64) __builtin_ia32_psignd ((__v2si)__X, (__v2si)__Y); +} + +#ifdef __OPTIMIZE__ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_alignr_epi8(__m128i __X, __m128i __Y, const int __N) +{ + return (__m128i) __builtin_ia32_palignr128 ((__v2di)__X, + (__v2di)__Y, __N * 8); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_alignr_pi8(__m64 __X, __m64 __Y, const int __N) +{ + return (__m64) __builtin_ia32_palignr ((__v1di)__X, + (__v1di)__Y, __N * 8); +} +#else +#define _mm_alignr_epi8(X, Y, N) \ + ((__m128i) __builtin_ia32_palignr128 ((__v2di)(__m128i)(X), \ + (__v2di)(__m128i)(Y), \ + (int)(N) * 8)) +#define _mm_alignr_pi8(X, Y, N) \ + ((__m64) __builtin_ia32_palignr ((__v1di)(__m64)(X), \ + (__v1di)(__m64)(Y), \ + (int)(N) * 8)) +#endif + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_abs_epi8 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pabsb128 ((__v16qi)__X); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_abs_epi16 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pabsw128 ((__v8hi)__X); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_abs_epi32 (__m128i __X) +{ + return (__m128i) __builtin_ia32_pabsd128 ((__v4si)__X); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_abs_pi8 (__m64 __X) +{ + return (__m64) __builtin_ia32_pabsb ((__v8qi)__X); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_abs_pi16 (__m64 __X) +{ + return (__m64) __builtin_ia32_pabsw ((__v4hi)__X); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_abs_pi32 (__m64 __X) +{ + return (__m64) __builtin_ia32_pabsd ((__v2si)__X); +} + +#ifdef __DISABLE_SSSE3__ +#undef __DISABLE_SSSE3__ +#pragma GCC pop_options +#endif /* __DISABLE_SSSE3__ */ + +#endif /* _TMMINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/unwind.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/unwind.h new file mode 100644 index 0000000000..77dd5a9020 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/unwind.h @@ -0,0 +1,291 @@ +/* Exception handling and frame unwind runtime interface routines. + Copyright (C) 2001-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public + License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +/* This is derived from the C++ ABI for IA-64. Where we diverge + for cross-architecture compatibility are noted with "@@@". */ + +#ifndef _UNWIND_H +#define _UNWIND_H + +#if defined (__SEH__) && !defined (__USING_SJLJ_EXCEPTIONS__) +/* Only for _GCC_specific_handler. */ +#include +#endif + +#ifndef HIDE_EXPORTS +#pragma GCC visibility push(default) +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* Level 1: Base ABI */ + +/* @@@ The IA-64 ABI uses uint64 throughout. Most places this is + inefficient for 32-bit and smaller machines. */ +typedef unsigned _Unwind_Word __attribute__((__mode__(__unwind_word__))); +typedef signed _Unwind_Sword __attribute__((__mode__(__unwind_word__))); +#if defined(__ia64__) && defined(__hpux__) +typedef unsigned _Unwind_Ptr __attribute__((__mode__(__word__))); +#else +typedef unsigned _Unwind_Ptr __attribute__((__mode__(__pointer__))); +#endif +typedef unsigned _Unwind_Internal_Ptr __attribute__((__mode__(__pointer__))); + +/* @@@ The IA-64 ABI uses a 64-bit word to identify the producer and + consumer of an exception. We'll go along with this for now even on + 32-bit machines. We'll need to provide some other option for + 16-bit machines and for machines with > 8 bits per byte. */ +typedef unsigned _Unwind_Exception_Class __attribute__((__mode__(__DI__))); + +/* The unwind interface uses reason codes in several contexts to + identify the reasons for failures or other actions. */ +typedef enum +{ + _URC_NO_REASON = 0, + _URC_FOREIGN_EXCEPTION_CAUGHT = 1, + _URC_FATAL_PHASE2_ERROR = 2, + _URC_FATAL_PHASE1_ERROR = 3, + _URC_NORMAL_STOP = 4, + _URC_END_OF_STACK = 5, + _URC_HANDLER_FOUND = 6, + _URC_INSTALL_CONTEXT = 7, + _URC_CONTINUE_UNWIND = 8 +} _Unwind_Reason_Code; + + +/* The unwind interface uses a pointer to an exception header object + as its representation of an exception being thrown. In general, the + full representation of an exception object is language- and + implementation-specific, but it will be prefixed by a header + understood by the unwind interface. */ + +struct _Unwind_Exception; + +typedef void (*_Unwind_Exception_Cleanup_Fn) (_Unwind_Reason_Code, + struct _Unwind_Exception *); + +struct _Unwind_Exception +{ + _Unwind_Exception_Class exception_class; + _Unwind_Exception_Cleanup_Fn exception_cleanup; + +#if !defined (__USING_SJLJ_EXCEPTIONS__) && defined (__SEH__) + _Unwind_Word private_[6]; +#else + _Unwind_Word private_1; + _Unwind_Word private_2; +#endif + + /* @@@ The IA-64 ABI says that this structure must be double-word aligned. + Taking that literally does not make much sense generically. Instead we + provide the maximum alignment required by any type for the machine. */ +} __attribute__((__aligned__)); + + +/* The ACTIONS argument to the personality routine is a bitwise OR of one + or more of the following constants. */ +typedef int _Unwind_Action; + +#define _UA_SEARCH_PHASE 1 +#define _UA_CLEANUP_PHASE 2 +#define _UA_HANDLER_FRAME 4 +#define _UA_FORCE_UNWIND 8 +#define _UA_END_OF_STACK 16 + +/* The target can override this macro to define any back-end-specific + attributes required for the lowest-level stack frame. */ +#ifndef LIBGCC2_UNWIND_ATTRIBUTE +#define LIBGCC2_UNWIND_ATTRIBUTE +#endif + +/* This is an opaque type used to refer to a system-specific data + structure used by the system unwinder. This context is created and + destroyed by the system, and passed to the personality routine + during unwinding. */ +struct _Unwind_Context; + +/* Raise an exception, passing along the given exception object. */ +extern _Unwind_Reason_Code LIBGCC2_UNWIND_ATTRIBUTE +_Unwind_RaiseException (struct _Unwind_Exception *); + +/* Raise an exception for forced unwinding. */ + +typedef _Unwind_Reason_Code (*_Unwind_Stop_Fn) + (int, _Unwind_Action, _Unwind_Exception_Class, + struct _Unwind_Exception *, struct _Unwind_Context *, void *); + +extern _Unwind_Reason_Code LIBGCC2_UNWIND_ATTRIBUTE +_Unwind_ForcedUnwind (struct _Unwind_Exception *, _Unwind_Stop_Fn, void *); + +/* Helper to invoke the exception_cleanup routine. */ +extern void _Unwind_DeleteException (struct _Unwind_Exception *); + +/* Resume propagation of an existing exception. This is used after + e.g. executing cleanup code, and not to implement rethrowing. */ +extern void LIBGCC2_UNWIND_ATTRIBUTE +_Unwind_Resume (struct _Unwind_Exception *); + +/* @@@ Resume propagation of a FORCE_UNWIND exception, or to rethrow + a normal exception that was handled. */ +extern _Unwind_Reason_Code LIBGCC2_UNWIND_ATTRIBUTE +_Unwind_Resume_or_Rethrow (struct _Unwind_Exception *); + +/* @@@ Use unwind data to perform a stack backtrace. The trace callback + is called for every stack frame in the call chain, but no cleanup + actions are performed. */ +typedef _Unwind_Reason_Code (*_Unwind_Trace_Fn) + (struct _Unwind_Context *, void *); + +extern _Unwind_Reason_Code LIBGCC2_UNWIND_ATTRIBUTE +_Unwind_Backtrace (_Unwind_Trace_Fn, void *); + +/* These functions are used for communicating information about the unwind + context (i.e. the unwind descriptors and the user register state) between + the unwind library and the personality routine and landing pad. Only + selected registers may be manipulated. */ + +extern _Unwind_Word _Unwind_GetGR (struct _Unwind_Context *, int); +extern void _Unwind_SetGR (struct _Unwind_Context *, int, _Unwind_Word); + +extern _Unwind_Ptr _Unwind_GetIP (struct _Unwind_Context *); +extern _Unwind_Ptr _Unwind_GetIPInfo (struct _Unwind_Context *, int *); +extern void _Unwind_SetIP (struct _Unwind_Context *, _Unwind_Ptr); + +/* @@@ Retrieve the CFA of the given context. */ +extern _Unwind_Word _Unwind_GetCFA (struct _Unwind_Context *); + +extern void *_Unwind_GetLanguageSpecificData (struct _Unwind_Context *); + +extern _Unwind_Ptr _Unwind_GetRegionStart (struct _Unwind_Context *); + + +/* The personality routine is the function in the C++ (or other language) + runtime library which serves as an interface between the system unwind + library and language-specific exception handling semantics. It is + specific to the code fragment described by an unwind info block, and + it is always referenced via the pointer in the unwind info block, and + hence it has no ABI-specified name. + + Note that this implies that two different C++ implementations can + use different names, and have different contents in the language + specific data area. Moreover, that the language specific data + area contains no version info because name of the function invoked + provides more effective versioning by detecting at link time the + lack of code to handle the different data format. */ + +typedef _Unwind_Reason_Code (*_Unwind_Personality_Fn) + (int, _Unwind_Action, _Unwind_Exception_Class, + struct _Unwind_Exception *, struct _Unwind_Context *); + +/* @@@ The following alternate entry points are for setjmp/longjmp + based unwinding. */ + +struct SjLj_Function_Context; +extern void _Unwind_SjLj_Register (struct SjLj_Function_Context *); +extern void _Unwind_SjLj_Unregister (struct SjLj_Function_Context *); + +extern _Unwind_Reason_Code LIBGCC2_UNWIND_ATTRIBUTE +_Unwind_SjLj_RaiseException (struct _Unwind_Exception *); +extern _Unwind_Reason_Code LIBGCC2_UNWIND_ATTRIBUTE +_Unwind_SjLj_ForcedUnwind (struct _Unwind_Exception *, _Unwind_Stop_Fn, void *); +extern void LIBGCC2_UNWIND_ATTRIBUTE +_Unwind_SjLj_Resume (struct _Unwind_Exception *); +extern _Unwind_Reason_Code LIBGCC2_UNWIND_ATTRIBUTE +_Unwind_SjLj_Resume_or_Rethrow (struct _Unwind_Exception *); + +/* @@@ The following provide access to the base addresses for text + and data-relative addressing in the LDSA. In order to stay link + compatible with the standard ABI for IA-64, we inline these. */ + +#ifdef __ia64__ +static inline _Unwind_Ptr +_Unwind_GetDataRelBase (struct _Unwind_Context *_C) +{ + /* The GP is stored in R1. */ + return _Unwind_GetGR (_C, 1); +} + +static inline _Unwind_Ptr +_Unwind_GetTextRelBase (struct _Unwind_Context *_C __attribute__ ((__unused__))) +{ + __builtin_abort (); + return 0; +} + +/* @@@ Retrieve the Backing Store Pointer of the given context. */ +extern _Unwind_Word _Unwind_GetBSP (struct _Unwind_Context *); +#else +extern _Unwind_Ptr _Unwind_GetDataRelBase (struct _Unwind_Context *); +extern _Unwind_Ptr _Unwind_GetTextRelBase (struct _Unwind_Context *); +#endif + +/* @@@ Given an address, return the entry point of the function that + contains it. */ +extern void * _Unwind_FindEnclosingFunction (void *pc); + +#ifndef __SIZEOF_LONG__ + #error "__SIZEOF_LONG__ macro not defined" +#endif + +#ifndef __SIZEOF_POINTER__ + #error "__SIZEOF_POINTER__ macro not defined" +#endif + + +/* leb128 type numbers have a potentially unlimited size. + The target of the following definitions of _sleb128_t and _uleb128_t + is to have efficient data types large enough to hold the leb128 type + numbers used in the unwind code. + Mostly these types will simply be defined to long and unsigned long + except when a unsigned long data type on the target machine is not + capable of storing a pointer. */ + +#if __SIZEOF_LONG__ >= __SIZEOF_POINTER__ + typedef long _sleb128_t; + typedef unsigned long _uleb128_t; +#elif __SIZEOF_LONG_LONG__ >= __SIZEOF_POINTER__ + typedef long long _sleb128_t; + typedef unsigned long long _uleb128_t; +#else +# error "What type shall we use for _sleb128_t?" +#endif + +#if defined (__SEH__) && !defined (__USING_SJLJ_EXCEPTIONS__) +/* Handles the mapping from SEH to GCC interfaces. */ +EXCEPTION_DISPOSITION _GCC_specific_handler (PEXCEPTION_RECORD, void *, + PCONTEXT, PDISPATCHER_CONTEXT, + _Unwind_Personality_Fn); +#endif + +#ifdef __cplusplus +} +#endif + +#ifndef HIDE_EXPORTS +#pragma GCC visibility pop +#endif + +#endif /* unwind.h */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/varargs.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/varargs.h new file mode 100644 index 0000000000..4b9803e71a --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/varargs.h @@ -0,0 +1,7 @@ +#ifndef _VARARGS_H +#define _VARARGS_H + +#error "GCC no longer implements ." +#error "Revise your code to use ." + +#endif diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/wmmintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/wmmintrin.h new file mode 100644 index 0000000000..53de853a56 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/wmmintrin.h @@ -0,0 +1,132 @@ +/* Copyright (C) 2008-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +/* Implemented from the specification included in the Intel C++ Compiler + User Guide and Reference, version 10.1. */ + +#ifndef _WMMINTRIN_H_INCLUDED +#define _WMMINTRIN_H_INCLUDED + +/* We need definitions from the SSE2 header file. */ +#include + +/* AES */ + +#ifndef __AES__ +#pragma GCC push_options +#pragma GCC target("aes") +#define __DISABLE_AES__ +#endif /* __AES__ */ + +/* Performs 1 round of AES decryption of the first m128i using + the second m128i as a round key. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_aesdec_si128 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_aesdec128 ((__v2di)__X, (__v2di)__Y); +} + +/* Performs the last round of AES decryption of the first m128i + using the second m128i as a round key. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_aesdeclast_si128 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_aesdeclast128 ((__v2di)__X, + (__v2di)__Y); +} + +/* Performs 1 round of AES encryption of the first m128i using + the second m128i as a round key. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_aesenc_si128 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_aesenc128 ((__v2di)__X, (__v2di)__Y); +} + +/* Performs the last round of AES encryption of the first m128i + using the second m128i as a round key. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_aesenclast_si128 (__m128i __X, __m128i __Y) +{ + return (__m128i) __builtin_ia32_aesenclast128 ((__v2di)__X, (__v2di)__Y); +} + +/* Performs the InverseMixColumn operation on the source m128i + and stores the result into m128i destination. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_aesimc_si128 (__m128i __X) +{ + return (__m128i) __builtin_ia32_aesimc128 ((__v2di)__X); +} + +/* Generates a m128i round key for the input m128i AES cipher key and + byte round constant. The second parameter must be a compile time + constant. */ +#ifdef __OPTIMIZE__ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_aeskeygenassist_si128 (__m128i __X, const int __C) +{ + return (__m128i) __builtin_ia32_aeskeygenassist128 ((__v2di)__X, __C); +} +#else +#define _mm_aeskeygenassist_si128(X, C) \ + ((__m128i) __builtin_ia32_aeskeygenassist128 ((__v2di)(__m128i)(X), \ + (int)(C))) +#endif + +#ifdef __DISABLE_AES__ +#undef __DISABLE_AES__ +#pragma GCC pop_options +#endif /* __DISABLE_AES__ */ + +/* PCLMUL */ + +#ifndef __PCLMUL__ +#pragma GCC push_options +#pragma GCC target("pclmul") +#define __DISABLE_PCLMUL__ +#endif /* __PCLMUL__ */ + +/* Performs carry-less integer multiplication of 64-bit halves of + 128-bit input operands. The third parameter inducates which 64-bit + haves of the input parameters v1 and v2 should be used. It must be + a compile time constant. */ +#ifdef __OPTIMIZE__ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_clmulepi64_si128 (__m128i __X, __m128i __Y, const int __I) +{ + return (__m128i) __builtin_ia32_pclmulqdq128 ((__v2di)__X, + (__v2di)__Y, __I); +} +#else +#define _mm_clmulepi64_si128(X, Y, I) \ + ((__m128i) __builtin_ia32_pclmulqdq128 ((__v2di)(__m128i)(X), \ + (__v2di)(__m128i)(Y), (int)(I))) +#endif + +#ifdef __DISABLE_PCLMUL__ +#undef __DISABLE_PCLMUL__ +#pragma GCC pop_options +#endif /* __DISABLE_PCLMUL__ */ + +#endif /* _WMMINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/x86intrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/x86intrin.h new file mode 100644 index 0000000000..780edbbe54 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/x86intrin.h @@ -0,0 +1,102 @@ +/* Copyright (C) 2008-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _X86INTRIN_H_INCLUDED +#define _X86INTRIN_H_INCLUDED + +#include + +#ifndef __iamcu__ + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +/* For including AVX instructions */ +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#endif /* __iamcu__ */ + +#include + +#ifndef __iamcu__ + +#include + +#include + +#include + +#include + +#include + +#include + +#include + +#endif /* __iamcu__ */ + +#endif /* _X86INTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/xmmintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/xmmintrin.h new file mode 100644 index 0000000000..5f58d24430 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/xmmintrin.h @@ -0,0 +1,1270 @@ +/* Copyright (C) 2002-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +/* Implemented from the specification included in the Intel C++ Compiler + User Guide and Reference, version 9.0. */ + +#ifndef _XMMINTRIN_H_INCLUDED +#define _XMMINTRIN_H_INCLUDED + +/* We need type definitions from the MMX header file. */ +#include + +/* Get _mm_malloc () and _mm_free (). */ +#include + +/* Constants for use with _mm_prefetch. */ +enum _mm_hint +{ + /* _MM_HINT_ET is _MM_HINT_T with set 3rd bit. */ + _MM_HINT_ET0 = 7, + _MM_HINT_ET1 = 6, + _MM_HINT_T0 = 3, + _MM_HINT_T1 = 2, + _MM_HINT_T2 = 1, + _MM_HINT_NTA = 0 +}; + +/* Loads one cache line from address P to a location "closer" to the + processor. The selector I specifies the type of prefetch operation. */ +#ifdef __OPTIMIZE__ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_prefetch (const void *__P, enum _mm_hint __I) +{ + __builtin_prefetch (__P, (__I & 0x4) >> 2, __I & 0x3); +} +#else +#define _mm_prefetch(P, I) \ + __builtin_prefetch ((P), ((I & 0x4) >> 2), (I & 0x3)) +#endif + +#ifndef __SSE__ +#pragma GCC push_options +#pragma GCC target("sse") +#define __DISABLE_SSE__ +#endif /* __SSE__ */ + +/* The Intel API is flexible enough that we must allow aliasing with other + vector types, and their scalar components. */ +typedef float __m128 __attribute__ ((__vector_size__ (16), __may_alias__)); + +/* Unaligned version of the same type. */ +typedef float __m128_u __attribute__ ((__vector_size__ (16), __may_alias__, __aligned__ (1))); + +/* Internal data types for implementing the intrinsics. */ +typedef float __v4sf __attribute__ ((__vector_size__ (16))); + +/* Create a selector for use with the SHUFPS instruction. */ +#define _MM_SHUFFLE(fp3,fp2,fp1,fp0) \ + (((fp3) << 6) | ((fp2) << 4) | ((fp1) << 2) | (fp0)) + +/* Bits in the MXCSR. */ +#define _MM_EXCEPT_MASK 0x003f +#define _MM_EXCEPT_INVALID 0x0001 +#define _MM_EXCEPT_DENORM 0x0002 +#define _MM_EXCEPT_DIV_ZERO 0x0004 +#define _MM_EXCEPT_OVERFLOW 0x0008 +#define _MM_EXCEPT_UNDERFLOW 0x0010 +#define _MM_EXCEPT_INEXACT 0x0020 + +#define _MM_MASK_MASK 0x1f80 +#define _MM_MASK_INVALID 0x0080 +#define _MM_MASK_DENORM 0x0100 +#define _MM_MASK_DIV_ZERO 0x0200 +#define _MM_MASK_OVERFLOW 0x0400 +#define _MM_MASK_UNDERFLOW 0x0800 +#define _MM_MASK_INEXACT 0x1000 + +#define _MM_ROUND_MASK 0x6000 +#define _MM_ROUND_NEAREST 0x0000 +#define _MM_ROUND_DOWN 0x2000 +#define _MM_ROUND_UP 0x4000 +#define _MM_ROUND_TOWARD_ZERO 0x6000 + +#define _MM_FLUSH_ZERO_MASK 0x8000 +#define _MM_FLUSH_ZERO_ON 0x8000 +#define _MM_FLUSH_ZERO_OFF 0x0000 + +/* Create an undefined vector. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_undefined_ps (void) +{ + __m128 __Y = __Y; + return __Y; +} + +/* Create a vector of zeros. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_setzero_ps (void) +{ + return __extension__ (__m128){ 0.0f, 0.0f, 0.0f, 0.0f }; +} + +/* Perform the respective operation on the lower SPFP (single-precision + floating-point) values of A and B; the upper three SPFP values are + passed through from A. */ + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_add_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_addss ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sub_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_subss ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mul_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_mulss ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_div_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_divss ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sqrt_ss (__m128 __A) +{ + return (__m128) __builtin_ia32_sqrtss ((__v4sf)__A); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rcp_ss (__m128 __A) +{ + return (__m128) __builtin_ia32_rcpss ((__v4sf)__A); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rsqrt_ss (__m128 __A) +{ + return (__m128) __builtin_ia32_rsqrtss ((__v4sf)__A); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_min_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_minss ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_max_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_maxss ((__v4sf)__A, (__v4sf)__B); +} + +/* Perform the respective operation on the four SPFP values in A and B. */ + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_add_ps (__m128 __A, __m128 __B) +{ + return (__m128) ((__v4sf)__A + (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sub_ps (__m128 __A, __m128 __B) +{ + return (__m128) ((__v4sf)__A - (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mul_ps (__m128 __A, __m128 __B) +{ + return (__m128) ((__v4sf)__A * (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_div_ps (__m128 __A, __m128 __B) +{ + return (__m128) ((__v4sf)__A / (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sqrt_ps (__m128 __A) +{ + return (__m128) __builtin_ia32_sqrtps ((__v4sf)__A); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rcp_ps (__m128 __A) +{ + return (__m128) __builtin_ia32_rcpps ((__v4sf)__A); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rsqrt_ps (__m128 __A) +{ + return (__m128) __builtin_ia32_rsqrtps ((__v4sf)__A); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_min_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_minps ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_max_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_maxps ((__v4sf)__A, (__v4sf)__B); +} + +/* Perform logical bit-wise operations on 128-bit values. */ + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_and_ps (__m128 __A, __m128 __B) +{ + return __builtin_ia32_andps (__A, __B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_andnot_ps (__m128 __A, __m128 __B) +{ + return __builtin_ia32_andnps (__A, __B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_or_ps (__m128 __A, __m128 __B) +{ + return __builtin_ia32_orps (__A, __B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_xor_ps (__m128 __A, __m128 __B) +{ + return __builtin_ia32_xorps (__A, __B); +} + +/* Perform a comparison on the lower SPFP values of A and B. If the + comparison is true, place a mask of all ones in the result, otherwise a + mask of zeros. The upper three SPFP values are passed through from A. */ + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpeqss ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmplt_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpltss ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmple_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpless ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_movss ((__v4sf) __A, + (__v4sf) + __builtin_ia32_cmpltss ((__v4sf) __B, + (__v4sf) + __A)); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpge_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_movss ((__v4sf) __A, + (__v4sf) + __builtin_ia32_cmpless ((__v4sf) __B, + (__v4sf) + __A)); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpneq_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpneqss ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpnlt_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpnltss ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpnle_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpnless ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpngt_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_movss ((__v4sf) __A, + (__v4sf) + __builtin_ia32_cmpnltss ((__v4sf) __B, + (__v4sf) + __A)); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpnge_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_movss ((__v4sf) __A, + (__v4sf) + __builtin_ia32_cmpnless ((__v4sf) __B, + (__v4sf) + __A)); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpord_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpordss ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpunord_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpunordss ((__v4sf)__A, (__v4sf)__B); +} + +/* Perform a comparison on the four SPFP values of A and B. For each + element, if the comparison is true, place a mask of all ones in the + result, otherwise a mask of zeros. */ + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpeq_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpeqps ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmplt_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpltps ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmple_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpleps ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpgt_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpgtps ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpge_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpgeps ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpneq_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpneqps ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpnlt_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpnltps ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpnle_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpnleps ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpngt_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpngtps ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpnge_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpngeps ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpord_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpordps ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmpunord_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_cmpunordps ((__v4sf)__A, (__v4sf)__B); +} + +/* Compare the lower SPFP values of A and B and return 1 if true + and 0 if false. */ + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comieq_ss (__m128 __A, __m128 __B) +{ + return __builtin_ia32_comieq ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comilt_ss (__m128 __A, __m128 __B) +{ + return __builtin_ia32_comilt ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comile_ss (__m128 __A, __m128 __B) +{ + return __builtin_ia32_comile ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comigt_ss (__m128 __A, __m128 __B) +{ + return __builtin_ia32_comigt ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comige_ss (__m128 __A, __m128 __B) +{ + return __builtin_ia32_comige ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comineq_ss (__m128 __A, __m128 __B) +{ + return __builtin_ia32_comineq ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_ucomieq_ss (__m128 __A, __m128 __B) +{ + return __builtin_ia32_ucomieq ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_ucomilt_ss (__m128 __A, __m128 __B) +{ + return __builtin_ia32_ucomilt ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_ucomile_ss (__m128 __A, __m128 __B) +{ + return __builtin_ia32_ucomile ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_ucomigt_ss (__m128 __A, __m128 __B) +{ + return __builtin_ia32_ucomigt ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_ucomige_ss (__m128 __A, __m128 __B) +{ + return __builtin_ia32_ucomige ((__v4sf)__A, (__v4sf)__B); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_ucomineq_ss (__m128 __A, __m128 __B) +{ + return __builtin_ia32_ucomineq ((__v4sf)__A, (__v4sf)__B); +} + +/* Convert the lower SPFP value to a 32-bit integer according to the current + rounding mode. */ +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtss_si32 (__m128 __A) +{ + return __builtin_ia32_cvtss2si ((__v4sf) __A); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_ss2si (__m128 __A) +{ + return _mm_cvtss_si32 (__A); +} + +#ifdef __x86_64__ +/* Convert the lower SPFP value to a 32-bit integer according to the + current rounding mode. */ + +/* Intel intrinsic. */ +extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtss_si64 (__m128 __A) +{ + return __builtin_ia32_cvtss2si64 ((__v4sf) __A); +} + +/* Microsoft intrinsic. */ +extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtss_si64x (__m128 __A) +{ + return __builtin_ia32_cvtss2si64 ((__v4sf) __A); +} +#endif + +/* Convert the two lower SPFP values to 32-bit integers according to the + current rounding mode. Return the integers in packed form. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtps_pi32 (__m128 __A) +{ + return (__m64) __builtin_ia32_cvtps2pi ((__v4sf) __A); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_ps2pi (__m128 __A) +{ + return _mm_cvtps_pi32 (__A); +} + +/* Truncate the lower SPFP value to a 32-bit integer. */ +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttss_si32 (__m128 __A) +{ + return __builtin_ia32_cvttss2si ((__v4sf) __A); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtt_ss2si (__m128 __A) +{ + return _mm_cvttss_si32 (__A); +} + +#ifdef __x86_64__ +/* Truncate the lower SPFP value to a 32-bit integer. */ + +/* Intel intrinsic. */ +extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttss_si64 (__m128 __A) +{ + return __builtin_ia32_cvttss2si64 ((__v4sf) __A); +} + +/* Microsoft intrinsic. */ +extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttss_si64x (__m128 __A) +{ + return __builtin_ia32_cvttss2si64 ((__v4sf) __A); +} +#endif + +/* Truncate the two lower SPFP values to 32-bit integers. Return the + integers in packed form. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvttps_pi32 (__m128 __A) +{ + return (__m64) __builtin_ia32_cvttps2pi ((__v4sf) __A); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtt_ps2pi (__m128 __A) +{ + return _mm_cvttps_pi32 (__A); +} + +/* Convert B to a SPFP value and insert it as element zero in A. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi32_ss (__m128 __A, int __B) +{ + return (__m128) __builtin_ia32_cvtsi2ss ((__v4sf) __A, __B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_si2ss (__m128 __A, int __B) +{ + return _mm_cvtsi32_ss (__A, __B); +} + +#ifdef __x86_64__ +/* Convert B to a SPFP value and insert it as element zero in A. */ + +/* Intel intrinsic. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi64_ss (__m128 __A, long long __B) +{ + return (__m128) __builtin_ia32_cvtsi642ss ((__v4sf) __A, __B); +} + +/* Microsoft intrinsic. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtsi64x_ss (__m128 __A, long long __B) +{ + return (__m128) __builtin_ia32_cvtsi642ss ((__v4sf) __A, __B); +} +#endif + +/* Convert the two 32-bit values in B to SPFP form and insert them + as the two lower elements in A. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtpi32_ps (__m128 __A, __m64 __B) +{ + return (__m128) __builtin_ia32_cvtpi2ps ((__v4sf) __A, (__v2si)__B); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvt_pi2ps (__m128 __A, __m64 __B) +{ + return _mm_cvtpi32_ps (__A, __B); +} + +/* Convert the four signed 16-bit values in A to SPFP form. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtpi16_ps (__m64 __A) +{ + __v4hi __sign; + __v2si __hisi, __losi; + __v4sf __zero, __ra, __rb; + + /* This comparison against zero gives us a mask that can be used to + fill in the missing sign bits in the unpack operations below, so + that we get signed values after unpacking. */ + __sign = __builtin_ia32_pcmpgtw ((__v4hi)0LL, (__v4hi)__A); + + /* Convert the four words to doublewords. */ + __losi = (__v2si) __builtin_ia32_punpcklwd ((__v4hi)__A, __sign); + __hisi = (__v2si) __builtin_ia32_punpckhwd ((__v4hi)__A, __sign); + + /* Convert the doublewords to floating point two at a time. */ + __zero = (__v4sf) _mm_setzero_ps (); + __ra = __builtin_ia32_cvtpi2ps (__zero, __losi); + __rb = __builtin_ia32_cvtpi2ps (__ra, __hisi); + + return (__m128) __builtin_ia32_movlhps (__ra, __rb); +} + +/* Convert the four unsigned 16-bit values in A to SPFP form. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtpu16_ps (__m64 __A) +{ + __v2si __hisi, __losi; + __v4sf __zero, __ra, __rb; + + /* Convert the four words to doublewords. */ + __losi = (__v2si) __builtin_ia32_punpcklwd ((__v4hi)__A, (__v4hi)0LL); + __hisi = (__v2si) __builtin_ia32_punpckhwd ((__v4hi)__A, (__v4hi)0LL); + + /* Convert the doublewords to floating point two at a time. */ + __zero = (__v4sf) _mm_setzero_ps (); + __ra = __builtin_ia32_cvtpi2ps (__zero, __losi); + __rb = __builtin_ia32_cvtpi2ps (__ra, __hisi); + + return (__m128) __builtin_ia32_movlhps (__ra, __rb); +} + +/* Convert the low four signed 8-bit values in A to SPFP form. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtpi8_ps (__m64 __A) +{ + __v8qi __sign; + + /* This comparison against zero gives us a mask that can be used to + fill in the missing sign bits in the unpack operations below, so + that we get signed values after unpacking. */ + __sign = __builtin_ia32_pcmpgtb ((__v8qi)0LL, (__v8qi)__A); + + /* Convert the four low bytes to words. */ + __A = (__m64) __builtin_ia32_punpcklbw ((__v8qi)__A, __sign); + + return _mm_cvtpi16_ps(__A); +} + +/* Convert the low four unsigned 8-bit values in A to SPFP form. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtpu8_ps(__m64 __A) +{ + __A = (__m64) __builtin_ia32_punpcklbw ((__v8qi)__A, (__v8qi)0LL); + return _mm_cvtpu16_ps(__A); +} + +/* Convert the four signed 32-bit values in A and B to SPFP form. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtpi32x2_ps(__m64 __A, __m64 __B) +{ + __v4sf __zero = (__v4sf) _mm_setzero_ps (); + __v4sf __sfa = __builtin_ia32_cvtpi2ps (__zero, (__v2si)__A); + __v4sf __sfb = __builtin_ia32_cvtpi2ps (__sfa, (__v2si)__B); + return (__m128) __builtin_ia32_movlhps (__sfa, __sfb); +} + +/* Convert the four SPFP values in A to four signed 16-bit integers. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtps_pi16(__m128 __A) +{ + __v4sf __hisf = (__v4sf)__A; + __v4sf __losf = __builtin_ia32_movhlps (__hisf, __hisf); + __v2si __hisi = __builtin_ia32_cvtps2pi (__hisf); + __v2si __losi = __builtin_ia32_cvtps2pi (__losf); + return (__m64) __builtin_ia32_packssdw (__hisi, __losi); +} + +/* Convert the four SPFP values in A to four signed 8-bit integers. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtps_pi8(__m128 __A) +{ + __v4hi __tmp = (__v4hi) _mm_cvtps_pi16 (__A); + return (__m64) __builtin_ia32_packsswb (__tmp, (__v4hi)0LL); +} + +/* Selects four specific SPFP values from A and B based on MASK. */ +#ifdef __OPTIMIZE__ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_shuffle_ps (__m128 __A, __m128 __B, int const __mask) +{ + return (__m128) __builtin_ia32_shufps ((__v4sf)__A, (__v4sf)__B, __mask); +} +#else +#define _mm_shuffle_ps(A, B, MASK) \ + ((__m128) __builtin_ia32_shufps ((__v4sf)(__m128)(A), \ + (__v4sf)(__m128)(B), (int)(MASK))) +#endif + +/* Selects and interleaves the upper two SPFP values from A and B. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpackhi_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_unpckhps ((__v4sf)__A, (__v4sf)__B); +} + +/* Selects and interleaves the lower two SPFP values from A and B. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_unpacklo_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_unpcklps ((__v4sf)__A, (__v4sf)__B); +} + +/* Sets the upper two SPFP values with 64-bits of data loaded from P; + the lower two values are passed through from A. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_loadh_pi (__m128 __A, __m64 const *__P) +{ + return (__m128) __builtin_ia32_loadhps ((__v4sf)__A, (const __v2sf *)__P); +} + +/* Stores the upper two SPFP values of A into P. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_storeh_pi (__m64 *__P, __m128 __A) +{ + __builtin_ia32_storehps ((__v2sf *)__P, (__v4sf)__A); +} + +/* Moves the upper two values of B into the lower two values of A. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movehl_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_movhlps ((__v4sf)__A, (__v4sf)__B); +} + +/* Moves the lower two values of B into the upper two values of A. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movelh_ps (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_movlhps ((__v4sf)__A, (__v4sf)__B); +} + +/* Sets the lower two SPFP values with 64-bits of data loaded from P; + the upper two values are passed through from A. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_loadl_pi (__m128 __A, __m64 const *__P) +{ + return (__m128) __builtin_ia32_loadlps ((__v4sf)__A, (const __v2sf *)__P); +} + +/* Stores the lower two SPFP values of A into P. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_storel_pi (__m64 *__P, __m128 __A) +{ + __builtin_ia32_storelps ((__v2sf *)__P, (__v4sf)__A); +} + +/* Creates a 4-bit mask from the most significant bits of the SPFP values. */ +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movemask_ps (__m128 __A) +{ + return __builtin_ia32_movmskps ((__v4sf)__A); +} + +/* Return the contents of the control register. */ +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_getcsr (void) +{ + return __builtin_ia32_stmxcsr (); +} + +/* Read exception bits from the control register. */ +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_MM_GET_EXCEPTION_STATE (void) +{ + return _mm_getcsr() & _MM_EXCEPT_MASK; +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_MM_GET_EXCEPTION_MASK (void) +{ + return _mm_getcsr() & _MM_MASK_MASK; +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_MM_GET_ROUNDING_MODE (void) +{ + return _mm_getcsr() & _MM_ROUND_MASK; +} + +extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_MM_GET_FLUSH_ZERO_MODE (void) +{ + return _mm_getcsr() & _MM_FLUSH_ZERO_MASK; +} + +/* Set the control register to I. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_setcsr (unsigned int __I) +{ + __builtin_ia32_ldmxcsr (__I); +} + +/* Set exception bits in the control register. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_MM_SET_EXCEPTION_STATE(unsigned int __mask) +{ + _mm_setcsr((_mm_getcsr() & ~_MM_EXCEPT_MASK) | __mask); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_MM_SET_EXCEPTION_MASK (unsigned int __mask) +{ + _mm_setcsr((_mm_getcsr() & ~_MM_MASK_MASK) | __mask); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_MM_SET_ROUNDING_MODE (unsigned int __mode) +{ + _mm_setcsr((_mm_getcsr() & ~_MM_ROUND_MASK) | __mode); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_MM_SET_FLUSH_ZERO_MODE (unsigned int __mode) +{ + _mm_setcsr((_mm_getcsr() & ~_MM_FLUSH_ZERO_MASK) | __mode); +} + +/* Create a vector with element 0 as F and the rest zero. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set_ss (float __F) +{ + return __extension__ (__m128)(__v4sf){ __F, 0.0f, 0.0f, 0.0f }; +} + +/* Create a vector with all four elements equal to F. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set1_ps (float __F) +{ + return __extension__ (__m128)(__v4sf){ __F, __F, __F, __F }; +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set_ps1 (float __F) +{ + return _mm_set1_ps (__F); +} + +/* Create a vector with element 0 as *P and the rest zero. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_load_ss (float const *__P) +{ + return _mm_set_ss (*__P); +} + +/* Create a vector with all four elements equal to *P. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_load1_ps (float const *__P) +{ + return _mm_set1_ps (*__P); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_load_ps1 (float const *__P) +{ + return _mm_load1_ps (__P); +} + +/* Load four SPFP values from P. The address must be 16-byte aligned. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_load_ps (float const *__P) +{ + return *(__m128 *)__P; +} + +/* Load four SPFP values from P. The address need not be 16-byte aligned. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_loadu_ps (float const *__P) +{ + return *(__m128_u *)__P; +} + +/* Load four SPFP values in reverse order. The address must be aligned. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_loadr_ps (float const *__P) +{ + __v4sf __tmp = *(__v4sf *)__P; + return (__m128) __builtin_ia32_shufps (__tmp, __tmp, _MM_SHUFFLE (0,1,2,3)); +} + +/* Create the vector [Z Y X W]. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_set_ps (const float __Z, const float __Y, const float __X, const float __W) +{ + return __extension__ (__m128)(__v4sf){ __W, __X, __Y, __Z }; +} + +/* Create the vector [W X Y Z]. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_setr_ps (float __Z, float __Y, float __X, float __W) +{ + return __extension__ (__m128)(__v4sf){ __Z, __Y, __X, __W }; +} + +/* Stores the lower SPFP value. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_store_ss (float *__P, __m128 __A) +{ + *__P = ((__v4sf)__A)[0]; +} + +extern __inline float __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cvtss_f32 (__m128 __A) +{ + return ((__v4sf)__A)[0]; +} + +/* Store four SPFP values. The address must be 16-byte aligned. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_store_ps (float *__P, __m128 __A) +{ + *(__m128 *)__P = __A; +} + +/* Store four SPFP values. The address need not be 16-byte aligned. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_storeu_ps (float *__P, __m128 __A) +{ + *(__m128_u *)__P = __A; +} + +/* Store the lower SPFP value across four words. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_store1_ps (float *__P, __m128 __A) +{ + __v4sf __va = (__v4sf)__A; + __v4sf __tmp = __builtin_ia32_shufps (__va, __va, _MM_SHUFFLE (0,0,0,0)); + _mm_storeu_ps (__P, __tmp); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_store_ps1 (float *__P, __m128 __A) +{ + _mm_store1_ps (__P, __A); +} + +/* Store four SPFP values in reverse order. The address must be aligned. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_storer_ps (float *__P, __m128 __A) +{ + __v4sf __va = (__v4sf)__A; + __v4sf __tmp = __builtin_ia32_shufps (__va, __va, _MM_SHUFFLE (0,1,2,3)); + _mm_store_ps (__P, __tmp); +} + +/* Sets the low SPFP value of A from the low value of B. */ +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_move_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_movss ((__v4sf)__A, (__v4sf)__B); +} + +/* Extracts one of the four words of A. The selector N must be immediate. */ +#ifdef __OPTIMIZE__ +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_extract_pi16 (__m64 const __A, int const __N) +{ + return __builtin_ia32_vec_ext_v4hi ((__v4hi)__A, __N); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pextrw (__m64 const __A, int const __N) +{ + return _mm_extract_pi16 (__A, __N); +} +#else +#define _mm_extract_pi16(A, N) \ + ((int) __builtin_ia32_vec_ext_v4hi ((__v4hi)(__m64)(A), (int)(N))) + +#define _m_pextrw(A, N) _mm_extract_pi16(A, N) +#endif + +/* Inserts word D into one of four words of A. The selector N must be + immediate. */ +#ifdef __OPTIMIZE__ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_insert_pi16 (__m64 const __A, int const __D, int const __N) +{ + return (__m64) __builtin_ia32_vec_set_v4hi ((__v4hi)__A, __D, __N); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pinsrw (__m64 const __A, int const __D, int const __N) +{ + return _mm_insert_pi16 (__A, __D, __N); +} +#else +#define _mm_insert_pi16(A, D, N) \ + ((__m64) __builtin_ia32_vec_set_v4hi ((__v4hi)(__m64)(A), \ + (int)(D), (int)(N))) + +#define _m_pinsrw(A, D, N) _mm_insert_pi16(A, D, N) +#endif + +/* Compute the element-wise maximum of signed 16-bit values. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_max_pi16 (__m64 __A, __m64 __B) +{ + return (__m64) __builtin_ia32_pmaxsw ((__v4hi)__A, (__v4hi)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pmaxsw (__m64 __A, __m64 __B) +{ + return _mm_max_pi16 (__A, __B); +} + +/* Compute the element-wise maximum of unsigned 8-bit values. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_max_pu8 (__m64 __A, __m64 __B) +{ + return (__m64) __builtin_ia32_pmaxub ((__v8qi)__A, (__v8qi)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pmaxub (__m64 __A, __m64 __B) +{ + return _mm_max_pu8 (__A, __B); +} + +/* Compute the element-wise minimum of signed 16-bit values. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_min_pi16 (__m64 __A, __m64 __B) +{ + return (__m64) __builtin_ia32_pminsw ((__v4hi)__A, (__v4hi)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pminsw (__m64 __A, __m64 __B) +{ + return _mm_min_pi16 (__A, __B); +} + +/* Compute the element-wise minimum of unsigned 8-bit values. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_min_pu8 (__m64 __A, __m64 __B) +{ + return (__m64) __builtin_ia32_pminub ((__v8qi)__A, (__v8qi)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pminub (__m64 __A, __m64 __B) +{ + return _mm_min_pu8 (__A, __B); +} + +/* Create an 8-bit mask of the signs of 8-bit values. */ +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_movemask_pi8 (__m64 __A) +{ + return __builtin_ia32_pmovmskb ((__v8qi)__A); +} + +extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pmovmskb (__m64 __A) +{ + return _mm_movemask_pi8 (__A); +} + +/* Multiply four unsigned 16-bit values in A by four unsigned 16-bit values + in B and produce the high 16 bits of the 32-bit results. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_mulhi_pu16 (__m64 __A, __m64 __B) +{ + return (__m64) __builtin_ia32_pmulhuw ((__v4hi)__A, (__v4hi)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pmulhuw (__m64 __A, __m64 __B) +{ + return _mm_mulhi_pu16 (__A, __B); +} + +/* Return a combination of the four 16-bit values in A. The selector + must be an immediate. */ +#ifdef __OPTIMIZE__ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_shuffle_pi16 (__m64 __A, int const __N) +{ + return (__m64) __builtin_ia32_pshufw ((__v4hi)__A, __N); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pshufw (__m64 __A, int const __N) +{ + return _mm_shuffle_pi16 (__A, __N); +} +#else +#define _mm_shuffle_pi16(A, N) \ + ((__m64) __builtin_ia32_pshufw ((__v4hi)(__m64)(A), (int)(N))) + +#define _m_pshufw(A, N) _mm_shuffle_pi16 (A, N) +#endif + +/* Conditionally store byte elements of A into P. The high bit of each + byte in the selector N determines whether the corresponding byte from + A is stored. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maskmove_si64 (__m64 __A, __m64 __N, char *__P) +{ + __builtin_ia32_maskmovq ((__v8qi)__A, (__v8qi)__N, __P); +} + +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_maskmovq (__m64 __A, __m64 __N, char *__P) +{ + _mm_maskmove_si64 (__A, __N, __P); +} + +/* Compute the rounded averages of the unsigned 8-bit values in A and B. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_avg_pu8 (__m64 __A, __m64 __B) +{ + return (__m64) __builtin_ia32_pavgb ((__v8qi)__A, (__v8qi)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pavgb (__m64 __A, __m64 __B) +{ + return _mm_avg_pu8 (__A, __B); +} + +/* Compute the rounded averages of the unsigned 16-bit values in A and B. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_avg_pu16 (__m64 __A, __m64 __B) +{ + return (__m64) __builtin_ia32_pavgw ((__v4hi)__A, (__v4hi)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_pavgw (__m64 __A, __m64 __B) +{ + return _mm_avg_pu16 (__A, __B); +} + +/* Compute the sum of the absolute differences of the unsigned 8-bit + values in A and B. Return the value in the lower 16-bit word; the + upper words are cleared. */ +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sad_pu8 (__m64 __A, __m64 __B) +{ + return (__m64) __builtin_ia32_psadbw ((__v8qi)__A, (__v8qi)__B); +} + +extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_m_psadbw (__m64 __A, __m64 __B) +{ + return _mm_sad_pu8 (__A, __B); +} + +/* Stores the data in A to the address P without polluting the caches. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_stream_pi (__m64 *__P, __m64 __A) +{ + __builtin_ia32_movntq ((unsigned long long *)__P, (unsigned long long)__A); +} + +/* Likewise. The address must be 16-byte aligned. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_stream_ps (float *__P, __m128 __A) +{ + __builtin_ia32_movntps (__P, (__v4sf)__A); +} + +/* Guarantees that every preceding store is globally visible before + any subsequent store. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sfence (void) +{ + __builtin_ia32_sfence (); +} + +/* Transpose the 4x4 matrix composed of row[0-3]. */ +#define _MM_TRANSPOSE4_PS(row0, row1, row2, row3) \ +do { \ + __v4sf __r0 = (row0), __r1 = (row1), __r2 = (row2), __r3 = (row3); \ + __v4sf __t0 = __builtin_ia32_unpcklps (__r0, __r1); \ + __v4sf __t1 = __builtin_ia32_unpcklps (__r2, __r3); \ + __v4sf __t2 = __builtin_ia32_unpckhps (__r0, __r1); \ + __v4sf __t3 = __builtin_ia32_unpckhps (__r2, __r3); \ + (row0) = __builtin_ia32_movlhps (__t0, __t1); \ + (row1) = __builtin_ia32_movhlps (__t1, __t0); \ + (row2) = __builtin_ia32_movlhps (__t2, __t3); \ + (row3) = __builtin_ia32_movhlps (__t3, __t2); \ +} while (0) + +/* For backward source compatibility. */ +# include + +#ifdef __DISABLE_SSE__ +#undef __DISABLE_SSE__ +#pragma GCC pop_options +#endif /* __DISABLE_SSE__ */ + +/* The execution of the next instruction is delayed by an implementation + specific amount of time. The instruction does not modify the + architectural state. This is after the pop_options pragma because + it does not require SSE support in the processor--the encoding is a + nop on processors that do not support it. */ +extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_pause (void) +{ + __builtin_ia32_pause (); +} + +#endif /* _XMMINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/xopintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/xopintrin.h new file mode 100644 index 0000000000..3a83e24932 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/xopintrin.h @@ -0,0 +1,844 @@ +/* Copyright (C) 2007-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _X86INTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _XOPMMINTRIN_H_INCLUDED +#define _XOPMMINTRIN_H_INCLUDED + +#include + +#ifndef __XOP__ +#pragma GCC push_options +#pragma GCC target("xop") +#define __DISABLE_XOP__ +#endif /* __XOP__ */ + +/* Integer multiply/add instructions. */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maccs_epi16(__m128i __A, __m128i __B, __m128i __C) +{ + return (__m128i) __builtin_ia32_vpmacssww ((__v8hi)__A,(__v8hi)__B, (__v8hi)__C); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_macc_epi16(__m128i __A, __m128i __B, __m128i __C) +{ + return (__m128i) __builtin_ia32_vpmacsww ((__v8hi)__A, (__v8hi)__B, (__v8hi)__C); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maccsd_epi16(__m128i __A, __m128i __B, __m128i __C) +{ + return (__m128i) __builtin_ia32_vpmacsswd ((__v8hi)__A, (__v8hi)__B, (__v4si)__C); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maccd_epi16(__m128i __A, __m128i __B, __m128i __C) +{ + return (__m128i) __builtin_ia32_vpmacswd ((__v8hi)__A, (__v8hi)__B, (__v4si)__C); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maccs_epi32(__m128i __A, __m128i __B, __m128i __C) +{ + return (__m128i) __builtin_ia32_vpmacssdd ((__v4si)__A, (__v4si)__B, (__v4si)__C); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_macc_epi32(__m128i __A, __m128i __B, __m128i __C) +{ + return (__m128i) __builtin_ia32_vpmacsdd ((__v4si)__A, (__v4si)__B, (__v4si)__C); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maccslo_epi32(__m128i __A, __m128i __B, __m128i __C) +{ + return (__m128i) __builtin_ia32_vpmacssdql ((__v4si)__A, (__v4si)__B, (__v2di)__C); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_macclo_epi32(__m128i __A, __m128i __B, __m128i __C) +{ + return (__m128i) __builtin_ia32_vpmacsdql ((__v4si)__A, (__v4si)__B, (__v2di)__C); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maccshi_epi32(__m128i __A, __m128i __B, __m128i __C) +{ + return (__m128i) __builtin_ia32_vpmacssdqh ((__v4si)__A, (__v4si)__B, (__v2di)__C); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_macchi_epi32(__m128i __A, __m128i __B, __m128i __C) +{ + return (__m128i) __builtin_ia32_vpmacsdqh ((__v4si)__A, (__v4si)__B, (__v2di)__C); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maddsd_epi16(__m128i __A, __m128i __B, __m128i __C) +{ + return (__m128i) __builtin_ia32_vpmadcsswd ((__v8hi)__A,(__v8hi)__B,(__v4si)__C); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_maddd_epi16(__m128i __A, __m128i __B, __m128i __C) +{ + return (__m128i) __builtin_ia32_vpmadcswd ((__v8hi)__A,(__v8hi)__B,(__v4si)__C); +} + +/* Packed Integer Horizontal Add and Subtract */ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_haddw_epi8(__m128i __A) +{ + return (__m128i) __builtin_ia32_vphaddbw ((__v16qi)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_haddd_epi8(__m128i __A) +{ + return (__m128i) __builtin_ia32_vphaddbd ((__v16qi)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_haddq_epi8(__m128i __A) +{ + return (__m128i) __builtin_ia32_vphaddbq ((__v16qi)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_haddd_epi16(__m128i __A) +{ + return (__m128i) __builtin_ia32_vphaddwd ((__v8hi)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_haddq_epi16(__m128i __A) +{ + return (__m128i) __builtin_ia32_vphaddwq ((__v8hi)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_haddq_epi32(__m128i __A) +{ + return (__m128i) __builtin_ia32_vphadddq ((__v4si)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_haddw_epu8(__m128i __A) +{ + return (__m128i) __builtin_ia32_vphaddubw ((__v16qi)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_haddd_epu8(__m128i __A) +{ + return (__m128i) __builtin_ia32_vphaddubd ((__v16qi)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_haddq_epu8(__m128i __A) +{ + return (__m128i) __builtin_ia32_vphaddubq ((__v16qi)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_haddd_epu16(__m128i __A) +{ + return (__m128i) __builtin_ia32_vphadduwd ((__v8hi)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_haddq_epu16(__m128i __A) +{ + return (__m128i) __builtin_ia32_vphadduwq ((__v8hi)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_haddq_epu32(__m128i __A) +{ + return (__m128i) __builtin_ia32_vphaddudq ((__v4si)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hsubw_epi8(__m128i __A) +{ + return (__m128i) __builtin_ia32_vphsubbw ((__v16qi)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hsubd_epi16(__m128i __A) +{ + return (__m128i) __builtin_ia32_vphsubwd ((__v8hi)__A); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_hsubq_epi32(__m128i __A) +{ + return (__m128i) __builtin_ia32_vphsubdq ((__v4si)__A); +} + +/* Vector conditional move and permute */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_cmov_si128(__m128i __A, __m128i __B, __m128i __C) +{ + return (__m128i) __builtin_ia32_vpcmov (__A, __B, __C); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_perm_epi8(__m128i __A, __m128i __B, __m128i __C) +{ + return (__m128i) __builtin_ia32_vpperm ((__v16qi)__A, (__v16qi)__B, (__v16qi)__C); +} + +/* Packed Integer Rotates and Shifts + Rotates - Non-Immediate form */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rot_epi8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vprotb ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rot_epi16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vprotw ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rot_epi32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vprotd ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_rot_epi64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vprotq ((__v2di)__A, (__v2di)__B); +} + +/* Rotates - Immediate form */ + +#ifdef __OPTIMIZE__ +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_roti_epi8(__m128i __A, const int __B) +{ + return (__m128i) __builtin_ia32_vprotbi ((__v16qi)__A, __B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_roti_epi16(__m128i __A, const int __B) +{ + return (__m128i) __builtin_ia32_vprotwi ((__v8hi)__A, __B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_roti_epi32(__m128i __A, const int __B) +{ + return (__m128i) __builtin_ia32_vprotdi ((__v4si)__A, __B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_roti_epi64(__m128i __A, const int __B) +{ + return (__m128i) __builtin_ia32_vprotqi ((__v2di)__A, __B); +} +#else +#define _mm_roti_epi8(A, N) \ + ((__m128i) __builtin_ia32_vprotbi ((__v16qi)(__m128i)(A), (int)(N))) +#define _mm_roti_epi16(A, N) \ + ((__m128i) __builtin_ia32_vprotwi ((__v8hi)(__m128i)(A), (int)(N))) +#define _mm_roti_epi32(A, N) \ + ((__m128i) __builtin_ia32_vprotdi ((__v4si)(__m128i)(A), (int)(N))) +#define _mm_roti_epi64(A, N) \ + ((__m128i) __builtin_ia32_vprotqi ((__v2di)(__m128i)(A), (int)(N))) +#endif + +/* Shifts */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_shl_epi8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpshlb ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_shl_epi16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpshlw ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_shl_epi32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpshld ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_shl_epi64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpshlq ((__v2di)__A, (__v2di)__B); +} + + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sha_epi8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpshab ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sha_epi16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpshaw ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sha_epi32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpshad ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_sha_epi64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpshaq ((__v2di)__A, (__v2di)__B); +} + +/* Compare and Predicate Generation + pcom (integer, unsigned bytes) */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comlt_epu8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomltub ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comle_epu8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomleub ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comgt_epu8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomgtub ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comge_epu8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomgeub ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comeq_epu8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomequb ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comneq_epu8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomnequb ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comfalse_epu8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomfalseub ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comtrue_epu8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomtrueub ((__v16qi)__A, (__v16qi)__B); +} + +/*pcom (integer, unsigned words) */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comlt_epu16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomltuw ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comle_epu16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomleuw ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comgt_epu16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomgtuw ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comge_epu16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomgeuw ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comeq_epu16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomequw ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comneq_epu16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomnequw ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comfalse_epu16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomfalseuw ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comtrue_epu16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomtrueuw ((__v8hi)__A, (__v8hi)__B); +} + +/*pcom (integer, unsigned double words) */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comlt_epu32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomltud ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comle_epu32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomleud ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comgt_epu32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomgtud ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comge_epu32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomgeud ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comeq_epu32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomequd ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comneq_epu32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomnequd ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comfalse_epu32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomfalseud ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comtrue_epu32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomtrueud ((__v4si)__A, (__v4si)__B); +} + +/*pcom (integer, unsigned quad words) */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comlt_epu64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomltuq ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comle_epu64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomleuq ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comgt_epu64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomgtuq ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comge_epu64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomgeuq ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comeq_epu64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomequq ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comneq_epu64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomnequq ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comfalse_epu64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomfalseuq ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comtrue_epu64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomtrueuq ((__v2di)__A, (__v2di)__B); +} + +/*pcom (integer, signed bytes) */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comlt_epi8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomltb ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comle_epi8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomleb ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comgt_epi8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomgtb ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comge_epi8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomgeb ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comeq_epi8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomeqb ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comneq_epi8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomneqb ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comfalse_epi8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomfalseb ((__v16qi)__A, (__v16qi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comtrue_epi8(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomtrueb ((__v16qi)__A, (__v16qi)__B); +} + +/*pcom (integer, signed words) */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comlt_epi16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomltw ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comle_epi16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomlew ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comgt_epi16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomgtw ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comge_epi16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomgew ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comeq_epi16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomeqw ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comneq_epi16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomneqw ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comfalse_epi16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomfalsew ((__v8hi)__A, (__v8hi)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comtrue_epi16(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomtruew ((__v8hi)__A, (__v8hi)__B); +} + +/*pcom (integer, signed double words) */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comlt_epi32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomltd ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comle_epi32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomled ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comgt_epi32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomgtd ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comge_epi32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomged ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comeq_epi32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomeqd ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comneq_epi32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomneqd ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comfalse_epi32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomfalsed ((__v4si)__A, (__v4si)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comtrue_epi32(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomtrued ((__v4si)__A, (__v4si)__B); +} + +/*pcom (integer, signed quad words) */ + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comlt_epi64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomltq ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comle_epi64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomleq ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comgt_epi64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomgtq ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comge_epi64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomgeq ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comeq_epi64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomeqq ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comneq_epi64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomneqq ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comfalse_epi64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomfalseq ((__v2di)__A, (__v2di)__B); +} + +extern __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_comtrue_epi64(__m128i __A, __m128i __B) +{ + return (__m128i) __builtin_ia32_vpcomtrueq ((__v2di)__A, (__v2di)__B); +} + +/* FRCZ */ + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_frcz_ps (__m128 __A) +{ + return (__m128) __builtin_ia32_vfrczps ((__v4sf)__A); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_frcz_pd (__m128d __A) +{ + return (__m128d) __builtin_ia32_vfrczpd ((__v2df)__A); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_frcz_ss (__m128 __A, __m128 __B) +{ + return (__m128) __builtin_ia32_movss ((__v4sf)__A, + (__v4sf) + __builtin_ia32_vfrczss ((__v4sf)__B)); +} + +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_frcz_sd (__m128d __A, __m128d __B) +{ + return (__m128d) __builtin_ia32_movsd ((__v2df)__A, + (__v2df) + __builtin_ia32_vfrczsd ((__v2df)__B)); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_frcz_ps (__m256 __A) +{ + return (__m256) __builtin_ia32_vfrczps256 ((__v8sf)__A); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_frcz_pd (__m256d __A) +{ + return (__m256d) __builtin_ia32_vfrczpd256 ((__v4df)__A); +} + +/* PERMIL2 */ + +#ifdef __OPTIMIZE__ +extern __inline __m128d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permute2_pd (__m128d __X, __m128d __Y, __m128i __C, const int __I) +{ + return (__m128d) __builtin_ia32_vpermil2pd ((__v2df)__X, + (__v2df)__Y, + (__v2di)__C, + __I); +} + +extern __inline __m256d __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permute2_pd (__m256d __X, __m256d __Y, __m256i __C, const int __I) +{ + return (__m256d) __builtin_ia32_vpermil2pd256 ((__v4df)__X, + (__v4df)__Y, + (__v4di)__C, + __I); +} + +extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm_permute2_ps (__m128 __X, __m128 __Y, __m128i __C, const int __I) +{ + return (__m128) __builtin_ia32_vpermil2ps ((__v4sf)__X, + (__v4sf)__Y, + (__v4si)__C, + __I); +} + +extern __inline __m256 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_mm256_permute2_ps (__m256 __X, __m256 __Y, __m256i __C, const int __I) +{ + return (__m256) __builtin_ia32_vpermil2ps256 ((__v8sf)__X, + (__v8sf)__Y, + (__v8si)__C, + __I); +} +#else +#define _mm_permute2_pd(X, Y, C, I) \ + ((__m128d) __builtin_ia32_vpermil2pd ((__v2df)(__m128d)(X), \ + (__v2df)(__m128d)(Y), \ + (__v2di)(__m128d)(C), \ + (int)(I))) + +#define _mm256_permute2_pd(X, Y, C, I) \ + ((__m256d) __builtin_ia32_vpermil2pd256 ((__v4df)(__m256d)(X), \ + (__v4df)(__m256d)(Y), \ + (__v4di)(__m256d)(C), \ + (int)(I))) + +#define _mm_permute2_ps(X, Y, C, I) \ + ((__m128) __builtin_ia32_vpermil2ps ((__v4sf)(__m128)(X), \ + (__v4sf)(__m128)(Y), \ + (__v4si)(__m128)(C), \ + (int)(I))) + +#define _mm256_permute2_ps(X, Y, C, I) \ + ((__m256) __builtin_ia32_vpermil2ps256 ((__v8sf)(__m256)(X), \ + (__v8sf)(__m256)(Y), \ + (__v8si)(__m256)(C), \ + (int)(I))) +#endif /* __OPTIMIZE__ */ + +#ifdef __DISABLE_XOP__ +#undef __DISABLE_XOP__ +#pragma GCC pop_options +#endif /* __DISABLE_XOP__ */ + +#endif /* _XOPMMINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/xsavecintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/xsavecintrin.h new file mode 100644 index 0000000000..a510a7e8e6 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/xsavecintrin.h @@ -0,0 +1,58 @@ +/* Copyright (C) 2014-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#if !defined _X86INTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _XSAVECINTRIN_H_INCLUDED +#define _XSAVECINTRIN_H_INCLUDED + +#ifndef __XSAVEC__ +#pragma GCC push_options +#pragma GCC target("xsavec") +#define __DISABLE_XSAVEC__ +#endif /* __XSAVEC__ */ + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_xsavec (void *__P, long long __M) +{ + __builtin_ia32_xsavec (__P, __M); +} + +#ifdef __x86_64__ +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_xsavec64 (void *__P, long long __M) +{ + __builtin_ia32_xsavec64 (__P, __M); +} +#endif + +#ifdef __DISABLE_XSAVEC__ +#undef __DISABLE_XSAVEC__ +#pragma GCC pop_options +#endif /* __DISABLE_XSAVEC__ */ + +#endif /* _XSAVECINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/xsaveintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/xsaveintrin.h new file mode 100644 index 0000000000..c136d30aa6 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/xsaveintrin.h @@ -0,0 +1,72 @@ +/* Copyright (C) 2012-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +/* #if !defined _X86INTRIN_H_INCLUDED && !defined _IMMINTRIN_H_INCLUDED */ +/* # error "Never use directly; include instead." */ +/* #endif */ + +#ifndef _XSAVEINTRIN_H_INCLUDED +#define _XSAVEINTRIN_H_INCLUDED + +#ifndef __XSAVE__ +#pragma GCC push_options +#pragma GCC target("xsave") +#define __DISABLE_XSAVE__ +#endif /* __XSAVE__ */ + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_xsave (void *__P, long long __M) +{ + __builtin_ia32_xsave (__P, __M); +} + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_xrstor (void *__P, long long __M) +{ + __builtin_ia32_xrstor (__P, __M); +} + +#ifdef __x86_64__ +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_xsave64 (void *__P, long long __M) +{ + __builtin_ia32_xsave64 (__P, __M); +} + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_xrstor64 (void *__P, long long __M) +{ + __builtin_ia32_xrstor64 (__P, __M); +} +#endif + +#ifdef __DISABLE_XSAVE__ +#undef __DISABLE_XSAVE__ +#pragma GCC pop_options +#endif /* __DISABLE_XSAVE__ */ + +#endif /* _XSAVEINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/xsaveoptintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/xsaveoptintrin.h new file mode 100644 index 0000000000..294fcee43a --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/xsaveoptintrin.h @@ -0,0 +1,58 @@ +/* Copyright (C) 2012-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +/* #if !defined _X86INTRIN_H_INCLUDED && !defined _IMMINTRIN_H_INCLUDED */ +/* # error "Never use directly; include instead." */ +/* #endif */ + +#ifndef _XSAVEOPTINTRIN_H_INCLUDED +#define _XSAVEOPTINTRIN_H_INCLUDED + +#ifndef __XSAVEOPT__ +#pragma GCC push_options +#pragma GCC target("xsaveopt") +#define __DISABLE_XSAVEOPT__ +#endif /* __XSAVEOPT__ */ + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_xsaveopt (void *__P, long long __M) +{ + __builtin_ia32_xsaveopt (__P, __M); +} + +#ifdef __x86_64__ +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_xsaveopt64 (void *__P, long long __M) +{ + __builtin_ia32_xsaveopt64 (__P, __M); +} +#endif + +#ifdef __DISABLE_XSAVEOPT__ +#undef __DISABLE_XSAVEOPT__ +#pragma GCC pop_options +#endif /* __DISABLE_XSAVEOPT__ */ + +#endif /* _XSAVEOPTINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/xsavesintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/xsavesintrin.h new file mode 100644 index 0000000000..b55a1834e2 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/xsavesintrin.h @@ -0,0 +1,72 @@ +/* Copyright (C) 2014-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#if !defined _X86INTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _XSAVESINTRIN_H_INCLUDED +#define _XSAVESINTRIN_H_INCLUDED + +#ifndef __XSAVES__ +#pragma GCC push_options +#pragma GCC target("xsaves") +#define __DISABLE_XSAVES__ +#endif /* __XSAVES__ */ + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_xsaves (void *__P, long long __M) +{ + __builtin_ia32_xsaves (__P, __M); +} + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_xrstors (void *__P, long long __M) +{ + __builtin_ia32_xrstors (__P, __M); +} + +#ifdef __x86_64__ +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_xrstors64 (void *__P, long long __M) +{ + __builtin_ia32_xrstors64 (__P, __M); +} + +extern __inline void +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_xsaves64 (void *__P, long long __M) +{ + __builtin_ia32_xsaves64 (__P, __M); +} +#endif + +#ifdef __DISABLE_XSAVES__ +#undef __DISABLE_XSAVES__ +#pragma GCC pop_options +#endif /* __DISABLE_XSAVES__ */ + +#endif /* _XSAVESINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/include/xtestintrin.h b/openair1/PHY/CODING/nrLDPC_decoder_full/include/xtestintrin.h new file mode 100644 index 0000000000..3c7425f868 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/include/xtestintrin.h @@ -0,0 +1,51 @@ +/* Copyright (C) 2012-2017 Free Software Foundation, Inc. + + This file is part of GCC. + + GCC is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3, or (at your option) + any later version. + + GCC is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + Under Section 7 of GPL version 3, you are granted additional + permissions described in the GCC Runtime Library Exception, version + 3.1, as published by the Free Software Foundation. + + You should have received a copy of the GNU General Public License and + a copy of the GCC Runtime Library Exception along with this program; + see the files COPYING3 and COPYING.RUNTIME respectively. If not, see + . */ + +#ifndef _IMMINTRIN_H_INCLUDED +# error "Never use directly; include instead." +#endif + +#ifndef _XTESTINTRIN_H_INCLUDED +#define _XTESTINTRIN_H_INCLUDED + +#ifndef __RTM__ +#pragma GCC push_options +#pragma GCC target("rtm") +#define __DISABLE_RTM__ +#endif /* __RTM__ */ + +/* Return non-zero if the instruction executes inside an RTM or HLE code + region. Return zero otherwise. */ +extern __inline int +__attribute__((__gnu_inline__, __always_inline__, __artificial__)) +_xtest (void) +{ + return __builtin_ia32_xtest (); +} + +#ifdef __DISABLE_RTM__ +#undef __DISABLE_RTM__ +#pragma GCC pop_options +#endif /* __DISABLE_RTM__ */ + +#endif /* _XTESTINTRIN_H_INCLUDED */ diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/nrLDPC_bnProc.h b/openair1/PHY/CODING/nrLDPC_decoder_full/nrLDPC_bnProc.h new file mode 100644 index 0000000000..b61ca82717 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/nrLDPC_bnProc.h @@ -0,0 +1,2801 @@ +/* + * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The OpenAirInterface Software Alliance licenses this file to You under + * the OAI Public License, Version 1.1 (the "License"); you may not use this file + * except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.openairinterface.org/?page_id=698 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *------------------------------------------------------------------------------- + * For more information about the OpenAirInterface (OAI) Software Alliance: + * contact@openairinterface.org + */ + +/*!\file nrLDPC_bnProc.h + * \brief Defines the functions for check node processing + * Version AVX512 +*/ + +#ifndef __NR_LDPC_BNPROC__H__ +#define __NR_LDPC_BNPROC__H__ + +/** + \brief Performs first part of BN processing on the BN processing buffer and stores the results in the LLR results buffer. + At every BN, the sum of the returned LLRs from the connected CNs and the LLR of the receiver input is computed. + \param p_lut Pointer to decoder LUTs + \param Z Lifting size +*/ +static inline void nrLDPC_bnProcPc(t_nrLDPC_lut* p_lut, t_nrLDPC_procBuf* p_procBuf, uint16_t Z) +{ + const uint8_t* lut_numBnInBnGroups = p_lut->numBnInBnGroups; + const uint32_t* lut_startAddrBnGroups = p_lut->startAddrBnGroups; + const uint16_t* lut_startAddrBnGroupsLlr = p_lut->startAddrBnGroupsLlr; + + int8_t* bnProcBuf = p_procBuf->bnProcBuf; + int8_t* bnProcBufRes = p_procBuf->bnProcBufRes; + int8_t* llrRes = p_procBuf->llrRes; + int8_t* llrProcBuf = p_procBuf->llrProcBuf; + + __m256i* p_bnProcBuf; + __m512i* p_bnProcBufRes; + __m256i* p_llrProcBuf; + __m512i* p_llrProcBuf256; + __m512i* p_llrRes; + + // Number of BNs in Groups + uint32_t M; + //uint32_t M32rem; + uint32_t i,j; + uint32_t k; + // Offset to each bit within a group in terms of 32 Byte + uint32_t cnOffsetInGroup; + uint8_t idxBnGroup = 0; + + __m512i zmm0, zmm1, zmmRes0, zmmRes1; + + // ===================================================================== + // Process group with 1 CN + + // There is always a BN group with 1 CN + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[0]*Z + 63)>>5; + + p_bnProcBuf = (__m256i*) &bnProcBuf [lut_startAddrBnGroups [idxBnGroup]]; + p_bnProcBufRes = (__m512i*) &bnProcBufRes [lut_startAddrBnGroups [idxBnGroup]]; + p_llrProcBuf = (__m256i*) &llrProcBuf [lut_startAddrBnGroupsLlr[idxBnGroup]]; + p_llrProcBuf256 = (__m512i*) &llrProcBuf [lut_startAddrBnGroupsLlr[idxBnGroup]]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0,j=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[1]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 16 Byte + cnOffsetInGroup = (lut_numBnInBnGroups[1]*NR_LDPC_ZMAX)>>4; + + // Set pointers to start of group 2 + p_bnProcBuf = (__m256i*) &bnProcBuf [lut_startAddrBnGroups [idxBnGroup]]; + p_llrProcBuf = (__m256i*) &llrProcBuf [lut_startAddrBnGroupsLlr[idxBnGroup]]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0,j=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[2]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 16 Byte + cnOffsetInGroup = (lut_numBnInBnGroups[2]*NR_LDPC_ZMAX)>>4; + + // Set pointers to start of group 3 + p_bnProcBuf = (__m256i*) &bnProcBuf [lut_startAddrBnGroups [idxBnGroup]]; + p_llrProcBuf = (__m256i*) &llrProcBuf [lut_startAddrBnGroupsLlr[idxBnGroup]]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0,j=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[3]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 16 Byte + cnOffsetInGroup = (lut_numBnInBnGroups[3]*NR_LDPC_ZMAX)>>4; + + // Set pointers to start of group 4 + p_bnProcBuf = (__m256i*) &bnProcBuf [lut_startAddrBnGroups [idxBnGroup]]; + p_llrProcBuf = (__m256i*) &llrProcBuf [lut_startAddrBnGroupsLlr[idxBnGroup]]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0,j=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[4]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 16 Byte + cnOffsetInGroup = (lut_numBnInBnGroups[4]*NR_LDPC_ZMAX)>>4; + + // Set pointers to start of group 5 + p_bnProcBuf = (__m256i*) &bnProcBuf [lut_startAddrBnGroups [idxBnGroup]]; + p_llrProcBuf = (__m256i*) &llrProcBuf [lut_startAddrBnGroupsLlr[idxBnGroup]]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0,j=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[5]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 16 Byte + cnOffsetInGroup = (lut_numBnInBnGroups[5]*NR_LDPC_ZMAX)>>4; + + // Set pointers to start of group 6 + p_bnProcBuf = (__m256i*) &bnProcBuf [lut_startAddrBnGroups [idxBnGroup]]; + p_llrProcBuf = (__m256i*) &llrProcBuf [lut_startAddrBnGroupsLlr[idxBnGroup]]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0,j=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[6]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 16 Byte + cnOffsetInGroup = (lut_numBnInBnGroups[6]*NR_LDPC_ZMAX)>>4; + + // Set pointers to start of group 7 + p_bnProcBuf = (__m256i*) &bnProcBuf [lut_startAddrBnGroups [idxBnGroup]]; + p_llrProcBuf = (__m256i*) &llrProcBuf [lut_startAddrBnGroupsLlr[idxBnGroup]]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0,j=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[7]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 16 Byte + cnOffsetInGroup = (lut_numBnInBnGroups[7]*NR_LDPC_ZMAX)>>4; + + // Set pointers to start of group 8 + p_bnProcBuf = (__m256i*) &bnProcBuf [lut_startAddrBnGroups [idxBnGroup]]; + p_llrProcBuf = (__m256i*) &llrProcBuf [lut_startAddrBnGroupsLlr[idxBnGroup]]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0,j=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[8]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 16 Byte + cnOffsetInGroup = (lut_numBnInBnGroups[8]*NR_LDPC_ZMAX)>>4; + + // Set pointers to start of group 9 + p_bnProcBuf = (__m256i*) &bnProcBuf [lut_startAddrBnGroups [idxBnGroup]]; + p_llrProcBuf = (__m256i*) &llrProcBuf [lut_startAddrBnGroupsLlr[idxBnGroup]]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0,j=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[9]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 16 Byte + cnOffsetInGroup = (lut_numBnInBnGroups[9]*NR_LDPC_ZMAX)>>4; + + // Set pointers to start of group 10 + p_bnProcBuf = (__m256i*) &bnProcBuf [lut_startAddrBnGroups [idxBnGroup]]; + p_llrProcBuf = (__m256i*) &llrProcBuf [lut_startAddrBnGroupsLlr[idxBnGroup]]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0,j=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[10]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 16 Byte + cnOffsetInGroup = (lut_numBnInBnGroups[10]*NR_LDPC_ZMAX)>>4; + + // Set pointers to start of group 11 + p_bnProcBuf = (__m256i*) &bnProcBuf [lut_startAddrBnGroups [idxBnGroup]]; + p_llrProcBuf = (__m256i*) &llrProcBuf [lut_startAddrBnGroupsLlr[idxBnGroup]]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0,j=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[11]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 16 Byte + cnOffsetInGroup = (lut_numBnInBnGroups[11]*NR_LDPC_ZMAX)>>4; + + // Set pointers to start of group 12 + p_bnProcBuf = (__m256i*) &bnProcBuf [lut_startAddrBnGroups [idxBnGroup]]; + p_llrProcBuf = (__m256i*) &llrProcBuf [lut_startAddrBnGroupsLlr[idxBnGroup]]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0,j=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[12]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 16 Byte + cnOffsetInGroup = (lut_numBnInBnGroups[12]*NR_LDPC_ZMAX)>>4; + + // Set pointers to start of group 13 + p_bnProcBuf = (__m256i*) &bnProcBuf [lut_startAddrBnGroups [idxBnGroup]]; + p_llrProcBuf = (__m256i*) &llrProcBuf [lut_startAddrBnGroupsLlr[idxBnGroup]]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0,j=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[13]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 16 Byte + cnOffsetInGroup = (lut_numBnInBnGroups[13]*NR_LDPC_ZMAX)>>4; + + // Set pointers to start of group 14 + p_bnProcBuf = (__m256i*) &bnProcBuf [lut_startAddrBnGroups [idxBnGroup]]; + p_llrProcBuf = (__m256i*) &llrProcBuf [lut_startAddrBnGroupsLlr[idxBnGroup]]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0,j=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[14]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 16 Byte + cnOffsetInGroup = (lut_numBnInBnGroups[14]*NR_LDPC_ZMAX)>>4; + + // Set pointers to start of group 15 + p_bnProcBuf = (__m256i*) &bnProcBuf [lut_startAddrBnGroups [idxBnGroup]]; + p_llrProcBuf = (__m256i*) &llrProcBuf [lut_startAddrBnGroupsLlr[idxBnGroup]]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0,j=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[15]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 16 Byte + cnOffsetInGroup = (lut_numBnInBnGroups[15]*NR_LDPC_ZMAX)>>4; + + // Set pointers to start of group 16 + p_bnProcBuf = (__m256i*) &bnProcBuf [lut_startAddrBnGroups [idxBnGroup]]; + p_llrProcBuf = (__m256i*) &llrProcBuf [lut_startAddrBnGroupsLlr[idxBnGroup]]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0,j=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[16]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 16 Byte + cnOffsetInGroup = (lut_numBnInBnGroups[16]*NR_LDPC_ZMAX)>>4; + + // Set pointers to start of group 16 + p_bnProcBuf = (__m256i*) &bnProcBuf [lut_startAddrBnGroups [idxBnGroup]]; + p_llrProcBuf = (__m256i*) &llrProcBuf [lut_startAddrBnGroupsLlr[idxBnGroup]]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0,j=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[17]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 16 Byte + cnOffsetInGroup = (lut_numBnInBnGroups[17]*NR_LDPC_ZMAX)>>4; + + // Set pointers to start of group 18 + p_bnProcBuf = (__m256i*) &bnProcBuf [lut_startAddrBnGroups [idxBnGroup]]; + p_llrProcBuf = (__m256i*) &llrProcBuf [lut_startAddrBnGroupsLlr[idxBnGroup]]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0,j=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[18]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 16 Byte + cnOffsetInGroup = (lut_numBnInBnGroups[18]*NR_LDPC_ZMAX)>>4; + + // Set pointers to start of group 19 + p_bnProcBuf = (__m256i*) &bnProcBuf [lut_startAddrBnGroups [idxBnGroup]]; + p_llrProcBuf = (__m256i*) &llrProcBuf [lut_startAddrBnGroupsLlr[idxBnGroup]]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0,j=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[19]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 16 Byte + cnOffsetInGroup = (lut_numBnInBnGroups[19]*NR_LDPC_ZMAX)>>4; + + // Set pointers to start of group 20 + p_bnProcBuf = (__m256i*) &bnProcBuf [lut_startAddrBnGroups [idxBnGroup]]; + p_llrProcBuf = (__m256i*) &llrProcBuf [lut_startAddrBnGroupsLlr[idxBnGroup]]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0,j=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[20]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 16 Byte + cnOffsetInGroup = (lut_numBnInBnGroups[20]*NR_LDPC_ZMAX)>>4; + + // Set pointers to start of group 21 + p_bnProcBuf = (__m256i*) &bnProcBuf [lut_startAddrBnGroups [idxBnGroup]]; + p_llrProcBuf = (__m256i*) &llrProcBuf [lut_startAddrBnGroupsLlr[idxBnGroup]]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0,j=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[21]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 16 Byte + cnOffsetInGroup = (lut_numBnInBnGroups[21]*NR_LDPC_ZMAX)>>4; + + // Set pointers to start of group 22 + p_bnProcBuf = (__m256i*) &bnProcBuf [lut_startAddrBnGroups [idxBnGroup]]; + p_llrProcBuf = (__m256i*) &llrProcBuf [lut_startAddrBnGroupsLlr[idxBnGroup]]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0,j=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[22]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 16 Byte + cnOffsetInGroup = (lut_numBnInBnGroups[22]*NR_LDPC_ZMAX)>>4; + + // Set pointers to start of group 23 + p_bnProcBuf = (__m256i*) &bnProcBuf [lut_startAddrBnGroups [idxBnGroup]]; + p_llrProcBuf = (__m256i*) &llrProcBuf [lut_startAddrBnGroupsLlr[idxBnGroup]]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0,j=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[23]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 16 Byte + cnOffsetInGroup = (lut_numBnInBnGroups[23]*NR_LDPC_ZMAX)>>4; + + // Set pointers to start of group 24 + p_bnProcBuf = (__m256i*) &bnProcBuf [lut_startAddrBnGroups [idxBnGroup]]; + p_llrProcBuf = (__m256i*) &llrProcBuf [lut_startAddrBnGroupsLlr[idxBnGroup]]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0,j=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[24]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 16 Byte + cnOffsetInGroup = (lut_numBnInBnGroups[24]*NR_LDPC_ZMAX)>>4; + + // Set pointers to start of group 25 + p_bnProcBuf = (__m256i*) &bnProcBuf [lut_startAddrBnGroups [idxBnGroup]]; + p_llrProcBuf = (__m256i*) &llrProcBuf [lut_startAddrBnGroupsLlr[idxBnGroup]]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0,j=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[25]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 16 Byte + cnOffsetInGroup = (lut_numBnInBnGroups[25]*NR_LDPC_ZMAX)>>4; + + // Set pointers to start of group 26 + p_bnProcBuf = (__m256i*) &bnProcBuf [lut_startAddrBnGroups [idxBnGroup]]; + p_llrProcBuf = (__m256i*) &llrProcBuf [lut_startAddrBnGroupsLlr[idxBnGroup]]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0,j=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[26]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 16 Byte + cnOffsetInGroup = (lut_numBnInBnGroups[26]*NR_LDPC_ZMAX)>>4; + + // Set pointers to start of group 27 + p_bnProcBuf = (__m256i*) &bnProcBuf [lut_startAddrBnGroups [idxBnGroup]]; + p_llrProcBuf = (__m256i*) &llrProcBuf [lut_startAddrBnGroupsLlr[idxBnGroup]]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0,j=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[27]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 16 Byte + cnOffsetInGroup = (lut_numBnInBnGroups[27]*NR_LDPC_ZMAX)>>4; + + // Set pointers to start of group 28 + p_bnProcBuf = (__m256i*) &bnProcBuf [lut_startAddrBnGroups [idxBnGroup]]; + p_llrProcBuf = (__m256i*) &llrProcBuf [lut_startAddrBnGroupsLlr[idxBnGroup]]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0,j=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[28]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 16 Byte + cnOffsetInGroup = (lut_numBnInBnGroups[28]*NR_LDPC_ZMAX)>>4; + + // Set pointers to start of group 29 + p_bnProcBuf = (__m256i*) &bnProcBuf [lut_startAddrBnGroups [idxBnGroup]]; + p_llrProcBuf = (__m256i*) &llrProcBuf [lut_startAddrBnGroupsLlr[idxBnGroup]]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0,j=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[29]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 16 Byte + cnOffsetInGroup = (lut_numBnInBnGroups[29]*NR_LDPC_ZMAX)>>4; + + // Set pointers to start of group 30 + p_bnProcBuf = (__m256i*) &bnProcBuf [lut_startAddrBnGroups [idxBnGroup]]; + p_llrProcBuf = (__m256i*) &llrProcBuf [lut_startAddrBnGroupsLlr[idxBnGroup]]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0,j=0; inumBnInBnGroups; + const uint64_t* lut_startAddrBnGroups = p_lut->startAddrBnGroups; + const uint16_t* lut_startAddrBnGroupsLlr = p_lut->startAddrBnGroupsLlr; + + int8_t* bnProcBuf = p_procBuf->bnProcBuf; + int8_t* bnProcBufRes = p_procBuf->bnProcBufRes; + int8_t* llrRes = p_procBuf->llrRes; + + __m512i* p_bnProcBuf; + __m512i* p_bnProcBufRes; + __m512i* p_llrRes; + __m512i* p_res; + + // Number of BNs in Groups + uint64_t M; + //uint64_t M64rem; + uint64_t i; + uint64_t k; + // Offset to each bit within a group in terms of 64 byte + uint64_t cnOffsetInGroup; + uint8_t idxBnGroup = 0; + + // ===================================================================== + // Process group with 1 CN + // Already done in bnProcBufPc + + // ===================================================================== + // Process group with 2 CNs + + if (lut_numBnInBnGroups[1] > 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[1]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 64 byte + cnOffsetInGroup = (lut_numBnInBnGroups[1]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 2 + p_bnProcBuf = (__m512i*) &bnProcBuf [lut_startAddrBnGroups[idxBnGroup]]; + p_bnProcBufRes = (__m512i*) &bnProcBufRes[lut_startAddrBnGroups[idxBnGroup]]; + + // Loop over CNs + for (k=0; k<2; k++) + { + p_res = &p_bnProcBufRes[k*cnOffsetInGroup]; + p_llrRes = (__m512i*) &llrRes[lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[2]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 64 byte + cnOffsetInGroup = (lut_numBnInBnGroups[2]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 3 + p_bnProcBuf = (__m512i*) &bnProcBuf [lut_startAddrBnGroups[idxBnGroup]]; + p_bnProcBufRes = (__m512i*) &bnProcBufRes[lut_startAddrBnGroups[idxBnGroup]]; + + // Loop over CNs + for (k=0; k<3; k++) + { + p_res = &p_bnProcBufRes[k*cnOffsetInGroup]; + p_llrRes = (__m512i*) &llrRes[lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[3]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 64 byte + cnOffsetInGroup = (lut_numBnInBnGroups[3]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 4 + p_bnProcBuf = (__m512i*) &bnProcBuf [lut_startAddrBnGroups[idxBnGroup]]; + p_bnProcBufRes = (__m512i*) &bnProcBufRes[lut_startAddrBnGroups[idxBnGroup]]; + + // Loop over CNs + for (k=0; k<4; k++) + { + p_res = &p_bnProcBufRes[k*cnOffsetInGroup]; + p_llrRes = (__m512i*) &llrRes[lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[4]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 64 byte + cnOffsetInGroup = (lut_numBnInBnGroups[4]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 5 + p_bnProcBuf = (__m512i*) &bnProcBuf [lut_startAddrBnGroups[idxBnGroup]]; + p_bnProcBufRes = (__m512i*) &bnProcBufRes[lut_startAddrBnGroups[idxBnGroup]]; + + // Loop over CNs + for (k=0; k<5; k++) + { + p_res = &p_bnProcBufRes[k*cnOffsetInGroup]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[5]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 64 byte + cnOffsetInGroup = (lut_numBnInBnGroups[5]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 6 + p_bnProcBuf = (__m512i*) &bnProcBuf [lut_startAddrBnGroups[idxBnGroup]]; + p_bnProcBufRes = (__m512i*) &bnProcBufRes[lut_startAddrBnGroups[idxBnGroup]]; + + // Loop over CNs + for (k=0; k<6; k++) + { + p_res = &p_bnProcBufRes[k*cnOffsetInGroup]; + p_llrRes = (__m512i*) &llrRes[lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[6]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 64 byte + cnOffsetInGroup = (lut_numBnInBnGroups[6]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 7 + p_bnProcBuf = (__m512i*) &bnProcBuf [lut_startAddrBnGroups[idxBnGroup]]; + p_bnProcBufRes = (__m512i*) &bnProcBufRes[lut_startAddrBnGroups[idxBnGroup]]; + + // Loop over CNs + for (k=0; k<7; k++) + { + p_res = &p_bnProcBufRes[k*cnOffsetInGroup]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[7]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 64 byte + cnOffsetInGroup = (lut_numBnInBnGroups[7]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 8 + p_bnProcBuf = (__m512i*) &bnProcBuf [lut_startAddrBnGroups[idxBnGroup]]; + p_bnProcBufRes = (__m512i*) &bnProcBufRes[lut_startAddrBnGroups[idxBnGroup]]; + + // Loop over CNs + for (k=0; k<8; k++) + { + p_res = &p_bnProcBufRes[k*cnOffsetInGroup]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[8]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 64 byte + cnOffsetInGroup = (lut_numBnInBnGroups[8]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 9 + p_bnProcBuf = (__m512i*) &bnProcBuf [lut_startAddrBnGroups[idxBnGroup]]; + p_bnProcBufRes = (__m512i*) &bnProcBufRes[lut_startAddrBnGroups[idxBnGroup]]; + + // Loop over CNs + for (k=0; k<9; k++) + { + p_res = &p_bnProcBufRes[k*cnOffsetInGroup]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[9]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 64 byte + cnOffsetInGroup = (lut_numBnInBnGroups[9]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 10 + p_bnProcBuf = (__m512i*) &bnProcBuf [lut_startAddrBnGroups[idxBnGroup]]; + p_bnProcBufRes = (__m512i*) &bnProcBufRes[lut_startAddrBnGroups[idxBnGroup]]; + + // Loop over CNs + for (k=0; k<10; k++) + { + p_res = &p_bnProcBufRes[k*cnOffsetInGroup]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[10]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 64 byte + cnOffsetInGroup = (lut_numBnInBnGroups[10]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 10 + p_bnProcBuf = (__m512i*) &bnProcBuf [lut_startAddrBnGroups[idxBnGroup]]; + p_bnProcBufRes = (__m512i*) &bnProcBufRes[lut_startAddrBnGroups[idxBnGroup]]; + + // Loop over CNs + for (k=0; k<11; k++) + { + p_res = &p_bnProcBufRes[k*cnOffsetInGroup]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[11]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 64 byte + cnOffsetInGroup = (lut_numBnInBnGroups[11]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 12 + p_bnProcBuf = (__m512i*) &bnProcBuf [lut_startAddrBnGroups[idxBnGroup]]; + p_bnProcBufRes = (__m512i*) &bnProcBufRes[lut_startAddrBnGroups[idxBnGroup]]; + + // Loop over CNs + for (k=0; k<12; k++) + { + p_res = &p_bnProcBufRes[k*cnOffsetInGroup]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[12]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 64 byte + cnOffsetInGroup = (lut_numBnInBnGroups[12]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 13 + p_bnProcBuf = (__m512i*) &bnProcBuf [lut_startAddrBnGroups[idxBnGroup]]; + p_bnProcBufRes = (__m512i*) &bnProcBufRes[lut_startAddrBnGroups[idxBnGroup]]; + + // Loop over CNs + for (k=0; k<13; k++) + { + p_res = &p_bnProcBufRes[k*cnOffsetInGroup]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[13]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 64 byte + cnOffsetInGroup = (lut_numBnInBnGroups[13]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 14 + p_bnProcBuf = (__m512i*) &bnProcBuf [lut_startAddrBnGroups[idxBnGroup]]; + p_bnProcBufRes = (__m512i*) &bnProcBufRes[lut_startAddrBnGroups[idxBnGroup]]; + + // Loop over CNs + for (k=0; k<14; k++) + { + p_res = &p_bnProcBufRes[k*cnOffsetInGroup]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[14]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 64 byte + cnOffsetInGroup = (lut_numBnInBnGroups[14]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 15 + p_bnProcBuf = (__m512i*) &bnProcBuf [lut_startAddrBnGroups[idxBnGroup]]; + p_bnProcBufRes = (__m512i*) &bnProcBufRes[lut_startAddrBnGroups[idxBnGroup]]; + + // Loop over CNs + for (k=0; k<15; k++) + { + p_res = &p_bnProcBufRes[k*cnOffsetInGroup]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[15]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 64 byte + cnOffsetInGroup = (lut_numBnInBnGroups[15]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 16 + p_bnProcBuf = (__m512i*) &bnProcBuf [lut_startAddrBnGroups[idxBnGroup]]; + p_bnProcBufRes = (__m512i*) &bnProcBufRes[lut_startAddrBnGroups[idxBnGroup]]; + + // Loop over CNs + for (k=0; k<16; k++) + { + p_res = &p_bnProcBufRes[k*cnOffsetInGroup]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[16]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 64 byte + cnOffsetInGroup = (lut_numBnInBnGroups[16]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 17 + p_bnProcBuf = (__m512i*) &bnProcBuf [lut_startAddrBnGroups[idxBnGroup]]; + p_bnProcBufRes = (__m512i*) &bnProcBufRes[lut_startAddrBnGroups[idxBnGroup]]; + + // Loop over CNs + for (k=0; k<17; k++) + { + p_res = &p_bnProcBufRes[k*cnOffsetInGroup]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[17]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 64 byte + cnOffsetInGroup = (lut_numBnInBnGroups[17]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 18 + p_bnProcBuf = (__m512i*) &bnProcBuf [lut_startAddrBnGroups[idxBnGroup]]; + p_bnProcBufRes = (__m512i*) &bnProcBufRes[lut_startAddrBnGroups[idxBnGroup]]; + + // Loop over CNs + for (k=0; k<18; k++) + { + p_res = &p_bnProcBufRes[k*cnOffsetInGroup]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[18]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 64 byte + cnOffsetInGroup = (lut_numBnInBnGroups[18]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 19 + p_bnProcBuf = (__m512i*) &bnProcBuf [lut_startAddrBnGroups[idxBnGroup]]; + p_bnProcBufRes = (__m512i*) &bnProcBufRes[lut_startAddrBnGroups[idxBnGroup]]; + + // Loop over CNs + for (k=0; k<19; k++) + { + p_res = &p_bnProcBufRes[k*cnOffsetInGroup]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[19]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 64 byte + cnOffsetInGroup = (lut_numBnInBnGroups[19]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 20 + p_bnProcBuf = (__m512i*) &bnProcBuf [lut_startAddrBnGroups[idxBnGroup]]; + p_bnProcBufRes = (__m512i*) &bnProcBufRes[lut_startAddrBnGroups[idxBnGroup]]; + + // Loop over CNs + for (k=0; k<20; k++) + { + p_res = &p_bnProcBufRes[k*cnOffsetInGroup]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[20]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 64 byte + cnOffsetInGroup = (lut_numBnInBnGroups[20]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 21 + p_bnProcBuf = (__m512i*) &bnProcBuf [lut_startAddrBnGroups[idxBnGroup]]; + p_bnProcBufRes = (__m512i*) &bnProcBufRes[lut_startAddrBnGroups[idxBnGroup]]; + + // Loop over CNs + for (k=0; k<21; k++) + { + p_res = &p_bnProcBufRes[k*cnOffsetInGroup]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[21]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 64 byte + cnOffsetInGroup = (lut_numBnInBnGroups[21]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 22 + p_bnProcBuf = (__m512i*) &bnProcBuf [lut_startAddrBnGroups[idxBnGroup]]; + p_bnProcBufRes = (__m512i*) &bnProcBufRes[lut_startAddrBnGroups[idxBnGroup]]; + + // Loop over CNs + for (k=0; k<22; k++) + { + p_res = &p_bnProcBufRes[k*cnOffsetInGroup]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[22]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 64 byte + cnOffsetInGroup = (lut_numBnInBnGroups[22]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 23 + p_bnProcBuf = (__m512i*) &bnProcBuf [lut_startAddrBnGroups[idxBnGroup]]; + p_bnProcBufRes = (__m512i*) &bnProcBufRes[lut_startAddrBnGroups[idxBnGroup]]; + + // Loop over CNs + for (k=0; k<23; k++) + { + p_res = &p_bnProcBufRes[k*cnOffsetInGroup]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[23]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 64 byte + cnOffsetInGroup = (lut_numBnInBnGroups[23]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 24 + p_bnProcBuf = (__m512i*) &bnProcBuf [lut_startAddrBnGroups[idxBnGroup]]; + p_bnProcBufRes = (__m512i*) &bnProcBufRes[lut_startAddrBnGroups[idxBnGroup]]; + + // Loop over CNs + for (k=0; k<24; k++) + { + p_res = &p_bnProcBufRes[k*cnOffsetInGroup]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[24]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 64 byte + cnOffsetInGroup = (lut_numBnInBnGroups[24]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 25 + p_bnProcBuf = (__m512i*) &bnProcBuf [lut_startAddrBnGroups[idxBnGroup]]; + p_bnProcBufRes = (__m512i*) &bnProcBufRes[lut_startAddrBnGroups[idxBnGroup]]; + + // Loop over CNs + for (k=0; k<25; k++) + { + p_res = &p_bnProcBufRes[k*cnOffsetInGroup]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[25]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 64 byte + cnOffsetInGroup = (lut_numBnInBnGroups[25]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 26 + p_bnProcBuf = (__m512i*) &bnProcBuf [lut_startAddrBnGroups[idxBnGroup]]; + p_bnProcBufRes = (__m512i*) &bnProcBufRes[lut_startAddrBnGroups[idxBnGroup]]; + + // Loop over CNs + for (k=0; k<26; k++) + { + p_res = &p_bnProcBufRes[k*cnOffsetInGroup]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[26]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 64 byte + cnOffsetInGroup = (lut_numBnInBnGroups[26]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 27 + p_bnProcBuf = (__m512i*) &bnProcBuf [lut_startAddrBnGroups[idxBnGroup]]; + p_bnProcBufRes = (__m512i*) &bnProcBufRes[lut_startAddrBnGroups[idxBnGroup]]; + + // Loop over CNs + for (k=0; k<27; k++) + { + p_res = &p_bnProcBufRes[k*cnOffsetInGroup]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[27]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 64 byte + cnOffsetInGroup = (lut_numBnInBnGroups[27]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 28 + p_bnProcBuf = (__m512i*) &bnProcBuf [lut_startAddrBnGroups[idxBnGroup]]; + p_bnProcBufRes = (__m512i*) &bnProcBufRes[lut_startAddrBnGroups[idxBnGroup]]; + + // Loop over CNs + for (k=0; k<28; k++) + { + p_res = &p_bnProcBufRes[k*cnOffsetInGroup]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[28]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 64 byte + cnOffsetInGroup = (lut_numBnInBnGroups[28]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 29 + p_bnProcBuf = (__m512i*) &bnProcBuf [lut_startAddrBnGroups[idxBnGroup]]; + p_bnProcBufRes = (__m512i*) &bnProcBufRes[lut_startAddrBnGroups[idxBnGroup]]; + + // Loop over CNs + for (k=0; k<29; k++) + { + p_res = &p_bnProcBufRes[k*cnOffsetInGroup]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0; i 0) + { + // If elements in group move to next address + idxBnGroup++; + + // Number of groups of 64 BNs for parallel processing + M = (lut_numBnInBnGroups[29]*Z + 63)>>5; + + // Set the offset to each CN within a group in terms of 64 byte + cnOffsetInGroup = (lut_numBnInBnGroups[29]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 30 + p_bnProcBuf = (__m512i*) &bnProcBuf [lut_startAddrBnGroups[idxBnGroup]]; + p_bnProcBufRes = (__m512i*) &bnProcBufRes[lut_startAddrBnGroups[idxBnGroup]]; + + // Loop over CNs + for (k=0; k<30; k++) + { + p_res = &p_bnProcBufRes[k*cnOffsetInGroup]; + p_llrRes = (__m512i*) &llrRes [lut_startAddrBnGroupsLlr[idxBnGroup]]; + + // Loop over BNs + for (i=0; i>5; + uint64_t Mr = numLLR&63; + + const __m512i* p_zeros = (__m512i*) zeros256_epi8; + const __m512i* p_ones = (__m512i*) ones256_epi8; + + for (i=0; i 0) + { + // Remaining LLRs that do not fit in multiples of 64 bytes + p_llrOut8 = (int8_t*) p_llrOut; + p_out8 = (int8_t*) p_out; + + for (i=0; i>5; + uint64_t Mr = numLLR&63; + const __m512i* p_shuffle = (__m512i*) constShuffle_256_epi8; + + for (i=0; i 0) + { + // Remaining LLRs that do not fit in multiples of 64 bytes + p_llrOut8 = (int8_t*) p_llrOut; + + for (i=0; i +#include "include/avx512fintrin.h" + +/** + \brief Performs CN processing for BG2 on the CN processing buffer and stores the results in the CN processing results buffer. + \param p_lut Pointer to decoder LUTs + \param p_procBuf Pointer to processing buffers + \param Z Lifting size +*/ + + //intrinsic sign avec AVX512 car __m512i _mm512_sign_epi16 (__m512i a, __m512i b) n'exite pas en AVX, c'est pour quoi je la rédefini ici + /* Emulate _mm512_sign_epi16() with instructions that exist in the AVX-512 instruction set */ + __m512i mm512_sign_epi16(__m512i a, __m512i b){ + b = _mm512_min_epi16(b, _mm512_set1_epi16(1)); + b = _mm512_max_epi16(b, _mm512_set1_epi16(-1)); + a = _mm512_mullo_epi16(a, b); + return a; + } + + +/* comment +static inline void nrLDPC_cnProc_BG2(t_nrLDPC_lut* p_lut, t_nrLDPC_procBuf* p_procBuf, uint16_t Z) +{ + const uint8_t* lut_numCnInCnGroups = p_lut->numCnInCnGroups; + const uint64_t* lut_startAddrCnGroups = p_lut->startAddrCnGroups; + + int8_t* cnProcBuf = p_procBuf->cnProcBuf; + int8_t* cnProcBufRes = p_procBuf->cnProcBufRes; + + __m512i* p_cnProcBuf; + __m512i* p_cnProcBufRes; + + // Number of CNs in Groups + uint64_t M; + uint64_t i; + uint64_t j; + uint64_t k; + // Offset to each bit within a group in terms of 64 bytes + uint64_t bitOffsetInGroup; + + __m512i zmm0, min, sgn; + __m512i* p_cnProcBufResBit; + + const __m512i* p_ones = (__m512i*) ones512_epi8; + const __m512i* p_maxLLR = (__m512i*) maxLLR512_epi8; + + + + + // LUT with offsets for bits that need to be processed + // 1. bit proc requires LLRs of 2. and 3. bit, 2.bits of 1. and 3. etc. + // Offsets are in units of bitOffsetInGroup + const uint8_t lut_idxCnProcG3[3][2] = {{36,72}, {0,72}, {0,36}}; + + // ===================================================================== + // Process group with 3 BNs + + if (lut_numCnInCnGroups[0] > 0) + { + // Number of groups of 64 CNS for parallel processing + // Ceil for values not divisible 64 + M = (lut_numCnInCnGroups[0]*Z + 63)>>5; + // Set the offset to each bit within a group in terms of 64 bytes + bitOffsetInGroup = (lut_numCnInCnGroups_BG2_R15[0]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 3 + p_cnProcBuf = (__m512i*) &cnProcBuf [lut_startAddrCnGroups[0]]; + p_cnProcBufRes = (__m512i*) &cnProcBufRes[lut_startAddrCnGroups[0]]; + + // Loop over every BN + for (j=0; j<3; j++) + { + // Set of results pointer to correct BN address + p_cnProcBufResBit = p_cnProcBufRes + (j*bitOffsetInGroup); + + // __m512i *pj0 = &p_cnProcBuf[lut_idxCnProcG3[j][0]]; + // __m512i *pj1 = &p_cnProcBuf[lut_idxCnProcG3[j][1]]; + + // Loop over CNs + for (i=0; i 0) + { + // Number of groups of 64 CNS for parallel processing + // Ceil for values not divisible 64 + M = (lut_numCnInCnGroups[1]*Z + 63)>>5; + // Set the offset to each bit within a group in terms of 64 bytes + bitOffsetInGroup = (lut_numCnInCnGroups_BG2_R15[1]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 4 + p_cnProcBuf = (__m512i*) &cnProcBuf [lut_startAddrCnGroups[1]]; + p_cnProcBufRes = (__m512i*) &cnProcBufRes[lut_startAddrCnGroups[1]]; + + // Loop over every BN + for (j=0; j<4; j++) + { + // Set of results pointer to correct BN address + p_cnProcBufResBit = p_cnProcBufRes + (j*bitOffsetInGroup); + + // Loop over CNs + for (i=0; i 0) + { + // Number of groups of 64 CNS for parallel processing + // Ceil for values not divisible 64 + M = (lut_numCnInCnGroups[2]*Z + 63)>>5; + // Set the offset to each bit within a group in terms of 64 bytes + bitOffsetInGroup = (lut_numCnInCnGroups_BG2_R15[2]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 5 + p_cnProcBuf = (__m512i*) &cnProcBuf [lut_startAddrCnGroups[2]]; + p_cnProcBufRes = (__m512i*) &cnProcBufRes[lut_startAddrCnGroups[2]]; + + // Loop over every BN + for (j=0; j<5; j++) + { + // Set of results pointer to correct BN address + p_cnProcBufResBit = p_cnProcBufRes + (j*bitOffsetInGroup); + + // Loop over CNs + for (i=0; i 0) + { + // Number of groups of 64 CNS for parallel processing + // Ceil for values not divisible 64 + M = (lut_numCnInCnGroups[3]*Z + 63)>>5; + // Set the offset to each bit within a group in terms of 64 bytes + bitOffsetInGroup = (lut_numCnInCnGroups_BG2_R15[3]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 6 + p_cnProcBuf = (__m512i*) &cnProcBuf [lut_startAddrCnGroups[3]]; + p_cnProcBufRes = (__m512i*) &cnProcBufRes[lut_startAddrCnGroups[3]]; + + // Loop over every BN + for (j=0; j<6; j++) + { + // Set of results pointer to correct BN address + p_cnProcBufResBit = p_cnProcBufRes + (j*bitOffsetInGroup); + + // Loop over CNs + for (i=0; i 0) + { + // Number of groups of 64 CNS for parallel processing + // Ceil for values not divisible 64 + M = (lut_numCnInCnGroups[4]*Z + 63)>>5; + // Set the offset to each bit within a group in terms of 64 bytes + bitOffsetInGroup = (lut_numCnInCnGroups_BG2_R15[4]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 8 + p_cnProcBuf = (__m512i*) &cnProcBuf [lut_startAddrCnGroups[4]]; + p_cnProcBufRes = (__m512i*) &cnProcBufRes[lut_startAddrCnGroups[4]]; + + // Loop over every BN + for (j=0; j<8; j++) + { + // Set of results pointer to correct BN address + p_cnProcBufResBit = p_cnProcBufRes + (j*bitOffsetInGroup); + + // Loop over CNs + for (i=0; i 0) + { + // Number of groups of 64 CNS for parallel processing + // Ceil for values not divisible 64 + M = (lut_numCnInCnGroups[5]*Z + 63)>>5; + // Set the offset to each bit within a group in terms of 64 bytes + bitOffsetInGroup = (lut_numCnInCnGroups_BG2_R15[5]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 10 + p_cnProcBuf = (__m512i*) &cnProcBuf [lut_startAddrCnGroups[5]]; + p_cnProcBufRes = (__m512i*) &cnProcBufRes[lut_startAddrCnGroups[5]]; + + // Loop over every BN + for (j=0; j<10; j++) + { + // Set of results pointer to correct BN address + p_cnProcBufResBit = p_cnProcBufRes + (j*bitOffsetInGroup); + + // Loop over CNs + for (i=0; inumCnInCnGroups; + const uint64_t* lut_startAddrCnGroups = p_lut->startAddrCnGroups; + + int8_t* cnProcBuf = p_procBuf->cnProcBuf; + int8_t* cnProcBufRes = p_procBuf->cnProcBufRes; + + __m512i* p_cnProcBuf; + __m512i* p_cnProcBufRes; + + // Number of CNs in Groups + uint64_t M; + uint64_t i; + uint64_t j; + uint64_t k; + // Offset to each bit within a group in terms of 64 bytes + uint64_t bitOffsetInGroup; + + __m512i zmm0, min, sgn; + __m512i* p_cnProcBufResBit; + + const __m512i* p_ones = (__m512i*) ones512_epi8; + const __m512i* p_maxLLR = (__m512i*) maxLLR512_epi8; + + // LUT with offsets for bits that need to be processed + // 1. bit proc requires LLRs of 2. and 3. bit, 2.bits of 1. and 3. etc. + // Offsets are in units of bitOffsetInGroup (1*384/64)=6 + + const uint8_t lut_idxCnProcG3[3][2] = {{6,12}, {0,12}, {0,6}}; + + // ===================================================================== + // Process group with 3 BNs + + if (lut_numCnInCnGroups[0] > 0) + { + // Number of groups of 64 CNS for parallel processing + // Ceil for values not divisible 64 + M = (lut_numCnInCnGroups[0]*Z + 63)>>5; + + // Set the offset to each bit within a group in terms of 64 bytes + bitOffsetInGroup = (lut_numCnInCnGroups_BG1_R13[0]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 3 + p_cnProcBuf = (__m512i*) &cnProcBuf [lut_startAddrCnGroups[0]]; + p_cnProcBufRes = (__m512i*) &cnProcBufRes[lut_startAddrCnGroups[0]]; + + // Loop over every BN + for (j=0; j<3; j++) + { + // Set of results pointer to correct BN address + p_cnProcBufResBit = p_cnProcBufRes + (j*bitOffsetInGroup); + + // Loop over CNs + for (i=0; i 0) + { + // Number of groups of 64 CNS for parallel processing + // Ceil for values not divisible 64 + M = (lut_numCnInCnGroups[1]*Z + 63)>>5; + + // Set the offset to each bit within a group in terms of 64 bytes + bitOffsetInGroup = (lut_numCnInCnGroups_BG1_R13[1]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 4 + p_cnProcBuf = (__m512i*) &cnProcBuf [lut_startAddrCnGroups[1]]; + p_cnProcBufRes = (__m512i*) &cnProcBufRes[lut_startAddrCnGroups[1]]; + + // Loop over every BN + for (j=0; j<4; j++) + { + // Set of results pointer to correct BN address + p_cnProcBufResBit = p_cnProcBufRes + (j*bitOffsetInGroup); + + // Loop over CNs + for (i=0; i 0) + { + // Number of groups of 64 CNS for parallel processing + // Ceil for values not divisible 64 + M = (lut_numCnInCnGroups[2]*Z + 63)>>5; + + // Set the offset to each bit within a group in terms of 64 bytes + bitOffsetInGroup = (lut_numCnInCnGroups_BG1_R13[2]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 5 + p_cnProcBuf = (__m512i*) &cnProcBuf [lut_startAddrCnGroups[2]]; + p_cnProcBufRes = (__m512i*) &cnProcBufRes[lut_startAddrCnGroups[2]]; + + // Loop over every BN + for (j=0; j<5; j++) + { + // Set of results pointer to correct BN address + p_cnProcBufResBit = p_cnProcBufRes + (j*bitOffsetInGroup); + + // Loop over CNs + for (i=0; i 0) + { + // Number of groups of 64 CNS for parallel processing + // Ceil for values not divisible 64 + M = (lut_numCnInCnGroups[3]*Z + 63)>>5; + + // Set the offset to each bit within a group in terms of 64 bytes + bitOffsetInGroup = (lut_numCnInCnGroups_BG1_R13[3]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 6 + p_cnProcBuf = (__m512i*) &cnProcBuf [lut_startAddrCnGroups[3]]; + p_cnProcBufRes = (__m512i*) &cnProcBufRes[lut_startAddrCnGroups[3]]; + + // Loop over every BN + for (j=0; j<6; j++) + { + // Set of results pointer to correct BN address + p_cnProcBufResBit = p_cnProcBufRes + (j*bitOffsetInGroup); + + // Loop over CNs + for (i=0; i 0) + { + // Number of groups of 64 CNS for parallel processing + // Ceil for values not divisible 64 + M = (lut_numCnInCnGroups[4]*Z + 63)>>5; + + // Set the offset to each bit within a group in terms of 64 bytes + bitOffsetInGroup = (lut_numCnInCnGroups_BG1_R13[4]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 7 + p_cnProcBuf = (__m512i*) &cnProcBuf [lut_startAddrCnGroups[4]]; + p_cnProcBufRes = (__m512i*) &cnProcBufRes[lut_startAddrCnGroups[4]]; + + // Loop over every BN + for (j=0; j<7; j++) + { + // Set of results pointer to correct BN address + p_cnProcBufResBit = p_cnProcBufRes + (j*bitOffsetInGroup); + + // Loop over CNs + for (i=0; i 0) + { + // Number of groups of 64 CNS for parallel processing + // Ceil for values not divisible 64 + M = (lut_numCnInCnGroups[5]*Z + 63)>>5; + + // Set the offset to each bit within a group in terms of 64 bytes + bitOffsetInGroup = (lut_numCnInCnGroups_BG1_R13[5]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 8 + p_cnProcBuf = (__m512i*) &cnProcBuf [lut_startAddrCnGroups[5]]; + p_cnProcBufRes = (__m512i*) &cnProcBufRes[lut_startAddrCnGroups[5]]; + + // Loop over every BN + for (j=0; j<8; j++) + { + // Set of results pointer to correct BN address + p_cnProcBufResBit = p_cnProcBufRes + (j*bitOffsetInGroup); + + // Loop over CNs + for (i=0; i 0) + { + // Number of groups of 64 CNS for parallel processing + // Ceil for values not divisible 64 + M = (lut_numCnInCnGroups[6]*Z + 63)>>5; + + // Set the offset to each bit within a group in terms of 64 bytes + bitOffsetInGroup = (lut_numCnInCnGroups_BG1_R13[6]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 9 + p_cnProcBuf = (__m512i*) &cnProcBuf [lut_startAddrCnGroups[6]]; + p_cnProcBufRes = (__m512i*) &cnProcBufRes[lut_startAddrCnGroups[6]]; + + // Loop over every BN + for (j=0; j<9; j++) + { + // Set of results pointer to correct BN address + p_cnProcBufResBit = p_cnProcBufRes + (j*bitOffsetInGroup); + + // Loop over CNs + for (i=0; i 0) + { + // Number of groups of 64 CNS for parallel processing + // Ceil for values not divisible 64 + M = (lut_numCnInCnGroups[7]*Z + 63)>>5; + + // Set the offset to each bit within a group in terms of 64 bytes + bitOffsetInGroup = (lut_numCnInCnGroups_BG1_R13[7]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 10 + p_cnProcBuf = (__m512i*) &cnProcBuf [lut_startAddrCnGroups[7]]; + p_cnProcBufRes = (__m512i*) &cnProcBufRes[lut_startAddrCnGroups[7]]; + + // Loop over every BN + for (j=0; j<10; j++) + { + // Set of results pointer to correct BN address + p_cnProcBufResBit = p_cnProcBufRes + (j*bitOffsetInGroup); + + // Loop over CNs + for (i=0; i 0) + { + // Number of groups of 64 CNS for parallel processing + // Ceil for values not divisible 64 + M = (lut_numCnInCnGroups[8]*Z + 63)>>5; + + // Set the offset to each bit within a group in terms of 64 bytes + bitOffsetInGroup = (lut_numCnInCnGroups_BG1_R13[8]*NR_LDPC_ZMAX)>>5; + + // Set pointers to start of group 19 + p_cnProcBuf = (__m512i*) &cnProcBuf [lut_startAddrCnGroups[8]]; + p_cnProcBufRes = (__m512i*) &cnProcBufRes[lut_startAddrCnGroups[8]]; + + // Loop over every BN + for (j=0; j<19; j++) + { + // Set of results pointer to correct BN address + p_cnProcBufResBit = p_cnProcBufRes + (j*bitOffsetInGroup); + + // Loop over CNs + for (i=0; inumCnInCnGroups; + const uint64_t* lut_startAddrCnGroups = p_lut->startAddrCnGroups; + + int8_t* cnProcBuf = p_procBuf->cnProcBuf; + int8_t* cnProcBufRes = p_procBuf->cnProcBufRes; + + __m512i* p_cnProcBuf; + __m512i* p_cnProcBufRes; + + // Number of CNs in Groups + uint64_t M; + uint64_t i; + uint64_t j; + uint64_t pcRes = 0; + uint64_t pcResSum = 0; + uint64_t Mrem; + uint64_t M64; + + __m512i zmm0, zmm1; + + // ===================================================================== + // Process group with 3 BNs + + if (lut_numCnInCnGroups[0] > 0) + { + // Reset results + pcResSum = 0; + + // Number of CNs in group + M = lut_numCnInCnGroups[0]*Z; + // Remainder modulo 32 + Mrem = M&63; + // Number of groups of 64 CNs for parallel processing + // Ceil for values not divisible by 32 + M64 = (M + 63)>>5; + + // Set pointers to start of group 3 + p_cnProcBuf = (__m512i*) &cnProcBuf [lut_startAddrCnGroups[0]]; + p_cnProcBufRes = (__m512i*) &cnProcBufRes[lut_startAddrCnGroups[0]]; + + // Loop over CNs + for (i=0; i<(M64-1); i++) + { + pcRes = 0; + // Loop over every BN + // Compute PC for 64 CNs at once + for (j=0; j<3; j++) + { + // BN offset is units of (1*384/64) = 6 + zmm0 = p_cnProcBuf [j*6 + i]; + zmm1 = p_cnProcBufRes[j*6 + i]; + + // Add BN and input LLR, extract the sign bit + // and add in GF(2) (xor) + pcRes ^= _mm512_movemask_epi8(_mm512_adds_epi8(ymm0,ymm1)); + } + + // If no error pcRes should be 0 + pcResSum |= pcRes; + } + + // Last 64 CNs might not be full valid 32 depending on Z + pcRes = 0; + // Loop over every BN + // Compute PC for 64 CNs at once + for (j=0; j<3; j++) + { + // BN offset is units of (1*384/64) = 6 + zmm0 = p_cnProcBuf [j*6 + i]; + zmm1 = p_cnProcBufRes[j*6 + i]; + + // Add BN and input LLR, extract the sign bit + // and add in GF(2) (xor) + pcRes ^= _mm512_movemask_epi8(_mm512_adds_epi8(zmm0,zmm1)); + } + + // If no error pcRes should be 0 + // Only use valid CNs + pcResSum |= (pcRes&(0xFFFFFFFF>>(64 - Mrem))); + + // If PC failed we can stop here + if (pcResSum > 0) + { + return pcResSum; + } + } + + // ===================================================================== + // Process group with 4 BNs + + if (lut_numCnInCnGroups[1] > 0) + { + // Reset results + pcResSum = 0; + + // Number of CNs in group + M = lut_numCnInCnGroups[1]*Z; + // Remainder modulo 32 + Mrem = M&63; + // Number of groups of 64 CNs for parallel processing + // Ceil for values not divisible by 32 + M64 = (M + 63)>>5; + + // Set pointers to start of group 4 + p_cnProcBuf = (__m512i*) &cnProcBuf [lut_startAddrCnGroups[1]]; + p_cnProcBufRes = (__m512i*) &cnProcBufRes[lut_startAddrCnGroups[1]]; + + // Loop over CNs + for (i=0; i<(M64-1); i++) + { + pcRes = 0; + // Loop over every BN + // Compute PC for 64 CNs at once + for (j=0; j<4; j++) + { + // BN offset is units of 5*384/64 = 30 + zmm0 = p_cnProcBuf [j*30 + i]; + zmm1 = p_cnProcBufRes[j*30 + i]; + + // Add BN and input LLR, extract the sign bit + // and add in GF(2) (xor) + pcRes ^= _mm512_movemask_epi8(_mm512_adds_epi8(zmm0,zmm1)); + } + + // If no error pcRes should be 0 + pcResSum |= pcRes; + } + + // Last 64 CNs might not be full valid 32 depending on Z + pcRes = 0; + // Loop over every BN + // Compute PC for 64 CNs at once + for (j=0; j<4; j++) + { + // BN offset is units of 5*384/64 = 30 + zmm0 = p_cnProcBuf [j*30 + i]; + zmm1 = p_cnProcBufRes[j*30 + i]; + + // Add BN and input LLR, extract the sign bit + // and add in GF(2) (xor) + pcRes ^= _mm512_movemask_epi8(_mm512_adds_epi8(zmm0,zmm1)); + } + + // If no error pcRes should be 0 + // Only use valid CNs + pcResSum |= (pcRes&(0xFFFFFFFF>>(64 - Mrem))); + + // If PC failed we can stop here + if (pcResSum > 0) + { + return pcResSum; + } + } + + // ===================================================================== + // Process group with 5 BNs + + if (lut_numCnInCnGroups[2] > 0) + { + // Reset results + pcResSum = 0; + + // Number of CNs in group + M = lut_numCnInCnGroups[2]*Z; + // Remainder modulo 32 + Mrem = M&63; + // Number of groups of 64 CNs for parallel processing + // Ceil for values not divisible by 32 + M64 = (M + 63)>>5; + + // Set pointers to start of group 5 + p_cnProcBuf = (__m512i*) &cnProcBuf [lut_startAddrCnGroups[2]]; + p_cnProcBufRes = (__m512i*) &cnProcBufRes[lut_startAddrCnGroups[2]]; + + // Loop over CNs + for (i=0; i<(M64-1); i++) + { + pcRes = 0; + // Loop over every BN + // Compute PC for 64 CNs at once + for (j=0; j<5; j++) + { + // BN offset is units of 18*384/64 = 108 + zmm0 = p_cnProcBuf [j*108 + i]; + zmm1 = p_cnProcBufRes[j*108 + i]; + + // Add BN and input LLR, extract the sign bit + // and add in GF(2) (xor) + pcRes ^= _mm512_movemask_epi8(_mm512_adds_epi8(ymm0,ymm1)); + } + + // If no error pcRes should be 0 + pcResSum |= pcRes; + } + + // Last 64 CNs might not be full valid 32 depending on Z + pcRes = 0; + + // Loop over every BN + // Compute PC for 64 CNs at once + for (j=0; j<5; j++) + { + // BN offset is units of 18*384/64 = 108 + zmm0 = p_cnProcBuf [j*108 + i]; + zmm1 = p_cnProcBufRes[j*108 + i]; + + // Add BN and input LLR, extract the sign bit + // and add in GF(2) (xor) + pcRes ^= _mm512_movemask_epi8(_mm512_adds_epi8(zmm0,zmm1)); + } + + // If no error pcRes should be 0 + // Only use valid CNs + pcResSum |= (pcRes&(0xFFFFFFFF>>(64 - Mrem))); + + // If PC failed we can stop here + if (pcResSum > 0) + { + return pcResSum; + } + } + + // ===================================================================== + // Process group with 6 BNs + + if (lut_numCnInCnGroups[3] > 0) + { + // Reset results + pcResSum = 0; + + // Number of CNs in group + M = lut_numCnInCnGroups[3]*Z; + // Remainder modulo 32 + Mrem = M&63; + // Number of groups of 64 CNs for parallel processing + // Ceil for values not divisible by 32 + M64 = (M + 63)>>5; + + // Set pointers to start of group 6 + p_cnProcBuf = (__m512i*) &cnProcBuf [lut_startAddrCnGroups[3]]; + p_cnProcBufRes = (__m512i*) &cnProcBufRes[lut_startAddrCnGroups[3]]; + + // Loop over CNs + for (i=0; i<(M64-1); i++) + { + pcRes = 0; + // Loop over every BN + // Compute PC for 64 CNs at once + for (j=0; j<6; j++) + { + // BN offset is units of 8*384/64 = 48 + zmm0 = p_cnProcBuf [j*48 + i]; + zmm1 = p_cnProcBufRes[j*48 + i]; + + // Add BN and input LLR, extract the sign bit + // and add in GF(2) (xor) + pcRes ^= _mm512_movemask_epi8(_mm512_adds_epi8(zmm0,zmm1)); + } + + // If no error pcRes should be 0 + pcResSum |= pcRes; + } + + // Last 64 CNs might not be full valid 32 depending on Z + pcRes = 0; + // Loop over every BN + // Compute PC for 64 CNs at once + for (j=0; j<6; j++) + { + // BN offset is units of 8*384/64 = 48 + zmm0 = p_cnProcBuf [j*48 + i]; + zmm1 = p_cnProcBufRes[j*48 + i]; + + // Add BN and input LLR, extract the sign bit + // and add in GF(2) (xor) + pcRes ^= _mm512_movemask_epi8(_mm512_adds_epi8(zmm0,zmm1)); + } + + // If no error pcRes should be 0 + // Only use valid CNs + pcResSum |= (pcRes&(0xFFFFFFFF>>(64 - Mrem))); + + // If PC failed we can stop here + if (pcResSum > 0) + { + return pcResSum; + } + } + + // ===================================================================== + // Process group with 7 BNs + + if (lut_numCnInCnGroups[4] > 0) + { + // Reset results + pcResSum = 0; + + // Number of CNs in group + M = lut_numCnInCnGroups[4]*Z; + // Remainder modulo 32 + Mrem = M&63; + // Number of groups of 64 CNs for parallel processing + // Ceil for values not divisible by 32 + M64 = (M + 63)>>5; + + // Set pointers to start of group 7 + p_cnProcBuf = (__m512i*) &cnProcBuf [lut_startAddrCnGroups[4]]; + p_cnProcBufRes = (__m512i*) &cnProcBufRes[lut_startAddrCnGroups[4]]; + + // Loop over CNs + for (i=0; i<(M64-1); i++) + { + pcRes = 0; + // Loop over every BN + // Compute PC for 64 CNs at once + for (j=0; j<7; j++) + { + // BN offset is units of 5*384/64 = 30 + zmm0 = p_cnProcBuf [j*30 + i]; + zmm1 = p_cnProcBufRes[j*30 + i]; + + // Add BN and input LLR, extract the sign bit + // and add in GF(2) (xor) + pcRes ^= _mm512_movemask_epi8(_mm512_adds_epi8(zmm0,zmm1)); + } + + // If no error pcRes should be 0 + pcResSum |= pcRes; + } + + // Last 64 CNs might not be full valid 32 depending on Z + pcRes = 0; + // Loop over every BN + // Compute PC for 64 CNs at once + for (j=0; j<7; j++) + { + // BN offset is units of 5*384/64 = 30 + zmm0 = p_cnProcBuf [j*30 + i]; + zmm1 = p_cnProcBufRes[j*30 + i]; + + // Add BN and input LLR, extract the sign bit + // and add in GF(2) (xor) + pcRes ^= _mm512_movemask_epi8(_mm512_adds_epi8(ymm0,ymm1)); + } + + // If no error pcRes should be 0 + // Only use valid CNs + pcResSum |= (pcRes&(0xFFFFFFFF>>(64 - Mrem))); + + // If PC failed we can stop here + if (pcResSum > 0) + { + return pcResSum; + } + } + + // ===================================================================== + // Process group with 8 BNs + + if (lut_numCnInCnGroups[5] > 0) + { + // Reset results + pcResSum = 0; + + // Number of CNs in group + M = lut_numCnInCnGroups[5]*Z; + // Remainder modulo 32 + Mrem = M&63; + // Number of groups of 64 CNs for parallel processing + // Ceil for values not divisible by 32 + M64 = (M + 63)>>5; + + // Set pointers to start of group 8 + p_cnProcBuf = (__m512i*) &cnProcBuf [lut_startAddrCnGroups[5]]; + p_cnProcBufRes = (__m512i*) &cnProcBufRes[lut_startAddrCnGroups[5]]; + + // Loop over CNs + for (i=0; i<(M64-1); i++) + { + pcRes = 0; + // Loop over every BN + // Compute PC for 64 CNs at once + for (j=0; j<8; j++) + { + // BN offset is units of 2*384/64 = 12 + zmm0 = p_cnProcBuf [j*12 + i]; + zmm1 = p_cnProcBufRes[j*12 + i]; + + // Add BN and input LLR, extract the sign bit + // and add in GF(2) (xor) + pcRes ^= _mm512_movemask_epi8(_mm512_adds_epi8(zmm0,zmm1)); + } + + // If no error pcRes should be 0 + pcResSum |= pcRes; + } + + // Last 64 CNs might not be full valid 32 depending on Z + pcRes = 0; + // Loop over every BN + // Compute PC for 64 CNs at once + for (j=0; j<8; j++) + { + // BN offset is units of 2*384/64 = 12 + zmm0 = p_cnProcBuf [j*12 + i]; + zmm1 = p_cnProcBufRes[j*12 + i]; + + // Add BN and input LLR, extract the sign bit + // and add in GF(2) (xor) + pcRes ^= _mm512_movemask_epi8(_mm512_adds_epi8(zmm0,zmm1)); + } + + // If no error pcRes should be 0 + // Only use valid CNs + pcResSum |= (pcRes&(0xFFFFFFFF>>(64 - Mrem))); + + // If PC failed we can stop here + if (pcResSum > 0) + { + return pcResSum; + } + } + + // ===================================================================== + // Process group with 9 BNs + + if (lut_numCnInCnGroups[6] > 0) + { + // Reset results + pcResSum = 0; + + // Number of CNs in group + M = lut_numCnInCnGroups[6]*Z; + // Remainder modulo 32 + Mrem = M&63; + // Number of groups of 64 CNs for parallel processing + // Ceil for values not divisible by 32 + M64 = (M + 63)>>5; + + // Set pointers to start of group 9 + p_cnProcBuf = (__m512i*) &cnProcBuf [lut_startAddrCnGroups[6]]; + p_cnProcBufRes = (__m512i*) &cnProcBufRes[lut_startAddrCnGroups[6]]; + + // Loop over CNs + for (i=0; i<(M64-1); i++) + { + pcRes = 0; + // Loop over every BN + // Compute PC for 64 CNs at once + for (j=0; j<9; j++) + { + // BN offset is units of 2*384/64 = 12 + zmm0 = p_cnProcBuf [j*12 + i]; + zmm1 = p_cnProcBufRes[j*12 + i]; + + // Add BN and input LLR, extract the sign bit + // and add in GF(2) (xor) + pcRes ^= _mm512_movemask_epi8(_mm512_adds_epi8(ymm0,ymm1)); + } + + // If no error pcRes should be 0 + pcResSum |= pcRes; + } + + // Last 64 CNs might not be full valid 32 depending on Z + pcRes = 0; + // Loop over every BN + // Compute PC for 64 CNs at once + for (j=0; j<9; j++) + { + // BN offset is units of 2*384/64 = 12 + zmm0 = p_cnProcBuf [j*12 + i]; + zmm1 = p_cnProcBufRes[j*12 + i]; + + // Add BN and input LLR, extract the sign bit + // and add in GF(2) (xor) + pcRes ^= _mm512_movemask_epi8(_mm512_adds_epi8(zmm0,zmm1)); + } + + // If no error pcRes should be 0 + // Only use valid CNs + pcResSum |= (pcRes&(0xFFFFFFFF>>(64 - Mrem))); + + // If PC failed we can stop here + if (pcResSum > 0) + { + return pcResSum; + } + } + + // ===================================================================== + // Process group with 10 BNs + + if (lut_numCnInCnGroups[7] > 0) + { + // Reset results + pcResSum = 0; + + // Number of CNs in group + M = lut_numCnInCnGroups[7]*Z; + // Remainder modulo 32 + Mrem = M&63; + // Number of groups of 64 CNs for parallel processing + // Ceil for values not divisible by 32 + M64 = (M + 63)>>5; + + // Set pointers to start of group 10 + p_cnProcBuf = (__m512i*) &cnProcBuf [lut_startAddrCnGroups[7]]; + p_cnProcBufRes = (__m512i*) &cnProcBufRes[lut_startAddrCnGroups[7]]; + + // Loop over CNs + for (i=0; i<(M64-1); i++) + { + pcRes = 0; + // Loop over every BN + // Compute PC for 64 CNs at once + for (j=0; j<10; j++) + { + // BN offset is units of 1*384/64 = 6 + zmm0 = p_cnProcBuf [j*6 + i]; + zmm1 = p_cnProcBufRes[j*6 + i]; + + // Add BN and input LLR, extract the sign bit + // and add in GF(2) (xor) + pcRes ^= _mm512_movemask_epi8(_mm512_adds_epi8(ymm0,ymm1)); + } + + // If no error pcRes should be 0 + pcResSum |= pcRes; + } + + // Last 64 CNs might not be full valid 32 depending on Z + pcRes = 0; + // Loop over every BN + // Compute PC for 64 CNs at once + for (j=0; j<10; j++) + { + // BN offset is units of 1*384/64 = 6 + ymm0 = p_cnProcBuf [j*6 + i]; + ymm1 = p_cnProcBufRes[j*6 + i]; + + // Add BN and input LLR, extract the sign bit + // and add in GF(2) (xor) + pcRes ^= _mm512_movemask_epi8(_mm512_adds_epi8(ymm0,ymm1)); + } + + // If no error pcRes should be 0 + // Only use valid CNs + pcResSum |= (pcRes&(0xFFFFFFFF>>(64 - Mrem))); + + // If PC failed we can stop here + if (pcResSum > 0) + { + return pcResSum; + } + } + + // ===================================================================== + // Process group with 19 BNs + + if (lut_numCnInCnGroups[8] > 0) + { + // Reset results + pcResSum = 0; + + // Number of CNs in group + M = lut_numCnInCnGroups[8]*Z; + // Remainder modulo 32 + Mrem = M&63; + // Number of groups of 64 CNs for parallel processing + // Ceil for values not divisible by 32 + M64 = (M + 63)>>5; + + // Set pointers to start of group 19 + p_cnProcBuf = (__m512i*) &cnProcBuf [lut_startAddrCnGroups[8]]; + p_cnProcBufRes = (__m512i*) &cnProcBufRes[lut_startAddrCnGroups[8]]; + + // Loop over CNs + for (i=0; i<(M64-1); i++) + { + pcRes = 0; + // Loop over every BN (Last BN is connected to multiple CNs) + // Compute PC for 64 CNs at once + for (j=0; j<19; j++) + { + // BN offset is units of 4*384/64 = 24 + zmm0 = p_cnProcBuf [j*24 + i]; + zmm1 = p_cnProcBufRes[j*24 + i]; + + // Add BN and input LLR, extract the sign bit + // and add in GF(2) (xor) + pcRes ^= _mm512_movemask_epi8(_mm512_adds_epi8(zmm0,zmm1)); + } + + // If no error pcRes should be 0 + pcResSum |= pcRes; + } + + // Last 64 CNs might not be full valid 32 depending on Z + pcRes = 0; + // Loop over every BN (Last BN is connected to multiple CNs) + // Compute PC for 64 CNs at once + for (j=0; j<19; j++) + { + // BN offset is units of 4*384/64 = 24 + zmm0 = p_cnProcBuf [j*24 + i]; + zmm1 = p_cnProcBufRes[j*24 + i]; + + // Add BN and input LLR, extract the sign bit + // and add in GF(2) (xor) + pcRes ^= _mm512_movemask_epi8(_mm512_adds_epi8(zmm0,zmm1)); + } + + // If no error pcRes should be 0 + // Only use valid CNs + pcResSum |= (pcRes&(0xFFFFFFFF>>(64 - Mrem))); + + // If PC failed we can stop here + if (pcResSum > 0) + { + return pcResSum; + } + } + + return pcResSum; +} + + + + + +/** + \brief Performs parity check for BG2 on the CN processing buffer. Stops as soon as error is detected. + \param p_lut Pointer to decoder LUTs + \param Z Lifting size + \return 32-bit parity check indicator +*/ + +/*** comment +static inline uint64_t nrLDPC_cnProcPc_BG2(t_nrLDPC_lut* p_lut, t_nrLDPC_procBuf* p_procBuf, uint16_t Z) +{ + const uint8_t* lut_numCnInCnGroups = p_lut->numCnInCnGroups; + const uint64_t* lut_startAddrCnGroups = p_lut->startAddrCnGroups; + + int8_t* cnProcBuf = p_procBuf->cnProcBuf; + int8_t* cnProcBufRes = p_procBuf->cnProcBufRes; + + __m512i* p_cnProcBuf; + __m512i* p_cnProcBufRes; + + // Number of CNs in Groups + uint64_t M; + uint64_t i; + uint64_t j; + uint64_t pcRes = 0; + uint64_t pcResSum = 0; + uint64_t Mrem; + uint64_ ; + + __m512i ymm0, ymm1; + + // ===================================================================== + // Process group with 3 BNs + + if (lut_numCnInCnGroups[0] > 0) + { + // Reset results + pcResSum = 0; + + // Number of CNs in group + M = lut_numCnInCnGroups[0]*Z; + // Remainder modulo 32 + Mrem = M&63; + // Number of groups of 64 CNs for parallel processing + // Ceil for values not divisible by 32 + M64 = (M + 63)>>5; + + // Set pointers to start of group 3 + p_cnProcBuf = (__m512i*) &cnProcBuf [lut_startAddrCnGroups[0]]; + p_cnProcBufRes = (__m512i*) &cnProcBufRes[lut_startAddrCnGroups[0]]; + + // Loop over CNs + for (i=0; i<(M32-1); i++) + { + pcRes = 0; + // Loop over every BN + // Compute PC for 64 CNs at once + for (j=0; j<3; j++) + { + // BN offset is units of (6*384/32) = 72 + ymm0 = p_cnProcBuf [j*72 + i]; + ymm1 = p_cnProcBufRes[j*72 + i]; + + // Add BN and input LLR, extract the sign bit + // and add in GF(2) (xor) + pcRes ^= _mm512_movemask_epi8(_mm512_adds_epi8(ymm0,ymm1)); + } + + // If no error pcRes should be 0 + pcResSum |= pcRes; + } + + // Last 64 CNs might not be full valid 32 depending on Z + pcRes = 0; + // Loop over every BN + // Compute PC for 64 CNs at once + for (j=0; j<3; j++) + { + // BN offset is units of (6*384/32) = 72 + ymm0 = p_cnProcBuf [j*72 + i]; + ymm1 = p_cnProcBufRes[j*72 + i]; + + // Add BN and input LLR, extract the sign bit + // and add in GF(2) (xor) + pcRes ^= _mm512_movemask_epi8(_mm512_adds_epi8(ymm0,ymm1)); + } + + // If no error pcRes should be 0 + // Only use valid CNs + pcResSum |= (pcRes&(0xFFFFFFFF>>(64 - Mrem))); + + // If PC failed we can stop here + if (pcResSum > 0) + { + return pcResSum; + } + } + + // ===================================================================== + // Process group with 4 BNs + + if (lut_numCnInCnGroups[1] > 0) + { + // Reset results + pcResSum = 0; + + // Number of CNs in group + M = lut_numCnInCnGroups[1]*Z; + // Remainder modulo 32 + Mrem = M&63; + // Number of groups of 64 CNs for parallel processing + // Ceil for values not divisible by 32 + M64 = (M + 63)>>5; + + // Set pointers to start of group 4 + p_cnProcBuf = (__m512i*) &cnProcBuf [lut_startAddrCnGroups[1]]; + p_cnProcBufRes = (__m512i*) &cnProcBufRes[lut_startAddrCnGroups[1]]; + + // Loop over CNs + for (i=0; i<(M32-1); i++) + { + pcRes = 0; + // Loop over every BN + // Compute PC for 64 CNs at once + for (j=0; j<4; j++) + { + // BN offset is units of 20*384/32 = 240 + ymm0 = p_cnProcBuf [j*240 + i]; + ymm1 = p_cnProcBufRes[j*240 + i]; + + // Add BN and input LLR, extract the sign bit + // and add in GF(2) (xor) + pcRes ^= _mm512_movemask_epi8(_mm512_adds_epi8(ymm0,ymm1)); + } + + // If no error pcRes should be 0 + pcResSum |= pcRes; + } + + // Last 64 CNs might not be full valid 32 depending on Z + pcRes = 0; + // Loop over every BN + // Compute PC for 64 CNs at once + for (j=0; j<4; j++) + { + // BN offset is units of 20*384/32 = 240 + ymm0 = p_cnProcBuf [j*240 + i]; + ymm1 = p_cnProcBufRes[j*240 + i]; + + // Add BN and input LLR, extract the sign bit + // and add in GF(2) (xor) + pcRes ^= _mm512_movemask_epi8(_mm512_adds_epi8(ymm0,ymm1)); + } + + // If no error pcRes should be 0 + // Only use valid CNs + pcResSum |= (pcRes&(0xFFFFFFFF>>(64 - Mrem))); + + // If PC failed we can stop here + if (pcResSum > 0) + { + return pcResSum; + } + } + + // ===================================================================== + // Process group with 5 BNs + + if (lut_numCnInCnGroups[2] > 0) + { + // Reset results + pcResSum = 0; + + // Number of CNs in group + M = lut_numCnInCnGroups[2]*Z; + // Remainder modulo 32 + Mrem = M&63; + // Number of groups of 64 CNs for parallel processing + // Ceil for values not divisible by 32 + M64 = (M + 63)>>5; + + // Set pointers to start of group 5 + p_cnProcBuf = (__m512i*) &cnProcBuf [lut_startAddrCnGroups[2]]; + p_cnProcBufRes = (__m512i*) &cnProcBufRes[lut_startAddrCnGroups[2]]; + + // Loop over CNs + for (i=0; i<(M32-1); i++) + { + pcRes = 0; + // Loop over every BN + // Compute PC for 64 CNs at once + for (j=0; j<5; j++) + { + // BN offset is units of 9*384/32 = 108 + ymm0 = p_cnProcBuf [j*108 + i]; + ymm1 = p_cnProcBufRes[j*108 + i]; + + // Add BN and input LLR, extract the sign bit + // and add in GF(2) (xor) + pcRes ^= _mm512_movemask_epi8(_mm512_adds_epi8(ymm0,ymm1)); + } + + // If no error pcRes should be 0 + pcResSum |= pcRes; + } + + // Last 64 CNs might not be full valid 32 depending on Z + pcRes = 0; + // Loop over every BN + // Compute PC for 64 CNs at once + for (j=0; j<5; j++) + { + // BN offset is units of 9*384/32 = 108 + ymm0 = p_cnProcBuf [j*108 + i]; + ymm1 = p_cnProcBufRes[j*108 + i]; + + // Add BN and input LLR, extract the sign bit + // and add in GF(2) (xor) + pcRes ^= _mm512_movemask_epi8(_mm512_adds_epi8(ymm0,ymm1)); + } + + // If no error pcRes should be 0 + // Only use valid CNs + pcResSum |= (pcRes&(0xFFFFFFFF>>(64 - Mrem))); + + // If PC failed we can stop here + if (pcResSum > 0) + { + return pcResSum; + } + } + + // ===================================================================== + // Process group with 6 BNs + + if (lut_numCnInCnGroups[3] > 0) + { + // Reset results + pcResSum = 0; + + // Number of CNs in group + M = lut_numCnInCnGroups[3]*Z; + // Remainder modulo 32 + Mrem = M&63; + // Number of groups of 64 CNs for parallel processing + // Ceil for values not divisible by 32 + M64 = (M + 63)>>5; + + // Set pointers to start of group 6 + p_cnProcBuf = (__m512i*) &cnProcBuf [lut_startAddrCnGroups[3]]; + p_cnProcBufRes = (__m512i*) &cnProcBufRes[lut_startAddrCnGroups[3]]; + + // Loop over CNs + for (i=0; i<(M32-1); i++) + { + pcRes = 0; + // Loop over every BN + // Compute PC for 64 CNs at once + for (j=0; j<6; j++) + { + // BN offset is units of 3*384/32 = 36 + ymm0 = p_cnProcBuf [j*36 + i]; + ymm1 = p_cnProcBufRes[j*36 + i]; + + // Add BN and input LLR, extract the sign bit + // and add in GF(2) (xor) + pcRes ^= _mm512_movemask_epi8(_mm512_adds_epi8(ymm0,ymm1)); + } + + // If no error pcRes should be 0 + pcResSum |= pcRes; + } + + // Last 64 CNs might not be full valid 32 depending on Z + pcRes = 0; + // Loop over every BN + // Compute PC for 64 CNs at once + for (j=0; j<6; j++) + { + // BN offset is units of 3*384/32 = 36 + ymm0 = p_cnProcBuf [j*36 + i]; + ymm1 = p_cnProcBufRes[j*36 + i]; + + // Add BN and input LLR, extract the sign bit + // and add in GF(2) (xor) + pcRes ^= _mm512_movemask_epi8(_mm512_adds_epi8(ymm0,ymm1)); + } + + // If no error pcRes should be 0 + // Only use valid CNs + pcResSum |= (pcRes&(0xFFFFFFFF>>(64 - Mrem))); + + // If PC failed we can stop here + if (pcResSum > 0) + { + return pcResSum; + } + } + + // ===================================================================== + // Process group with 8 BNs + + if (lut_numCnInCnGroups[4] > 0) + { + // Reset results + pcResSum = 0; + + // Number of CNs in group + M = lut_numCnInCnGroups[4]*Z; + // Remainder modulo 32 + Mrem = M&63; + // Number of groups of 64 CNs for parallel processing + // Ceil for values not divisible by 32 + M64 = (M + 63)>>5; + + // Set pointers to start of group 8 + p_cnProcBuf = (__m512i*) &cnProcBuf [lut_startAddrCnGroups[4]]; + p_cnProcBufRes = (__m512i*) &cnProcBufRes[lut_startAddrCnGroups[4]]; + + // Loop over CNs + for (i=0; i<(M32-1); i++) + { + pcRes = 0; + // Loop over every BN + // Compute PC for 64 CNs at once + for (j=0; j<8; j++) + { + // BN offset is units of 2*384/32 = 24 + ymm0 = p_cnProcBuf [j*24 + i]; + ymm1 = p_cnProcBufRes[j*24 + i]; + + // Add BN and input LLR, extract the sign bit + // and add in GF(2) (xor) + pcRes ^= _mm512_movemask_epi8(_mm512_adds_epi8(ymm0,ymm1)); + } + + // If no error pcRes should be 0 + pcResSum |= pcRes; + } + + // Last 64 CNs might not be full valid 32 depending on Z + pcRes = 0; + // Loop over every BN + // Compute PC for 64 CNs at once + for (j=0; j<8; j++) + { + // BN offset is units of 2*384/32 = 24 + ymm0 = p_cnProcBuf [j*24 + i]; + ymm1 = p_cnProcBufRes[j*24 + i]; + + // Add BN and input LLR, extract the sign bit + // and add in GF(2) (xor) + pcRes ^= _mm512_movemask_epi8(_mm512_adds_epi8(ymm0,ymm1)); + } + + // If no error pcRes should be 0 + // Only use valid CNs + pcResSum |= (pcRes&(0xFFFFFFFF>>(64 - Mrem))); + + // If PC failed we can stop here + if (pcResSum > 0) + { + return pcResSum; + } + } + + // ===================================================================== + // Process group with 10 BNs + + if (lut_numCnInCnGroups[5] > 0) + { + // Reset results + pcResSum = 0; + + // Number of CNs in group + M = lut_numCnInCnGroups[5]*Z; + // Remainder modulo 32 + Mrem = M&63; + // Number of groups of 64 CNs for parallel processing + // Ceil for values not divisible by 32 + M64 = (M + 63)>>5; + + // Set pointers to start of group 10 + p_cnProcBuf = (__m512i*) &cnProcBuf [lut_startAddrCnGroups[5]]; + p_cnProcBufRes = (__m512i*) &cnProcBufRes[lut_startAddrCnGroups[5]]; + + // Loop over CNs + for (i=0; i<(M32-1); i++) + { + pcRes = 0; + // Loop over every BN + // Compute PC for 64 CNs at once + for (j=0; j<10; j++) + { + // BN offset is units of 2*384/32 = 24 + ymm0 = p_cnProcBuf [j*24 + i]; + ymm1 = p_cnProcBufRes[j*24 + i]; + + // Add BN and input LLR, extract the sign bit + // and add in GF(2) (xor) + pcRes ^= _mm512_movemask_epi8(_mm512_adds_epi8(ymm0,ymm1)); + } + + // If no error pcRes should be 0 + pcResSum |= pcRes; + } + + // Last 64 CNs might not be full valid 32 depending on Z + pcRes = 0; + // Loop over every BN + // Compute PC for 64 CNs at once + for (j=0; j<10; j++) + { + // BN offset is units of 2*384/32 = 24 + ymm0 = p_cnProcBuf [j*24 + i]; + ymm1 = p_cnProcBufRes[j*24 + i]; + + // Add BN and input LLR, extract the sign bit + // and add in GF(2) (xor) + pcRes ^= _mm512_movemask_epi8(_mm512_adds_epi8(ymm0,ymm1)); + } + + // If no error pcRes should be 0 + // Only use valid CNs + pcResSum |= (pcRes&(0xFFFFFFFF>>(64 - Mrem))); + + // If PC failed we can stop here + if (pcResSum > 0) + { + return pcResSum; + } + } + + return pcResSum; +} +*/ +#endif + diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/nrLDPC_decoder.c b/openair1/PHY/CODING/nrLDPC_decoder_full/nrLDPC_decoder.c new file mode 100644 index 0000000000..9ff57acf22 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/nrLDPC_decoder.c @@ -0,0 +1,496 @@ +/* + * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The OpenAirInterface Software Alliance licenses this file to You under + * the OAI Public License, Version 1.1 (the "License"); you may not use this file + * except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.openairinterface.org/?page_id=698 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *------------------------------------------------------------------------------- + * For more information about the OpenAirInterface (OAI) Software Alliance: + * contact@openairinterface.org + */ + +/*!\file nrLDPC_decoder.c + * \brief Defines the LDPC decoder + * \author Sebastian Wagner (TCL Communications) Email: + * \date 30-09-2019 + * \version 2.0 + * \note + * \warning + */ + +#include +//#include +#include "nrLDPCdecoder_defs.h" +#include "nrLDPC_types.h" +#include "nrLDPC_init.h" +#include "nrLDPC_mPass.h" +#include "nrLDPC_cnProc.h" +#include "nrLDPC_bnProc.h" + +//#include "include/immintrin.h" +#include "include/avx512fintrin.h" +#include "include/avx2intrin.h" + +#define NR_LDPC_ENABLE_PARITY_CHECK +#define NR_LDPC_PROFILER_DETAIL + +#ifdef NR_LDPC_DEBUG_MODE +#include "nrLDPC_tools/nrLDPC_debug.h" +#endif + +static inline uint32_t nrLDPC_decoder_core(int8_t* p_llr, int8_t* p_out, t_nrLDPC_procBuf* p_procBuf, uint32_t numLLR, t_nrLDPC_lut* p_lut, t_nrLDPC_dec_params* p_decParams, t_nrLDPC_time_stats* p_profiler); + +int32_t nrLDPC_decod(t_nrLDPC_dec_params* p_decParams, int8_t* p_llr, int8_t* p_out, t_nrLDPC_procBuf* p_procBuf, t_nrLDPC_time_stats* p_profiler) +{ + uint32_t numLLR; + uint32_t numIter = 0; + t_nrLDPC_lut lut; + t_nrLDPC_lut* p_lut = &lut; + + //printf("p_procBuf->cnProcBuf = %p\n", p_procBuf->cnProcBuf); + + // Initialize decoder core(s) with correct LUTs + numLLR = nrLDPC_init(p_decParams, p_lut); + + // Launch LDPC decoder core for one segment + numIter = nrLDPC_decoder_core(p_llr, p_out, p_procBuf, numLLR, p_lut, p_decParams, p_profiler); + + return numIter; +} + +/** + \brief Performs LDPC decoding of one code block + \param p_llr Input LLRs + \param p_out Output vector + \param numLLR Number of LLRs + \param p_lut Pointer to decoder LUTs + \param p_decParams LDPC decoder parameters + \param p_profiler LDPC profiler statistics +*/ +static inline uint32_t nrLDPC_decoder_core(int8_t* p_llr, int8_t* p_out, t_nrLDPC_procBuf* p_procBuf, uint32_t numLLR, t_nrLDPC_lut* p_lut, t_nrLDPC_dec_params* p_decParams, t_nrLDPC_time_stats* p_profiler) +{ + uint16_t Z = p_decParams->Z; + uint8_t BG = p_decParams->BG; + uint8_t numMaxIter = p_decParams->numMaxIter; + e_nrLDPC_outMode outMode = p_decParams->outMode; + + // Minimum number of iterations is 1 + // 0 iterations means hard-decision on input LLRs + uint32_t i = 1; + // Initialize with parity check fail != 0 + int32_t pcRes = 1; + int8_t* p_llrOut; + + if (outMode == nrLDPC_outMode_LLRINT8) + { + p_llrOut = p_out; + } + else + { + // Use LLR processing buffer as temporary output buffer + p_llrOut = p_procBuf->llrProcBuf; + // Clear llrProcBuf + memset(p_llrOut,0, NR_LDPC_MAX_NUM_LLR*sizeof(int8_t)); + } + + + // Initialization +#ifdef NR_LDPC_PROFILER_DETAIL + start_meas(&p_profiler->llr2llrProcBuf); +#endif + nrLDPC_llr2llrProcBuf(p_lut, p_llr, p_procBuf, Z, BG); +#ifdef NR_LDPC_PROFILER_DETAIL + stop_meas(&p_profiler->llr2llrProcBuf); +#endif + +#ifdef NR_LDPC_DEBUG_MODE + nrLDPC_debug_initBuffer2File(nrLDPC_buffers_LLR_PROC); + nrLDPC_debug_writeBuffer2File(nrLDPC_buffers_LLR_PROC, p_procBuf); +#endif + +#ifdef NR_LDPC_PROFILER_DETAIL + start_meas(&p_profiler->llr2CnProcBuf); +#endif + if (BG == 1) + { + nrLDPC_llr2CnProcBuf_BG1(p_lut, p_llr, p_procBuf, Z); + } + else + { + nrLDPC_llr2CnProcBuf_BG2(p_lut, p_llr, p_procBuf, Z); + } +#ifdef NR_LDPC_PROFILER_DETAIL + stop_meas(&p_profiler->llr2CnProcBuf); +#endif + +#ifdef NR_LDPC_DEBUG_MODE + nrLDPC_debug_initBuffer2File(nrLDPC_buffers_CN_PROC); + nrLDPC_debug_writeBuffer2File(nrLDPC_buffers_CN_PROC, p_procBuf); +#endif + + // First iteration + + // CN processing +#ifdef NR_LDPC_PROFILER_DETAIL + start_meas(&p_profiler->cnProc); +#endif + if (BG == 1) + { + nrLDPC_cnProc_BG1(p_lut, p_procBuf, Z); + } + else + { + nrLDPC_cnProc_BG2(p_lut, p_procBuf, Z); + } +#ifdef NR_LDPC_PROFILER_DETAIL + stop_meas(&p_profiler->cnProc); +#endif + +#ifdef NR_LDPC_DEBUG_MODE + nrLDPC_debug_initBuffer2File(nrLDPC_buffers_CN_PROC_RES); + nrLDPC_debug_writeBuffer2File(nrLDPC_buffers_CN_PROC_RES, p_procBuf); +#endif + +#ifdef NR_LDPC_PROFILER_DETAIL + start_meas(&p_profiler->cn2bnProcBuf); +#endif + if (BG == 1) + { + nrLDPC_cn2bnProcBuf_BG1(p_lut, p_procBuf, Z); + } + else + { + nrLDPC_cn2bnProcBuf_BG2(p_lut, p_procBuf, Z); + } +#ifdef NR_LDPC_PROFILER_DETAIL + stop_meas(&p_profiler->cn2bnProcBuf); +#endif + +#ifdef NR_LDPC_DEBUG_MODE + nrLDPC_debug_initBuffer2File(nrLDPC_buffers_BN_PROC); + nrLDPC_debug_writeBuffer2File(nrLDPC_buffers_BN_PROC, p_procBuf); +#endif + + // BN processing +#ifdef NR_LDPC_PROFILER_DETAIL + start_meas(&p_profiler->bnProcPc); +#endif + nrLDPC_bnProcPc(p_lut, p_procBuf, Z); +#ifdef NR_LDPC_PROFILER_DETAIL + stop_meas(&p_profiler->bnProcPc); +#endif + +#ifdef NR_LDPC_DEBUG_MODE + nrLDPC_debug_initBuffer2File(nrLDPC_buffers_LLR_RES); + nrLDPC_debug_writeBuffer2File(nrLDPC_buffers_LLR_RES, p_procBuf); +#endif + +#ifdef NR_LDPC_PROFILER_DETAIL + start_meas(&p_profiler->bnProc); +#endif + nrLDPC_bnProc(p_lut, p_procBuf, Z); +#ifdef NR_LDPC_PROFILER_DETAIL + stop_meas(&p_profiler->bnProc); +#endif + +#ifdef NR_LDPC_DEBUG_MODE + nrLDPC_debug_initBuffer2File(nrLDPC_buffers_BN_PROC_RES); + nrLDPC_debug_writeBuffer2File(nrLDPC_buffers_BN_PROC_RES, p_procBuf); +#endif + + // BN results to CN processing buffer +#ifdef NR_LDPC_PROFILER_DETAIL + start_meas(&p_profiler->bn2cnProcBuf); +#endif + if (BG == 1) + { + nrLDPC_bn2cnProcBuf_BG1(p_lut, p_procBuf, Z); + } + else + { + nrLDPC_bn2cnProcBuf_BG2(p_lut, p_procBuf, Z); + } +#ifdef NR_LDPC_PROFILER_DETAIL + stop_meas(&p_profiler->bn2cnProcBuf); +#endif + +#ifdef NR_LDPC_DEBUG_MODE + nrLDPC_debug_writeBuffer2File(nrLDPC_buffers_CN_PROC, p_procBuf); +#endif + + // Parity Check not necessary here since it will fail + // because first 2 cols/BNs in BG are punctured and cannot be + // estimated after only one iteration + + // First iteration finished + + while ( (i < (numMaxIter-1)) && (pcRes != 0) ) + { + // Increase iteration counter + i++; + + // CN processing +#ifdef NR_LDPC_PROFILER_DETAIL + start_meas(&p_profiler->cnProc); +#endif + if (BG == 1) + { + nrLDPC_cnProc_BG1(p_lut, p_procBuf, Z); + } + else + { + nrLDPC_cnProc_BG2(p_lut, p_procBuf, Z); + } +#ifdef NR_LDPC_PROFILER_DETAIL + stop_meas(&p_profiler->cnProc); +#endif + +#ifdef NR_LDPC_DEBUG_MODE + nrLDPC_debug_writeBuffer2File(nrLDPC_buffers_CN_PROC_RES, p_procBuf); +#endif + + // Send CN results back to BNs +#ifdef NR_LDPC_PROFILER_DETAIL + start_meas(&p_profiler->cn2bnProcBuf); +#endif + if (BG == 1) + { + nrLDPC_cn2bnProcBuf_BG1(p_lut, p_procBuf, Z); + } + else + { + nrLDPC_cn2bnProcBuf_BG2(p_lut, p_procBuf, Z); + } +#ifdef NR_LDPC_PROFILER_DETAIL + stop_meas(&p_profiler->cn2bnProcBuf); +#endif + +#ifdef NR_LDPC_DEBUG_MODE + nrLDPC_debug_writeBuffer2File(nrLDPC_buffers_BN_PROC, p_procBuf); +#endif + + // BN Processing +#ifdef NR_LDPC_PROFILER_DETAIL + start_meas(&p_profiler->bnProcPc); +#endif + nrLDPC_bnProcPc(p_lut, p_procBuf, Z); +#ifdef NR_LDPC_PROFILER_DETAIL + stop_meas(&p_profiler->bnProcPc); +#endif + +#ifdef NR_LDPC_DEBUG_MODE + nrLDPC_debug_writeBuffer2File(nrLDPC_buffers_LLR_RES, p_procBuf); +#endif + +#ifdef NR_LDPC_PROFILER_DETAIL + start_meas(&p_profiler->bnProc); +#endif + nrLDPC_bnProc(p_lut, p_procBuf, Z); +#ifdef NR_LDPC_PROFILER_DETAIL + stop_meas(&p_profiler->bnProc); +#endif + +#ifdef NR_LDPC_DEBUG_MODE + nrLDPC_debug_writeBuffer2File(nrLDPC_buffers_BN_PROC_RES, p_procBuf); +#endif + + // BN results to CN processing buffer +#ifdef NR_LDPC_PROFILER_DETAIL + start_meas(&p_profiler->bn2cnProcBuf); +#endif + if (BG == 1) + { + nrLDPC_bn2cnProcBuf_BG1(p_lut, p_procBuf, Z); + } + else + { + nrLDPC_bn2cnProcBuf_BG2(p_lut, p_procBuf, Z); + } +#ifdef NR_LDPC_PROFILER_DETAIL + stop_meas(&p_profiler->bn2cnProcBuf); +#endif + +#ifdef NR_LDPC_DEBUG_MODE + nrLDPC_debug_writeBuffer2File(nrLDPC_buffers_CN_PROC, p_procBuf); +#endif + + // Parity Check +#ifdef NR_LDPC_ENABLE_PARITY_CHECK +#ifdef NR_LDPC_PROFILER_DETAIL + start_meas(&p_profiler->cnProcPc); +#endif + if (BG == 1) + { + pcRes = nrLDPC_cnProcPc_BG1(p_lut, p_procBuf, Z); + } + else + { + pcRes = nrLDPC_cnProcPc_BG2(p_lut, p_procBuf, Z); + } +#ifdef NR_LDPC_PROFILER_DETAIL + stop_meas(&p_profiler->cnProcPc); +#endif +#endif + } + + // Last iteration + if ( (i < numMaxIter) && (pcRes != 0) ) + { + // Increase iteration counter + i++; + + // CN processing +#ifdef NR_LDPC_PROFILER_DETAIL + start_meas(&p_profiler->cnProc); +#endif + if (BG == 1) + { + nrLDPC_cnProc_BG1(p_lut, p_procBuf, Z); + } + else + { + nrLDPC_cnProc_BG2(p_lut, p_procBuf, Z); + } +#ifdef NR_LDPC_PROFILER_DETAIL + stop_meas(&p_profiler->cnProc); +#endif + +#ifdef NR_LDPC_DEBUG_MODE + nrLDPC_debug_writeBuffer2File(nrLDPC_buffers_CN_PROC_RES, p_procBuf); +#endif + + // Send CN results back to BNs +#ifdef NR_LDPC_PROFILER_DETAIL + start_meas(&p_profiler->cn2bnProcBuf); +#endif + if (BG == 1) + { + nrLDPC_cn2bnProcBuf_BG1(p_lut, p_procBuf, Z); + } + else + { + nrLDPC_cn2bnProcBuf_BG2(p_lut, p_procBuf, Z); + } +#ifdef NR_LDPC_PROFILER_DETAIL + stop_meas(&p_profiler->cn2bnProcBuf); +#endif + +#ifdef NR_LDPC_DEBUG_MODE + nrLDPC_debug_writeBuffer2File(nrLDPC_buffers_BN_PROC, p_procBuf); +#endif + + // BN Processing +#ifdef NR_LDPC_PROFILER_DETAIL + start_meas(&p_profiler->bnProcPc); +#endif + nrLDPC_bnProcPc(p_lut, p_procBuf, Z); +#ifdef NR_LDPC_PROFILER_DETAIL + stop_meas(&p_profiler->bnProcPc); +#endif + +#ifdef NR_LDPC_DEBUG_MODE + nrLDPC_debug_writeBuffer2File(nrLDPC_buffers_LLR_RES, p_procBuf); +#endif + + // If parity check not enabled, no need to send the BN proc results + // back to CNs +#ifdef NR_LDPC_ENABLE_PARITY_CHECK +#ifdef NR_LDPC_PROFILER_DETAIL + start_meas(&p_profiler->bnProc); +#endif + nrLDPC_bnProc(p_lut, p_procBuf, Z); +#ifdef NR_LDPC_PROFILER_DETAIL + stop_meas(&p_profiler->bnProc); +#endif + +#ifdef NR_LDPC_DEBUG_MODE + nrLDPC_debug_writeBuffer2File(nrLDPC_buffers_BN_PROC_RES, p_procBuf); +#endif + + // BN results to CN processing buffer +#ifdef NR_LDPC_PROFILER_DETAIL + start_meas(&p_profiler->bn2cnProcBuf); +#endif + if (BG == 1) + { + nrLDPC_bn2cnProcBuf_BG1(p_lut, p_procBuf, Z); + } + else + { + nrLDPC_bn2cnProcBuf_BG2(p_lut, p_procBuf, Z); + } +#ifdef NR_LDPC_PROFILER_DETAIL + stop_meas(&p_profiler->bn2cnProcBuf); +#endif + +#ifdef NR_LDPC_DEBUG_MODE + nrLDPC_debug_writeBuffer2File(nrLDPC_buffers_CN_PROC, p_procBuf); +#endif + + // Parity Check +#ifdef NR_LDPC_PROFILER_DETAIL + start_meas(&p_profiler->cnProcPc); +#endif + if (BG == 1) + { + pcRes = nrLDPC_cnProcPc_BG1(p_lut, p_procBuf, Z); + } + else + { + pcRes = nrLDPC_cnProcPc_BG2(p_lut, p_procBuf, Z); + } +#ifdef NR_LDPC_PROFILER_DETAIL + stop_meas(&p_profiler->cnProcPc); +#endif +#endif + } + + // If maximum number of iterations reached an PC still fails increase number of iterations + // Thus, i > numMaxIter indicates that PC has failed + +#ifdef NR_LDPC_ENABLE_PARITY_CHECK + if (pcRes != 0) + { + i++; + } +#endif + + // Assign results from processing buffer to output +#ifdef NR_LDPC_PROFILER_DETAIL + start_meas(&p_profiler->llrRes2llrOut); +#endif + nrLDPC_llrRes2llrOut(p_lut, p_llrOut, p_procBuf, Z, BG); +#ifdef NR_LDPC_PROFILER_DETAIL + stop_meas(&p_profiler->llrRes2llrOut); +#endif + + // Hard-decision +#ifdef NR_LDPC_PROFILER_DETAIL + start_meas(&p_profiler->llr2bit); +#endif + if (outMode == nrLDPC_outMode_BIT) + { + nrLDPC_llr2bitPacked(p_out, p_llrOut, numLLR); + } + else if (outMode == nrLDPC_outMode_BITINT8) + { + nrLDPC_llr2bit(p_out, p_llrOut, numLLR); + } + +#ifdef NR_LDPC_PROFILER_DETAIL + stop_meas(&p_profiler->llr2bit); +#endif + + return i; +} + diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/nrLDPC_init.h b/openair1/PHY/CODING/nrLDPC_decoder_full/nrLDPC_init.h new file mode 100644 index 0000000000..ea031dc2da --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/nrLDPC_init.h @@ -0,0 +1,1640 @@ +/* + * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The OpenAirInterface Software Alliance licenses this file to You under + * the OAI Public License, Version 1.1 (the "License"); you may not use this file + * except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.openairinterface.org/?page_id=698 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *------------------------------------------------------------------------------- + * For more information about the OpenAirInterface (OAI) Software Alliance: + * contact@openairinterface.org + */ + +/*!\file nrLDPC_init.h + * \brief Defines the function to initialize the LDPC decoder and sets correct LUTs. + * \author Sebastian Wagner (TCL Communications) Email: + * \date 30-09-2019 + * \version 1.0 + * \note + * \warning + */ + +#ifndef __NR_LDPC_INIT__H__ +#define __NR_LDPC_INIT__H__ + +#include "nrLDPC_lut.h" +#include "nrLDPCdecoder_defs.h" + +/** + \brief Initializes the decoder and sets correct LUTs + \param p_decParams Pointer to decoder parameters + \param p_lut Pointer to decoder LUTs + \return Number of LLR values +*/ +static inline uint32_t nrLDPC_init(t_nrLDPC_dec_params* p_decParams, t_nrLDPC_lut* p_lut) +{ + uint32_t numLLR = 0; + uint8_t BG = p_decParams->BG; + uint16_t Z = p_decParams->Z; + uint8_t R = p_decParams->R; + + if (BG == 2) + { + // LUT that only depend on BG + p_lut->startAddrCnGroups = lut_startAddrCnGroups_BG2; + + p_lut->posBnInCnProcBuf[0] = (const uint8_t**) posBnInCnProcBuf_BG2_CNG3; + p_lut->posBnInCnProcBuf[1] = (const uint8_t**) posBnInCnProcBuf_BG2_CNG4; + p_lut->posBnInCnProcBuf[2] = (const uint8_t**) posBnInCnProcBuf_BG2_CNG5; + p_lut->posBnInCnProcBuf[3] = (const uint8_t**) posBnInCnProcBuf_BG2_CNG6; + p_lut->posBnInCnProcBuf[4] = (const uint8_t**) posBnInCnProcBuf_BG2_CNG8; + p_lut->posBnInCnProcBuf[5] = (const uint8_t**) posBnInCnProcBuf_BG2_CNG10; + p_lut->posBnInCnProcBuf[6] = NULL; + p_lut->posBnInCnProcBuf[7] = NULL; + p_lut->posBnInCnProcBuf[8] = NULL; + + // LUT that only depend on R + if (R == 15) + { + p_lut->startAddrBnProcBuf[0] = (const uint32_t**) startAddrBnProcBuf_BG2_R15_CNG3; + p_lut->startAddrBnProcBuf[1] = (const uint32_t**) startAddrBnProcBuf_BG2_R15_CNG4; + p_lut->startAddrBnProcBuf[2] = (const uint32_t**) startAddrBnProcBuf_BG2_R15_CNG5; + p_lut->startAddrBnProcBuf[3] = (const uint32_t**) startAddrBnProcBuf_BG2_R15_CNG6; + p_lut->startAddrBnProcBuf[4] = (const uint32_t**) startAddrBnProcBuf_BG2_R15_CNG8; + p_lut->startAddrBnProcBuf[5] = (const uint32_t**) startAddrBnProcBuf_BG2_R15_CNG10; + p_lut->startAddrBnProcBuf[6] = NULL; + p_lut->startAddrBnProcBuf[7] = NULL; + p_lut->startAddrBnProcBuf[8] = NULL; + + p_lut->bnPosBnProcBuf[0] = (const uint8_t**) bnPosBnProcBuf_BG2_R15_CNG3; + p_lut->bnPosBnProcBuf[1] = (const uint8_t**) bnPosBnProcBuf_BG2_R15_CNG4; + p_lut->bnPosBnProcBuf[2] = (const uint8_t**) bnPosBnProcBuf_BG2_R15_CNG5; + p_lut->bnPosBnProcBuf[3] = (const uint8_t**) bnPosBnProcBuf_BG2_R15_CNG6; + p_lut->bnPosBnProcBuf[4] = (const uint8_t**) bnPosBnProcBuf_BG2_R15_CNG8; + p_lut->bnPosBnProcBuf[5] = (const uint8_t**) bnPosBnProcBuf_BG2_R15_CNG10; + p_lut->bnPosBnProcBuf[6] = NULL; + p_lut->bnPosBnProcBuf[7] = NULL; + p_lut->bnPosBnProcBuf[8] = NULL; + + p_lut->llr2llrProcBufAddr = llr2llrProcBufAddr_BG2_R15; + p_lut->llr2llrProcBufBnPos = llr2llrProcBufBnPos_BG2_R15; + + p_lut->numCnInCnGroups = lut_numCnInCnGroups_BG2_R15; + p_lut->numBnInBnGroups = lut_numBnInBnGroups_BG2_R15; + p_lut->startAddrBnGroups = lut_startAddrBnGroups_BG2_R15; + p_lut->startAddrBnGroupsLlr = lut_startAddrBnGroupsLlr_BG2_R15; + + numLLR = NR_LDPC_NCOL_BG2_R15*Z; + } + else if (R == 13) + { + p_lut->startAddrBnProcBuf[0] = NULL; + p_lut->startAddrBnProcBuf[1] = (const uint32_t**) startAddrBnProcBuf_BG2_R13_CNG4; + p_lut->startAddrBnProcBuf[2] = (const uint32_t**) startAddrBnProcBuf_BG2_R13_CNG5; + p_lut->startAddrBnProcBuf[3] = (const uint32_t**) startAddrBnProcBuf_BG2_R13_CNG6; + p_lut->startAddrBnProcBuf[4] = (const uint32_t**) startAddrBnProcBuf_BG2_R13_CNG8; + p_lut->startAddrBnProcBuf[5] = (const uint32_t**) startAddrBnProcBuf_BG2_R13_CNG10; + p_lut->startAddrBnProcBuf[6] = NULL; + p_lut->startAddrBnProcBuf[7] = NULL; + p_lut->startAddrBnProcBuf[8] = NULL; + + p_lut->bnPosBnProcBuf[0] = NULL; + p_lut->bnPosBnProcBuf[1] = (const uint8_t**) bnPosBnProcBuf_BG2_R13_CNG4; + p_lut->bnPosBnProcBuf[2] = (const uint8_t**) bnPosBnProcBuf_BG2_R13_CNG5; + p_lut->bnPosBnProcBuf[3] = (const uint8_t**) bnPosBnProcBuf_BG2_R13_CNG6; + p_lut->bnPosBnProcBuf[4] = (const uint8_t**) bnPosBnProcBuf_BG2_R13_CNG8; + p_lut->bnPosBnProcBuf[5] = (const uint8_t**) bnPosBnProcBuf_BG2_R13_CNG10; + p_lut->bnPosBnProcBuf[6] = NULL; + p_lut->bnPosBnProcBuf[7] = NULL; + p_lut->bnPosBnProcBuf[8] = NULL; + + p_lut->llr2llrProcBufAddr = llr2llrProcBufAddr_BG2_R13; + p_lut->llr2llrProcBufBnPos = llr2llrProcBufBnPos_BG2_R13; + + p_lut->numCnInCnGroups = lut_numCnInCnGroups_BG2_R13; + p_lut->numBnInBnGroups = lut_numBnInBnGroups_BG2_R13; + p_lut->startAddrBnGroups = lut_startAddrBnGroups_BG2_R13; + p_lut->startAddrBnGroupsLlr = lut_startAddrBnGroupsLlr_BG2_R13; + + numLLR = NR_LDPC_NCOL_BG2_R13*Z; + } + else if (R == 23) + { + p_lut->startAddrBnProcBuf[0] = NULL; + p_lut->startAddrBnProcBuf[1] = (const uint32_t**) startAddrBnProcBuf_BG2_R23_CNG4; + p_lut->startAddrBnProcBuf[2] = NULL; + p_lut->startAddrBnProcBuf[3] = (const uint32_t**) startAddrBnProcBuf_BG2_R23_CNG6; + p_lut->startAddrBnProcBuf[4] = (const uint32_t**) startAddrBnProcBuf_BG2_R23_CNG8; + p_lut->startAddrBnProcBuf[5] = (const uint32_t**) startAddrBnProcBuf_BG2_R23_CNG10; + p_lut->startAddrBnProcBuf[6] = NULL; + p_lut->startAddrBnProcBuf[7] = NULL; + p_lut->startAddrBnProcBuf[8] = NULL; + + p_lut->bnPosBnProcBuf[0] = NULL; + p_lut->bnPosBnProcBuf[1] = (const uint8_t**) bnPosBnProcBuf_BG2_R23_CNG4; + p_lut->bnPosBnProcBuf[2] = NULL; + p_lut->bnPosBnProcBuf[3] = (const uint8_t**) bnPosBnProcBuf_BG2_R23_CNG6; + p_lut->bnPosBnProcBuf[4] = (const uint8_t**) bnPosBnProcBuf_BG2_R23_CNG8; + p_lut->bnPosBnProcBuf[5] = (const uint8_t**) bnPosBnProcBuf_BG2_R23_CNG10; + p_lut->bnPosBnProcBuf[6] = NULL; + p_lut->bnPosBnProcBuf[7] = NULL; + p_lut->bnPosBnProcBuf[8] = NULL; + + p_lut->llr2llrProcBufAddr = llr2llrProcBufAddr_BG2_R23; + p_lut->llr2llrProcBufBnPos = llr2llrProcBufBnPos_BG2_R23; + + p_lut->numCnInCnGroups = lut_numCnInCnGroups_BG2_R23; + p_lut->numBnInBnGroups = lut_numBnInBnGroups_BG2_R23; + p_lut->startAddrBnGroups = lut_startAddrBnGroups_BG2_R23; + p_lut->startAddrBnGroupsLlr = lut_startAddrBnGroupsLlr_BG2_R23; + + numLLR = NR_LDPC_NCOL_BG2_R23*Z; + } + + // LUT that depend on Z and R + switch (Z) + { + case 2: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z2_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z2_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z2_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z2_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z2_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z2_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 3: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z3_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z3_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z3_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z3_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z3_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z3_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 4: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z4_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z4_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z4_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z4_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z4_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z4_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 5: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z5_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z5_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z5_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z5_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z5_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z5_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 6: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z6_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z6_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z6_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z6_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z6_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z6_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 7: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z7_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z7_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z7_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z7_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z7_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z7_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 8: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z8_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z8_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z8_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z8_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z8_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z8_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 9: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z9_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z9_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z9_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z9_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z9_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z9_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 10: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z10_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z10_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z10_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z10_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z10_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z10_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 11: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z11_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z11_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z11_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z11_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z11_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z11_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 12: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z12_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z12_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z12_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z12_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z12_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z12_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 13: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z13_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z13_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z13_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z13_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z13_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z13_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 14: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z14_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z14_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z14_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z14_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z14_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z14_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 15: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z15_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z15_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z15_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z15_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z15_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z15_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 16: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z16_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z16_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z16_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z16_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z16_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z16_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 18: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z18_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z18_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z18_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z18_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z18_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z18_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 20: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z20_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z20_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z20_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z20_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z20_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z20_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 22: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z22_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z22_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z22_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z22_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z22_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z22_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 24: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z24_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z24_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z24_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z24_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z24_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z24_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 26: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z26_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z26_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z26_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z26_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z26_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z26_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 28: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z28_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z28_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z28_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z28_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z28_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z28_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 30: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z30_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z30_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z30_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z30_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z30_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z30_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 32: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z32_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z32_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z32_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z32_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z32_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z32_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 36: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z36_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z36_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z36_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z36_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z36_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z36_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 40: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z40_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z40_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z40_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z40_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z40_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z40_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 44: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z44_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z44_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z44_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z44_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z44_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z44_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 48: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z48_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z48_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z48_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z48_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z48_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z48_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 52: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z52_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z52_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z52_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z52_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z52_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z52_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 56: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z56_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z56_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z56_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z56_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z56_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z56_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 60: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z60_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z60_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z60_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z60_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z60_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z60_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 64: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z64_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z64_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z64_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z64_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z64_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z64_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 72: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z72_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z72_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z72_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z72_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z72_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z72_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 80: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z80_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z80_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z80_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z80_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z80_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z80_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 88: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z88_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z88_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z88_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z88_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z88_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z88_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 96: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z96_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z96_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z96_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z96_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z96_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z96_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 104: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z104_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z104_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z104_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z104_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z104_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z104_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 112: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z112_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z112_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z112_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z112_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z112_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z112_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 120: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z120_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z120_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z120_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z120_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z120_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z120_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 128: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z128_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z128_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z128_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z128_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z128_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z128_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 144: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z144_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z144_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z144_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z144_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z144_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z144_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 160: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z160_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z160_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z160_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z160_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z160_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z160_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 176: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z176_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z176_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z176_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z176_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z176_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z176_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 192: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z192_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z192_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z192_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z192_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z192_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z192_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 208: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z208_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z208_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z208_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z208_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z208_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z208_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 224: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z224_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z224_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z224_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z224_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z224_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z224_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 240: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z240_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z240_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z240_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z240_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z240_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z240_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 256: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z256_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z256_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z256_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z256_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z256_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z256_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 288: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z288_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z288_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z288_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z288_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z288_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z288_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 320: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z320_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z320_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z320_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z320_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z320_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z320_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 352: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z352_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z352_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z352_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z352_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z352_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z352_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + case 384: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG2_Z384_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG2_Z384_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG2_Z384_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG2_Z384_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG2_Z384_CNG8; + p_lut->circShift[5] = (const uint16_t**) circShift_BG2_Z384_CNG10; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + break; + } + default: + { + p_lut->circShift[0] = NULL; + p_lut->circShift[1] = NULL; + p_lut->circShift[2] = NULL; + p_lut->circShift[3] = NULL; + p_lut->circShift[4] = NULL; + p_lut->circShift[5] = NULL; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + } + } + } + else + { // BG == 1 + // LUT that only depend on BG + p_lut->startAddrCnGroups = lut_startAddrCnGroups_BG1; + + p_lut->posBnInCnProcBuf[0] = (const uint8_t**) posBnInCnProcBuf_BG1_CNG3; + p_lut->posBnInCnProcBuf[1] = (const uint8_t**) posBnInCnProcBuf_BG1_CNG4; + p_lut->posBnInCnProcBuf[2] = (const uint8_t**) posBnInCnProcBuf_BG1_CNG5; + p_lut->posBnInCnProcBuf[3] = (const uint8_t**) posBnInCnProcBuf_BG1_CNG6; + p_lut->posBnInCnProcBuf[4] = (const uint8_t**) posBnInCnProcBuf_BG1_CNG7; + p_lut->posBnInCnProcBuf[5] = (const uint8_t**) posBnInCnProcBuf_BG1_CNG8; + p_lut->posBnInCnProcBuf[6] = (const uint8_t**) posBnInCnProcBuf_BG1_CNG9; + p_lut->posBnInCnProcBuf[7] = (const uint8_t**) posBnInCnProcBuf_BG1_CNG10; + p_lut->posBnInCnProcBuf[8] = (const uint8_t**) posBnInCnProcBuf_BG1_CNG19; + + // LUT that only depend on R + if (R == 13) + { + p_lut->startAddrBnProcBuf[0] = (const uint32_t**) startAddrBnProcBuf_BG1_R13_CNG3; + p_lut->startAddrBnProcBuf[1] = (const uint32_t**) startAddrBnProcBuf_BG1_R13_CNG4; + p_lut->startAddrBnProcBuf[2] = (const uint32_t**) startAddrBnProcBuf_BG1_R13_CNG5; + p_lut->startAddrBnProcBuf[3] = (const uint32_t**) startAddrBnProcBuf_BG1_R13_CNG6; + p_lut->startAddrBnProcBuf[4] = (const uint32_t**) startAddrBnProcBuf_BG1_R13_CNG7; + p_lut->startAddrBnProcBuf[5] = (const uint32_t**) startAddrBnProcBuf_BG1_R13_CNG8; + p_lut->startAddrBnProcBuf[6] = (const uint32_t**) startAddrBnProcBuf_BG1_R13_CNG9; + p_lut->startAddrBnProcBuf[7] = (const uint32_t**) startAddrBnProcBuf_BG1_R13_CNG10; + p_lut->startAddrBnProcBuf[8] = (const uint32_t**) startAddrBnProcBuf_BG1_R13_CNG19; + + p_lut->bnPosBnProcBuf[0] = (const uint8_t**) NULL; + p_lut->bnPosBnProcBuf[1] = (const uint8_t**) bnPosBnProcBuf_BG1_R13_CNG4; + p_lut->bnPosBnProcBuf[2] = (const uint8_t**) bnPosBnProcBuf_BG1_R13_CNG5; + p_lut->bnPosBnProcBuf[3] = (const uint8_t**) bnPosBnProcBuf_BG1_R13_CNG6; + p_lut->bnPosBnProcBuf[4] = (const uint8_t**) bnPosBnProcBuf_BG1_R13_CNG7; + p_lut->bnPosBnProcBuf[5] = (const uint8_t**) bnPosBnProcBuf_BG1_R13_CNG8; + p_lut->bnPosBnProcBuf[6] = (const uint8_t**) bnPosBnProcBuf_BG1_R13_CNG9; + p_lut->bnPosBnProcBuf[7] = (const uint8_t**) bnPosBnProcBuf_BG1_R13_CNG10; + p_lut->bnPosBnProcBuf[8] = (const uint8_t**) bnPosBnProcBuf_BG1_R13_CNG19; + + p_lut->llr2llrProcBufAddr = llr2llrProcBufAddr_BG1_R13; + p_lut->llr2llrProcBufBnPos = llr2llrProcBufBnPos_BG1_R13; + + p_lut->numCnInCnGroups = lut_numCnInCnGroups_BG1_R13; + p_lut->numBnInBnGroups = lut_numBnInBnGroups_BG1_R13; + p_lut->startAddrBnGroups = lut_startAddrBnGroups_BG1_R13; + p_lut->startAddrBnGroupsLlr = lut_startAddrBnGroupsLlr_BG1_R13; + + numLLR = NR_LDPC_NCOL_BG1_R13*Z; + } + else if (R == 23) + { + p_lut->startAddrBnProcBuf[0] = (const uint32_t**) startAddrBnProcBuf_BG1_R23_CNG3; + p_lut->startAddrBnProcBuf[1] = (const uint32_t**) NULL; + p_lut->startAddrBnProcBuf[2] = (const uint32_t**) NULL; + p_lut->startAddrBnProcBuf[3] = (const uint32_t**) NULL; + p_lut->startAddrBnProcBuf[4] = (const uint32_t**) startAddrBnProcBuf_BG1_R23_CNG7; + p_lut->startAddrBnProcBuf[5] = (const uint32_t**) startAddrBnProcBuf_BG1_R23_CNG8; + p_lut->startAddrBnProcBuf[6] = (const uint32_t**) startAddrBnProcBuf_BG1_R23_CNG9; + p_lut->startAddrBnProcBuf[7] = (const uint32_t**) startAddrBnProcBuf_BG1_R23_CNG10; + p_lut->startAddrBnProcBuf[8] = (const uint32_t**) startAddrBnProcBuf_BG1_R23_CNG19; + + p_lut->bnPosBnProcBuf[0] = (const uint8_t**) NULL; + p_lut->bnPosBnProcBuf[1] = (const uint8_t**) NULL; + p_lut->bnPosBnProcBuf[2] = (const uint8_t**) NULL; + p_lut->bnPosBnProcBuf[3] = (const uint8_t**) NULL; + p_lut->bnPosBnProcBuf[4] = (const uint8_t**) bnPosBnProcBuf_BG1_R23_CNG7; + p_lut->bnPosBnProcBuf[5] = (const uint8_t**) bnPosBnProcBuf_BG1_R23_CNG8; + p_lut->bnPosBnProcBuf[6] = (const uint8_t**) bnPosBnProcBuf_BG1_R23_CNG9; + p_lut->bnPosBnProcBuf[7] = (const uint8_t**) bnPosBnProcBuf_BG1_R23_CNG10; + p_lut->bnPosBnProcBuf[8] = (const uint8_t**) bnPosBnProcBuf_BG1_R23_CNG19; + + p_lut->llr2llrProcBufAddr = llr2llrProcBufAddr_BG1_R23; + p_lut->llr2llrProcBufBnPos = llr2llrProcBufBnPos_BG1_R23; + + p_lut->numCnInCnGroups = lut_numCnInCnGroups_BG1_R23; + p_lut->numBnInBnGroups = lut_numBnInBnGroups_BG1_R23; + p_lut->startAddrBnGroups = lut_startAddrBnGroups_BG1_R23; + p_lut->startAddrBnGroupsLlr = lut_startAddrBnGroupsLlr_BG1_R23; + + numLLR = NR_LDPC_NCOL_BG1_R23*Z; + } + else if (R == 89) + { + p_lut->startAddrBnProcBuf[0] = (const uint32_t**) startAddrBnProcBuf_BG1_R89_CNG3; + p_lut->startAddrBnProcBuf[1] = (const uint32_t**) NULL; + p_lut->startAddrBnProcBuf[2] = (const uint32_t**) NULL; + p_lut->startAddrBnProcBuf[3] = (const uint32_t**) NULL; + p_lut->startAddrBnProcBuf[4] = (const uint32_t**) NULL; + p_lut->startAddrBnProcBuf[5] = (const uint32_t**) NULL; + p_lut->startAddrBnProcBuf[6] = (const uint32_t**) NULL; + p_lut->startAddrBnProcBuf[7] = (const uint32_t**) NULL; + p_lut->startAddrBnProcBuf[8] = (const uint32_t**) startAddrBnProcBuf_BG1_R89_CNG19; + + p_lut->bnPosBnProcBuf[0] = (const uint8_t**) NULL; + p_lut->bnPosBnProcBuf[1] = (const uint8_t**) NULL; + p_lut->bnPosBnProcBuf[2] = (const uint8_t**) NULL; + p_lut->bnPosBnProcBuf[3] = (const uint8_t**) NULL; + p_lut->bnPosBnProcBuf[4] = (const uint8_t**) NULL; + p_lut->bnPosBnProcBuf[5] = (const uint8_t**) NULL; + p_lut->bnPosBnProcBuf[6] = (const uint8_t**) NULL; + p_lut->bnPosBnProcBuf[7] = (const uint8_t**) NULL; + p_lut->bnPosBnProcBuf[8] = (const uint8_t**) bnPosBnProcBuf_BG1_R89_CNG19; + + p_lut->llr2llrProcBufAddr = llr2llrProcBufAddr_BG1_R89; + p_lut->llr2llrProcBufBnPos = llr2llrProcBufBnPos_BG1_R89; + + p_lut->numCnInCnGroups = lut_numCnInCnGroups_BG1_R89; + p_lut->numBnInBnGroups = lut_numBnInBnGroups_BG1_R89; + p_lut->startAddrBnGroups = lut_startAddrBnGroups_BG1_R89; + p_lut->startAddrBnGroupsLlr = lut_startAddrBnGroupsLlr_BG1_R89; + + numLLR = NR_LDPC_NCOL_BG1_R89*Z; + } + + // LUT that depend on Z and R + switch (Z) + { + case 2: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z2_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z2_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z2_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z2_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z2_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z2_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z2_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z2_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z2_CNG19; + break; + } + case 3: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z3_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z3_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z3_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z3_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z3_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z3_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z3_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z3_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z3_CNG19; + break; + } + case 4: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z4_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z4_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z4_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z4_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z4_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z4_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z4_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z4_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z4_CNG19; + break; + } + case 5: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z5_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z5_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z5_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z5_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z5_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z5_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z5_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z5_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z5_CNG19; + break; + } + case 6: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z6_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z6_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z6_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z6_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z6_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z6_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z6_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z6_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z6_CNG19; + break; + } + case 7: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z7_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z7_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z7_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z7_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z7_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z7_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z7_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z7_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z7_CNG19; + break; + } + case 8: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z8_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z8_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z8_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z8_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z8_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z8_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z8_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z8_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z8_CNG19; + break; + } + case 9: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z9_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z9_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z9_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z9_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z9_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z9_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z9_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z9_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z9_CNG19; + break; + } + case 10: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z10_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z10_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z10_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z10_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z10_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z10_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z10_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z10_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z10_CNG19; + break; + } + case 11: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z11_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z11_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z11_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z11_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z11_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z11_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z11_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z11_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z11_CNG19; + break; + } + case 12: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z12_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z12_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z12_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z12_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z12_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z12_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z12_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z12_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z12_CNG19; + break; + } + case 13: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z13_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z13_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z13_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z13_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z13_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z13_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z13_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z13_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z13_CNG19; + break; + } + case 14: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z14_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z14_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z14_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z14_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z14_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z14_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z14_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z14_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z14_CNG19; + break; + } + case 15: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z15_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z15_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z15_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z15_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z15_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z15_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z15_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z15_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z15_CNG19; + break; + } + case 16: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z16_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z16_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z16_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z16_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z16_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z16_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z16_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z16_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z16_CNG19; + break; + } + case 18: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z18_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z18_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z18_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z18_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z18_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z18_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z18_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z18_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z18_CNG19; + break; + } + case 20: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z20_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z20_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z20_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z20_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z20_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z20_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z20_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z20_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z20_CNG19; + break; + } + case 22: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z22_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z22_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z22_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z22_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z22_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z22_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z22_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z22_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z22_CNG19; + break; + } + case 24: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z24_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z24_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z24_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z24_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z24_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z24_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z24_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z24_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z24_CNG19; + break; + } + case 26: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z26_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z26_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z26_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z26_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z26_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z26_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z26_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z26_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z26_CNG19; + break; + } + case 28: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z28_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z28_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z28_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z28_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z28_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z28_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z28_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z28_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z28_CNG19; + break; + } + case 30: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z30_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z30_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z30_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z30_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z30_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z30_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z30_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z30_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z30_CNG19; + break; + } + case 32: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z32_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z32_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z32_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z32_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z32_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z32_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z32_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z32_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z32_CNG19; + break; + } + case 36: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z36_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z36_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z36_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z36_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z36_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z36_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z36_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z36_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z36_CNG19; + break; + } + case 40: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z40_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z40_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z40_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z40_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z40_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z40_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z40_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z40_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z40_CNG19; + break; + } + case 44: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z44_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z44_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z44_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z44_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z44_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z44_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z44_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z44_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z44_CNG19; + break; + } + case 48: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z48_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z48_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z48_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z48_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z48_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z48_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z48_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z48_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z48_CNG19; + break; + } + case 52: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z52_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z52_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z52_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z52_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z52_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z52_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z52_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z52_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z52_CNG19; + break; + } + case 56: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z56_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z56_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z56_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z56_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z56_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z56_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z56_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z56_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z56_CNG19; + break; + } + case 60: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z60_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z60_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z60_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z60_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z60_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z60_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z60_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z60_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z60_CNG19; + break; + } + case 64: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z64_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z64_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z64_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z64_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z64_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z64_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z64_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z64_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z64_CNG19; + break; + } + case 72: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z72_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z72_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z72_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z72_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z72_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z72_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z72_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z72_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z72_CNG19; + break; + } + case 80: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z80_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z80_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z80_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z80_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z80_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z80_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z80_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z80_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z80_CNG19; + break; + } + case 88: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z88_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z88_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z88_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z88_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z88_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z88_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z88_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z88_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z88_CNG19; + break; + } + case 96: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z96_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z96_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z96_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z96_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z96_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z96_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z96_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z96_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z96_CNG19; + break; + } + case 104: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z104_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z104_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z104_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z104_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z104_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z104_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z104_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z104_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z104_CNG19; + break; + } + case 112: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z112_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z112_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z112_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z112_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z112_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z112_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z112_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z112_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z112_CNG19; + break; + } + case 120: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z120_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z120_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z120_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z120_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z120_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z120_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z120_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z120_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z120_CNG19; + break; + } + case 128: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z128_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z128_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z128_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z128_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z128_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z128_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z128_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z128_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z128_CNG19; + break; + } + case 144: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z144_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z144_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z144_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z144_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z144_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z144_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z144_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z144_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z144_CNG19; + break; + } + case 160: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z160_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z160_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z160_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z160_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z160_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z160_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z160_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z160_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z160_CNG19; + break; + } + case 176: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z176_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z176_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z176_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z176_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z176_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z176_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z176_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z176_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z176_CNG19; + break; + } + case 192: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z192_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z192_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z192_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z192_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z192_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z192_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z192_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z192_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z192_CNG19; + break; + } + case 208: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z208_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z208_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z208_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z208_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z208_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z208_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z208_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z208_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z208_CNG19; + break; + } + case 224: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z224_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z224_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z224_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z224_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z224_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z224_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z224_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z224_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z224_CNG19; + break; + } + case 240: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z240_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z240_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z240_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z240_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z240_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z240_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z240_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z240_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z240_CNG19; + break; + } + case 256: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z256_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z256_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z256_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z256_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z256_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z256_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z256_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z256_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z256_CNG19; + break; + } + case 288: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z288_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z288_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z288_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z288_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z288_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z288_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z288_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z288_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z288_CNG19; + break; + } + case 320: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z320_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z320_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z320_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z320_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z320_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z320_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z320_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z320_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z320_CNG19; + break; + } + case 352: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z352_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z352_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z352_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z352_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z352_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z352_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z352_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z352_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z352_CNG19; + break; + } + case 384: + { + p_lut->circShift[0] = (const uint16_t**) circShift_BG1_Z384_CNG3; + p_lut->circShift[1] = (const uint16_t**) circShift_BG1_Z384_CNG4; + p_lut->circShift[2] = (const uint16_t**) circShift_BG1_Z384_CNG5; + p_lut->circShift[3] = (const uint16_t**) circShift_BG1_Z384_CNG6; + p_lut->circShift[4] = (const uint16_t**) circShift_BG1_Z384_CNG7; + p_lut->circShift[5] = (const uint16_t**) circShift_BG1_Z384_CNG8; + p_lut->circShift[6] = (const uint16_t**) circShift_BG1_Z384_CNG9; + p_lut->circShift[7] = (const uint16_t**) circShift_BG1_Z384_CNG10; + p_lut->circShift[8] = (const uint16_t**) circShift_BG1_Z384_CNG19; + break; + } + default: + { + p_lut->circShift[0] = NULL; + p_lut->circShift[1] = NULL; + p_lut->circShift[2] = NULL; + p_lut->circShift[3] = NULL; + p_lut->circShift[4] = NULL; + p_lut->circShift[5] = NULL; + p_lut->circShift[6] = NULL; + p_lut->circShift[7] = NULL; + p_lut->circShift[8] = NULL; + } + } + } + + return numLLR; +} + +#endif diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/nrLDPC_init_mem.h b/openair1/PHY/CODING/nrLDPC_decoder_full/nrLDPC_init_mem.h new file mode 100644 index 0000000000..0eaa116782 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/nrLDPC_init_mem.h @@ -0,0 +1,85 @@ +/* + * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The OpenAirInterface Software Alliance licenses this file to You under + * the OAI Public License, Version 1.1 (the "License"); you may not use this file + * except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.openairinterface.org/?page_id=698 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *------------------------------------------------------------------------------- + * For more information about the OpenAirInterface (OAI) Software Alliance: + * contact@openairinterface.org + */ + +/*!\file nrLDPC_init_mem.h + * \brief Defines the function to initialize the LDPC decoder and sets correct LUTs. + * \author Sebastian Wagner (TCL Communications) Email: + * \date 07-12-2018 + * \version 1.0 + * \note + * \warning + */ + +#ifndef __NR_LDPC_INIT_MEM__H__ +#define __NR_LDPC_INIT_MEM__H__ + +#include +#include "nrLDPC_types.h" + +#ifndef malloc64_clear +/** + \brief Allocates 32 byte aligned memory and initializes to zero + \param size Input size in bytes + \return Pointer to memory +*/ +static inline void* malloc64_clear(size_t size) +{ + void* ptr = (void*) memalign(64, size+64); + memset(ptr, 0, size); + return ptr; +} +#endif + +/** + \brief Allocates and initializes the internal decoder processing buffers + \param p_decParams Pointer to decoder parameters + \param p_lut Pointer to decoder LUTs + \return Number of LLR values +*/ +static inline t_nrLDPC_procBuf* nrLDPC_init_mem(void) +{ + t_nrLDPC_procBuf* p_procBuf = (t_nrLDPC_procBuf*) malloc64_clear(sizeof(t_nrLDPC_procBuf)); + + if (p_procBuf) + { + p_procBuf->cnProcBuf = (int8_t*) malloc64_clear(NR_LDPC_SIZE_CN_PROC_BUF*sizeof(int8_t)); + p_procBuf->cnProcBufRes = (int8_t*) malloc64_clear(NR_LDPC_SIZE_CN_PROC_BUF*sizeof(int8_t)); + p_procBuf->bnProcBuf = (int8_t*) malloc64_clear(NR_LDPC_SIZE_BN_PROC_BUF*sizeof(int8_t)); + p_procBuf->bnProcBufRes = (int8_t*) malloc64_clear(NR_LDPC_SIZE_BN_PROC_BUF*sizeof(int8_t)); + p_procBuf->llrRes = (int8_t*) malloc64_clear(NR_LDPC_MAX_NUM_LLR *sizeof(int8_t)); + p_procBuf->llrProcBuf = (int8_t*) malloc64_clear(NR_LDPC_MAX_NUM_LLR *sizeof(int8_t)); + } + + return(p_procBuf); +} + +static inline void nrLDPC_free_mem(t_nrLDPC_procBuf* p_procBuf) +{ + free(p_procBuf->cnProcBuf); + free(p_procBuf->cnProcBufRes); + free(p_procBuf->bnProcBuf); + free(p_procBuf->bnProcBufRes); + free(p_procBuf->llrRes); + free(p_procBuf->llrProcBuf); + + free(p_procBuf); +} +#endif diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/nrLDPC_lut.h b/openair1/PHY/CODING/nrLDPC_decoder_full/nrLDPC_lut.h new file mode 100644 index 0000000000..df80e11d18 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/nrLDPC_lut.h @@ -0,0 +1,1004 @@ +/* + * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The OpenAirInterface Software Alliance licenses this file to You under + * the OAI Public License, Version 1.1 (the "License"); you may not use this file + * except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.openairinterface.org/?page_id=698 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *------------------------------------------------------------------------------- + * For more information about the OpenAirInterface (OAI) Software Alliance: + * contact@openairinterface.org + */ + +/*!\file nrLDPC_lut.h + * \brief Header file defining all look-up tables + * \author Sebastian Wagner (TCL Communications) Email: + * \date 30-09-2019 + * \version 2.0 + * \note + * \warning + */ + +#ifndef __NR_LDPC_LUT__H__ +#define __NR_LDPC_LUT__H__ + +// Circular shifts of base graph +// There are 8 sets of circular shifts depending on Z +// However, each shift is modulo Z +// BG1 +// Group of CNs with 3 BNs (CNG3) +// e.g. first and second value correspond to row 4 col 0 and 1 in the BG table (modulo Z) +static const uint16_t circShift_BG1_Z2_CNG3[3][1] = {{1},{0},{0}}; +static const uint16_t circShift_BG1_Z3_CNG3[3][1] = {{2},{1},{0}}; +static const uint16_t circShift_BG1_Z5_CNG3[3][1] = {{3},{0},{0}}; +static const uint16_t circShift_BG1_Z7_CNG3[3][1] = {{2},{3},{0}}; +static const uint16_t circShift_BG1_Z9_CNG3[3][1] = {{3},{1},{0}}; +static const uint16_t circShift_BG1_Z11_CNG3[3][1] = {{9},{3},{0}}; +static const uint16_t circShift_BG1_Z13_CNG3[3][1] = {{11},{9},{0}}; +static const uint16_t circShift_BG1_Z15_CNG3[3][1] = {{4},{1},{0}}; +static const uint16_t circShift_BG1_Z4_CNG3[3][1] = {{1},{2},{0}}; +static const uint16_t circShift_BG1_Z6_CNG3[3][1] = {{2},{1},{0}}; +static const uint16_t circShift_BG1_Z10_CNG3[3][1] = {{3},{5},{0}}; +static const uint16_t circShift_BG1_Z14_CNG3[3][1] = {{2},{10},{0}}; +static const uint16_t circShift_BG1_Z18_CNG3[3][1] = {{12},{1},{0}}; +static const uint16_t circShift_BG1_Z22_CNG3[3][1] = {{20},{14},{0}}; +static const uint16_t circShift_BG1_Z26_CNG3[3][1] = {{24},{22},{0}}; +static const uint16_t circShift_BG1_Z30_CNG3[3][1] = {{4},{1},{0}}; +static const uint16_t circShift_BG1_Z8_CNG3[3][1] = {{5},{6},{0}}; +static const uint16_t circShift_BG1_Z12_CNG3[3][1] = {{8},{1},{0}}; +static const uint16_t circShift_BG1_Z20_CNG3[3][1] = {{13},{5},{0}}; +static const uint16_t circShift_BG1_Z28_CNG3[3][1] = {{2},{10},{0}}; +static const uint16_t circShift_BG1_Z36_CNG3[3][1] = {{30},{19},{0}}; +static const uint16_t circShift_BG1_Z44_CNG3[3][1] = {{42},{36},{0}}; +static const uint16_t circShift_BG1_Z52_CNG3[3][1] = {{24},{48},{0}}; +static const uint16_t circShift_BG1_Z60_CNG3[3][1] = {{4},{31},{0}}; +static const uint16_t circShift_BG1_Z16_CNG3[3][1] = {{13},{6},{0}}; +static const uint16_t circShift_BG1_Z24_CNG3[3][1] = {{20},{13},{0}}; +static const uint16_t circShift_BG1_Z40_CNG3[3][1] = {{33},{5},{0}}; +static const uint16_t circShift_BG1_Z56_CNG3[3][1] = {{2},{10},{0}}; +static const uint16_t circShift_BG1_Z72_CNG3[3][1] = {{30},{19},{0}}; +static const uint16_t circShift_BG1_Z88_CNG3[3][1] = {{42},{80},{0}}; +static const uint16_t circShift_BG1_Z104_CNG3[3][1] = {{24},{100},{0}}; +static const uint16_t circShift_BG1_Z120_CNG3[3][1] = {{64},{91},{0}}; +static const uint16_t circShift_BG1_Z32_CNG3[3][1] = {{29},{6},{0}}; +static const uint16_t circShift_BG1_Z48_CNG3[3][1] = {{44},{37},{0}}; +static const uint16_t circShift_BG1_Z80_CNG3[3][1] = {{73},{45},{0}}; +static const uint16_t circShift_BG1_Z112_CNG3[3][1] = {{58},{10},{0}}; +static const uint16_t circShift_BG1_Z144_CNG3[3][1] = {{102},{91},{0}}; +static const uint16_t circShift_BG1_Z176_CNG3[3][1] = {{42},{80},{0}}; +static const uint16_t circShift_BG1_Z208_CNG3[3][1] = {{24},{204},{0}}; +static const uint16_t circShift_BG1_Z240_CNG3[3][1] = {{64},{211},{0}}; +static const uint16_t circShift_BG1_Z64_CNG3[3][1] = {{29},{38},{0}}; +static const uint16_t circShift_BG1_Z96_CNG3[3][1] = {{44},{85},{0}}; +static const uint16_t circShift_BG1_Z160_CNG3[3][1] = {{73},{45},{0}}; +static const uint16_t circShift_BG1_Z224_CNG3[3][1] = {{170},{10},{0}}; +static const uint16_t circShift_BG1_Z288_CNG3[3][1] = {{246},{235},{0}}; +static const uint16_t circShift_BG1_Z352_CNG3[3][1] = {{42},{256},{0}}; +static const uint16_t circShift_BG1_Z128_CNG3[3][1] = {{29},{102},{0}}; +static const uint16_t circShift_BG1_Z192_CNG3[3][1] = {{140},{181},{0}}; +static const uint16_t circShift_BG1_Z320_CNG3[3][1] = {{233},{205},{0}}; +static const uint16_t circShift_BG1_Z256_CNG3[3][1] = {{157},{102},{0}}; +static const uint16_t circShift_BG1_Z384_CNG3[3][1] = {{332},{181},{0}}; + +// Group of CNs with 4 BNs (CNG4) ->[Num BNs][Num of CN with 4 BNs] +static const uint16_t circShift_BG1_Z2_CNG4[4][5] = {{1, 0, 1, 1, 1},{0, 0, 1, 1, 1},{0, 0, 1, 1, 1},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z3_CNG4[4][5] = {{2, 2, 1, 2, 0},{2, 1, 1, 2, 2},{2, 1, 0, 2, 0},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z5_CNG4[4][5] = {{0, 3, 2, 1, 1},{4, 0, 0, 0, 3},{4, 0, 2, 1, 1},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z7_CNG4[4][5] = {{5, 4, 4, 2, 2},{1, 6, 3, 0, 2},{6, 5, 5, 1, 1},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z9_CNG4[4][5] = {{0, 0, 0, 0, 6},{0, 0, 0, 0, 0},{0, 0, 2, 8, 0},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z11_CNG4[4][5] = {{3, 0, 7, 2, 5},{7, 5, 6, 6, 1},{4, 8, 3, 6, 4},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z13_CNG4[4][5] = {{7, 4, 10, 7, 12},{10, 2, 6, 10, 6},{9, 5, 12, 6, 10},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z15_CNG4[4][5] = {{0, 14, 4, 12, 12},{7, 9, 13, 5, 9},{9, 11, 0, 11, 3},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z4_CNG4[4][5] = {{1, 2, 1, 1, 1},{0, 0, 1, 1, 3},{2, 2, 1, 1, 3},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z6_CNG4[4][5] = {{2, 5, 1, 5, 3},{2, 4, 1, 2, 5},{5, 1, 0, 2, 3},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z10_CNG4[4][5] = {{0, 8, 2, 6, 1},{9, 5, 0, 5, 8},{4, 5, 7, 6, 6},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z14_CNG4[4][5] = {{12, 11, 11, 2, 2},{1, 13, 3, 7, 9},{6, 12, 12, 8, 1},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z18_CNG4[4][5] = {{9, 0, 0, 9, 6},{0, 0, 0, 0, 0},{0, 0, 2, 17, 0},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z22_CNG4[4][5] = {{3, 11, 18, 2, 16},{18, 16, 17, 17, 1},{4, 19, 3, 17, 15},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z26_CNG4[4][5] = {{20, 4, 10, 7, 25},{10, 2, 6, 10, 19},{22, 5, 12, 6, 23},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z30_CNG4[4][5] = {{15, 29, 4, 12, 27},{22, 24, 13, 20, 24},{9, 11, 0, 11, 3},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z8_CNG4[4][5] = {{1, 6, 5, 5, 5},{0, 4, 1, 5, 7},{2, 2, 5, 1, 7},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z12_CNG4[4][5] = {{2, 5, 7, 5, 3},{2, 10, 1, 2, 5},{5, 7, 0, 2, 3},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z20_CNG4[4][5] = {{10, 18, 12, 6, 1},{9, 15, 0, 5, 8},{4, 15, 17, 6, 6},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z28_CNG4[4][5] = {{12, 25, 11, 2, 16},{1, 27, 17, 7, 9},{6, 26, 12, 22, 1},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z36_CNG4[4][5] = {{9, 0, 0, 27, 24},{0, 0, 0, 0, 0},{0, 0, 2, 35, 0},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z44_CNG4[4][5] = {{3, 11, 40, 24, 38},{18, 16, 17, 39, 23},{4, 41, 3, 17, 15},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z52_CNG4[4][5] = {{46, 30, 10, 7, 25},{36, 2, 32, 10, 45},{22, 31, 12, 6, 49},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z60_CNG4[4][5] = {{45, 59, 4, 42, 57},{52, 24, 43, 50, 54},{9, 41, 30, 11, 33},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z16_CNG4[4][5] = {{9, 6, 13, 5, 5},{8, 12, 9, 13, 7},{2, 10, 5, 9, 7},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z24_CNG4[4][5] = {{2, 5, 7, 17, 15},{2, 10, 13, 14, 5},{5, 19, 0, 2, 15},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z40_CNG4[4][5] = {{10, 18, 32, 26, 21},{29, 15, 0, 25, 28},{24, 35, 37, 6, 6},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z56_CNG4[4][5] = {{40, 25, 11, 2, 16},{29, 27, 45, 35, 9},{6, 26, 40, 22, 29},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z72_CNG4[4][5] = {{45, 0, 0, 27, 24},{36, 0, 0, 0, 0},{0, 0, 2, 35, 0},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z88_CNG4[4][5] = {{3, 55, 40, 24, 82},{62, 60, 61, 83, 67},{4, 85, 47, 17, 59},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z104_CNG4[4][5] = {{98, 82, 10, 59, 77},{36, 2, 84, 10, 45},{22, 31, 12, 58, 49},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z120_CNG4[4][5] = {{45, 59, 4, 102, 57},{112, 84, 103, 50, 114},{9, 41, 30, 71, 93},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z32_CNG4[4][5] = {{25, 6, 29, 21, 21},{8, 28, 9, 29, 23},{2, 26, 21, 9, 7},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z48_CNG4[4][5] = {{2, 29, 31, 17, 39},{2, 34, 37, 14, 5},{5, 19, 24, 26, 15},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z80_CNG4[4][5] = {{50, 58, 32, 66, 21},{29, 15, 0, 65, 68},{64, 35, 37, 46, 46},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z112_CNG4[4][5] = {{96, 81, 67, 2, 72},{29, 83, 45, 91, 9},{62, 26, 96, 78, 29},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z144_CNG4[4][5] = {{45, 72, 72, 27, 24},{36, 0, 0, 0, 0},{72, 0, 2, 35, 0},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z176_CNG4[4][5] = {{91, 143, 128, 112, 82},{150, 60, 61, 83, 67},{92, 85, 135, 17, 59},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z208_CNG4[4][5] = {{98, 82, 10, 163, 181},{140, 2, 84, 10, 45},{22, 135, 12, 162, 153},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z240_CNG4[4][5] = {{165, 59, 4, 222, 177},{232, 204, 103, 170, 114},{9, 161, 30, 71, 93},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z64_CNG4[4][5] = {{25, 6, 29, 21, 21},{40, 28, 9, 29, 23},{2, 58, 21, 9, 39},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z96_CNG4[4][5] = {{2, 77, 79, 17, 39},{2, 82, 37, 14, 53},{5, 19, 24, 26, 15},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z160_CNG4[4][5] = {{50, 138, 32, 66, 101},{29, 15, 80, 65, 68},{144, 115, 37, 126, 126},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z224_CNG4[4][5] = {{208, 81, 67, 114, 184},{141, 195, 45, 91, 121},{174, 138, 96, 78, 29},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z288_CNG4[4][5] = {{45, 72, 216, 27, 168},{36, 144, 144, 0, 0},{72, 0, 2, 35, 144},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z352_CNG4[4][5] = {{91, 319, 304, 288, 82},{326, 236, 237, 83, 67},{268, 85, 135, 17, 235},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z128_CNG4[4][5] = {{25, 70, 29, 21, 21},{104, 92, 9, 29, 23},{66, 122, 21, 9, 39},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z192_CNG4[4][5] = {{2, 77, 175, 113, 135},{2, 82, 37, 14, 149},{101, 115, 120, 26, 15},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z320_CNG4[4][5] = {{210, 298, 32, 226, 101},{29, 15, 80, 65, 228},{304, 115, 197, 126, 126},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z256_CNG4[4][5] = {{25, 198, 157, 149, 149},{104, 220, 137, 157, 151},{194, 122, 149, 137, 167},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z384_CNG4[4][5] = {{194, 269, 175, 113, 135},{194, 82, 37, 14, 149},{101, 115, 312, 218, 15},{0, 0, 0, 0, 0}}; + +// Group of CNs with 5 BNs (CNG5) ->[Num BNs][Num of CN with 5 BNs] +static const uint16_t circShift_BG1_Z2_CNG5[5][18] = {{0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1},{0, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1},{0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1},{0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z3_CNG5[5][18] = {{0, 0, 0, 2, 0, 0, 0, 1, 0, 2, 2, 2, 0, 2, 0, 1, 2, 2},{2, 2, 2, 0, 1, 2, 1, 2, 0, 2, 1, 2, 1, 0, 0, 2, 0, 0},{2, 1, 2, 1, 1, 0, 2, 0, 1, 1, 1, 2, 0, 1, 1, 1, 0, 1},{1, 0, 0, 1, 1, 0, 0, 1, 2, 0, 0, 2, 0, 1, 0, 0, 0, 1},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z5_CNG5[5][18] = {{2, 4, 2, 0, 1, 2, 1, 2, 0, 0, 2, 2, 4, 1, 2, 4, 3, 4},{0, 0, 0, 0, 3, 3, 3, 3, 2, 0, 1, 3, 4, 4, 2, 2, 4, 2},{0, 3, 1, 3, 0, 2, 0, 1, 1, 0, 0, 0, 3, 4, 2, 4, 1, 4},{1, 0, 1, 2, 4, 4, 0, 2, 2, 3, 4, 0, 0, 1, 3, 2, 2, 0},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z7_CNG5[5][18] = {{4, 4, 1, 2, 6, 3, 6, 0, 3, 2, 5, 3, 5, 4, 5, 2, 3, 0},{0, 0, 5, 1, 6, 4, 4, 0, 6, 0, 4, 5, 3, 6, 0, 5, 1, 4},{3, 0, 2, 2, 0, 4, 0, 2, 2, 0, 5, 5, 0, 3, 0, 4, 1, 2},{1, 3, 4, 1, 2, 5, 4, 2, 2, 4, 3, 5, 6, 3, 2, 6, 0, 5},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z9_CNG5[5][18] = {{0, 0, 7, 0, 5, 0, 0, 0, 0, 0, 8, 1, 0, 1, 0, 0, 7, 0},{0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8, 7, 0, 0, 0, 0, 0},{0, 0, 0, 0, 2, 0, 0, 4, 0, 4, 0, 0, 0, 0, 3, 0, 0, 0},{5, 3, 3, 7, 8, 6, 2, 1, 0, 0, 0, 7, 0, 0, 0, 3, 0, 3},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z11_CNG5[5][18] = {{5, 7, 4, 6, 3, 9, 6, 2, 10, 1, 5, 1, 3, 10, 5, 2, 1, 2},{3, 6, 2, 7, 1, 5, 5, 0, 1, 7, 1, 1, 4, 9, 0, 0, 3, 2},{5, 5, 1, 6, 7, 1, 7, 4, 8, 9, 2, 8, 2, 0, 4, 3, 9, 7},{3, 7, 10, 0, 2, 8, 3, 2, 6, 2, 10, 4, 1, 4, 8, 8, 2, 8},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z13_CNG5[5][18] = {{5, 6, 4, 8, 4, 1, 2, 4, 6, 10, 4, 4, 6, 0, 4, 2, 1, 4},{6, 5, 9, 12, 1, 6, 10, 10, 9, 4, 10, 6, 5, 12, 2, 10, 7, 6},{0, 2, 3, 10, 1, 2, 8, 8, 11, 6, 2, 12, 8, 3, 5, 12, 8, 12},{1, 9, 2, 2, 2, 6, 7, 12, 7, 1, 12, 3, 4, 9, 6, 11, 7, 3},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z15_CNG5[5][18] = {{10, 5, 12, 13, 2, 7, 5, 13, 13, 4, 11, 6, 11, 1, 10, 8, 7, 11},{2, 7, 6, 11, 13, 2, 4, 14, 5, 12, 12, 9, 12, 0, 8, 9, 7, 1},{6, 12, 11, 4, 5, 13, 11, 10, 1, 7, 7, 10, 12, 11, 3, 5, 4, 7},{13, 4, 1, 8, 10, 13, 0, 3, 1, 5, 0, 10, 10, 5, 7, 9, 5, 13},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z4_CNG5[5][18] = {{0, 0, 3, 3, 0, 2, 0, 3, 1, 2, 3, 1, 1, 3, 1, 3, 3, 3},{2, 3, 0, 3, 1, 0, 1, 1, 0, 3, 3, 1, 1, 3, 3, 1, 3, 1},{0, 2, 0, 3, 1, 0, 0, 0, 3, 1, 0, 3, 3, 1, 1, 3, 1, 3},{2, 0, 2, 1, 3, 2, 1, 1, 1, 3, 3, 3, 0, 3, 0, 3, 3, 1},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z6_CNG5[5][18] = {{0, 0, 0, 5, 0, 0, 3, 4, 0, 5, 2, 2, 0, 5, 0, 4, 5, 2},{5, 5, 5, 3, 1, 5, 1, 5, 3, 2, 4, 5, 1, 3, 3, 2, 0, 0},{5, 1, 5, 4, 4, 3, 2, 0, 1, 1, 4, 2, 3, 1, 4, 1, 0, 1},{4, 3, 0, 4, 4, 0, 0, 4, 5, 3, 0, 2, 3, 4, 3, 0, 0, 4},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z10_CNG5[5][18] = {{7, 9, 2, 0, 1, 7, 1, 2, 0, 0, 7, 7, 9, 1, 2, 4, 8, 4},{0, 0, 5, 0, 3, 8, 3, 8, 2, 0, 6, 8, 9, 9, 7, 7, 9, 7},{5, 3, 6, 8, 0, 7, 5, 6, 1, 0, 5, 5, 8, 4, 2, 4, 6, 9},{6, 5, 6, 7, 4, 9, 5, 2, 7, 3, 4, 5, 0, 1, 8, 7, 2, 0},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z14_CNG5[5][18] = {{11, 4, 1, 9, 6, 10, 6, 7, 10, 2, 12, 10, 12, 11, 12, 9, 10, 0},{0, 7, 12, 8, 13, 11, 4, 7, 13, 7, 4, 5, 3, 6, 7, 5, 8, 4},{3, 0, 9, 2, 7, 11, 0, 9, 2, 7, 12, 12, 0, 3, 7, 4, 8, 9},{8, 10, 11, 1, 2, 12, 11, 2, 2, 11, 10, 5, 6, 10, 2, 13, 7, 12},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z18_CNG5[5][18] = {{0, 0, 7, 0, 5, 0, 0, 0, 0, 0, 17, 1, 0, 10, 9, 0, 16, 0},{9, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 8, 16, 0, 0, 0, 9, 0},{0, 0, 0, 0, 11, 0, 0, 4, 0, 4, 0, 0, 0, 0, 3, 0, 0, 0},{5, 3, 3, 7, 8, 6, 11, 1, 0, 0, 0, 7, 0, 0, 9, 3, 0, 3},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z22_CNG5[5][18] = {{16, 18, 15, 17, 14, 9, 17, 13, 21, 12, 16, 12, 3, 10, 16, 13, 12, 13},{3, 6, 13, 18, 1, 16, 16, 11, 1, 18, 12, 12, 15, 20, 0, 11, 3, 2},{16, 5, 1, 6, 18, 1, 7, 15, 8, 9, 2, 19, 13, 11, 15, 3, 9, 7},{3, 18, 10, 11, 13, 19, 14, 2, 17, 13, 10, 4, 12, 4, 8, 8, 13, 8},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z26_CNG5[5][18] = {{18, 19, 4, 21, 4, 14, 2, 4, 6, 10, 4, 4, 6, 13, 4, 2, 1, 4},{6, 18, 22, 12, 1, 6, 10, 23, 22, 4, 23, 19, 18, 12, 2, 23, 7, 6},{0, 2, 16, 10, 14, 2, 8, 21, 11, 6, 15, 12, 21, 16, 5, 12, 21, 12},{1, 22, 2, 15, 15, 6, 20, 12, 20, 14, 25, 3, 17, 22, 6, 24, 20, 3},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z30_CNG5[5][18] = {{25, 5, 27, 13, 2, 22, 20, 13, 13, 19, 11, 21, 11, 1, 25, 23, 22, 11},{17, 7, 21, 11, 13, 2, 19, 29, 20, 12, 27, 24, 27, 0, 8, 9, 7, 1},{6, 27, 11, 19, 20, 28, 26, 25, 1, 22, 22, 25, 12, 26, 18, 5, 19, 7},{28, 19, 1, 8, 25, 13, 15, 18, 1, 5, 0, 10, 10, 20, 22, 24, 5, 28},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z8_CNG5[5][18] = {{0, 4, 7, 3, 0, 6, 0, 7, 5, 2, 7, 1, 5, 7, 5, 7, 7, 3},{6, 3, 0, 3, 5, 4, 1, 1, 0, 3, 7, 5, 1, 7, 3, 5, 3, 5},{4, 2, 4, 7, 5, 4, 0, 4, 7, 1, 4, 7, 3, 5, 5, 3, 5, 3},{6, 0, 6, 5, 7, 6, 1, 5, 1, 3, 7, 7, 0, 3, 0, 7, 3, 5},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z12_CNG5[5][18] = {{6, 0, 0, 11, 6, 0, 3, 4, 6, 11, 2, 8, 6, 5, 6, 4, 5, 8},{11, 5, 5, 9, 7, 5, 1, 11, 9, 8, 4, 11, 1, 9, 9, 2, 0, 6},{5, 1, 11, 4, 4, 3, 8, 6, 7, 1, 10, 2, 9, 1, 10, 7, 6, 7},{10, 3, 0, 4, 10, 6, 6, 4, 11, 9, 0, 2, 9, 10, 9, 0, 0, 10},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z20_CNG5[5][18] = {{17, 9, 12, 10, 11, 7, 11, 2, 10, 10, 7, 7, 19, 11, 2, 14, 8, 14},{0, 10, 15, 10, 13, 8, 3, 8, 2, 0, 16, 18, 19, 19, 17, 7, 9, 7},{15, 13, 16, 18, 10, 17, 5, 6, 11, 0, 5, 15, 18, 4, 12, 4, 16, 19},{16, 5, 6, 7, 14, 9, 15, 2, 17, 13, 4, 5, 0, 1, 8, 7, 2, 0},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z28_CNG5[5][18] = {{25, 4, 1, 23, 6, 10, 6, 7, 24, 2, 26, 24, 26, 11, 12, 23, 10, 0},{0, 7, 26, 8, 13, 11, 18, 21, 13, 21, 18, 5, 17, 6, 21, 19, 22, 4},{3, 0, 9, 16, 21, 25, 14, 9, 2, 21, 12, 26, 0, 17, 21, 4, 22, 9},{8, 10, 25, 15, 2, 12, 11, 16, 2, 25, 10, 19, 6, 10, 2, 27, 21, 12},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z36_CNG5[5][18] = {{0, 0, 25, 0, 23, 0, 0, 0, 0, 0, 17, 19, 0, 10, 9, 0, 16, 18},{9, 0, 0, 0, 0, 0, 18, 0, 31, 0, 0, 26, 16, 0, 0, 0, 27, 0},{0, 0, 0, 0, 11, 0, 0, 22, 0, 4, 0, 0, 0, 0, 21, 0, 0, 0},{5, 21, 3, 25, 26, 6, 29, 19, 18, 18, 0, 25, 0, 0, 9, 3, 18, 21},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z44_CNG5[5][18] = {{16, 18, 15, 17, 14, 9, 39, 13, 43, 12, 16, 12, 3, 32, 16, 35, 34, 35},{25, 6, 35, 18, 1, 16, 16, 33, 1, 40, 12, 34, 15, 20, 0, 33, 3, 24},{16, 5, 23, 6, 40, 23, 7, 15, 30, 9, 24, 19, 35, 33, 37, 25, 9, 29},{25, 40, 10, 11, 35, 19, 36, 2, 39, 13, 10, 4, 12, 26, 8, 8, 35, 8},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z52_CNG5[5][18] = {{44, 45, 4, 21, 4, 40, 2, 4, 6, 10, 4, 4, 6, 39, 4, 2, 1, 4},{6, 18, 22, 38, 1, 32, 36, 49, 22, 30, 49, 45, 44, 12, 28, 23, 7, 6},{26, 28, 16, 36, 40, 2, 8, 21, 11, 6, 41, 12, 21, 42, 5, 38, 47, 38},{27, 48, 28, 41, 41, 6, 20, 38, 20, 40, 51, 29, 43, 48, 32, 24, 46, 3},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z60_CNG5[5][18] = {{25, 5, 27, 43, 32, 52, 50, 13, 43, 19, 41, 51, 11, 1, 55, 53, 22, 11},{47, 7, 21, 11, 43, 32, 19, 29, 50, 12, 57, 54, 27, 30, 8, 9, 7, 31},{6, 57, 11, 49, 20, 58, 26, 55, 31, 52, 22, 25, 42, 26, 48, 35, 49, 7},{58, 19, 1, 38, 25, 43, 45, 48, 31, 5, 0, 40, 40, 20, 52, 24, 5, 28},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z16_CNG5[5][18] = {{0, 12, 7, 3, 0, 6, 8, 15, 13, 2, 15, 1, 5, 7, 13, 7, 7, 11},{14, 3, 8, 3, 5, 12, 9, 1, 0, 11, 7, 5, 9, 7, 11, 13, 3, 13},{12, 10, 4, 7, 5, 4, 8, 12, 7, 9, 4, 15, 3, 13, 5, 11, 13, 3},{14, 8, 6, 13, 15, 6, 9, 13, 9, 3, 15, 7, 8, 3, 0, 7, 11, 13},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z24_CNG5[5][18] = {{6, 0, 0, 23, 6, 12, 15, 4, 6, 11, 14, 8, 18, 17, 18, 4, 17, 8},{11, 17, 17, 9, 19, 5, 13, 23, 9, 8, 4, 23, 1, 9, 21, 2, 12, 6},{17, 13, 23, 4, 4, 3, 20, 18, 7, 1, 10, 2, 9, 1, 10, 19, 18, 19},{22, 3, 0, 16, 10, 6, 18, 16, 23, 9, 0, 2, 21, 22, 9, 0, 0, 10},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z40_CNG5[5][18] = {{17, 9, 12, 30, 11, 27, 11, 22, 10, 10, 27, 7, 19, 31, 22, 34, 28, 34},{20, 10, 15, 30, 13, 28, 23, 28, 22, 20, 16, 38, 19, 19, 37, 7, 29, 27},{15, 13, 16, 38, 10, 37, 25, 26, 31, 20, 5, 15, 18, 24, 32, 4, 16, 19},{36, 25, 6, 27, 34, 29, 15, 2, 17, 33, 4, 5, 0, 21, 8, 7, 22, 20},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z56_CNG5[5][18] = {{53, 32, 1, 51, 34, 38, 34, 7, 24, 2, 26, 24, 54, 11, 12, 23, 38, 28},{0, 35, 54, 8, 41, 11, 18, 49, 41, 49, 18, 5, 45, 34, 21, 47, 22, 4},{3, 0, 9, 44, 49, 53, 42, 9, 2, 49, 12, 26, 0, 17, 49, 4, 22, 9},{36, 10, 25, 15, 2, 12, 11, 16, 30, 25, 38, 19, 6, 10, 2, 55, 49, 12},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z72_CNG5[5][18] = {{0, 36, 61, 0, 59, 0, 0, 0, 0, 0, 53, 55, 0, 46, 9, 0, 52, 18},{45, 0, 0, 0, 0, 36, 18, 0, 67, 0, 0, 62, 16, 0, 0, 0, 27, 0},{0, 0, 0, 0, 11, 0, 36, 22, 36, 4, 36, 0, 0, 0, 21, 0, 0, 0},{41, 21, 39, 61, 62, 42, 29, 19, 18, 18, 0, 61, 0, 0, 45, 39, 54, 57},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z88_CNG5[5][18] = {{16, 18, 15, 61, 14, 9, 83, 13, 87, 56, 60, 12, 47, 76, 60, 35, 34, 79},{69, 6, 79, 62, 1, 16, 16, 33, 1, 40, 56, 34, 15, 20, 0, 77, 3, 68},{60, 5, 23, 50, 40, 67, 7, 15, 74, 53, 24, 19, 35, 33, 37, 25, 53, 29},{25, 40, 54, 11, 79, 19, 36, 46, 83, 13, 54, 4, 12, 70, 8, 8, 79, 8},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z104_CNG5[5][18] = {{44, 45, 4, 21, 4, 92, 2, 4, 6, 10, 4, 4, 6, 91, 4, 2, 1, 4},{58, 18, 74, 38, 1, 32, 88, 49, 22, 30, 49, 45, 96, 64, 28, 75, 59, 6},{26, 28, 16, 88, 40, 2, 8, 21, 63, 6, 93, 64, 73, 94, 5, 38, 99, 38},{27, 100, 28, 93, 93, 6, 20, 90, 20, 92, 51, 81, 43, 100, 84, 24, 98, 3},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z120_CNG5[5][18] = {{25, 65, 27, 43, 32, 112, 50, 73, 103, 79, 41, 111, 11, 1, 115, 53, 22, 71},{47, 7, 21, 11, 43, 32, 79, 29, 110, 12, 117, 54, 87, 90, 68, 69, 7, 91},{6, 117, 11, 49, 80, 118, 26, 55, 31, 52, 22, 25, 42, 26, 48, 95, 49, 67},{58, 79, 61, 98, 85, 103, 105, 108, 91, 65, 60, 100, 100, 20, 52, 24, 5, 28},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z32_CNG5[5][18] = {{0, 28, 23, 3, 0, 22, 24, 31, 29, 2, 31, 1, 5, 7, 13, 7, 23, 11},{14, 19, 8, 19, 5, 12, 9, 17, 16, 27, 7, 5, 9, 23, 11, 13, 3, 29},{28, 10, 20, 23, 21, 20, 24, 12, 7, 9, 4, 15, 19, 29, 21, 11, 13, 3},{30, 24, 22, 29, 31, 6, 9, 29, 25, 19, 31, 7, 24, 3, 0, 23, 11, 13},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z48_CNG5[5][18] = {{30, 24, 24, 23, 30, 12, 15, 4, 6, 35, 38, 32, 18, 41, 18, 4, 17, 32},{11, 41, 17, 33, 19, 5, 37, 23, 9, 8, 4, 47, 25, 33, 45, 26, 36, 30},{41, 13, 47, 28, 4, 3, 20, 18, 31, 25, 10, 2, 9, 1, 10, 43, 18, 19},{22, 27, 24, 40, 34, 30, 42, 16, 47, 9, 24, 26, 21, 22, 9, 0, 24, 34},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z80_CNG5[5][18] = {{17, 9, 12, 30, 11, 27, 11, 62, 50, 10, 27, 47, 19, 71, 22, 74, 68, 74},{20, 50, 55, 30, 53, 68, 23, 68, 22, 20, 56, 78, 19, 19, 77, 47, 69, 67},{55, 53, 16, 78, 50, 37, 25, 66, 31, 60, 5, 55, 18, 64, 32, 44, 16, 19},{76, 25, 46, 67, 74, 29, 55, 2, 57, 33, 44, 45, 0, 21, 48, 47, 22, 20},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z112_CNG5[5][18] = {{53, 88, 1, 107, 34, 38, 34, 63, 80, 2, 82, 80, 110, 11, 12, 23, 94, 84},{0, 91, 54, 64, 41, 11, 18, 49, 97, 49, 74, 61, 45, 90, 77, 103, 22, 4},{3, 56, 65, 100, 105, 53, 98, 65, 58, 49, 68, 26, 0, 73, 49, 60, 22, 9},{36, 10, 81, 15, 2, 68, 11, 16, 30, 25, 38, 75, 6, 10, 2, 55, 49, 12},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z144_CNG5[5][18] = {{72, 36, 61, 0, 131, 0, 0, 0, 0, 0, 53, 55, 72, 46, 9, 0, 52, 18},{45, 0, 0, 0, 0, 36, 90, 0, 67, 0, 0, 134, 16, 0, 0, 0, 99, 0},{72, 0, 0, 0, 11, 0, 108, 22, 36, 76, 108, 0, 0, 0, 21, 0, 0, 0},{113, 21, 39, 133, 62, 42, 29, 19, 18, 18, 0, 61, 0, 0, 117, 39, 126, 57},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z176_CNG5[5][18] = {{104, 18, 103, 149, 102, 97, 171, 101, 175, 56, 60, 100, 135, 164, 60, 123, 34, 79},{157, 6, 79, 150, 1, 104, 16, 121, 89, 128, 144, 34, 15, 20, 88, 77, 3, 68},{60, 5, 111, 50, 40, 67, 95, 103, 162, 141, 112, 19, 35, 33, 37, 25, 53, 117},{113, 128, 54, 99, 167, 107, 36, 46, 83, 101, 54, 92, 12, 70, 96, 96, 167, 96},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z208_CNG5[5][18] = {{44, 45, 4, 21, 4, 92, 2, 4, 6, 10, 4, 4, 6, 91, 4, 2, 1, 4},{58, 18, 74, 142, 1, 136, 88, 49, 126, 30, 153, 45, 200, 64, 28, 75, 163, 6},{130, 132, 16, 192, 40, 106, 112, 125, 63, 6, 197, 168, 177, 198, 109, 142, 99, 142},{131, 100, 28, 197, 93, 6, 20, 194, 20, 92, 155, 185, 43, 100, 188, 128, 98, 3},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z240_CNG5[5][18] = {{25, 185, 27, 163, 32, 232, 170, 73, 103, 199, 161, 231, 11, 121, 115, 53, 22, 191},{47, 127, 141, 131, 43, 32, 199, 149, 110, 132, 237, 174, 207, 90, 188, 189, 127, 211},{126, 117, 11, 169, 200, 118, 26, 175, 151, 172, 142, 145, 42, 26, 168, 215, 49, 187},{178, 199, 181, 98, 205, 103, 105, 108, 211, 65, 180, 100, 100, 140, 52, 24, 125, 148},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z64_CNG5[5][18] = {{0, 28, 23, 3, 0, 22, 24, 31, 29, 2, 63, 33, 37, 39, 45, 39, 23, 11},{14, 19, 8, 51, 37, 44, 9, 49, 48, 59, 39, 5, 41, 23, 11, 45, 35, 29},{60, 42, 52, 23, 53, 20, 56, 44, 7, 41, 36, 15, 51, 29, 21, 11, 45, 35},{30, 24, 54, 61, 63, 6, 9, 61, 57, 19, 31, 39, 56, 35, 0, 23, 11, 45},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z96_CNG5[5][18] = {{30, 24, 72, 71, 30, 60, 63, 4, 6, 35, 38, 32, 18, 89, 66, 52, 17, 80},{11, 89, 17, 81, 19, 5, 37, 23, 9, 8, 52, 47, 25, 81, 93, 26, 36, 78},{41, 61, 95, 76, 52, 51, 68, 66, 79, 73, 10, 2, 9, 1, 58, 43, 18, 67},{22, 27, 24, 40, 82, 78, 90, 64, 95, 9, 24, 74, 21, 22, 9, 0, 72, 82},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z160_CNG5[5][18] = {{17, 89, 12, 110, 11, 27, 91, 62, 50, 10, 27, 47, 99, 151, 102, 154, 68, 74},{20, 50, 135, 110, 133, 148, 23, 148, 22, 20, 136, 158, 19, 19, 77, 47, 69, 67},{55, 133, 96, 158, 50, 117, 105, 66, 111, 140, 5, 55, 18, 64, 32, 124, 16, 99},{156, 105, 46, 67, 74, 29, 135, 2, 57, 33, 44, 125, 0, 21, 48, 47, 102, 100},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z224_CNG5[5][18] = {{53, 88, 1, 107, 146, 150, 34, 175, 192, 114, 82, 192, 222, 123, 12, 23, 206, 84},{0, 203, 166, 176, 153, 11, 130, 49, 209, 49, 186, 173, 157, 90, 77, 215, 22, 4},{3, 168, 65, 212, 217, 53, 210, 177, 58, 161, 68, 26, 0, 73, 49, 60, 134, 9},{148, 122, 81, 127, 114, 68, 123, 128, 30, 137, 150, 187, 6, 10, 114, 167, 161, 12},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z288_CNG5[5][18] = {{72, 180, 205, 0, 275, 0, 0, 144, 0, 0, 197, 199, 216, 190, 153, 0, 52, 18},{189, 0, 0, 0, 0, 180, 90, 144, 211, 0, 0, 278, 16, 0, 0, 0, 243, 0},{72, 0, 0, 0, 155, 0, 252, 166, 36, 76, 108, 0, 0, 0, 165, 0, 0, 0},{257, 165, 183, 277, 62, 42, 173, 19, 162, 18, 0, 205, 0, 0, 117, 183, 270, 57},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z352_CNG5[5][18] = {{280, 18, 279, 325, 102, 273, 171, 101, 351, 56, 60, 100, 135, 164, 236, 123, 210, 79},{157, 6, 255, 326, 1, 104, 16, 297, 265, 304, 320, 210, 15, 196, 264, 77, 3, 244},{236, 181, 111, 226, 40, 243, 95, 279, 338, 141, 112, 195, 35, 209, 37, 25, 53, 293},{113, 304, 54, 99, 167, 107, 212, 222, 83, 101, 54, 268, 188, 246, 272, 272, 167, 272},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z128_CNG5[5][18] = {{64, 28, 23, 67, 0, 86, 88, 95, 93, 2, 127, 33, 37, 39, 45, 39, 23, 11},{14, 19, 8, 115, 37, 108, 73, 49, 112, 59, 39, 69, 105, 23, 11, 45, 35, 29},{60, 42, 116, 87, 53, 84, 120, 44, 71, 41, 36, 79, 51, 29, 21, 11, 45, 35},{30, 24, 54, 61, 63, 6, 9, 61, 121, 83, 31, 103, 120, 35, 0, 23, 11, 45},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z192_CNG5[5][18] = {{30, 24, 72, 71, 30, 60, 159, 100, 102, 131, 38, 128, 18, 185, 66, 52, 113, 80},{11, 89, 17, 81, 19, 5, 37, 23, 9, 8, 148, 143, 121, 177, 93, 122, 132, 78},{41, 61, 191, 76, 52, 147, 68, 66, 175, 169, 10, 2, 105, 97, 154, 139, 114, 163},{22, 27, 120, 136, 82, 78, 90, 64, 95, 105, 120, 74, 21, 22, 105, 96, 168, 82},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z320_CNG5[5][18] = {{177, 249, 172, 270, 11, 27, 91, 222, 210, 170, 187, 207, 259, 151, 102, 154, 228, 234},{20, 50, 295, 110, 293, 308, 23, 308, 22, 20, 296, 158, 179, 179, 77, 47, 69, 227},{55, 133, 96, 318, 50, 117, 105, 66, 271, 140, 5, 55, 178, 64, 192, 124, 176, 259},{316, 105, 46, 67, 234, 29, 135, 162, 217, 33, 44, 285, 160, 181, 208, 207, 102, 260},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z256_CNG5[5][18] = {{64, 156, 23, 195, 128, 86, 216, 95, 221, 2, 127, 161, 37, 167, 173, 167, 151, 139},{142, 147, 136, 243, 165, 236, 73, 177, 112, 187, 167, 197, 105, 151, 139, 173, 163, 157},{188, 170, 116, 215, 181, 84, 120, 172, 199, 41, 164, 207, 51, 157, 149, 139, 173, 163},{158, 152, 182, 61, 63, 6, 9, 61, 121, 211, 159, 103, 120, 163, 0, 151, 139, 173},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z384_CNG5[5][18] = {{30, 24, 72, 71, 222, 252, 159, 100, 102, 323, 230, 320, 210, 185, 258, 52, 113, 80},{11, 89, 17, 81, 19, 5, 229, 215, 201, 8, 148, 335, 313, 177, 93, 314, 132, 78},{233, 61, 383, 76, 244, 147, 260, 258, 175, 361, 202, 2, 297, 289, 346, 139, 114, 163},{22, 27, 312, 136, 274, 78, 90, 256, 287, 105, 312, 266, 21, 214, 297, 288, 168, 274},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; + +// Group of CNs with 6 BNs (CNG6) ->[Num BNs][Num of CN with 6 BNs] +static const uint16_t circShift_BG1_Z2_CNG6[6][8] = {{1, 0, 1, 0, 0, 1, 1, 0},{0, 1, 0, 1, 1, 0, 1, 0},{1, 1, 1, 0, 0, 1, 1, 0},{1, 1, 1, 1, 1, 1, 1, 0},{0, 0, 0, 1, 0, 0, 1, 0},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z3_CNG6[6][8] = {{1, 1, 2, 1, 1, 1, 1, 1},{0, 2, 0, 1, 0, 2, 0, 2},{2, 0, 0, 1, 2, 0, 1, 1},{1, 1, 1, 0, 2, 2, 0, 0},{1, 0, 0, 1, 2, 2, 1, 0},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z5_CNG6[6][8] = {{4, 4, 2, 0, 4, 1, 4, 4},{2, 0, 2, 4, 1, 2, 0, 0},{3, 0, 3, 1, 1, 0, 2, 0},{0, 0, 1, 1, 0, 0, 3, 2},{2, 0, 3, 0, 1, 0, 3, 1},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z7_CNG6[6][8] = {{4, 0, 0, 3, 1, 0, 3, 0},{0, 3, 5, 5, 6, 0, 5, 3},{2, 1, 4, 4, 5, 2, 3, 1},{0, 0, 0, 3, 2, 4, 1, 4},{1, 3, 4, 4, 1, 1, 1, 5},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z9_CNG6[6][8] = {{6, 0, 0, 8, 0, 4, 1, 2},{0, 4, 2, 7, 0, 0, 1, 1},{0, 0, 0, 0, 1, 0, 5, 0},{2, 0, 0, 7, 0, 0, 0, 5},{0, 0, 0, 0, 0, 3, 0, 0},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z11_CNG6[6][8] = {{3, 3, 0, 3, 10, 1, 8, 5},{9, 7, 8, 0, 0, 10, 0, 5},{1, 2, 5, 4, 4, 1, 5, 7},{6, 1, 6, 5, 5, 2, 0, 2},{4, 4, 3, 7, 6, 8, 10, 7},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z13_CNG6[6][8] = {{0, 10, 0, 1, 12, 6, 6, 9},{4, 3, 2, 2, 6, 12, 8, 8},{1, 4, 4, 11, 10, 2, 5, 9},{3, 5, 3, 3, 2, 5, 3, 2},{7, 0, 12, 5, 9, 4, 4, 6},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z15_CNG6[6][8] = {{10, 3, 3, 3, 14, 2, 2, 2},{5, 2, 7, 5, 3, 7, 2, 13},{4, 9, 1, 0, 9, 8, 5, 2},{13, 5, 9, 8, 3, 2, 5, 6},{0, 10, 9, 14, 1, 10, 6, 13},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z4_CNG6[6][8] = {{1, 0, 3, 2, 0, 3, 1, 0},{0, 1, 0, 1, 1, 2, 1, 2},{3, 1, 3, 0, 0, 1, 1, 0},{1, 3, 1, 3, 3, 3, 3, 0},{2, 2, 0, 3, 0, 0, 3, 2},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z6_CNG6[6][8] = {{1, 1, 2, 4, 1, 1, 1, 4},{3, 2, 3, 1, 3, 2, 0, 2},{2, 3, 3, 4, 2, 0, 4, 1},{1, 1, 4, 0, 2, 5, 3, 3},{4, 3, 3, 4, 5, 5, 1, 0},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z10_CNG6[6][8] = {{9, 9, 7, 0, 4, 1, 9, 9},{2, 0, 7, 4, 1, 2, 5, 0},{3, 5, 8, 1, 1, 0, 2, 0},{0, 5, 1, 1, 0, 0, 3, 7},{7, 0, 8, 5, 1, 0, 3, 1},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z14_CNG6[6][8] = {{11, 0, 0, 3, 8, 7, 10, 7},{0, 10, 12, 12, 6, 0, 5, 3},{2, 1, 4, 4, 5, 9, 10, 8},{7, 7, 7, 3, 2, 4, 8, 11},{8, 3, 4, 4, 8, 1, 8, 12},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z18_CNG6[6][8] = {{15, 0, 9, 17, 0, 13, 10, 2},{0, 13, 11, 7, 0, 9, 10, 1},{0, 9, 0, 0, 10, 0, 14, 0},{11, 0, 0, 7, 0, 0, 9, 14},{0, 0, 0, 0, 0, 12, 0, 0},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z22_CNG6[6][8] = {{3, 14, 0, 3, 21, 1, 8, 16},{9, 7, 8, 0, 0, 21, 0, 16},{1, 2, 5, 4, 15, 1, 16, 7},{6, 1, 6, 5, 16, 2, 11, 2},{15, 4, 14, 7, 6, 8, 21, 18},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z26_CNG6[6][8] = {{0, 10, 13, 1, 12, 6, 6, 9},{17, 16, 2, 15, 6, 12, 8, 21},{1, 4, 4, 11, 10, 15, 18, 9},{16, 18, 3, 16, 2, 5, 16, 2},{20, 0, 25, 18, 9, 4, 17, 6},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z30_CNG6[6][8] = {{10, 18, 3, 3, 29, 17, 17, 2},{5, 2, 22, 5, 18, 7, 2, 28},{4, 24, 16, 0, 9, 23, 5, 2},{28, 5, 9, 23, 18, 17, 5, 6},{0, 25, 9, 14, 16, 10, 6, 28},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z8_CNG6[6][8] = {{1, 0, 7, 2, 4, 7, 1, 0},{0, 1, 4, 1, 1, 2, 1, 6},{7, 1, 3, 0, 0, 1, 5, 4},{1, 3, 1, 3, 7, 7, 3, 4},{6, 2, 0, 3, 0, 0, 3, 6},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z12_CNG6[6][8] = {{1, 1, 8, 10, 1, 7, 1, 10},{9, 2, 3, 7, 9, 2, 6, 2},{2, 9, 9, 4, 8, 0, 4, 7},{7, 1, 10, 0, 2, 5, 9, 3},{10, 9, 3, 4, 5, 11, 1, 6},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z20_CNG6[6][8] = {{19, 9, 17, 0, 4, 1, 19, 9},{2, 0, 7, 14, 1, 2, 15, 0},{3, 5, 8, 11, 1, 0, 12, 10},{0, 15, 11, 1, 10, 10, 3, 7},{17, 0, 8, 15, 1, 10, 3, 1},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z28_CNG6[6][8] = {{25, 14, 0, 3, 8, 21, 24, 21},{0, 24, 26, 26, 6, 14, 19, 17},{16, 15, 4, 4, 19, 9, 24, 8},{21, 21, 7, 3, 2, 18, 22, 25},{22, 17, 4, 18, 8, 15, 22, 12},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z36_CNG6[6][8] = {{15, 18, 9, 17, 0, 13, 28, 20},{0, 13, 29, 7, 0, 9, 10, 19},{0, 27, 0, 0, 10, 18, 14, 0},{11, 18, 0, 25, 0, 0, 9, 14},{0, 0, 18, 0, 0, 12, 18, 0},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z44_CNG6[6][8] = {{3, 14, 22, 3, 43, 1, 30, 38},{31, 29, 8, 22, 22, 21, 0, 38},{1, 24, 27, 26, 15, 23, 38, 29},{6, 23, 28, 5, 38, 24, 11, 24},{37, 26, 14, 29, 28, 30, 21, 18},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z52_CNG6[6][8] = {{0, 36, 39, 1, 12, 6, 6, 9},{43, 42, 28, 41, 6, 12, 34, 21},{1, 4, 30, 11, 10, 15, 44, 35},{16, 18, 3, 16, 28, 5, 42, 28},{46, 26, 51, 44, 35, 30, 43, 6},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z60_CNG6[6][8] = {{40, 18, 3, 3, 59, 17, 17, 32},{5, 32, 52, 35, 48, 7, 2, 58},{34, 24, 46, 0, 39, 23, 35, 2},{58, 5, 39, 23, 18, 47, 5, 36},{30, 25, 9, 14, 16, 10, 36, 58},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z16_CNG6[6][8] = {{1, 0, 7, 10, 12, 7, 9, 0},{8, 1, 4, 9, 9, 10, 9, 6},{7, 1, 11, 8, 8, 9, 13, 12},{9, 3, 1, 11, 15, 15, 3, 4},{14, 10, 0, 3, 0, 0, 11, 14},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z24_CNG6[6][8] = {{1, 13, 20, 10, 1, 19, 13, 10},{9, 2, 15, 19, 21, 14, 6, 14},{2, 9, 9, 16, 8, 0, 16, 19},{19, 1, 22, 12, 2, 5, 21, 3},{10, 9, 15, 4, 5, 11, 1, 18},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z40_CNG6[6][8] = {{39, 29, 17, 20, 24, 21, 39, 9},{22, 20, 27, 14, 21, 2, 15, 0},{23, 5, 8, 11, 21, 0, 12, 30},{0, 35, 11, 21, 30, 10, 3, 27},{37, 20, 28, 15, 1, 10, 23, 1},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z56_CNG6[6][8] = {{25, 42, 0, 31, 8, 49, 24, 49},{0, 52, 54, 54, 6, 42, 19, 45},{16, 43, 32, 32, 47, 9, 52, 8},{49, 21, 7, 31, 30, 46, 50, 25},{22, 45, 4, 18, 8, 15, 50, 12},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z72_CNG6[6][8] = {{15, 54, 9, 17, 0, 49, 64, 20},{0, 13, 65, 7, 0, 9, 46, 55},{0, 27, 0, 0, 46, 18, 50, 0},{47, 54, 0, 25, 0, 0, 9, 50},{36, 0, 18, 0, 0, 12, 18, 0},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z88_CNG6[6][8] = {{47, 14, 22, 47, 87, 1, 74, 38},{75, 29, 52, 22, 22, 65, 0, 82},{1, 68, 71, 70, 59, 67, 82, 73},{6, 67, 28, 5, 82, 68, 55, 24},{81, 26, 14, 73, 28, 30, 21, 18},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z104_CNG6[6][8] = {{52, 88, 91, 1, 12, 6, 6, 9},{43, 94, 80, 41, 6, 12, 86, 21},{1, 56, 30, 63, 62, 15, 96, 87},{16, 18, 3, 68, 80, 5, 42, 28},{46, 78, 51, 44, 87, 30, 95, 6},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z120_CNG6[6][8] = {{100, 78, 63, 63, 59, 77, 77, 32},{65, 32, 112, 95, 108, 67, 2, 58},{34, 84, 106, 60, 39, 83, 95, 2},{58, 5, 99, 23, 18, 47, 65, 36},{30, 85, 9, 14, 76, 10, 96, 58},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z32_CNG6[6][8] = {{17, 0, 7, 10, 28, 23, 25, 16},{24, 17, 4, 9, 9, 26, 25, 22},{23, 17, 27, 8, 8, 25, 13, 12},{25, 19, 1, 27, 31, 15, 3, 20},{30, 26, 16, 19, 0, 0, 11, 30},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z48_CNG6[6][8] = {{25, 13, 20, 34, 1, 43, 13, 10},{33, 2, 15, 19, 21, 14, 6, 14},{26, 9, 33, 40, 8, 24, 40, 43},{19, 1, 22, 36, 2, 5, 21, 3},{34, 9, 39, 4, 5, 11, 1, 42},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z80_CNG6[6][8] = {{39, 69, 17, 20, 64, 61, 79, 49},{62, 60, 67, 54, 21, 2, 15, 40},{63, 45, 48, 51, 21, 40, 52, 30},{0, 35, 51, 61, 30, 50, 43, 27},{37, 60, 68, 55, 41, 10, 23, 41},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z112_CNG6[6][8] = {{81, 42, 56, 87, 8, 105, 80, 49},{56, 52, 110, 110, 6, 98, 19, 45},{72, 43, 88, 88, 103, 9, 108, 64},{105, 77, 63, 31, 86, 102, 50, 81},{78, 101, 4, 74, 8, 71, 106, 12},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z144_CNG6[6][8] = {{87, 126, 9, 17, 0, 121, 64, 92},{0, 13, 137, 7, 0, 81, 46, 55},{72, 99, 0, 0, 118, 90, 122, 0},{47, 54, 0, 97, 0, 0, 9, 122},{36, 0, 18, 0, 0, 84, 18, 0},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z176_CNG6[6][8] = {{135, 14, 110, 47, 87, 89, 162, 38},{75, 117, 52, 110, 110, 65, 88, 170},{89, 156, 71, 70, 147, 155, 170, 73},{94, 155, 116, 5, 82, 68, 143, 112},{81, 114, 14, 73, 28, 30, 109, 18},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z208_CNG6[6][8] = {{52, 88, 91, 1, 12, 6, 6, 9},{147, 198, 184, 41, 6, 12, 86, 125},{1, 160, 30, 167, 166, 15, 96, 191},{16, 122, 3, 68, 184, 5, 42, 28},{46, 182, 155, 148, 191, 30, 199, 6},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z240_CNG6[6][8] = {{220, 78, 183, 183, 179, 77, 197, 32},{185, 152, 112, 215, 108, 187, 122, 178},{154, 84, 106, 180, 159, 203, 215, 2},{178, 5, 219, 143, 138, 167, 65, 156},{150, 205, 129, 14, 196, 130, 216, 58},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z64_CNG6[6][8] = {{49, 0, 7, 42, 60, 23, 57, 48},{56, 49, 36, 41, 9, 58, 57, 22},{23, 49, 59, 8, 8, 25, 45, 44},{57, 51, 1, 27, 63, 47, 3, 52},{62, 26, 16, 19, 32, 32, 43, 30},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z96_CNG6[6][8] = {{25, 13, 68, 34, 49, 91, 13, 10},{81, 50, 15, 67, 21, 14, 6, 62},{74, 57, 81, 88, 56, 72, 40, 43},{19, 1, 70, 36, 50, 53, 21, 51},{82, 57, 87, 4, 5, 59, 1, 42},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z160_CNG6[6][8] = {{39, 69, 97, 100, 64, 141, 79, 129},{142, 140, 147, 134, 21, 2, 15, 120},{143, 45, 128, 131, 101, 40, 132, 110},{0, 115, 51, 141, 110, 130, 123, 27},{37, 140, 68, 135, 41, 10, 103, 121},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z224_CNG6[6][8] = {{81, 154, 56, 199, 8, 105, 192, 49},{56, 164, 110, 110, 6, 210, 131, 157},{72, 43, 200, 200, 103, 121, 220, 64},{217, 189, 63, 143, 198, 214, 50, 193},{78, 101, 4, 186, 8, 183, 106, 124},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z288_CNG6[6][8] = {{231, 270, 153, 161, 0, 265, 64, 236},{0, 13, 137, 151, 0, 81, 46, 199},{216, 99, 0, 0, 118, 90, 266, 0},{47, 54, 0, 241, 144, 144, 9, 266},{36, 0, 162, 144, 0, 228, 18, 0},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z352_CNG6[6][8] = {{311, 190, 110, 47, 87, 89, 162, 38},{251, 293, 228, 286, 110, 65, 264, 170},{265, 332, 247, 246, 147, 155, 346, 249},{94, 331, 116, 181, 258, 244, 143, 288},{81, 114, 190, 73, 204, 30, 109, 194},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z128_CNG6[6][8] = {{49, 64, 7, 42, 60, 23, 121, 112},{120, 49, 36, 105, 73, 58, 121, 86},{23, 49, 59, 8, 72, 89, 109, 108},{57, 51, 1, 27, 127, 47, 3, 116},{62, 26, 16, 19, 96, 32, 43, 94},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z192_CNG6[6][8] = {{121, 13, 68, 130, 145, 187, 13, 106},{177, 146, 111, 163, 21, 14, 102, 158},{74, 57, 81, 88, 152, 72, 136, 43},{115, 97, 166, 132, 50, 149, 21, 147},{178, 57, 183, 4, 5, 59, 97, 42},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z320_CNG6[6][8] = {{39, 69, 257, 260, 64, 301, 79, 289},{302, 140, 147, 294, 181, 162, 175, 280},{303, 45, 128, 291, 101, 40, 132, 110},{160, 115, 51, 141, 270, 130, 283, 187},{37, 300, 228, 295, 41, 10, 103, 281},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z256_CNG6[6][8] = {{177, 64, 7, 42, 60, 151, 249, 112},{248, 49, 164, 233, 73, 186, 121, 86},{151, 49, 59, 8, 72, 217, 109, 236},{185, 51, 1, 155, 127, 47, 131, 116},{62, 154, 144, 147, 224, 160, 171, 222},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z384_CNG6[6][8] = {{313, 13, 260, 130, 145, 187, 205, 298},{177, 338, 303, 163, 213, 206, 102, 158},{266, 57, 81, 280, 344, 264, 328, 235},{115, 289, 358, 132, 242, 341, 213, 339},{370, 57, 375, 4, 197, 59, 97, 234},{0, 0, 0, 0, 0, 0, 0, 0}}; + +// Group of CNs with 7 BNs (CNG7) ->[Num BNs][Num of CN with 7 BNs] +static const uint16_t circShift_BG1_Z2_CNG7[7][5] = {{0, 0, 0, 0, 0},{0, 1, 0, 1, 0},{1, 1, 1, 0, 1},{1, 0, 0, 1, 1},{1, 1, 0, 0, 1},{0, 0, 1, 1, 1},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z3_CNG7[7][5] = {{0, 2, 2, 1, 1},{2, 0, 0, 2, 2},{1, 1, 0, 2, 0},{0, 0, 1, 2, 0},{2, 2, 2, 2, 1},{0, 0, 2, 0, 2},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z5_CNG7[7][5] = {{2, 4, 4, 3, 4},{3, 0, 0, 4, 0},{2, 1, 4, 4, 0},{0, 0, 3, 1, 0},{0, 1, 0, 4, 4},{1, 1, 2, 3, 1},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z7_CNG7[7][5] = {{3, 5, 2, 0, 6},{6, 4, 0, 0, 1},{6, 6, 3, 1, 5},{2, 5, 3, 2, 6},{3, 5, 0, 2, 6},{4, 5, 0, 1, 4},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z9_CNG7[7][5] = {{7, 7, 0, 0, 8},{0, 1, 0, 6, 0},{1, 1, 1, 1, 3},{4, 6, 7, 7, 0},{5, 0, 4, 0, 5},{2, 3, 0, 7, 1},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z11_CNG7[7][5] = {{3, 10, 2, 0, 0},{4, 0, 8, 3, 7},{4, 5, 10, 2, 4},{0, 0, 0, 2, 4},{6, 5, 0, 0, 0},{6, 3, 10, 1, 3},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z13_CNG7[7][5] = {{2, 9, 6, 1, 4},{10, 12, 7, 7, 6},{8, 0, 8, 1, 3},{10, 3, 10, 0, 0},{3, 12, 0, 1, 1},{11, 5, 12, 2, 7},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z15_CNG7[7][5] = {{2, 11, 6, 4, 9},{11, 1, 5, 9, 3},{14, 10, 4, 2, 10},{2, 5, 0, 5, 8},{1, 10, 8, 12, 7},{3, 9, 5, 1, 4},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z4_CNG7[7][5] = {{0, 2, 0, 2, 0},{0, 1, 2, 3, 0},{3, 3, 1, 2, 1},{3, 0, 0, 3, 3},{3, 1, 2, 0, 3},{0, 2, 3, 1, 3},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z6_CNG7[7][5] = {{3, 5, 5, 4, 1},{2, 3, 0, 2, 2},{4, 4, 0, 5, 0},{3, 3, 4, 5, 0},{2, 5, 2, 5, 1},{0, 0, 2, 0, 5},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z10_CNG7[7][5] = {{2, 4, 9, 8, 9},{8, 0, 5, 9, 0},{7, 1, 9, 4, 0},{0, 5, 8, 1, 0},{5, 1, 5, 9, 4},{6, 1, 2, 8, 1},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z14_CNG7[7][5] = {{3, 12, 2, 0, 6},{6, 11, 7, 7, 8},{6, 6, 10, 1, 5},{2, 5, 3, 9, 13},{10, 12, 0, 9, 13},{4, 5, 0, 1, 11},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z18_CNG7[7][5] = {{7, 16, 9, 0, 8},{9, 1, 0, 6, 0},{10, 10, 10, 1, 3},{4, 15, 7, 16, 0},{14, 0, 4, 0, 14},{2, 3, 9, 7, 10},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z22_CNG7[7][5] = {{3, 21, 13, 0, 0},{15, 11, 19, 14, 18},{4, 5, 10, 13, 15},{11, 11, 0, 2, 15},{17, 5, 0, 0, 0},{17, 3, 10, 12, 3},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z26_CNG7[7][5] = {{15, 22, 6, 1, 17},{10, 25, 20, 20, 6},{8, 0, 21, 14, 3},{23, 16, 23, 0, 0},{3, 12, 0, 1, 1},{24, 18, 25, 2, 20},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z30_CNG7[7][5] = {{17, 26, 6, 4, 9},{26, 1, 5, 24, 18},{29, 10, 4, 2, 10},{2, 20, 0, 5, 23},{16, 25, 8, 12, 22},{18, 24, 20, 16, 19},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z8_CNG7[7][5] = {{4, 2, 0, 6, 0},{4, 5, 2, 7, 0},{7, 7, 5, 6, 1},{7, 0, 0, 7, 7},{7, 1, 2, 0, 3},{0, 2, 7, 5, 3},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z12_CNG7[7][5] = {{9, 5, 5, 10, 1},{2, 3, 6, 8, 2},{4, 10, 6, 5, 6},{9, 3, 4, 5, 6},{2, 11, 2, 11, 7},{6, 6, 2, 0, 5},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z20_CNG7[7][5] = {{12, 14, 9, 18, 9},{8, 0, 15, 19, 10},{7, 11, 9, 14, 0},{10, 15, 8, 1, 10},{5, 1, 5, 19, 4},{16, 11, 12, 18, 11},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z28_CNG7[7][5] = {{17, 26, 2, 14, 6},{20, 25, 7, 7, 8},{20, 6, 24, 15, 19},{16, 19, 3, 23, 13},{10, 26, 0, 23, 13},{18, 19, 14, 15, 25},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z36_CNG7[7][5] = {{25, 34, 9, 0, 26},{9, 1, 18, 6, 0},{10, 28, 28, 1, 3},{4, 15, 7, 16, 0},{32, 18, 22, 0, 32},{20, 21, 27, 7, 28},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z44_CNG7[7][5] = {{3, 43, 35, 22, 0},{15, 33, 41, 14, 40},{4, 27, 10, 13, 15},{33, 11, 22, 24, 37},{39, 5, 0, 22, 0},{39, 3, 32, 34, 25},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z52_CNG7[7][5] = {{41, 22, 6, 1, 17},{36, 25, 20, 46, 6},{8, 0, 47, 14, 29},{49, 16, 49, 26, 26},{3, 12, 0, 1, 1},{24, 18, 51, 2, 46},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z60_CNG7[7][5] = {{17, 56, 6, 4, 39},{26, 31, 35, 24, 18},{29, 10, 4, 2, 40},{32, 50, 0, 35, 53},{46, 55, 38, 12, 22},{18, 24, 20, 16, 49},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z16_CNG7[7][5] = {{12, 2, 0, 14, 8},{12, 5, 10, 7, 0},{15, 7, 5, 14, 1},{15, 0, 0, 15, 15},{7, 1, 10, 0, 11},{8, 10, 7, 5, 3},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z24_CNG7[7][5] = {{9, 5, 5, 22, 1},{14, 3, 18, 8, 2},{4, 10, 6, 17, 18},{21, 15, 16, 17, 6},{2, 23, 2, 11, 7},{18, 18, 2, 12, 5},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z40_CNG7[7][5] = {{12, 14, 29, 38, 29},{8, 0, 35, 19, 10},{7, 11, 9, 14, 20},{10, 35, 8, 21, 10},{25, 1, 25, 19, 24},{36, 31, 12, 18, 11},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z56_CNG7[7][5] = {{17, 26, 30, 14, 34},{20, 53, 7, 7, 8},{48, 6, 24, 43, 19},{44, 19, 3, 23, 41},{38, 26, 28, 51, 41},{46, 47, 14, 43, 25},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z72_CNG7[7][5] = {{25, 34, 9, 0, 26},{45, 1, 18, 42, 0},{10, 28, 28, 37, 39},{40, 51, 7, 16, 36},{32, 18, 22, 0, 68},{20, 57, 27, 7, 64},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z88_CNG7[7][5] = {{3, 87, 35, 22, 0},{15, 77, 41, 58, 84},{48, 27, 54, 13, 15},{33, 55, 22, 68, 81},{39, 49, 0, 66, 0},{39, 3, 76, 78, 25},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z104_CNG7[7][5] = {{41, 22, 6, 1, 69},{88, 77, 20, 98, 6},{8, 52, 99, 14, 81},{49, 16, 49, 26, 78},{55, 12, 0, 1, 53},{76, 70, 103, 2, 46},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z120_CNG7[7][5] = {{77, 116, 66, 4, 39},{26, 31, 95, 24, 18},{89, 70, 4, 62, 100},{32, 110, 60, 95, 53},{46, 115, 98, 72, 22},{18, 84, 80, 76, 49},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z32_CNG7[7][5] = {{28, 2, 0, 14, 8},{12, 21, 10, 23, 0},{31, 7, 21, 14, 1},{31, 0, 0, 31, 31},{7, 17, 10, 16, 11},{8, 26, 7, 5, 19},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z48_CNG7[7][5] = {{9, 5, 29, 46, 1},{14, 3, 42, 8, 2},{28, 34, 30, 41, 18},{45, 15, 40, 41, 30},{2, 47, 2, 11, 7},{18, 18, 26, 12, 29},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z80_CNG7[7][5] = {{12, 14, 69, 78, 69},{8, 0, 75, 59, 50},{47, 51, 9, 54, 60},{50, 75, 48, 61, 50},{25, 1, 25, 19, 24},{76, 71, 52, 18, 51},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z112_CNG7[7][5] = {{17, 82, 30, 14, 90},{76, 53, 63, 63, 8},{104, 62, 24, 99, 19},{100, 19, 3, 79, 97},{38, 82, 28, 51, 97},{46, 103, 70, 43, 81},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z144_CNG7[7][5] = {{25, 34, 81, 0, 26},{45, 1, 18, 42, 0},{10, 28, 100, 109, 39},{40, 123, 7, 16, 108},{104, 90, 94, 0, 68},{20, 57, 99, 79, 64},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z176_CNG7[7][5] = {{3, 175, 123, 22, 0},{103, 77, 41, 146, 172},{48, 27, 142, 101, 15},{121, 55, 110, 156, 81},{39, 49, 0, 66, 0},{39, 91, 76, 78, 113},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z208_CNG7[7][5] = {{145, 126, 6, 1, 173},{88, 77, 20, 202, 6},{112, 156, 203, 118, 81},{153, 16, 153, 130, 182},{159, 12, 104, 1, 53},{76, 70, 207, 2, 46},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z240_CNG7[7][5] = {{77, 116, 186, 124, 39},{146, 151, 215, 144, 138},{209, 70, 124, 182, 220},{32, 230, 180, 95, 173},{166, 115, 98, 72, 142},{18, 84, 80, 76, 49},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z64_CNG7[7][5] = {{28, 34, 32, 14, 40},{44, 21, 42, 55, 32},{31, 39, 21, 14, 1},{31, 32, 32, 63, 63},{39, 49, 42, 16, 11},{40, 58, 7, 37, 51},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z96_CNG7[7][5] = {{9, 5, 77, 46, 49},{62, 51, 90, 56, 2},{28, 82, 78, 41, 18},{45, 15, 40, 89, 30},{2, 95, 50, 59, 55},{18, 66, 74, 12, 77},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z160_CNG7[7][5] = {{12, 14, 69, 78, 69},{88, 80, 75, 139, 130},{47, 51, 9, 54, 60},{50, 75, 48, 61, 130},{25, 1, 105, 19, 24},{76, 151, 52, 98, 51},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z224_CNG7[7][5] = {{17, 82, 142, 14, 90},{76, 165, 175, 175, 120},{104, 174, 136, 211, 131},{100, 19, 3, 191, 209},{150, 194, 28, 51, 209},{158, 103, 182, 43, 81},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z288_CNG7[7][5] = {{169, 178, 225, 0, 170},{189, 1, 162, 186, 0},{154, 28, 244, 253, 183},{184, 267, 151, 16, 108},{104, 234, 238, 0, 68},{164, 201, 243, 79, 64},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z352_CNG7[7][5] = {{3, 175, 123, 22, 176},{103, 253, 217, 322, 348},{224, 27, 142, 277, 15},{297, 231, 110, 156, 81},{215, 49, 176, 66, 176},{39, 267, 76, 78, 113},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z128_CNG7[7][5] = {{92, 98, 32, 78, 40},{44, 21, 42, 55, 96},{31, 39, 21, 78, 65},{31, 32, 32, 127, 63},{39, 49, 106, 16, 75},{104, 58, 7, 101, 51},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z192_CNG7[7][5] = {{9, 101, 77, 142, 49},{62, 147, 186, 56, 2},{124, 82, 174, 137, 18},{141, 111, 40, 89, 126},{98, 191, 50, 155, 55},{114, 162, 74, 12, 77},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z320_CNG7[7][5] = {{12, 14, 229, 78, 229},{88, 80, 235, 299, 290},{207, 211, 169, 54, 60},{50, 75, 48, 61, 130},{25, 161, 105, 179, 184},{76, 311, 52, 258, 51},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z256_CNG7[7][5] = {{220, 98, 160, 206, 40},{44, 149, 42, 55, 96},{159, 167, 21, 206, 65},{31, 160, 32, 127, 63},{167, 49, 234, 16, 75},{104, 58, 7, 229, 179},{0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z384_CNG7[7][5] = {{9, 101, 77, 142, 241},{62, 339, 186, 248, 2},{316, 274, 174, 137, 210},{333, 111, 232, 89, 318},{290, 383, 50, 347, 55},{114, 354, 74, 12, 269},{0, 0, 0, 0, 0}}; + +// Group of CNs with 8 BNs (CNG8) ->[Num BNs][Num of CN with 8 BNs] +static const uint16_t circShift_BG1_Z2_CNG8[8][2] = {{1, 1},{0, 1},{0, 1},{1, 0},{0, 0},{1, 0},{1, 0},{0, 0}}; +static const uint16_t circShift_BG1_Z3_CNG8[8][2] = {{0, 0},{2, 0},{1, 2},{1, 2},{1, 2},{0, 1},{1, 1},{0, 0}}; +static const uint16_t circShift_BG1_Z5_CNG8[8][2] = {{3, 1},{2, 2},{0, 0},{3, 4},{1, 2},{2, 3},{4, 0},{0, 0}}; +static const uint16_t circShift_BG1_Z7_CNG8[8][2] = {{3, 3},{3, 4},{2, 2},{3, 0},{0, 4},{4, 0},{6, 5},{0, 0}}; +static const uint16_t circShift_BG1_Z9_CNG8[8][2] = {{0, 1},{1, 5},{0, 4},{4, 1},{2, 3},{7, 3},{0, 8},{0, 0}}; +static const uint16_t circShift_BG1_Z11_CNG8[8][2] = {{10, 3},{9, 3},{9, 2},{3, 10},{9, 1},{5, 8},{8, 5},{0, 0}}; +static const uint16_t circShift_BG1_Z13_CNG8[8][2] = {{3, 2},{9, 12},{11, 6},{0, 3},{12, 0},{5, 5},{7, 7},{0, 0}}; +static const uint16_t circShift_BG1_Z15_CNG8[8][2] = {{2, 0},{6, 10},{2, 14},{8, 1},{0, 2},{0, 0},{0, 2},{0, 0}}; +static const uint16_t circShift_BG1_Z4_CNG8[8][2] = {{1, 1},{0, 1},{2, 3},{3, 2},{0, 2},{3, 0},{3, 2},{0, 0}}; +static const uint16_t circShift_BG1_Z6_CNG8[8][2] = {{3, 0},{2, 0},{1, 2},{4, 5},{1, 2},{3, 4},{1, 1},{0, 0}}; +static const uint16_t circShift_BG1_Z10_CNG8[8][2] = {{3, 6},{2, 7},{0, 0},{8, 9},{1, 7},{7, 3},{9, 0},{0, 0}}; +static const uint16_t circShift_BG1_Z14_CNG8[8][2] = {{10, 10},{3, 11},{2, 2},{10, 7},{0, 4},{4, 0},{13, 5},{0, 0}}; +static const uint16_t circShift_BG1_Z18_CNG8[8][2] = {{9, 1},{1, 5},{0, 4},{13, 1},{2, 3},{7, 3},{0, 8},{0, 0}}; +static const uint16_t circShift_BG1_Z22_CNG8[8][2] = {{21, 3},{20, 14},{9, 2},{14, 21},{9, 12},{16, 8},{19, 16},{0, 0}}; +static const uint16_t circShift_BG1_Z26_CNG8[8][2] = {{3, 2},{22, 12},{24, 19},{13, 16},{25, 0},{5, 5},{20, 20},{0, 0}}; +static const uint16_t circShift_BG1_Z30_CNG8[8][2] = {{2, 15},{21, 25},{17, 14},{23, 1},{0, 2},{0, 15},{0, 17},{0, 0}}; +static const uint16_t circShift_BG1_Z8_CNG8[8][2] = {{5, 5},{4, 1},{2, 3},{7, 6},{4, 6},{3, 4},{3, 6},{0, 0}}; +static const uint16_t circShift_BG1_Z12_CNG8[8][2] = {{3, 0},{2, 6},{7, 8},{10, 11},{1, 8},{3, 10},{1, 7},{0, 0}}; +static const uint16_t circShift_BG1_Z20_CNG8[8][2] = {{3, 16},{12, 7},{10, 10},{18, 9},{1, 17},{7, 3},{19, 0},{0, 0}}; +static const uint16_t circShift_BG1_Z28_CNG8[8][2] = {{24, 24},{3, 11},{2, 2},{24, 7},{14, 4},{18, 0},{13, 5},{0, 0}}; +static const uint16_t circShift_BG1_Z36_CNG8[8][2] = {{9, 19},{1, 23},{0, 22},{31, 1},{2, 21},{7, 3},{0, 26},{0, 0}}; +static const uint16_t circShift_BG1_Z44_CNG8[8][2] = {{43, 25},{42, 14},{31, 24},{14, 43},{31, 34},{38, 30},{19, 16},{0, 0}}; +static const uint16_t circShift_BG1_Z52_CNG8[8][2] = {{29, 28},{48, 38},{24, 19},{13, 16},{51, 0},{5, 5},{20, 20},{0, 0}}; +static const uint16_t circShift_BG1_Z60_CNG8[8][2] = {{2, 45},{51, 55},{47, 14},{23, 1},{30, 32},{0, 45},{0, 47},{0, 0}}; +static const uint16_t circShift_BG1_Z16_CNG8[8][2] = {{13, 13},{12, 9},{2, 3},{7, 6},{12, 14},{11, 12},{3, 6},{0, 0}}; +static const uint16_t circShift_BG1_Z24_CNG8[8][2] = {{3, 0},{14, 6},{19, 8},{22, 23},{1, 20},{3, 22},{13, 19},{0, 0}}; +static const uint16_t circShift_BG1_Z40_CNG8[8][2] = {{3, 16},{12, 27},{10, 10},{38, 9},{1, 17},{27, 3},{39, 0},{0, 0}}; +static const uint16_t circShift_BG1_Z56_CNG8[8][2] = {{52, 52},{3, 11},{30, 2},{24, 35},{14, 32},{18, 28},{41, 33},{0, 0}}; +static const uint16_t circShift_BG1_Z72_CNG8[8][2] = {{45, 55},{37, 23},{0, 58},{67, 37},{38, 57},{7, 39},{0, 26},{0, 0}}; +static const uint16_t circShift_BG1_Z88_CNG8[8][2] = {{43, 25},{42, 58},{75, 24},{58, 87},{31, 78},{82, 74},{19, 16},{0, 0}}; +static const uint16_t circShift_BG1_Z104_CNG8[8][2] = {{81, 80},{100, 90},{24, 19},{65, 16},{103, 0},{57, 5},{72, 20},{0, 0}}; +static const uint16_t circShift_BG1_Z120_CNG8[8][2] = {{2, 45},{51, 115},{47, 14},{23, 1},{90, 32},{60, 45},{60, 107},{0, 0}}; +static const uint16_t circShift_BG1_Z32_CNG8[8][2] = {{13, 13},{12, 9},{2, 19},{7, 22},{28, 14},{27, 28},{19, 22},{0, 0}}; +static const uint16_t circShift_BG1_Z48_CNG8[8][2] = {{3, 0},{14, 6},{19, 8},{22, 47},{1, 44},{3, 46},{13, 19},{0, 0}}; +static const uint16_t circShift_BG1_Z80_CNG8[8][2] = {{3, 16},{52, 67},{50, 50},{78, 49},{41, 17},{27, 43},{39, 40},{0, 0}}; +static const uint16_t circShift_BG1_Z112_CNG8[8][2] = {{52, 52},{59, 11},{86, 2},{80, 35},{70, 32},{18, 84},{41, 89},{0, 0}}; +static const uint16_t circShift_BG1_Z144_CNG8[8][2] = {{117, 55},{37, 23},{72, 130},{139, 37},{110, 129},{79, 39},{0, 26},{0, 0}}; +static const uint16_t circShift_BG1_Z176_CNG8[8][2] = {{43, 25},{130, 146},{75, 24},{146, 175},{119, 166},{82, 162},{107, 16},{0, 0}}; +static const uint16_t circShift_BG1_Z208_CNG8[8][2] = {{185, 184},{100, 194},{24, 123},{65, 16},{207, 104},{161, 109},{72, 124},{0, 0}}; +static const uint16_t circShift_BG1_Z240_CNG8[8][2] = {{2, 45},{171, 115},{47, 134},{143, 1},{210, 152},{180, 165},{180, 107},{0, 0}}; +static const uint16_t circShift_BG1_Z64_CNG8[8][2] = {{13, 13},{44, 41},{2, 19},{39, 54},{28, 14},{59, 60},{51, 22},{0, 0}}; +static const uint16_t circShift_BG1_Z96_CNG8[8][2] = {{3, 48},{14, 6},{19, 8},{70, 47},{49, 92},{51, 46},{61, 19},{0, 0}}; +static const uint16_t circShift_BG1_Z160_CNG8[8][2] = {{83, 16},{132, 147},{50, 130},{158, 129},{41, 17},{107, 43},{119, 120},{0, 0}}; +static const uint16_t circShift_BG1_Z224_CNG8[8][2] = {{164, 52},{59, 11},{86, 2},{80, 35},{182, 32},{130, 84},{153, 201},{0, 0}}; +static const uint16_t circShift_BG1_Z288_CNG8[8][2] = {{261, 55},{181, 23},{72, 274},{283, 181},{254, 273},{79, 39},{144, 26},{0, 0}}; +static const uint16_t circShift_BG1_Z352_CNG8[8][2] = {{219, 25},{130, 322},{251, 200},{322, 351},{295, 166},{258, 338},{283, 192},{0, 0}}; +static const uint16_t circShift_BG1_Z128_CNG8[8][2] = {{77, 77},{108, 41},{66, 83},{103, 54},{28, 78},{123, 124},{115, 22},{0, 0}}; +static const uint16_t circShift_BG1_Z192_CNG8[8][2] = {{3, 48},{14, 102},{115, 8},{166, 47},{49, 188},{51, 142},{157, 115},{0, 0}}; +static const uint16_t circShift_BG1_Z320_CNG8[8][2] = {{83, 16},{292, 147},{50, 290},{318, 289},{201, 177},{267, 43},{279, 280},{0, 0}}; +static const uint16_t circShift_BG1_Z256_CNG8[8][2] = {{205, 77},{236, 41},{194, 83},{231, 182},{28, 78},{123, 252},{115, 22},{0, 0}}; +static const uint16_t circShift_BG1_Z384_CNG8[8][2] = {{195, 48},{14, 102},{115, 8},{166, 47},{241, 188},{51, 334},{157, 115},{0, 0}}; + +// Group of CNs with 9 BNs (CNG9) ->[Num BNs][Num of CN with 9 BNs] +static const uint16_t circShift_BG1_Z2_CNG9[9][2] = {{1, 1},{0, 0},{0, 1},{1, 1},{0, 0},{1, 0},{1, 1},{1, 0},{0, 0}}; +static const uint16_t circShift_BG1_Z3_CNG9[9][2] = {{2, 0},{2, 1},{1, 0},{0, 1},{2, 0},{1, 0},{0, 0},{0, 1},{0, 0}}; +static const uint16_t circShift_BG1_Z5_CNG9[9][2] = {{4, 4},{1, 4},{3, 1},{3, 1},{2, 1},{2, 2},{3, 0},{0, 4},{0, 0}}; +static const uint16_t circShift_BG1_Z7_CNG9[9][2] = {{4, 2},{0, 2},{1, 3},{0, 0},{0, 5},{2, 3},{3, 4},{4, 1},{0, 0}}; +static const uint16_t circShift_BG1_Z9_CNG9[9][2] = {{8, 0},{0, 6},{7, 3},{0, 8},{5, 0},{6, 6},{7, 4},{0, 2},{0, 0}}; +static const uint16_t circShift_BG1_Z11_CNG9[9][2] = {{8, 2},{7, 0},{0, 7},{0, 1},{7, 4},{7, 9},{9, 1},{8, 7},{0, 0}}; +static const uint16_t circShift_BG1_Z13_CNG9[9][2] = {{6, 6},{1, 10},{7, 2},{11, 1},{9, 6},{11, 4},{12, 8},{0, 4},{0, 0}}; +static const uint16_t circShift_BG1_Z15_CNG9[9][2] = {{4, 4},{7, 12},{8, 11},{10, 11},{2, 2},{12, 2},{7, 2},{13, 10},{0, 0}}; +static const uint16_t circShift_BG1_Z4_CNG9[9][2] = {{3, 3},{2, 2},{0, 1},{3, 1},{0, 2},{3, 2},{1, 1},{3, 0},{0, 0}}; +static const uint16_t circShift_BG1_Z6_CNG9[9][2] = {{2, 0},{5, 4},{1, 3},{3, 1},{2, 3},{1, 3},{0, 3},{3, 4},{0, 0}}; +static const uint16_t circShift_BG1_Z10_CNG9[9][2] = {{9, 9},{1, 4},{3, 6},{3, 6},{2, 1},{2, 7},{8, 5},{5, 9},{0, 0}}; +static const uint16_t circShift_BG1_Z14_CNG9[9][2] = {{4, 9},{7, 9},{1, 3},{7, 7},{7, 5},{9, 3},{10, 11},{4, 8},{0, 0}}; +static const uint16_t circShift_BG1_Z18_CNG9[9][2] = {{8, 0},{0, 15},{7, 3},{0, 8},{5, 9},{15, 6},{7, 4},{0, 11},{0, 0}}; +static const uint16_t circShift_BG1_Z22_CNG9[9][2] = {{8, 2},{7, 0},{0, 7},{11, 1},{18, 4},{18, 20},{20, 1},{19, 7},{0, 0}}; +static const uint16_t circShift_BG1_Z26_CNG9[9][2] = {{6, 6},{1, 10},{7, 15},{24, 1},{22, 19},{24, 4},{12, 8},{13, 17},{0, 0}}; +static const uint16_t circShift_BG1_Z30_CNG9[9][2] = {{19, 19},{22, 12},{23, 26},{10, 11},{2, 17},{27, 2},{22, 2},{13, 25},{0, 0}}; +static const uint16_t circShift_BG1_Z8_CNG9[9][2] = {{7, 7},{6, 6},{4, 5},{3, 5},{4, 6},{3, 6},{5, 5},{3, 0},{0, 0}}; +static const uint16_t circShift_BG1_Z12_CNG9[9][2] = {{2, 6},{5, 4},{1, 9},{3, 1},{8, 9},{1, 3},{6, 3},{9, 10},{0, 0}}; +static const uint16_t circShift_BG1_Z20_CNG9[9][2] = {{9, 9},{1, 4},{13, 16},{13, 6},{12, 11},{2, 7},{18, 15},{15, 9},{0, 0}}; +static const uint16_t circShift_BG1_Z28_CNG9[9][2] = {{18, 9},{7, 9},{1, 17},{21, 21},{7, 5},{23, 17},{24, 25},{4, 22},{0, 0}}; +static const uint16_t circShift_BG1_Z36_CNG9[9][2] = {{8, 18},{0, 15},{25, 21},{18, 26},{23, 9},{33, 24},{7, 4},{0, 29},{0, 0}}; +static const uint16_t circShift_BG1_Z44_CNG9[9][2] = {{30, 24},{29, 0},{22, 29},{11, 23},{40, 4},{18, 20},{20, 23},{41, 29},{0, 0}}; +static const uint16_t circShift_BG1_Z52_CNG9[9][2] = {{6, 6},{27, 10},{7, 41},{50, 1},{48, 45},{24, 4},{38, 8},{39, 17},{0, 0}}; +static const uint16_t circShift_BG1_Z60_CNG9[9][2] = {{19, 49},{22, 12},{23, 26},{40, 41},{32, 17},{27, 32},{52, 32},{13, 25},{0, 0}}; +static const uint16_t circShift_BG1_Z16_CNG9[9][2] = {{7, 7},{6, 6},{12, 13},{3, 5},{4, 14},{11, 14},{13, 13},{3, 8},{0, 0}}; +static const uint16_t circShift_BG1_Z24_CNG9[9][2] = {{14, 6},{17, 16},{1, 9},{15, 13},{20, 9},{13, 15},{18, 15},{9, 10},{0, 0}}; +static const uint16_t circShift_BG1_Z40_CNG9[9][2] = {{9, 29},{21, 4},{13, 36},{13, 6},{32, 31},{22, 27},{18, 15},{35, 9},{0, 0}}; +static const uint16_t circShift_BG1_Z56_CNG9[9][2] = {{46, 9},{7, 37},{1, 45},{21, 49},{7, 33},{51, 17},{24, 53},{4, 50},{0, 0}}; +static const uint16_t circShift_BG1_Z72_CNG9[9][2] = {{8, 18},{0, 15},{25, 21},{18, 62},{59, 45},{33, 60},{7, 4},{36, 65},{0, 0}}; +static const uint16_t circShift_BG1_Z88_CNG9[9][2] = {{30, 68},{73, 0},{66, 29},{11, 23},{84, 4},{62, 64},{20, 23},{41, 73},{0, 0}}; +static const uint16_t circShift_BG1_Z104_CNG9[9][2] = {{6, 6},{27, 10},{59, 41},{50, 53},{48, 97},{24, 4},{38, 60},{91, 69},{0, 0}}; +static const uint16_t circShift_BG1_Z120_CNG9[9][2] = {{79, 49},{22, 12},{23, 86},{100, 101},{92, 17},{87, 92},{52, 92},{13, 85},{0, 0}}; +static const uint16_t circShift_BG1_Z32_CNG9[9][2] = {{23, 7},{22, 22},{28, 13},{3, 21},{20, 14},{11, 14},{29, 29},{19, 24},{0, 0}}; +static const uint16_t circShift_BG1_Z48_CNG9[9][2] = {{38, 30},{17, 40},{1, 33},{15, 37},{44, 9},{13, 15},{18, 15},{33, 34},{0, 0}}; +static const uint16_t circShift_BG1_Z80_CNG9[9][2] = {{49, 29},{21, 4},{53, 36},{13, 46},{72, 71},{62, 27},{58, 55},{75, 49},{0, 0}}; +static const uint16_t circShift_BG1_Z112_CNG9[9][2] = {{46, 9},{7, 37},{1, 101},{21, 105},{63, 89},{51, 73},{24, 109},{4, 106},{0, 0}}; +static const uint16_t circShift_BG1_Z144_CNG9[9][2] = {{80, 18},{0, 15},{25, 93},{90, 134},{59, 45},{33, 132},{7, 76},{108, 65},{0, 0}}; +static const uint16_t circShift_BG1_Z176_CNG9[9][2] = {{118, 156},{73, 88},{154, 117},{99, 111},{172, 92},{150, 152},{108, 23},{129, 161},{0, 0}}; +static const uint16_t circShift_BG1_Z208_CNG9[9][2] = {{6, 6},{27, 10},{163, 145},{50, 53},{48, 201},{24, 4},{38, 164},{91, 173},{0, 0}}; +static const uint16_t circShift_BG1_Z240_CNG9[9][2] = {{199, 169},{22, 12},{23, 206},{100, 221},{92, 17},{207, 212},{52, 92},{13, 205},{0, 0}}; +static const uint16_t circShift_BG1_Z64_CNG9[9][2] = {{55, 39},{22, 54},{28, 45},{3, 21},{52, 14},{11, 14},{29, 61},{19, 24},{0, 0}}; +static const uint16_t circShift_BG1_Z96_CNG9[9][2] = {{86, 78},{65, 40},{1, 33},{63, 37},{92, 57},{61, 15},{18, 63},{33, 82},{0, 0}}; +static const uint16_t circShift_BG1_Z160_CNG9[9][2] = {{129, 29},{21, 84},{133, 36},{13, 126},{72, 151},{142, 107},{138, 135},{75, 49},{0, 0}}; +static const uint16_t circShift_BG1_Z224_CNG9[9][2] = {{158, 9},{119, 37},{113, 213},{21, 105},{63, 89},{51, 185},{136, 109},{116, 218},{0, 0}}; +static const uint16_t circShift_BG1_Z288_CNG9[9][2] = {{80, 162},{144, 159},{169, 93},{90, 134},{59, 45},{177, 132},{151, 76},{108, 209},{0, 0}}; +static const uint16_t circShift_BG1_Z352_CNG9[9][2] = {{294, 156},{73, 88},{330, 293},{99, 111},{172, 92},{150, 152},{284, 23},{305, 337},{0, 0}}; +static const uint16_t circShift_BG1_Z128_CNG9[9][2] = {{55, 103},{22, 54},{28, 109},{67, 21},{116, 14},{11, 14},{29, 61},{83, 88},{0, 0}}; +static const uint16_t circShift_BG1_Z192_CNG9[9][2] = {{86, 174},{65, 40},{1, 129},{159, 133},{92, 57},{61, 111},{18, 63},{33, 82},{0, 0}}; +static const uint16_t circShift_BG1_Z320_CNG9[9][2] = {{289, 189},{21, 244},{293, 36},{13, 286},{232, 151},{302, 267},{138, 135},{235, 209},{0, 0}}; +static const uint16_t circShift_BG1_Z256_CNG9[9][2] = {{183, 103},{22, 182},{28, 109},{67, 21},{244, 142},{11, 14},{157, 61},{211, 216},{0, 0}}; +static const uint16_t circShift_BG1_Z384_CNG9[9][2] = {{278, 366},{257, 232},{1, 321},{351, 133},{92, 57},{253, 303},{18, 63},{225, 82},{0, 0}}; + +// Group of CNs with 10 BNs (CNG10) ->[Num BNs][Num of CN with 10 BNs] +static const uint16_t circShift_BG1_Z2_CNG10[10][1] = {{0},{0},{1},{1},{0},{0},{1},{1},{0},{0}}; +static const uint16_t circShift_BG1_Z3_CNG10[10][1] = {{1},{2},{0},{0},{0},{2},{2},{1},{2},{0}}; +static const uint16_t circShift_BG1_Z5_CNG10[10][1] = {{0},{3},{0},{1},{1},{0},{4},{0},{4},{0}}; +static const uint16_t circShift_BG1_Z7_CNG10[10][1] = {{5},{4},{4},{0},{1},{4},{2},{2},{5},{0}}; +static const uint16_t circShift_BG1_Z9_CNG10[10][1] = {{0},{0},{0},{5},{8},{1},{6},{5},{0},{0}}; +static const uint16_t circShift_BG1_Z11_CNG10[10][1] = {{7},{9},{0},{4},{4},{6},{4},{6},{2},{0}}; +static const uint16_t circShift_BG1_Z13_CNG10[10][1] = {{3},{2},{1},{11},{5},{8},{10},{9},{12},{0}}; +static const uint16_t circShift_BG1_Z15_CNG10[10][1] = {{1},{0},{6},{13},{12},{10},{4},{3},{8},{0}}; +static const uint16_t circShift_BG1_Z4_CNG10[10][1] = {{0},{0},{3},{3},{2},{0},{1},{1},{2},{0}}; +static const uint16_t circShift_BG1_Z6_CNG10[10][1] = {{1},{5},{3},{0},{3},{2},{2},{1},{2},{0}}; +static const uint16_t circShift_BG1_Z10_CNG10[10][1] = {{5},{3},{0},{1},{6},{0},{9},{5},{4},{0}}; +static const uint16_t circShift_BG1_Z14_CNG10[10][1] = {{5},{11},{4},{7},{8},{11},{2},{2},{5},{0}}; +static const uint16_t circShift_BG1_Z18_CNG10[10][1] = {{0},{0},{0},{5},{8},{10},{15},{14},{0},{0}}; +static const uint16_t circShift_BG1_Z22_CNG10[10][1] = {{18},{9},{0},{4},{4},{17},{15},{6},{2},{0}}; +static const uint16_t circShift_BG1_Z26_CNG10[10][1] = {{16},{2},{1},{11},{18},{21},{23},{9},{12},{0}}; +static const uint16_t circShift_BG1_Z30_CNG10[10][1] = {{1},{15},{21},{13},{27},{25},{19},{3},{8},{0}}; +static const uint16_t circShift_BG1_Z8_CNG10[10][1] = {{0},{4},{7},{3},{6},{4},{5},{1},{2},{0}}; +static const uint16_t circShift_BG1_Z12_CNG10[10][1] = {{7},{11},{9},{6},{3},{8},{8},{7},{2},{0}}; +static const uint16_t circShift_BG1_Z20_CNG10[10][1] = {{15},{13},{10},{11},{16},{10},{9},{5},{14},{0}}; +static const uint16_t circShift_BG1_Z28_CNG10[10][1] = {{5},{11},{4},{21},{8},{11},{2},{16},{5},{0}}; +static const uint16_t circShift_BG1_Z36_CNG10[10][1] = {{18},{0},{0},{5},{26},{10},{15},{14},{18},{0}}; +static const uint16_t circShift_BG1_Z44_CNG10[10][1] = {{40},{31},{22},{4},{4},{17},{15},{6},{24},{0}}; +static const uint16_t circShift_BG1_Z52_CNG10[10][1] = {{16},{2},{27},{37},{44},{47},{49},{35},{12},{0}}; +static const uint16_t circShift_BG1_Z60_CNG10[10][1] = {{1},{45},{21},{43},{57},{25},{19},{33},{38},{0}}; +static const uint16_t circShift_BG1_Z16_CNG10[10][1] = {{0},{4},{7},{3},{6},{4},{13},{1},{10},{0}}; +static const uint16_t circShift_BG1_Z24_CNG10[10][1] = {{19},{11},{21},{18},{15},{8},{8},{19},{2},{0}}; +static const uint16_t circShift_BG1_Z40_CNG10[10][1] = {{15},{13},{10},{31},{16},{30},{29},{5},{34},{0}}; +static const uint16_t circShift_BG1_Z56_CNG10[10][1] = {{33},{39},{4},{49},{36},{39},{2},{44},{33},{0}}; +static const uint16_t circShift_BG1_Z72_CNG10[10][1] = {{54},{0},{36},{41},{26},{46},{15},{14},{54},{0}}; +static const uint16_t circShift_BG1_Z88_CNG10[10][1] = {{84},{75},{22},{48},{48},{17},{59},{50},{68},{0}}; +static const uint16_t circShift_BG1_Z104_CNG10[10][1] = {{68},{2},{27},{37},{96},{99},{101},{35},{12},{0}}; +static const uint16_t circShift_BG1_Z120_CNG10[10][1] = {{61},{105},{21},{103},{57},{25},{79},{33},{38},{0}}; +static const uint16_t circShift_BG1_Z32_CNG10[10][1] = {{16},{4},{7},{19},{6},{4},{13},{17},{26},{0}}; +static const uint16_t circShift_BG1_Z48_CNG10[10][1] = {{19},{35},{21},{18},{39},{32},{32},{19},{26},{0}}; +static const uint16_t circShift_BG1_Z80_CNG10[10][1] = {{55},{53},{50},{71},{56},{30},{29},{5},{74},{0}}; +static const uint16_t circShift_BG1_Z112_CNG10[10][1] = {{33},{95},{4},{105},{92},{39},{58},{44},{89},{0}}; +static const uint16_t circShift_BG1_Z144_CNG10[10][1] = {{54},{0},{108},{41},{98},{46},{15},{86},{54},{0}}; +static const uint16_t circShift_BG1_Z176_CNG10[10][1] = {{172},{75},{22},{136},{48},{17},{59},{138},{68},{0}}; +static const uint16_t circShift_BG1_Z208_CNG10[10][1] = {{172},{2},{131},{141},{96},{99},{101},{35},{116},{0}}; +static const uint16_t circShift_BG1_Z240_CNG10[10][1] = {{181},{105},{141},{223},{177},{145},{199},{153},{38},{0}}; +static const uint16_t circShift_BG1_Z64_CNG10[10][1] = {{48},{4},{7},{19},{38},{36},{45},{49},{26},{0}}; +static const uint16_t circShift_BG1_Z96_CNG10[10][1] = {{19},{83},{69},{18},{39},{32},{80},{67},{74},{0}}; +static const uint16_t circShift_BG1_Z160_CNG10[10][1] = {{135},{133},{130},{71},{136},{110},{109},{85},{154},{0}}; +static const uint16_t circShift_BG1_Z224_CNG10[10][1] = {{33},{95},{4},{217},{204},{39},{58},{44},{201},{0}}; +static const uint16_t circShift_BG1_Z288_CNG10[10][1] = {{54},{0},{252},{41},{98},{46},{15},{230},{54},{0}}; +static const uint16_t circShift_BG1_Z352_CNG10[10][1] = {{348},{75},{22},{312},{224},{17},{59},{314},{244},{0}}; +static const uint16_t circShift_BG1_Z128_CNG10[10][1] = {{112},{4},{7},{83},{102},{36},{109},{113},{90},{0}}; +static const uint16_t circShift_BG1_Z192_CNG10[10][1] = {{115},{179},{165},{18},{39},{32},{176},{67},{170},{0}}; +static const uint16_t circShift_BG1_Z320_CNG10[10][1] = {{295},{133},{130},{231},{296},{110},{269},{245},{154},{0}}; +static const uint16_t circShift_BG1_Z256_CNG10[10][1] = {{112},{4},{7},{211},{102},{164},{109},{241},{90},{0}}; +static const uint16_t circShift_BG1_Z384_CNG10[10][1] = {{307},{179},{165},{18},{39},{224},{368},{67},{170},{0}}; + +// Group of CNs with 19 BNs (CNG19) ->[Num BNs][Num of CN with 19 BNs] +static const uint16_t circShift_BG1_Z2_CNG19[19][4] = {{0, 0, 0, 1},{1, 1, 1, 1},{0, 1, 1, 0},{1, 0, 1, 0},{0, 1, 1, 0},{0, 0, 1, 1},{1, 0, 1, 1},{1, 1, 1, 0},{0, 0, 1, 0},{1, 0, 1, 0},{1, 1, 0, 1},{1, 0, 1, 1},{1, 0, 1, 0},{0, 1, 1, 0},{1, 1, 1, 0},{1, 0, 1, 0},{1, 0, 1, 0},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z3_CNG19[19][4] = {{1, 1, 1, 0},{1, 1, 1, 0},{2, 1, 1, 0},{0, 0, 2, 2},{1, 0, 1, 1},{0, 1, 2, 0},{2, 1, 0, 2},{0, 1, 1, 0},{1, 1, 0, 2},{2, 0, 0, 0},{0, 1, 2, 2},{2, 0, 1, 2},{1, 0, 2, 1},{2, 0, 2, 0},{0, 1, 0, 1},{0, 1, 1, 0},{1, 0, 1, 0},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z5_CNG19[19][4] = {{3, 3, 3, 0},{0, 4, 2, 3},{3, 2, 0, 0},{4, 1, 0, 2},{0, 1, 3, 1},{4, 3, 2, 0},{0, 4, 2, 4},{2, 0, 0, 1},{0, 4, 1, 3},{4, 0, 1, 3},{3, 1, 0, 4},{3, 1, 3, 3},{0, 2, 1, 4},{3, 3, 4, 3},{1, 0, 1, 2},{4, 1, 0, 4},{2, 0, 1, 3},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z7_CNG19[19][4] = {{6, 1, 4, 5},{2, 3, 0, 4},{3, 4, 3, 4},{0, 4, 1, 5},{4, 0, 5, 3},{3, 3, 4, 2},{0, 0, 4, 2},{2, 3, 6, 6},{6, 3, 2, 1},{0, 2, 0, 3},{5, 2, 0, 0},{0, 2, 4, 6},{0, 6, 0, 5},{1, 5, 2, 4},{2, 3, 5, 1},{6, 5, 6, 2},{4, 0, 6, 4},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z9_CNG19[19][4] = {{4, 8, 6, 7},{0, 0, 5, 1},{8, 7, 4, 4},{6, 4, 7, 0},{3, 7, 0, 1},{4, 4, 4, 6},{2, 4, 2, 8},{0, 0, 0, 5},{8, 1, 0, 0},{0, 6, 3, 3},{7, 0, 2, 0},{8, 8, 4, 7},{0, 5, 0, 8},{5, 3, 0, 3},{0, 3, 8, 0},{1, 2, 2, 5},{0, 0, 0, 0},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z11_CNG19[19][4] = {{8, 0, 6, 9},{8, 5, 2, 6},{2, 8, 4, 5},{0, 8, 5, 4},{9, 4, 1, 7},{0, 2, 1, 1},{1, 5, 7, 3},{8, 6, 6, 7},{1, 2, 8, 4},{9, 0, 6, 4},{3, 4, 0, 5},{9, 0, 0, 9},{6, 4, 4, 7},{7, 1, 10, 6},{2, 9, 9, 0},{3, 5, 1, 0},{1, 0, 7, 0},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z13_CNG19[19][4] = {{0, 9, 2, 4},{0, 11, 11, 6},{0, 7, 8, 7},{0, 0, 11, 9},{0, 10, 4, 10},{0, 0, 6, 8},{0, 0, 9, 6},{0, 2, 12, 8},{0, 3, 5, 5},{0, 8, 0, 11},{0, 0, 12, 9},{0, 6, 2, 5},{0, 4, 7, 1},{0, 0, 8, 4},{0, 12, 10, 3},{0, 5, 3, 2},{0, 1, 1, 12},{0, 0, 0, 0},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z15_CNG19[19][4] = {{0, 6, 9, 8},{2, 11, 4, 8},{6, 1, 0, 12},{14, 11, 1, 10},{9, 8, 11, 13},{8, 2, 12, 6},{8, 7, 14, 6},{0, 11, 2, 1},{10, 11, 9, 6},{8, 9, 8, 7},{0, 5, 6, 9},{0, 10, 11, 2},{7, 3, 3, 5},{10, 12, 6, 2},{0, 13, 5, 10},{0, 6, 7, 7},{2, 0, 5, 9},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z4_CNG19[19][4] = {{2, 2, 2, 1},{1, 3, 3, 1},{2, 1, 1, 0},{3, 0, 3, 0},{0, 3, 1, 2},{2, 2, 1, 3},{3, 0, 1, 3},{1, 1, 1, 0},{2, 0, 3, 2},{3, 2, 3, 2},{1, 1, 2, 3},{3, 0, 1, 3},{3, 2, 1, 0},{2, 3, 1, 0},{3, 3, 1, 0},{3, 0, 3, 0},{3, 0, 1, 0},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z6_CNG19[19][4] = {{1, 4, 1, 0},{1, 4, 4, 3},{2, 1, 4, 0},{3, 0, 2, 5},{1, 0, 4, 1},{0, 1, 5, 3},{5, 1, 3, 2},{0, 4, 4, 0},{1, 1, 3, 5},{5, 0, 3, 3},{3, 1, 2, 5},{5, 3, 4, 2},{4, 0, 5, 4},{2, 0, 5, 0},{0, 1, 0, 1},{0, 4, 1, 3},{4, 0, 1, 3},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z10_CNG19[19][4] = {{3, 3, 8, 0},{5, 4, 7, 8},{3, 7, 0, 0},{9, 1, 0, 7},{0, 1, 8, 1},{9, 3, 7, 5},{5, 4, 2, 9},{2, 0, 0, 1},{5, 9, 1, 3},{4, 0, 6, 3},{3, 6, 5, 4},{8, 6, 3, 3},{0, 2, 1, 4},{3, 3, 4, 8},{6, 0, 6, 7},{9, 1, 0, 9},{2, 0, 6, 8},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z14_CNG19[19][4] = {{13, 1, 11, 5},{2, 3, 7, 4},{10, 11, 3, 11},{7, 4, 8, 5},{4, 7, 12, 3},{10, 3, 4, 2},{0, 7, 11, 2},{9, 3, 13, 6},{6, 10, 9, 1},{7, 9, 0, 3},{5, 9, 7, 7},{0, 9, 4, 13},{0, 6, 7, 12},{1, 12, 2, 4},{2, 3, 5, 1},{6, 5, 13, 2},{11, 0, 6, 4},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z18_CNG19[19][4] = {{13, 17, 6, 7},{0, 0, 5, 1},{8, 7, 4, 4},{6, 4, 7, 0},{3, 16, 9, 10},{4, 4, 4, 6},{11, 4, 2, 17},{0, 9, 9, 5},{8, 1, 0, 0},{0, 15, 3, 3},{7, 0, 2, 0},{17, 8, 13, 7},{0, 14, 0, 8},{5, 3, 0, 3},{0, 12, 17, 0},{1, 11, 2, 5},{9, 0, 0, 0},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z22_CNG19[19][4] = {{8, 11, 6, 9},{8, 5, 13, 6},{13, 8, 15, 5},{0, 8, 16, 15},{9, 4, 1, 7},{11, 2, 1, 12},{1, 16, 18, 3},{8, 6, 17, 18},{1, 13, 8, 4},{9, 11, 17, 15},{3, 4, 0, 5},{9, 0, 0, 9},{17, 15, 15, 7},{18, 1, 21, 17},{13, 9, 9, 11},{3, 16, 12, 11},{12, 0, 7, 11},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z26_CNG19[19][4] = {{0, 22, 2, 4},{0, 11, 11, 6},{0, 20, 21, 7},{0, 0, 24, 9},{0, 10, 4, 23},{0, 0, 19, 21},{0, 0, 22, 6},{0, 2, 12, 21},{0, 16, 18, 5},{0, 8, 13, 24},{0, 0, 25, 9},{0, 6, 2, 5},{0, 4, 7, 14},{0, 0, 21, 17},{0, 12, 23, 16},{0, 5, 16, 2},{0, 1, 14, 12},{0, 0, 0, 0},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z30_CNG19[19][4] = {{15, 6, 9, 8},{17, 26, 4, 23},{6, 16, 15, 12},{14, 11, 1, 10},{24, 8, 26, 13},{23, 2, 27, 6},{23, 22, 29, 6},{15, 26, 2, 1},{25, 11, 24, 6},{8, 9, 8, 22},{15, 5, 6, 24},{15, 25, 11, 17},{7, 3, 3, 20},{10, 27, 6, 2},{0, 13, 5, 25},{15, 6, 7, 22},{17, 0, 20, 9},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z8_CNG19[19][4] = {{2, 2, 2, 1},{5, 7, 7, 1},{2, 5, 1, 4},{7, 4, 7, 4},{4, 7, 5, 6},{2, 6, 5, 3},{3, 0, 5, 3},{5, 5, 1, 0},{6, 4, 7, 6},{7, 6, 7, 6},{1, 5, 6, 3},{3, 4, 1, 3},{7, 6, 1, 0},{6, 3, 5, 4},{3, 7, 5, 4},{7, 4, 3, 0},{7, 0, 5, 4},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z12_CNG19[19][4] = {{7, 4, 1, 0},{7, 4, 10, 3},{2, 1, 4, 0},{9, 0, 8, 11},{1, 0, 4, 7},{0, 7, 5, 9},{5, 7, 3, 8},{0, 10, 4, 0},{1, 7, 3, 5},{5, 6, 9, 3},{9, 1, 8, 5},{11, 3, 4, 8},{10, 6, 5, 4},{2, 6, 11, 0},{0, 7, 0, 7},{6, 4, 1, 3},{10, 0, 1, 9},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z20_CNG19[19][4] = {{13, 3, 8, 0},{15, 14, 7, 8},{3, 7, 0, 10},{9, 1, 0, 17},{0, 1, 18, 1},{19, 13, 7, 15},{15, 4, 2, 19},{2, 0, 0, 1},{15, 9, 11, 3},{4, 0, 6, 13},{13, 16, 15, 4},{18, 6, 3, 13},{10, 12, 1, 4},{13, 3, 4, 8},{16, 0, 6, 17},{9, 1, 10, 19},{12, 0, 16, 8},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z28_CNG19[19][4] = {{27, 1, 11, 5},{16, 17, 7, 18},{10, 11, 3, 25},{7, 18, 8, 5},{18, 7, 12, 17},{10, 17, 18, 2},{0, 21, 11, 16},{9, 3, 13, 6},{20, 10, 9, 15},{21, 9, 14, 17},{19, 23, 21, 7},{14, 9, 18, 13},{14, 6, 21, 12},{1, 26, 2, 18},{2, 3, 5, 15},{20, 19, 27, 2},{25, 0, 6, 18},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z36_CNG19[19][4] = {{31, 35, 6, 7},{18, 18, 23, 1},{8, 7, 4, 22},{6, 4, 25, 0},{3, 16, 27, 10},{4, 4, 22, 24},{29, 22, 2, 17},{0, 9, 27, 5},{8, 1, 0, 18},{0, 15, 3, 21},{7, 0, 2, 0},{17, 8, 13, 7},{0, 14, 0, 26},{23, 3, 0, 21},{0, 12, 17, 0},{1, 11, 20, 5},{9, 0, 18, 0},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z44_CNG19[19][4] = {{30, 33, 6, 9},{30, 5, 35, 6},{35, 8, 37, 5},{22, 30, 38, 15},{31, 4, 23, 7},{33, 24, 1, 34},{23, 38, 40, 3},{30, 6, 17, 18},{1, 35, 30, 26},{31, 33, 39, 37},{25, 26, 22, 5},{9, 22, 22, 9},{39, 37, 37, 29},{40, 1, 43, 39},{35, 9, 31, 11},{3, 38, 34, 33},{12, 0, 7, 33},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z52_CNG19[19][4] = {{0, 22, 28, 4},{0, 11, 37, 6},{0, 20, 21, 33},{0, 0, 24, 9},{0, 10, 4, 49},{0, 0, 45, 21},{0, 0, 48, 6},{0, 2, 38, 47},{0, 16, 18, 31},{0, 8, 39, 50},{0, 0, 51, 35},{0, 6, 28, 5},{0, 30, 33, 40},{0, 0, 47, 17},{0, 12, 23, 16},{0, 31, 42, 2},{0, 1, 14, 38},{0, 0, 0, 0},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z60_CNG19[19][4] = {{15, 36, 9, 8},{47, 56, 4, 23},{6, 16, 45, 42},{14, 41, 31, 40},{24, 8, 56, 43},{23, 32, 57, 6},{53, 52, 59, 36},{45, 56, 32, 1},{25, 11, 24, 36},{8, 9, 8, 22},{15, 35, 6, 24},{15, 25, 41, 47},{37, 33, 33, 20},{40, 27, 36, 32},{30, 43, 5, 55},{45, 36, 7, 52},{17, 0, 50, 39},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z16_CNG19[19][4] = {{10, 2, 10, 9},{5, 15, 15, 9},{2, 5, 9, 4},{15, 12, 15, 4},{4, 7, 5, 6},{10, 14, 13, 3},{11, 8, 5, 3},{5, 13, 1, 8},{14, 12, 15, 6},{15, 6, 7, 6},{9, 13, 14, 11},{3, 4, 1, 3},{7, 14, 1, 0},{14, 11, 5, 12},{3, 15, 13, 4},{15, 12, 11, 0},{15, 0, 5, 12},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z24_CNG19[19][4] = {{19, 4, 13, 12},{19, 4, 10, 15},{2, 1, 16, 0},{9, 0, 20, 11},{13, 0, 16, 7},{0, 19, 17, 9},{5, 19, 3, 8},{0, 10, 16, 12},{13, 7, 15, 17},{17, 6, 9, 15},{21, 1, 8, 5},{23, 3, 16, 20},{10, 18, 5, 16},{2, 18, 11, 12},{12, 19, 0, 7},{18, 16, 13, 15},{10, 0, 13, 21},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z40_CNG19[19][4] = {{33, 23, 28, 20},{15, 14, 7, 8},{23, 27, 0, 30},{9, 21, 0, 37},{0, 1, 38, 21},{39, 13, 27, 15},{15, 4, 2, 39},{2, 0, 0, 1},{15, 9, 31, 23},{4, 20, 26, 13},{13, 36, 15, 4},{18, 26, 3, 13},{30, 32, 21, 4},{33, 3, 24, 28},{16, 20, 6, 37},{29, 21, 30, 39},{32, 0, 36, 28},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z56_CNG19[19][4] = {{55, 29, 39, 33},{16, 45, 35, 18},{38, 39, 31, 53},{35, 46, 8, 5},{18, 7, 12, 45},{10, 45, 18, 30},{0, 21, 39, 16},{37, 31, 41, 34},{48, 38, 9, 43},{21, 37, 14, 45},{47, 23, 21, 35},{14, 9, 46, 13},{14, 6, 21, 40},{29, 26, 30, 18},{30, 31, 5, 43},{48, 19, 55, 30},{25, 0, 34, 46},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z72_CNG19[19][4] = {{67, 35, 42, 43},{54, 18, 23, 1},{44, 7, 4, 22},{42, 40, 61, 36},{3, 16, 27, 10},{4, 4, 58, 60},{29, 58, 2, 53},{0, 45, 63, 41},{44, 37, 0, 18},{0, 15, 3, 57},{43, 0, 2, 36},{17, 8, 13, 43},{0, 14, 0, 26},{23, 3, 0, 21},{0, 12, 53, 0},{1, 47, 56, 41},{45, 0, 18, 36},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z88_CNG19[19][4] = {{30, 77, 50, 9},{30, 49, 35, 6},{79, 8, 37, 49},{66, 74, 38, 15},{31, 4, 23, 51},{77, 24, 1, 78},{67, 38, 40, 3},{74, 50, 61, 18},{1, 79, 30, 70},{75, 77, 39, 81},{25, 70, 22, 5},{53, 66, 22, 9},{83, 81, 37, 73},{40, 45, 43, 39},{79, 9, 75, 55},{47, 38, 34, 33},{12, 0, 7, 33},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z104_CNG19[19][4] = {{0, 22, 28, 4},{0, 11, 37, 6},{0, 20, 21, 33},{0, 0, 76, 9},{0, 10, 4, 49},{0, 0, 45, 21},{0, 0, 48, 6},{0, 2, 38, 47},{0, 16, 18, 83},{0, 60, 91, 50},{0, 0, 51, 87},{0, 6, 28, 5},{0, 30, 85, 92},{0, 0, 47, 69},{0, 64, 75, 16},{0, 31, 42, 2},{0, 1, 66, 38},{0, 0, 0, 0},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z120_CNG19[19][4] = {{15, 96, 69, 8},{107, 116, 4, 23},{6, 16, 105, 42},{14, 101, 31, 100},{84, 8, 116, 43},{83, 92, 117, 66},{53, 52, 59, 96},{105, 56, 92, 1},{85, 11, 24, 96},{8, 69, 68, 22},{75, 95, 6, 24},{15, 85, 101, 47},{97, 33, 33, 80},{100, 87, 96, 32},{90, 43, 5, 115},{105, 96, 67, 52},{17, 0, 110, 99},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z32_CNG19[19][4] = {{26, 2, 10, 25},{5, 15, 15, 25},{2, 21, 25, 20},{31, 28, 31, 20},{4, 7, 21, 22},{10, 30, 29, 3},{27, 8, 5, 19},{5, 13, 17, 8},{14, 28, 31, 22},{31, 6, 7, 22},{9, 13, 14, 27},{3, 4, 1, 19},{23, 14, 1, 16},{30, 27, 21, 12},{3, 31, 13, 20},{15, 28, 27, 16},{31, 0, 21, 12},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z48_CNG19[19][4] = {{19, 28, 13, 36},{19, 28, 10, 39},{2, 25, 40, 0},{33, 0, 44, 35},{37, 0, 16, 7},{24, 43, 17, 9},{29, 43, 27, 8},{0, 34, 16, 36},{13, 7, 15, 17},{17, 6, 33, 39},{21, 25, 8, 5},{23, 3, 40, 20},{10, 18, 5, 16},{2, 18, 35, 12},{36, 43, 0, 31},{42, 16, 13, 39},{10, 0, 13, 21},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z80_CNG19[19][4] = {{73, 63, 68, 60},{15, 54, 7, 48},{23, 27, 0, 30},{49, 21, 40, 37},{0, 1, 38, 61},{39, 53, 67, 15},{15, 4, 42, 79},{2, 0, 40, 41},{55, 49, 71, 63},{4, 60, 26, 13},{53, 76, 55, 4},{58, 26, 43, 53},{30, 72, 61, 44},{33, 3, 24, 28},{16, 20, 6, 77},{29, 61, 70, 79},{32, 0, 36, 68},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z112_CNG19[19][4] = {{111, 29, 95, 89},{16, 45, 91, 18},{94, 39, 31, 53},{91, 46, 64, 5},{74, 7, 68, 45},{10, 45, 74, 30},{0, 21, 95, 16},{93, 87, 41, 34},{104, 94, 65, 43},{21, 93, 70, 101},{103, 79, 77, 35},{14, 9, 102, 69},{70, 6, 77, 96},{29, 82, 86, 74},{86, 31, 5, 99},{104, 75, 111, 30},{81, 0, 90, 46},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z144_CNG19[19][4] = {{67, 35, 114, 43},{54, 18, 23, 1},{44, 79, 76, 22},{42, 112, 133, 108},{75, 16, 99, 82},{4, 76, 58, 132},{29, 58, 74, 53},{0, 117, 63, 41},{116, 109, 0, 18},{72, 15, 3, 57},{115, 72, 74, 36},{89, 8, 85, 115},{0, 14, 0, 98},{95, 3, 72, 21},{72, 12, 125, 0},{73, 119, 56, 113},{117, 0, 90, 108},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z176_CNG19[19][4] = {{118, 77, 50, 97},{118, 49, 35, 94},{167, 96, 37, 49},{154, 162, 126, 103},{31, 92, 111, 139},{165, 112, 89, 166},{67, 126, 128, 91},{74, 50, 61, 106},{1, 167, 118, 70},{163, 77, 127, 169},{25, 158, 110, 93},{53, 66, 110, 9},{171, 81, 125, 73},{128, 133, 131, 39},{167, 9, 163, 143},{47, 126, 34, 121},{12, 0, 7, 121},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z208_CNG19[19][4] = {{0, 22, 132, 4},{0, 11, 37, 6},{0, 124, 21, 33},{0, 0, 180, 113},{0, 10, 4, 49},{0, 0, 149, 21},{0, 0, 48, 6},{0, 2, 38, 151},{0, 16, 122, 83},{0, 60, 195, 154},{0, 0, 155, 87},{0, 6, 28, 5},{0, 30, 85, 92},{0, 0, 47, 173},{0, 168, 179, 120},{0, 31, 42, 2},{0, 105, 66, 142},{0, 0, 0, 0},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z240_CNG19[19][4] = {{135, 96, 189, 128},{227, 236, 4, 23},{126, 136, 225, 162},{134, 221, 151, 220},{84, 128, 236, 43},{83, 92, 117, 186},{53, 172, 179, 96},{225, 56, 92, 1},{205, 11, 24, 216},{128, 189, 68, 22},{75, 95, 6, 24},{135, 85, 101, 167},{217, 153, 33, 200},{220, 87, 96, 32},{90, 163, 125, 235},{105, 216, 67, 172},{137, 0, 230, 219},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z64_CNG19[19][4] = {{58, 2, 42, 57},{5, 47, 47, 25},{34, 53, 57, 20},{31, 60, 63, 20},{36, 7, 53, 22},{10, 30, 29, 3},{59, 40, 37, 51},{37, 45, 49, 8},{46, 28, 31, 22},{63, 38, 39, 54},{9, 45, 14, 27},{3, 4, 33, 19},{23, 14, 33, 48},{62, 27, 53, 12},{35, 63, 13, 52},{47, 28, 59, 16},{31, 0, 53, 12},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z96_CNG19[19][4] = {{19, 76, 13, 84},{19, 76, 58, 87},{50, 73, 40, 0},{81, 0, 44, 83},{85, 48, 64, 7},{24, 43, 65, 57},{29, 43, 75, 56},{0, 82, 64, 36},{13, 7, 63, 17},{17, 54, 33, 39},{69, 25, 8, 53},{23, 3, 88, 20},{10, 66, 53, 16},{50, 18, 35, 12},{84, 43, 48, 79},{42, 16, 13, 39},{58, 0, 13, 69},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z160_CNG19[19][4] = {{73, 143, 68, 60},{15, 134, 7, 48},{103, 27, 80, 30},{49, 101, 120, 37},{80, 1, 38, 61},{39, 133, 67, 15},{15, 4, 42, 79},{2, 80, 40, 121},{55, 129, 71, 143},{4, 140, 106, 93},{133, 76, 135, 4},{138, 106, 123, 53},{110, 72, 141, 44},{113, 83, 24, 28},{16, 100, 86, 77},{29, 141, 70, 159},{32, 0, 116, 68},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z224_CNG19[19][4] = {{223, 141, 207, 201},{16, 45, 203, 18},{94, 151, 31, 165},{91, 46, 176, 5},{74, 119, 180, 45},{10, 157, 186, 142},{0, 133, 95, 16},{205, 87, 153, 34},{216, 206, 177, 155},{21, 93, 70, 213},{215, 79, 77, 147},{14, 9, 214, 69},{70, 118, 77, 96},{141, 194, 198, 74},{198, 31, 117, 99},{104, 187, 223, 30},{81, 0, 90, 158},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z288_CNG19[19][4] = {{211, 179, 258, 187},{198, 162, 167, 145},{188, 223, 220, 166},{186, 256, 133, 108},{219, 160, 243, 82},{4, 76, 202, 132},{29, 202, 218, 197},{144, 117, 63, 41},{116, 109, 0, 162},{216, 15, 3, 57},{115, 72, 74, 36},{233, 152, 229, 115},{144, 158, 0, 242},{95, 147, 216, 165},{216, 156, 269, 0},{73, 119, 200, 113},{261, 0, 234, 108},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z352_CNG19[19][4] = {{294, 77, 226, 97},{118, 225, 35, 94},{167, 96, 213, 49},{330, 338, 302, 279},{207, 268, 111, 139},{165, 112, 265, 166},{243, 302, 128, 91},{250, 50, 237, 106},{1, 167, 294, 246},{339, 253, 127, 345},{201, 334, 110, 269},{53, 242, 286, 185},{347, 257, 125, 249},{304, 133, 131, 215},{167, 9, 163, 143},{47, 302, 210, 121},{188, 0, 7, 121},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z128_CNG19[19][4] = {{122, 2, 106, 121},{69, 111, 111, 89},{98, 117, 57, 84},{31, 124, 63, 20},{100, 71, 117, 22},{10, 94, 93, 3},{59, 104, 101, 115},{101, 45, 49, 8},{110, 92, 95, 86},{63, 102, 39, 118},{9, 109, 14, 91},{67, 4, 97, 83},{23, 14, 97, 112},{62, 27, 117, 76},{35, 127, 77, 116},{111, 28, 123, 16},{31, 0, 117, 12},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z192_CNG19[19][4] = {{115, 76, 13, 84},{19, 76, 58, 87},{50, 73, 136, 0},{177, 96, 140, 83},{181, 144, 64, 7},{24, 139, 161, 153},{125, 139, 75, 56},{96, 178, 160, 132},{109, 103, 63, 113},{17, 150, 129, 39},{165, 25, 8, 149},{23, 99, 88, 20},{106, 162, 53, 112},{50, 114, 131, 108},{180, 139, 48, 79},{138, 112, 13, 39},{154, 0, 13, 165},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z320_CNG19[19][4] = {{73, 303, 68, 220},{15, 294, 7, 208},{103, 27, 80, 30},{49, 261, 280, 197},{240, 161, 38, 61},{39, 133, 227, 175},{15, 4, 202, 79},{162, 80, 200, 281},{215, 129, 71, 303},{164, 300, 106, 253},{133, 76, 295, 164},{298, 266, 283, 53},{110, 72, 301, 44},{113, 83, 184, 28},{16, 260, 246, 77},{189, 301, 230, 319},{32, 0, 276, 68},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z256_CNG19[19][4] = {{250, 2, 106, 121},{69, 239, 111, 89},{226, 117, 185, 84},{159, 124, 63, 20},{100, 71, 117, 150},{10, 222, 93, 131},{59, 104, 229, 243},{229, 173, 177, 136},{110, 220, 95, 86},{191, 102, 39, 246},{9, 109, 142, 219},{195, 132, 225, 211},{23, 142, 225, 240},{190, 155, 245, 76},{35, 255, 205, 244},{239, 28, 251, 144},{31, 0, 117, 12},{1, 0, 0, 1},{0, 0, 0, 0}}; +static const uint16_t circShift_BG1_Z384_CNG19[19][4] = {{307, 76, 205, 276},{19, 76, 250, 87},{50, 73, 328, 0},{369, 288, 332, 275},{181, 144, 256, 199},{216, 331, 161, 153},{317, 331, 267, 56},{288, 178, 160, 132},{109, 295, 63, 305},{17, 342, 129, 231},{357, 217, 200, 341},{215, 99, 88, 212},{106, 354, 53, 304},{242, 114, 131, 300},{180, 331, 240, 271},{330, 112, 205, 39},{346, 0, 13, 357},{1, 0, 0, 1},{0, 0, 0, 0}}; + +// BG2 +// CNG3 +static const uint16_t circShift_BG2_Z2_CNG3[3][6] = {{0, 0, 0, 0, 0, 1},{1, 0, 1, 0, 1, 1},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z3_CNG3[3][6] = {{2, 0, 1, 2, 0, 1},{1, 0, 2, 2, 2, 1},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z4_CNG3[3][6] = {{2, 0, 0, 2, 2, 3},{3, 0, 3, 0, 1, 3},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z5_CNG3[3][6] = {{0, 0, 0, 0, 0, 0},{4, 0, 2, 4, 0, 4},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z6_CNG3[3][6] = {{2, 3, 1, 2, 3, 1},{4, 3, 2, 5, 2, 1},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z7_CNG3[3][6] = {{0, 1, 6, 3, 0, 5},{1, 3, 6, 5, 0, 5},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z8_CNG3[3][6] = {{6, 4, 0, 2, 2, 3},{7, 4, 7, 4, 1, 7},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z9_CNG3[3][6] = {{0, 2, 4, 7, 8, 7},{6, 0, 1, 7, 4, 2},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z10_CNG3[3][6] = {{0, 0, 0, 0, 0, 0},{4, 5, 2, 4, 0, 4},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z11_CNG3[3][6] = {{7, 6, 9, 7, 7, 2},{9, 4, 5, 7, 5, 5},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z12_CNG3[3][6] = {{8, 9, 7, 2, 3, 1},{4, 9, 2, 5, 2, 7},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z13_CNG3[3][6] = {{7, 2, 12, 11, 11, 12},{9, 12, 11, 1, 7, 11},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z14_CNG3[3][6] = {{7, 1, 13, 10, 7, 12},{1, 10, 6, 5, 0, 12},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z15_CNG3[3][6] = {{0, 6, 4, 10, 6, 4},{14, 5, 1, 12, 1, 9},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z16_CNG3[3][6] = {{14, 4, 8, 2, 10, 11},{15, 12, 7, 12, 1, 7},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z18_CNG3[3][6] = {{0, 2, 13, 7, 8, 7},{6, 9, 10, 7, 13, 2},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z20_CNG3[3][6] = {{0, 0, 0, 0, 0, 0},{4, 5, 12, 14, 0, 4},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z22_CNG3[3][6] = {{18, 6, 20, 7, 7, 13},{9, 4, 16, 7, 16, 16},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z24_CNG3[3][6] = {{20, 21, 7, 14, 3, 13},{4, 21, 2, 17, 2, 7},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z26_CNG3[3][6] = {{20, 15, 12, 24, 11, 25},{22, 12, 11, 1, 7, 11},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z28_CNG3[3][6] = {{21, 15, 27, 24, 7, 12},{1, 10, 6, 5, 14, 12},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z30_CNG3[3][6] = {{15, 6, 4, 25, 21, 4},{14, 5, 1, 27, 16, 24},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z32_CNG3[3][6] = {{30, 4, 8, 18, 10, 27},{31, 28, 23, 28, 1, 23},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z36_CNG3[3][6] = {{18, 20, 13, 25, 8, 25},{24, 27, 10, 25, 31, 20},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z40_CNG3[3][6] = {{0, 0, 0, 0, 0, 0},{4, 25, 32, 34, 20, 24},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z44_CNG3[3][6] = {{18, 28, 42, 7, 29, 35},{31, 4, 16, 29, 16, 38},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z48_CNG3[3][6] = {{20, 45, 7, 14, 3, 37},{4, 21, 2, 17, 26, 31},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z52_CNG3[3][6] = {{46, 41, 12, 50, 11, 25},{22, 38, 37, 27, 33, 37},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z56_CNG3[3][6] = {{49, 43, 27, 52, 35, 40},{1, 38, 6, 5, 14, 12},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z60_CNG3[3][6] = {{15, 6, 4, 25, 21, 34},{44, 35, 1, 57, 46, 54},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z64_CNG3[3][6] = {{30, 36, 8, 18, 42, 27},{63, 28, 23, 28, 1, 23},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z72_CNG3[3][6] = {{54, 56, 13, 61, 8, 25},{60, 63, 10, 25, 67, 56},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z80_CNG3[3][6] = {{0, 0, 0, 0, 0, 0},{4, 65, 32, 74, 20, 64},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z88_CNG3[3][6] = {{18, 72, 42, 51, 29, 79},{75, 48, 16, 73, 60, 38},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z96_CNG3[3][6] = {{20, 45, 7, 14, 3, 37},{52, 21, 50, 17, 74, 31},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z104_CNG3[3][6] = {{98, 93, 64, 50, 11, 77},{22, 90, 89, 27, 85, 89},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z112_CNG3[3][6] = {{49, 99, 27, 108, 35, 40},{1, 94, 6, 61, 70, 12},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z120_CNG3[3][6] = {{75, 66, 64, 25, 81, 34},{44, 95, 61, 57, 46, 114},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z128_CNG3[3][6] = {{94, 100, 8, 18, 106, 91},{63, 28, 23, 28, 1, 23},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z144_CNG3[3][6] = {{54, 128, 13, 133, 80, 97},{132, 63, 10, 25, 139, 56},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z160_CNG3[3][6] = {{0, 0, 0, 0, 0, 0},{4, 65, 32, 154, 20, 144},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z176_CNG3[3][6] = {{18, 72, 42, 139, 117, 167},{163, 136, 104, 161, 148, 38},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z192_CNG3[3][6] = {{20, 45, 103, 110, 3, 37},{52, 21, 50, 17, 170, 31},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z208_CNG3[3][6] = {{202, 197, 168, 50, 115, 181},{126, 194, 193, 27, 189, 193},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z224_CNG3[3][6] = {{49, 211, 27, 108, 147, 40},{1, 94, 118, 61, 182, 12},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z240_CNG3[3][6] = {{195, 66, 64, 25, 201, 154},{44, 95, 181, 57, 46, 114},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z256_CNG3[3][6] = {{222, 228, 8, 18, 106, 219},{63, 156, 151, 28, 1, 151},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z288_CNG3[3][6] = {{54, 128, 13, 133, 80, 97},{132, 63, 10, 25, 139, 56},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z320_CNG3[3][6] = {{0, 0, 0, 0, 0, 0},{4, 65, 32, 154, 20, 144},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z352_CNG3[3][6] = {{18, 72, 42, 139, 117, 167},{163, 136, 104, 161, 148, 38},{0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z384_CNG3[3][6] = {{20, 45, 103, 110, 3, 37},{52, 21, 50, 17, 170, 31},{0, 0, 0, 0, 0, 0}}; +// CNG4 +static const uint16_t circShift_BG2_Z2_CNG4[4][20] = {{1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 1},{0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1, 1},{1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z3_CNG4[4][20] = {{0, 1, 0, 2, 0, 1, 1, 1, 1, 2, 1, 0, 0, 2, 2, 1, 0, 0, 1, 0},{2, 1, 0, 0, 0, 0, 1, 1, 2, 0, 0, 2, 2, 0, 1, 1, 1, 0, 2, 1},{2, 2, 2, 0, 1, 0, 0, 1, 2, 1, 0, 2, 1, 0, 0, 2, 1, 0, 1, 0},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z4_CNG4[4][20] = {{3, 2, 3, 3, 0, 3, 2, 0, 3, 2, 2, 2, 0, 3, 1, 0, 3, 3, 0, 1},{2, 2, 3, 3, 2, 0, 1, 2, 3, 3, 1, 0, 0, 1, 0, 2, 2, 0, 3, 1},{3, 2, 2, 1, 2, 1, 1, 2, 2, 0, 3, 2, 3, 0, 3, 2, 2, 2, 0, 2},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z5_CNG4[4][20] = {{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},{1, 0, 3, 3, 3, 3, 3, 2, 0, 4, 1, 0, 3, 4, 2, 3, 3, 4, 1, 2},{2, 2, 3, 4, 1, 4, 4, 3, 3, 2, 0, 2, 3, 3, 1, 2, 4, 3, 3, 0},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z6_CNG4[4][20] = {{0, 4, 3, 2, 0, 4, 4, 1, 4, 2, 4, 0, 3, 5, 5, 1, 0, 3, 1, 3},{2, 4, 3, 3, 0, 0, 1, 1, 2, 3, 3, 2, 5, 3, 4, 1, 1, 0, 5, 1},{5, 2, 5, 3, 4, 0, 3, 1, 5, 4, 0, 5, 4, 0, 0, 2, 4, 3, 1, 0},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z7_CNG4[4][20] = {{4, 0, 4, 4, 5, 0, 6, 3, 2, 6, 6, 3, 1, 1, 0, 1, 2, 1, 0, 1},{2, 1, 0, 4, 2, 5, 6, 1, 5, 4, 2, 0, 5, 5, 4, 0, 6, 6, 0, 3},{3, 5, 4, 5, 2, 2, 1, 4, 1, 0, 4, 5, 4, 6, 6, 6, 4, 0, 1, 4},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z8_CNG4[4][20] = {{3, 6, 7, 3, 4, 7, 2, 4, 7, 6, 2, 2, 4, 3, 1, 4, 7, 7, 0, 1},{6, 6, 7, 7, 2, 4, 1, 2, 3, 3, 5, 4, 4, 5, 0, 6, 2, 4, 3, 5},{7, 6, 6, 5, 2, 1, 5, 2, 6, 0, 7, 6, 3, 4, 7, 2, 6, 2, 0, 6},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z9_CNG4[4][20] = {{6, 7, 6, 4, 8, 7, 2, 1, 5, 3, 7, 5, 0, 1, 6, 4, 1, 2, 6, 3},{6, 6, 5, 6, 7, 1, 4, 3, 7, 6, 5, 8, 6, 7, 2, 3, 7, 5, 1, 2},{6, 7, 5, 2, 7, 5, 6, 0, 2, 2, 7, 8, 2, 6, 3, 8, 1, 7, 3, 1},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z10_CNG4[4][20] = {{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},{6, 5, 3, 3, 8, 8, 8, 7, 5, 9, 6, 0, 8, 9, 7, 3, 3, 4, 6, 2},{7, 7, 8, 9, 6, 4, 4, 3, 8, 7, 0, 2, 3, 8, 6, 2, 9, 8, 3, 5},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z11_CNG4[4][20] = {{9, 9, 1, 5, 8, 2, 7, 2, 0, 3, 9, 6, 6, 0, 5, 7, 2, 10, 6, 0},{1, 9, 0, 4, 9, 3, 8, 1, 0, 9, 2, 7, 5, 8, 9, 0, 7, 6, 2, 6},{6, 7, 4, 7, 0, 2, 8, 7, 7, 6, 6, 10, 7, 8, 7, 8, 8, 6, 5, 8},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z12_CNG4[4][20] = {{0, 10, 3, 8, 6, 10, 4, 1, 10, 2, 10, 0, 9, 5, 5, 1, 0, 9, 7, 3},{2, 10, 9, 3, 6, 6, 1, 7, 2, 9, 3, 8, 11, 9, 4, 7, 7, 0, 11, 7},{5, 8, 11, 9, 10, 0, 9, 7, 11, 4, 0, 5, 4, 0, 6, 2, 10, 9, 7, 0},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z13_CNG4[4][20] = {{4, 10, 5, 5, 12, 6, 1, 10, 0, 9, 1, 6, 7, 2, 1, 2, 1, 12, 11, 9},{1, 8, 12, 4, 8, 7, 9, 5, 1, 5, 4, 0, 4, 9, 11, 11, 4, 1, 7, 2},{0, 9, 10, 11, 10, 0, 10, 10, 0, 10, 7, 7, 10, 11, 12, 10, 9, 4, 10, 11},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z14_CNG4[4][20] = {{4, 7, 4, 4, 12, 7, 6, 10, 2, 13, 6, 10, 1, 1, 7, 1, 9, 8, 0, 8},{2, 1, 7, 4, 2, 12, 6, 1, 12, 4, 2, 7, 12, 5, 11, 7, 13, 13, 7, 3},{3, 12, 11, 12, 2, 2, 1, 11, 8, 7, 11, 12, 11, 13, 13, 13, 4, 0, 8, 11},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z15_CNG4[4][20] = {{14, 11, 4, 8, 8, 9, 12, 13, 5, 6, 7, 14, 1, 4, 10, 13, 12, 2, 3, 8},{5, 4, 2, 7, 1, 9, 2, 13, 4, 4, 9, 14, 2, 0, 0, 7, 11, 0, 7, 5},{0, 0, 12, 7, 0, 13, 6, 11, 6, 10, 1, 11, 1, 6, 10, 1, 0, 9, 0, 5},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z16_CNG4[4][20] = {{3, 14, 15, 3, 12, 7, 10, 12, 7, 14, 2, 2, 4, 3, 9, 12, 15, 15, 0, 1},{6, 14, 15, 15, 2, 4, 9, 10, 11, 11, 5, 12, 4, 5, 8, 6, 2, 12, 11, 5},{7, 6, 14, 5, 2, 9, 13, 2, 14, 8, 7, 6, 11, 4, 15, 10, 6, 2, 8, 6},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z18_CNG4[4][20] = {{6, 16, 15, 13, 17, 7, 2, 1, 14, 12, 16, 14, 9, 1, 6, 13, 1, 11, 6, 12},{6, 15, 14, 6, 16, 10, 13, 3, 7, 6, 5, 17, 6, 7, 11, 3, 16, 14, 1, 11},{15, 16, 5, 2, 16, 5, 6, 9, 2, 11, 7, 17, 2, 6, 3, 8, 1, 16, 12, 1},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z20_CNG4[4][20] = {{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},{16, 5, 13, 13, 18, 18, 8, 17, 15, 9, 6, 0, 8, 19, 17, 3, 13, 4, 16, 2},{17, 7, 8, 19, 6, 14, 4, 13, 18, 7, 10, 2, 13, 18, 6, 2, 19, 18, 3, 15},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z22_CNG4[4][20] = {{20, 9, 12, 5, 19, 13, 18, 2, 0, 3, 20, 6, 17, 11, 16, 7, 2, 21, 6, 0},{1, 9, 11, 15, 9, 3, 8, 12, 0, 20, 2, 7, 5, 19, 20, 0, 7, 6, 2, 6},{17, 7, 4, 18, 0, 13, 19, 18, 18, 17, 17, 10, 7, 8, 18, 19, 19, 17, 16, 8},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z24_CNG4[4][20] = {{0, 22, 15, 20, 18, 10, 4, 13, 10, 14, 22, 12, 21, 5, 5, 1, 12, 21, 7, 3},{2, 22, 21, 15, 6, 18, 13, 7, 14, 9, 15, 8, 11, 9, 16, 7, 7, 12, 11, 7},{5, 8, 11, 9, 22, 12, 9, 19, 23, 16, 0, 17, 4, 12, 18, 2, 22, 9, 19, 12},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z26_CNG4[4][20] = {{17, 23, 18, 18, 25, 6, 14, 10, 0, 9, 14, 6, 7, 15, 1, 15, 1, 25, 11, 9},{1, 21, 25, 4, 21, 20, 9, 18, 14, 5, 4, 0, 17, 9, 11, 11, 17, 1, 7, 15},{13, 22, 23, 24, 10, 0, 10, 23, 13, 10, 20, 7, 23, 11, 25, 23, 9, 17, 23, 11},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z28_CNG4[4][20] = {{4, 7, 18, 4, 12, 7, 6, 24, 16, 13, 20, 24, 15, 1, 7, 1, 9, 22, 14, 8},{16, 15, 21, 4, 16, 26, 20, 15, 26, 4, 16, 21, 12, 5, 25, 7, 13, 13, 7, 17},{17, 12, 25, 26, 16, 2, 1, 25, 22, 7, 25, 26, 25, 13, 27, 27, 18, 14, 8, 25},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z30_CNG4[4][20] = {{29, 26, 4, 8, 8, 9, 27, 28, 5, 21, 7, 29, 16, 4, 25, 28, 12, 17, 18, 23},{20, 19, 2, 7, 1, 24, 17, 13, 4, 19, 24, 14, 2, 15, 15, 22, 11, 15, 7, 5},{0, 15, 12, 7, 0, 28, 6, 11, 21, 10, 16, 26, 16, 21, 25, 16, 15, 9, 0, 20},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z32_CNG4[4][20] = {{19, 14, 31, 19, 28, 23, 26, 12, 23, 14, 2, 18, 4, 19, 25, 12, 31, 15, 0, 1},{22, 30, 15, 15, 2, 20, 9, 10, 11, 11, 5, 12, 4, 21, 8, 6, 2, 12, 11, 5},{7, 6, 14, 21, 18, 25, 29, 18, 14, 8, 7, 6, 11, 4, 31, 26, 6, 2, 24, 22},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z36_CNG4[4][20] = {{6, 34, 33, 31, 17, 7, 2, 19, 32, 30, 34, 32, 27, 1, 24, 13, 1, 11, 6, 12},{24, 33, 14, 24, 34, 28, 31, 3, 7, 6, 5, 17, 6, 25, 29, 21, 34, 32, 1, 11},{15, 16, 23, 20, 34, 5, 6, 9, 20, 29, 7, 35, 2, 6, 21, 26, 1, 34, 30, 19},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z40_CNG4[4][20] = {{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},{16, 25, 33, 33, 18, 38, 28, 17, 35, 29, 6, 20, 8, 19, 37, 23, 13, 24, 36, 2},{37, 7, 8, 19, 6, 34, 24, 33, 38, 7, 30, 2, 13, 38, 6, 2, 19, 18, 23, 15},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z44_CNG4[4][20] = {{42, 9, 34, 5, 41, 13, 40, 24, 22, 25, 20, 28, 17, 11, 38, 29, 24, 21, 28, 0},{23, 31, 11, 15, 31, 3, 8, 34, 0, 20, 24, 29, 5, 41, 20, 22, 7, 6, 24, 6},{39, 29, 4, 40, 22, 13, 41, 40, 18, 17, 17, 10, 29, 30, 40, 41, 19, 17, 38, 8},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z48_CNG4[4][20] = {{24, 22, 39, 20, 42, 10, 28, 13, 10, 38, 22, 36, 21, 29, 29, 25, 36, 45, 7, 3},{26, 22, 45, 15, 6, 42, 13, 31, 38, 9, 15, 8, 35, 33, 40, 7, 7, 36, 11, 7},{29, 8, 11, 33, 46, 12, 9, 19, 47, 16, 24, 17, 28, 12, 18, 26, 46, 9, 19, 12},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z52_CNG4[4][20] = {{43, 49, 18, 18, 25, 6, 40, 10, 0, 9, 14, 6, 7, 41, 1, 41, 1, 25, 37, 35},{27, 21, 51, 4, 47, 20, 35, 18, 14, 5, 30, 0, 17, 35, 11, 11, 17, 1, 7, 41},{13, 22, 49, 50, 36, 0, 10, 23, 13, 36, 46, 7, 23, 37, 25, 23, 35, 17, 23, 11},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z56_CNG4[4][20] = {{32, 35, 46, 4, 12, 35, 6, 24, 44, 41, 48, 52, 15, 29, 35, 1, 37, 50, 42, 8},{16, 43, 49, 32, 16, 26, 20, 43, 54, 32, 44, 21, 12, 33, 53, 7, 41, 13, 35, 45},{45, 40, 53, 54, 44, 30, 29, 25, 22, 7, 25, 54, 53, 13, 55, 27, 46, 42, 36, 25},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z60_CNG4[4][20] = {{29, 56, 4, 38, 8, 39, 57, 58, 5, 21, 7, 59, 46, 4, 25, 58, 42, 47, 18, 53},{20, 49, 2, 37, 31, 24, 47, 13, 34, 19, 24, 14, 2, 15, 45, 52, 41, 45, 7, 35},{0, 45, 12, 37, 30, 28, 6, 11, 51, 10, 16, 56, 46, 21, 55, 46, 45, 9, 0, 50},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z64_CNG4[4][20] = {{51, 14, 63, 51, 28, 23, 26, 12, 23, 46, 34, 50, 4, 19, 57, 12, 31, 47, 0, 1},{22, 30, 47, 47, 2, 20, 41, 42, 43, 11, 37, 44, 36, 21, 40, 38, 2, 44, 11, 37},{7, 38, 14, 21, 50, 57, 29, 18, 46, 8, 7, 38, 43, 36, 63, 26, 38, 34, 56, 54},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z72_CNG4[4][20] = {{42, 70, 33, 67, 17, 7, 2, 19, 68, 30, 34, 32, 63, 37, 60, 49, 1, 47, 6, 48},{24, 69, 50, 24, 34, 28, 31, 3, 43, 42, 5, 17, 6, 25, 65, 57, 70, 32, 1, 47},{51, 16, 59, 56, 70, 41, 6, 9, 56, 29, 43, 71, 2, 42, 21, 26, 1, 70, 30, 19},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z80_CNG4[4][20] = {{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},{56, 65, 33, 73, 18, 78, 68, 17, 75, 69, 46, 20, 8, 19, 77, 63, 13, 24, 36, 2},{77, 7, 48, 19, 6, 74, 24, 33, 78, 7, 30, 42, 13, 78, 6, 2, 19, 58, 63, 55},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z88_CNG4[4][20] = {{86, 53, 34, 49, 85, 13, 40, 68, 22, 25, 64, 28, 17, 11, 38, 73, 24, 21, 72, 44},{67, 31, 11, 15, 31, 47, 52, 78, 44, 20, 24, 73, 49, 41, 64, 66, 7, 6, 68, 6},{83, 73, 4, 40, 22, 57, 85, 40, 62, 61, 61, 54, 29, 74, 84, 41, 19, 17, 82, 8},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z96_CNG4[4][20] = {{72, 22, 39, 68, 90, 58, 76, 61, 10, 86, 70, 84, 69, 77, 77, 25, 84, 93, 7, 51},{74, 70, 93, 63, 6, 42, 61, 79, 86, 57, 15, 8, 83, 81, 88, 55, 55, 36, 11, 7},{29, 56, 11, 81, 46, 60, 57, 67, 95, 64, 72, 17, 28, 12, 18, 74, 94, 57, 67, 60},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z104_CNG4[4][20] = {{43, 101, 18, 70, 25, 6, 92, 10, 52, 9, 14, 6, 59, 93, 53, 93, 53, 77, 89, 87},{27, 73, 51, 4, 99, 20, 35, 18, 66, 57, 82, 0, 69, 87, 63, 63, 69, 53, 59, 93},{13, 22, 49, 102, 36, 52, 10, 23, 13, 88, 46, 59, 75, 89, 77, 75, 87, 69, 75, 63},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z112_CNG4[4][20] = {{88, 35, 46, 4, 68, 35, 6, 80, 44, 41, 104, 108, 71, 29, 91, 1, 37, 106, 98, 8},{16, 43, 105, 88, 16, 26, 20, 43, 54, 88, 100, 21, 12, 89, 53, 63, 97, 69, 35, 101},{101, 40, 109, 110, 44, 86, 29, 81, 22, 63, 81, 110, 109, 69, 55, 83, 46, 42, 36, 81},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z120_CNG4[4][20] = {{29, 56, 4, 38, 8, 39, 117, 118, 5, 81, 7, 59, 46, 64, 85, 58, 42, 47, 78, 53},{20, 49, 2, 97, 91, 84, 107, 13, 94, 19, 24, 14, 2, 15, 105, 112, 41, 45, 67, 95},{60, 105, 72, 37, 90, 88, 6, 11, 111, 10, 16, 116, 106, 21, 55, 106, 105, 69, 60, 110},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z128_CNG4[4][20] = {{51, 14, 63, 51, 92, 87, 26, 76, 23, 46, 98, 114, 4, 19, 57, 12, 31, 111, 0, 1},{86, 94, 111, 47, 66, 20, 105, 42, 107, 11, 101, 44, 36, 85, 40, 38, 66, 44, 75, 101},{71, 102, 14, 85, 50, 57, 29, 82, 110, 8, 7, 38, 107, 36, 63, 26, 38, 34, 120, 118},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z144_CNG4[4][20] = {{42, 70, 33, 139, 17, 79, 2, 91, 68, 30, 106, 32, 135, 37, 60, 121, 1, 119, 6, 48},{24, 69, 122, 96, 106, 28, 103, 75, 115, 42, 77, 89, 6, 25, 137, 129, 70, 32, 73, 47},{51, 88, 131, 128, 142, 41, 78, 81, 56, 101, 43, 71, 2, 114, 93, 26, 1, 142, 102, 19},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z160_CNG4[4][20] = {{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},{136, 65, 113, 73, 18, 158, 148, 17, 75, 69, 126, 20, 88, 19, 157, 63, 93, 24, 36, 2},{157, 87, 48, 99, 86, 154, 104, 33, 158, 87, 110, 122, 13, 78, 6, 82, 19, 138, 143, 55},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z176_CNG4[4][20] = {{86, 53, 122, 137, 173, 13, 128, 156, 110, 113, 64, 116, 105, 11, 126, 73, 112, 109, 160, 132},{67, 31, 11, 103, 31, 135, 52, 166, 132, 108, 24, 73, 137, 41, 152, 154, 7, 6, 156, 6},{83, 161, 4, 40, 22, 145, 173, 40, 150, 61, 149, 142, 29, 162, 172, 129, 19, 105, 82, 8},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z192_CNG4[4][20] = {{72, 118, 39, 68, 186, 58, 76, 157, 106, 182, 70, 84, 165, 173, 77, 25, 84, 93, 103, 147},{74, 70, 93, 63, 6, 42, 61, 175, 86, 153, 111, 8, 179, 177, 184, 151, 151, 132, 107, 7},{29, 152, 11, 81, 46, 156, 153, 67, 95, 64, 168, 17, 124, 12, 18, 170, 190, 57, 163, 60},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z208_CNG4[4][20] = {{43, 101, 18, 174, 129, 110, 196, 10, 52, 113, 14, 110, 163, 197, 157, 197, 157, 181, 193, 191},{27, 177, 155, 108, 203, 124, 35, 122, 170, 161, 186, 0, 173, 191, 167, 167, 173, 157, 163, 197},{117, 22, 49, 102, 140, 52, 114, 23, 13, 88, 46, 163, 179, 193, 181, 179, 191, 173, 179, 167},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z224_CNG4[4][20] = {{200, 147, 46, 116, 68, 35, 6, 80, 156, 153, 216, 108, 71, 29, 91, 1, 37, 106, 98, 120},{16, 43, 217, 200, 16, 138, 20, 43, 54, 88, 212, 21, 12, 201, 165, 175, 97, 181, 35, 101},{101, 152, 109, 110, 156, 86, 141, 81, 134, 63, 193, 110, 109, 69, 55, 83, 46, 154, 36, 81},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z240_CNG4[4][20] = {{29, 176, 124, 38, 128, 39, 117, 238, 5, 81, 7, 179, 46, 184, 85, 178, 42, 167, 78, 53},{140, 169, 122, 217, 211, 84, 227, 13, 94, 19, 144, 14, 2, 135, 225, 112, 41, 45, 67, 215},{180, 225, 72, 157, 210, 88, 6, 11, 111, 130, 16, 116, 106, 141, 175, 106, 105, 189, 180, 230},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z256_CNG4[4][20] = {{179, 142, 63, 51, 220, 87, 26, 76, 23, 46, 98, 242, 132, 147, 57, 140, 31, 239, 0, 129},{214, 94, 111, 175, 194, 20, 105, 42, 235, 139, 101, 44, 164, 85, 40, 38, 66, 172, 75, 229},{71, 230, 14, 213, 50, 185, 29, 210, 238, 8, 135, 166, 235, 36, 63, 154, 38, 34, 120, 118},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z288_CNG4[4][20] = {{42, 70, 33, 139, 17, 79, 2, 91, 68, 30, 106, 32, 135, 37, 60, 121, 1, 119, 6, 48},{24, 69, 122, 96, 106, 28, 103, 75, 115, 42, 77, 89, 6, 25, 137, 129, 70, 32, 73, 47},{51, 88, 131, 128, 142, 41, 78, 81, 56, 101, 43, 71, 2, 114, 93, 26, 1, 142, 102, 19},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z320_CNG4[4][20] = {{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},{136, 65, 113, 73, 18, 158, 148, 17, 75, 69, 126, 20, 88, 19, 157, 63, 93, 24, 36, 2},{157, 87, 48, 99, 86, 154, 104, 33, 158, 87, 110, 122, 13, 78, 6, 82, 19, 138, 143, 55},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z352_CNG4[4][20] = {{86, 53, 122, 137, 173, 13, 128, 156, 110, 113, 64, 116, 105, 11, 126, 73, 112, 109, 160, 132},{67, 31, 11, 103, 31, 135, 52, 166, 132, 108, 24, 73, 137, 41, 152, 154, 7, 6, 156, 6},{83, 161, 4, 40, 22, 145, 173, 40, 150, 61, 149, 142, 29, 162, 172, 129, 19, 105, 82, 8},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z384_CNG4[4][20] = {{72, 118, 39, 68, 186, 58, 76, 157, 106, 182, 70, 84, 165, 173, 77, 25, 84, 93, 103, 147},{74, 70, 93, 63, 6, 42, 61, 175, 86, 153, 111, 8, 179, 177, 184, 151, 151, 132, 107, 7},{29, 152, 11, 81, 46, 156, 153, 67, 95, 64, 168, 17, 124, 12, 18, 170, 190, 57, 163, 60},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +// CNG5 +static const uint16_t circShift_BG2_Z2_CNG5[5][9] = {{1, 1, 1, 1, 1, 1, 0, 1, 1},{1, 1, 0, 0, 1, 0, 0, 1, 0},{1, 0, 0, 0, 1, 0, 0, 0, 1},{1, 1, 0, 0, 0, 0, 0, 0, 0},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z3_CNG5[5][9] = {{1, 2, 2, 1, 1, 0, 2, 1, 0},{0, 2, 2, 2, 1, 0, 2, 2, 1},{2, 1, 0, 2, 0, 0, 0, 1, 1},{1, 1, 1, 1, 1, 1, 0, 1, 2},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z4_CNG5[5][9] = {{3, 3, 3, 3, 3, 3, 2, 1, 3},{1, 1, 0, 2, 1, 2, 0, 3, 0},{1, 0, 2, 2, 3, 0, 2, 0, 1},{3, 1, 0, 2, 0, 2, 0, 2, 0},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z5_CNG5[5][9] = {{0, 0, 0, 0, 0, 0, 0, 0, 0},{2, 2, 2, 2, 3, 4, 4, 0, 0},{1, 1, 4, 2, 2, 1, 4, 3, 1},{0, 0, 2, 1, 2, 3, 3, 2, 4},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z6_CNG5[5][9] = {{4, 5, 2, 1, 1, 3, 2, 1, 0},{0, 2, 2, 5, 4, 3, 5, 5, 4},{5, 4, 0, 5, 3, 3, 3, 1, 4},{4, 4, 4, 4, 1, 4, 0, 1, 2},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z7_CNG5[5][9] = {{2, 6, 1, 2, 1, 5, 6, 6, 1},{2, 3, 5, 1, 4, 4, 6, 6, 2},{2, 1, 5, 3, 5, 1, 3, 4, 3},{6, 0, 1, 0, 4, 6, 2, 6, 1},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z8_CNG5[5][9] = {{3, 3, 3, 3, 3, 3, 6, 5, 7},{5, 1, 4, 2, 1, 6, 4, 7, 0},{5, 0, 2, 6, 3, 0, 2, 0, 1},{7, 5, 0, 6, 0, 2, 0, 2, 4},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z9_CNG5[5][9] = {{7, 1, 1, 4, 8, 3, 3, 7, 6},{4, 5, 3, 1, 6, 0, 7, 1, 2},{6, 2, 8, 8, 4, 1, 2, 1, 5},{4, 6, 1, 2, 7, 8, 6, 7, 7},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z10_CNG5[5][9] = {{0, 0, 0, 0, 0, 0, 0, 0, 0},{7, 7, 7, 2, 8, 9, 4, 0, 5},{1, 6, 4, 2, 7, 1, 9, 3, 1},{5, 0, 2, 6, 7, 3, 8, 7, 4},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z11_CNG5[5][9] = {{5, 1, 3, 7, 0, 1, 2, 3, 7},{10, 3, 10, 3, 2, 4, 1, 5, 8},{0, 9, 2, 5, 8, 6, 7, 9, 5},{9, 1, 9, 6, 1, 0, 9, 2, 7},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z12_CNG5[5][9] = {{4, 11, 8, 1, 7, 3, 2, 7, 0},{0, 8, 8, 5, 10, 9, 11, 5, 10},{5, 10, 6, 11, 9, 3, 9, 7, 4},{10, 4, 10, 10, 7, 4, 6, 1, 8},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z13_CNG5[5][9] = {{4, 8, 0, 6, 0, 8, 5, 8, 5},{1, 8, 3, 1, 7, 5, 6, 3, 4},{10, 8, 9, 4, 0, 2, 12, 10, 11},{3, 12, 8, 4, 5, 7, 9, 5, 6},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z14_CNG5[5][9] = {{2, 6, 1, 9, 8, 5, 6, 6, 8},{2, 10, 12, 1, 11, 4, 6, 6, 2},{2, 8, 12, 3, 12, 8, 10, 11, 3},{13, 0, 1, 0, 4, 13, 2, 6, 8},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z15_CNG5[5][9] = {{2, 8, 13, 3, 7, 5, 1, 11, 7},{13, 7, 4, 12, 7, 8, 8, 6, 4},{3, 13, 3, 2, 9, 10, 0, 10, 11},{8, 7, 6, 9, 12, 7, 8, 11, 2},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z16_CNG5[5][9] = {{11, 11, 11, 3, 3, 11, 14, 13, 7},{13, 9, 12, 2, 1, 14, 12, 15, 0},{13, 0, 2, 6, 3, 8, 2, 0, 9},{7, 5, 8, 14, 8, 2, 0, 10, 4},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z18_CNG5[5][9] = {{7, 10, 10, 4, 17, 12, 12, 16, 15},{4, 5, 12, 1, 15, 9, 7, 10, 2},{6, 11, 8, 8, 4, 10, 11, 10, 14},{13, 6, 10, 11, 7, 8, 6, 7, 16},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z20_CNG5[5][9] = {{0, 0, 0, 0, 0, 0, 0, 0, 0},{17, 17, 7, 12, 18, 19, 4, 0, 15},{11, 16, 4, 2, 17, 11, 9, 13, 11},{5, 0, 2, 6, 7, 3, 18, 7, 14},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z22_CNG5[5][9] = {{16, 1, 3, 7, 11, 12, 2, 14, 18},{10, 3, 21, 3, 13, 4, 1, 16, 19},{11, 20, 2, 5, 8, 17, 18, 9, 5},{20, 1, 9, 17, 12, 11, 9, 13, 7},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z24_CNG5[5][9] = {{4, 11, 8, 1, 19, 15, 14, 19, 0},{12, 8, 20, 5, 22, 9, 23, 17, 10},{17, 22, 6, 11, 21, 15, 9, 7, 4},{10, 4, 10, 22, 19, 16, 6, 13, 8},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z26_CNG5[5][9] = {{4, 21, 13, 6, 13, 21, 5, 8, 18},{1, 8, 3, 1, 20, 5, 19, 16, 17},{23, 21, 9, 17, 0, 2, 12, 23, 24},{3, 25, 8, 4, 5, 20, 9, 5, 6},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z28_CNG5[5][9] = {{2, 6, 15, 9, 8, 19, 20, 6, 22},{2, 10, 26, 1, 11, 18, 20, 6, 16},{16, 8, 26, 3, 12, 22, 10, 25, 17},{27, 0, 15, 0, 4, 13, 2, 6, 8},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z30_CNG5[5][9] = {{17, 8, 28, 18, 22, 20, 16, 26, 7},{28, 7, 4, 27, 22, 23, 23, 6, 19},{18, 28, 18, 17, 24, 25, 15, 10, 11},{8, 22, 21, 9, 12, 22, 8, 26, 17},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z32_CNG5[5][9] = {{11, 11, 11, 19, 19, 11, 30, 29, 7},{13, 25, 12, 2, 17, 14, 28, 15, 16},{29, 0, 18, 6, 3, 8, 18, 0, 9},{23, 21, 24, 30, 8, 18, 0, 26, 20},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z36_CNG5[5][9] = {{25, 10, 28, 4, 35, 12, 12, 34, 15},{4, 5, 30, 1, 15, 9, 7, 28, 20},{24, 29, 8, 26, 22, 28, 29, 28, 32},{13, 24, 28, 11, 25, 8, 6, 25, 34},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z40_CNG5[5][9] = {{0, 0, 0, 0, 0, 0, 0, 0, 0},{17, 17, 7, 32, 18, 39, 24, 20, 35},{11, 36, 4, 22, 17, 31, 29, 13, 11},{5, 20, 2, 26, 27, 23, 18, 7, 14},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z44_CNG5[5][9] = {{16, 23, 3, 29, 11, 34, 2, 36, 40},{10, 25, 43, 3, 13, 26, 23, 38, 41},{11, 42, 24, 27, 8, 17, 18, 9, 5},{20, 23, 9, 39, 34, 33, 31, 13, 29},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z48_CNG5[5][9] = {{28, 11, 32, 1, 19, 39, 14, 19, 24},{36, 8, 44, 29, 22, 33, 23, 41, 10},{41, 22, 30, 35, 21, 39, 9, 7, 4},{34, 4, 10, 22, 19, 16, 6, 37, 8},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z52_CNG5[5][9] = {{30, 21, 39, 32, 39, 21, 5, 8, 18},{27, 8, 29, 1, 20, 31, 45, 16, 17},{49, 21, 35, 43, 0, 2, 38, 49, 50},{29, 51, 8, 30, 5, 20, 35, 5, 6},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z56_CNG5[5][9] = {{2, 6, 43, 37, 36, 19, 48, 34, 50},{30, 38, 26, 1, 39, 46, 20, 6, 44},{16, 8, 54, 31, 40, 22, 38, 53, 17},{27, 0, 43, 28, 4, 41, 2, 6, 8},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z60_CNG5[5][9] = {{47, 38, 58, 48, 22, 50, 16, 26, 37},{58, 37, 34, 57, 52, 23, 53, 6, 19},{48, 28, 48, 47, 24, 55, 15, 10, 41},{8, 52, 51, 39, 42, 22, 38, 26, 17},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z64_CNG5[5][9] = {{11, 11, 11, 19, 51, 11, 62, 29, 7},{13, 57, 44, 2, 17, 14, 60, 15, 48},{61, 0, 18, 38, 3, 8, 50, 32, 9},{55, 53, 56, 30, 40, 50, 0, 58, 20},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z72_CNG5[5][9] = {{25, 46, 28, 4, 71, 48, 48, 70, 15},{40, 41, 30, 37, 51, 9, 43, 28, 56},{24, 29, 44, 62, 58, 28, 65, 28, 32},{49, 24, 64, 47, 25, 8, 42, 61, 70},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z80_CNG5[5][9] = {{0, 0, 0, 0, 0, 0, 0, 0, 0},{17, 17, 7, 32, 58, 79, 24, 20, 35},{51, 76, 4, 22, 57, 31, 29, 13, 51},{5, 20, 2, 26, 27, 63, 18, 7, 54},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z88_CNG5[5][9] = {{16, 23, 3, 29, 11, 78, 46, 36, 84},{54, 25, 87, 3, 57, 70, 23, 38, 85},{11, 86, 24, 27, 8, 17, 62, 53, 5},{64, 23, 53, 39, 78, 77, 75, 57, 29},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z96_CNG5[5][9] = {{28, 59, 32, 49, 19, 87, 62, 67, 24},{36, 8, 92, 29, 22, 81, 23, 41, 58},{89, 22, 78, 35, 21, 39, 9, 55, 52},{82, 52, 58, 70, 67, 64, 6, 85, 56},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z104_CNG5[5][9] = {{82, 21, 39, 32, 91, 21, 57, 60, 70},{27, 60, 29, 53, 20, 31, 97, 68, 17},{101, 73, 35, 95, 52, 54, 38, 49, 50},{81, 51, 8, 82, 5, 72, 35, 57, 6},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z112_CNG5[5][9] = {{2, 62, 99, 37, 36, 75, 48, 90, 106},{30, 38, 26, 1, 95, 46, 20, 6, 44},{72, 8, 110, 31, 40, 22, 94, 109, 73},{83, 56, 99, 28, 4, 97, 2, 6, 64},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z120_CNG5[5][9] = {{47, 38, 58, 48, 82, 50, 76, 26, 37},{118, 97, 94, 57, 112, 23, 53, 66, 19},{48, 88, 48, 47, 84, 55, 75, 70, 101},{68, 112, 51, 99, 42, 82, 98, 86, 17},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z128_CNG5[5][9] = {{75, 11, 11, 83, 115, 75, 126, 29, 71},{77, 57, 108, 2, 17, 14, 124, 15, 112},{61, 0, 82, 38, 3, 8, 114, 32, 9},{119, 117, 56, 94, 104, 114, 64, 122, 84},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z144_CNG5[5][9] = {{97, 46, 28, 76, 143, 48, 120, 142, 87},{40, 41, 30, 37, 51, 9, 43, 28, 56},{24, 101, 116, 62, 130, 28, 65, 100, 104},{49, 96, 64, 47, 97, 8, 42, 133, 70},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z160_CNG5[5][9] = {{0, 0, 0, 0, 0, 0, 0, 0, 0},{97, 17, 7, 112, 138, 79, 24, 100, 35},{51, 156, 4, 102, 57, 111, 109, 13, 51},{85, 20, 2, 26, 27, 143, 18, 7, 134},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z176_CNG5[5][9] = {{104, 111, 91, 29, 11, 78, 134, 36, 84},{142, 25, 175, 91, 145, 158, 23, 38, 173},{99, 174, 24, 27, 8, 17, 62, 53, 93},{64, 23, 141, 127, 166, 165, 163, 145, 29},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z192_CNG5[5][9] = {{28, 59, 32, 49, 19, 87, 158, 67, 120},{132, 104, 92, 125, 118, 177, 23, 137, 154},{185, 22, 174, 35, 21, 135, 9, 55, 52},{178, 52, 154, 166, 163, 64, 6, 85, 56},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z208_CNG5[5][9] = {{186, 125, 39, 32, 91, 125, 57, 164, 70},{27, 60, 29, 53, 20, 31, 201, 172, 17},{205, 177, 35, 95, 52, 54, 142, 49, 50},{81, 51, 8, 186, 109, 176, 35, 161, 6},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z224_CNG5[5][9] = {{2, 174, 99, 37, 36, 75, 48, 90, 106},{30, 150, 138, 113, 95, 158, 132, 6, 44},{184, 8, 110, 143, 40, 134, 206, 221, 185},{83, 56, 99, 140, 116, 97, 2, 6, 176},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z240_CNG5[5][9] = {{167, 38, 178, 48, 82, 170, 196, 146, 37},{238, 217, 214, 57, 232, 23, 173, 66, 139},{48, 208, 168, 167, 204, 175, 195, 190, 221},{68, 232, 51, 219, 162, 202, 218, 86, 17},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z256_CNG5[5][9] = {{203, 11, 11, 83, 115, 203, 254, 29, 71},{205, 185, 236, 2, 145, 142, 124, 143, 240},{61, 0, 210, 38, 3, 8, 114, 160, 9},{247, 117, 56, 222, 232, 242, 64, 122, 84},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z288_CNG5[5][9] = {{97, 46, 28, 76, 143, 48, 120, 142, 87},{40, 41, 30, 37, 51, 9, 43, 28, 56},{24, 101, 116, 62, 130, 28, 65, 100, 104},{49, 96, 64, 47, 97, 8, 42, 133, 70},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z320_CNG5[5][9] = {{0, 0, 0, 0, 0, 0, 0, 0, 0},{97, 17, 7, 112, 138, 79, 24, 100, 35},{51, 156, 4, 102, 57, 111, 109, 13, 51},{85, 20, 2, 26, 27, 143, 18, 7, 134},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z352_CNG5[5][9] = {{104, 111, 91, 29, 11, 78, 134, 36, 84},{142, 25, 175, 91, 145, 158, 23, 38, 173},{99, 174, 24, 27, 8, 17, 62, 53, 93},{64, 23, 141, 127, 166, 165, 163, 145, 29},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint16_t circShift_BG2_Z384_CNG5[5][9] = {{28, 59, 32, 49, 19, 87, 158, 67, 120},{132, 104, 92, 125, 118, 177, 23, 137, 154},{185, 22, 174, 35, 21, 135, 9, 55, 52},{178, 52, 154, 166, 163, 64, 6, 85, 56},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +// CNG6 +static const uint16_t circShift_BG2_Z2_CNG6[6][3] = {{1, 1, 1},{1, 0, 1},{0, 1, 0},{1, 0, 1},{1, 0, 0},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z3_CNG6[6][3] = {{1, 0, 2},{2, 2, 0},{1, 1, 1},{2, 1, 0},{0, 1, 2},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z4_CNG6[6][3] = {{3, 3, 1},{1, 0, 3},{2, 1, 0},{3, 0, 3},{3, 2, 0},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z5_CNG6[6][3] = {{0, 0, 0},{1, 4, 0},{2, 4, 3},{1, 0, 1},{4, 3, 3},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z6_CNG6[6][3] = {{4, 3, 2},{2, 2, 0},{1, 4, 4},{2, 1, 0},{0, 4, 5},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z7_CNG6[6][3] = {{3, 4, 5},{5, 6, 6},{2, 3, 0},{2, 5, 3},{4, 6, 6},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z8_CNG6[6][3] = {{7, 3, 1},{1, 4, 3},{2, 5, 4},{7, 4, 3},{7, 6, 4},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z9_CNG6[6][3] = {{4, 1, 7},{5, 6, 0},{3, 8, 8},{2, 7, 0},{8, 4, 2},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z10_CNG6[6][3] = {{0, 0, 0},{1, 4, 5},{2, 9, 8},{1, 5, 6},{4, 8, 8},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z11_CNG6[6][3] = {{2, 3, 4},{7, 0, 4},{2, 10, 2},{4, 1, 3},{5, 7, 0},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z12_CNG6[6][3] = {{10, 9, 8},{8, 8, 6},{1, 4, 4},{8, 1, 6},{0, 4, 11},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z13_CNG6[6][3] = {{6, 7, 7},{10, 8, 0},{10, 3, 0},{2, 12, 3},{10, 12, 6},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z14_CNG6[6][3] = {{3, 11, 5},{12, 13, 13},{2, 3, 7},{9, 12, 10},{11, 13, 13},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z15_CNG6[6][3] = {{1, 2, 11},{11, 12, 6},{4, 7, 12},{13, 4, 7},{12, 11, 4},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z16_CNG6[6][3] = {{7, 11, 1},{9, 4, 3},{2, 13, 12},{15, 12, 3},{7, 14, 4},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z18_CNG6[6][3] = {{4, 1, 7},{14, 15, 9},{12, 17, 17},{11, 7, 0},{17, 13, 11},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z20_CNG6[6][3] = {{0, 0, 0},{11, 4, 5},{2, 19, 8},{1, 5, 16},{4, 8, 18},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z22_CNG6[6][3] = {{13, 3, 4},{18, 0, 15},{13, 10, 2},{15, 1, 3},{16, 7, 11},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z24_CNG6[6][3] = {{10, 9, 8},{20, 20, 18},{1, 4, 16},{8, 1, 6},{0, 16, 23},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z26_CNG6[6][3] = {{6, 7, 7},{23, 8, 0},{10, 16, 13},{2, 12, 3},{10, 12, 6},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z28_CNG6[6][3] = {{17, 11, 19},{26, 27, 13},{2, 3, 21},{23, 26, 10},{25, 13, 13},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z30_CNG6[6][3] = {{1, 17, 26},{11, 12, 6},{19, 22, 27},{28, 4, 22},{27, 26, 4},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z32_CNG6[6][3] = {{7, 27, 1},{9, 4, 19},{2, 13, 12},{31, 28, 3},{7, 30, 20},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z36_CNG6[6][3] = {{4, 1, 25},{32, 15, 27},{12, 35, 35},{29, 25, 0},{35, 31, 29},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z40_CNG6[6][3] = {{0, 0, 0},{11, 4, 5},{22, 19, 28},{21, 5, 16},{24, 28, 38},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z44_CNG6[6][3] = {{35, 3, 4},{40, 22, 37},{35, 10, 24},{15, 23, 3},{16, 29, 11},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z48_CNG6[6][3] = {{10, 33, 32},{44, 44, 42},{25, 4, 16},{32, 1, 6},{0, 40, 47},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z52_CNG6[6][3] = {{32, 7, 7},{49, 34, 26},{36, 42, 39},{28, 12, 3},{10, 12, 6},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z56_CNG6[6][3] = {{17, 11, 47},{26, 55, 13},{2, 31, 49},{51, 54, 38},{25, 41, 13},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z60_CNG6[6][3] = {{1, 17, 26},{41, 12, 6},{49, 52, 27},{28, 4, 52},{27, 56, 34},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z64_CNG6[6][3] = {{39, 27, 1},{41, 36, 19},{2, 45, 12},{31, 28, 3},{39, 30, 52},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z72_CNG6[6][3] = {{40, 37, 25},{68, 15, 63},{12, 35, 35},{65, 61, 36},{71, 67, 65},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z80_CNG6[6][3] = {{0, 0, 0},{51, 44, 45},{62, 19, 68},{61, 45, 16},{64, 68, 78},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z88_CNG6[6][3] = {{79, 47, 48},{84, 66, 37},{35, 10, 24},{15, 67, 47},{60, 29, 55},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z96_CNG6[6][3] = {{10, 33, 80},{44, 92, 90},{25, 4, 16},{80, 49, 6},{48, 88, 47},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z104_CNG6[6][3] = {{32, 7, 59},{49, 34, 78},{36, 94, 39},{28, 64, 3},{62, 12, 58},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z112_CNG6[6][3] = {{73, 11, 103},{26, 55, 13},{58, 31, 105},{107, 110, 38},{81, 97, 69},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z120_CNG6[6][3] = {{1, 17, 86},{41, 72, 66},{49, 52, 87},{88, 4, 52},{87, 56, 34},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z128_CNG6[6][3] = {{103, 27, 1},{41, 100, 19},{66, 45, 12},{31, 28, 3},{103, 30, 116},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z144_CNG6[6][3] = {{40, 109, 97},{140, 87, 135},{84, 107, 35},{137, 133, 108},{71, 139, 65},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z160_CNG6[6][3] = {{0, 0, 0},{131, 124, 45},{142, 99, 148},{141, 45, 96},{64, 148, 78},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z176_CNG6[6][3] = {{79, 47, 48},{84, 154, 125},{35, 10, 24},{103, 155, 47},{60, 29, 55},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z192_CNG6[6][3] = {{10, 129, 80},{44, 92, 186},{121, 100, 16},{80, 49, 102},{48, 184, 143},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z208_CNG6[6][3] = {{136, 7, 163},{49, 34, 78},{36, 198, 143},{132, 168, 107},{62, 12, 58},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z224_CNG6[6][3] = {{185, 123, 103},{138, 55, 13},{170, 31, 105},{219, 222, 150},{193, 209, 181},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z240_CNG6[6][3] = {{121, 137, 86},{41, 72, 186},{169, 172, 87},{88, 124, 172},{207, 56, 154},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z256_CNG6[6][3] = {{231, 155, 129},{41, 228, 147},{194, 45, 140},{159, 28, 3},{103, 158, 116},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z288_CNG6[6][3] = {{40, 109, 97},{140, 87, 135},{84, 107, 35},{137, 133, 108},{71, 139, 65},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z320_CNG6[6][3] = {{0, 0, 0},{131, 124, 45},{142, 99, 148},{141, 45, 96},{64, 148, 78},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z352_CNG6[6][3] = {{79, 47, 48},{84, 154, 125},{35, 10, 24},{103, 155, 47},{60, 29, 55},{0, 0, 0}}; +static const uint16_t circShift_BG2_Z384_CNG6[6][3] = {{10, 129, 80},{44, 92, 186},{121, 100, 16},{80, 49, 102},{48, 184, 143},{0, 0, 0}}; +// CNG8 +static const uint16_t circShift_BG2_Z2_CNG8[8][2] = {{1, 1},{1, 0},{0, 0},{0, 0},{1, 0},{1, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z3_CNG8[8][2] = {{0, 1},{1, 0},{1, 0},{0, 2},{2, 0},{1, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z4_CNG8[8][2] = {{1, 1},{1, 2},{0, 0},{2, 0},{1, 0},{1, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z5_CNG8[8][2] = {{0, 0},{0, 4},{0, 4},{0, 4},{0, 3},{0, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z6_CNG8[8][2] = {{0, 1},{1, 0},{4, 3},{0, 2},{5, 0},{4, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z7_CNG8[8][2] = {{2, 5},{5, 5},{2, 4},{6, 2},{4, 0},{1, 0},{1, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z8_CNG8[8][2] = {{1, 1},{5, 2},{4, 4},{2, 4},{5, 0},{5, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z9_CNG8[8][2] = {{3, 5},{8, 7},{8, 2},{8, 2},{7, 3},{1, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z10_CNG8[8][2] = {{0, 0},{0, 4},{0, 9},{0, 9},{0, 8},{0, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z11_CNG8[8][2] = {{2, 10},{0, 3},{3, 8},{3, 5},{7, 10},{2, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z12_CNG8[8][2] = {{6, 1},{1, 6},{10, 9},{6, 2},{11, 6},{4, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z13_CNG8[8][2] = {{0, 9},{6, 7},{7, 8},{9, 1},{1, 2},{0, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z14_CNG8[8][2] = {{2, 12},{12, 5},{9, 4},{13, 9},{11, 7},{8, 0},{1, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z15_CNG8[8][2] = {{10, 14},{11, 1},{11, 3},{6, 8},{8, 5},{7, 0},{1, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z16_CNG8[8][2] = {{9, 1},{5, 2},{12, 12},{10, 4},{13, 0},{13, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z18_CNG8[8][2] = {{3, 5},{8, 16},{17, 2},{17, 2},{7, 3},{1, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z20_CNG8[8][2] = {{0, 0},{0, 14},{0, 19},{0, 9},{0, 8},{0, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z22_CNG8[8][2] = {{2, 10},{11, 14},{14, 19},{3, 16},{18, 10},{13, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z24_CNG8[8][2] = {{6, 1},{1, 18},{22, 21},{18, 14},{23, 18},{4, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z26_CNG8[8][2] = {{13, 22},{19, 7},{20, 21},{9, 1},{14, 2},{13, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z28_CNG8[8][2] = {{16, 12},{26, 19},{23, 18},{13, 23},{11, 7},{8, 0},{1, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z30_CNG8[8][2] = {{25, 14},{11, 1},{11, 3},{21, 23},{23, 5},{22, 0},{1, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z32_CNG8[8][2] = {{9, 17},{21, 18},{12, 12},{26, 20},{29, 16},{13, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z36_CNG8[8][2] = {{3, 23},{26, 34},{17, 20},{35, 2},{7, 3},{19, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z40_CNG8[8][2] = {{0, 20},{0, 14},{0, 19},{0, 9},{0, 28},{0, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z44_CNG8[8][2] = {{24, 10},{11, 36},{14, 19},{3, 38},{40, 10},{35, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z48_CNG8[8][2] = {{30, 25},{1, 18},{22, 21},{18, 14},{23, 18},{28, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z52_CNG8[8][2] = {{39, 22},{19, 7},{20, 47},{9, 27},{40, 28},{13, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z56_CNG8[8][2] = {{16, 40},{54, 19},{23, 46},{13, 23},{39, 35},{8, 0},{1, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z60_CNG8[8][2] = {{25, 14},{11, 31},{11, 3},{21, 53},{23, 35},{52, 0},{1, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z64_CNG8[8][2] = {{9, 17},{53, 50},{12, 44},{26, 52},{61, 48},{13, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z72_CNG8[8][2] = {{3, 23},{26, 34},{53, 20},{35, 38},{43, 39},{55, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z80_CNG8[8][2] = {{0, 20},{0, 14},{0, 19},{0, 9},{0, 28},{0, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z88_CNG8[8][2] = {{68, 10},{55, 80},{14, 19},{3, 82},{40, 54},{35, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z96_CNG8[8][2] = {{78, 25},{1, 18},{70, 21},{66, 14},{71, 18},{76, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z104_CNG8[8][2] = {{39, 22},{19, 59},{72, 47},{61, 79},{92, 28},{13, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z112_CNG8[8][2] = {{72, 40},{110, 19},{23, 46},{69, 79},{95, 91},{8, 0},{1, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z120_CNG8[8][2] = {{25, 74},{11, 31},{71, 3},{21, 53},{23, 35},{112, 0},{1, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z128_CNG8[8][2] = {{9, 81},{117, 114},{76, 44},{26, 52},{61, 112},{77, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z144_CNG8[8][2] = {{3, 95},{26, 106},{53, 92},{35, 110},{115, 111},{127, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z160_CNG8[8][2] = {{0, 20},{0, 94},{0, 99},{0, 9},{0, 108},{0, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z176_CNG8[8][2] = {{156, 98},{143, 168},{14, 107},{3, 82},{40, 142},{123, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z192_CNG8[8][2] = {{174, 25},{97, 114},{166, 117},{66, 110},{71, 114},{172, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z208_CNG8[8][2] = {{143, 126},{19, 163},{176, 47},{165, 183},{196, 132},{13, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z224_CNG8[8][2] = {{72, 152},{110, 131},{23, 46},{181, 191},{95, 91},{8, 0},{1, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z240_CNG8[8][2] = {{145, 74},{131, 31},{71, 3},{21, 53},{23, 155},{112, 0},{1, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z256_CNG8[8][2] = {{9, 81},{117, 114},{204, 44},{26, 52},{189, 240},{205, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z288_CNG8[8][2] = {{3, 95},{26, 106},{53, 92},{35, 110},{115, 111},{127, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z320_CNG8[8][2] = {{0, 20},{0, 94},{0, 99},{0, 9},{0, 108},{0, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z352_CNG8[8][2] = {{156, 98},{143, 168},{14, 107},{3, 82},{40, 142},{123, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z384_CNG8[8][2] = {{174, 25},{97, 114},{166, 117},{66, 110},{71, 114},{172, 1},{0, 0},{0, 0}}; +// CNG10 +static const uint16_t circShift_BG2_Z2_CNG10[10][2] = {{1, 0},{0, 0},{1, 0},{1, 0},{0, 1},{0, 0},{0, 0},{0, 0},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z3_CNG10[10][2] = {{0, 1},{0, 1},{0, 2},{2, 0},{1, 0},{1, 1},{2, 1},{0, 0},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z4_CNG10[10][2] = {{3, 0},{2, 2},{1, 2},{1, 0},{2, 1},{0, 2},{0, 2},{0, 0},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z5_CNG10[10][2] = {{2, 3},{4, 0},{0, 2},{0, 1},{3, 2},{0, 2},{0, 3},{0, 1},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z6_CNG10[10][2] = {{3, 4},{0, 1},{0, 5},{2, 0},{1, 3},{1, 4},{5, 4},{3, 0},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z7_CNG10[10][2] = {{4, 3},{2, 6},{3, 1},{2, 5},{3, 5},{4, 5},{1, 2},{3, 0},{0, 1},{0, 0}}; +static const uint16_t circShift_BG2_Z8_CNG10[10][2] = {{7, 0},{6, 2},{5, 6},{5, 0},{2, 1},{4, 6},{0, 2},{4, 0},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z9_CNG10[10][2] = {{1, 3},{4, 4},{5, 4},{3, 4},{3, 1},{8, 4},{6, 2},{5, 7},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z10_CNG10[10][2] = {{7, 8},{4, 5},{0, 2},{0, 6},{8, 2},{0, 7},{0, 3},{5, 6},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z11_CNG10[10][2] = {{6, 9},{10, 9},{8, 9},{1, 6},{0, 6},{4, 1},{6, 6},{1, 6},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z12_CNG10[10][2] = {{3, 4},{0, 7},{0, 5},{8, 0},{7, 3},{7, 10},{5, 4},{3, 6},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z13_CNG10[10][2] = {{5, 10},{1, 9},{3, 5},{11, 7},{3, 8},{4, 3},{1, 9},{11, 4},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z14_CNG10[10][2] = {{11, 3},{2, 6},{3, 8},{2, 12},{3, 12},{4, 12},{1, 2},{3, 7},{0, 1},{0, 0}}; +static const uint16_t circShift_BG2_Z15_CNG10[10][2] = {{7, 14},{9, 6},{3, 5},{12, 5},{6, 9},{8, 4},{9, 13},{2, 0},{0, 1},{0, 0}}; +static const uint16_t circShift_BG2_Z16_CNG10[10][2] = {{7, 8},{6, 10},{13, 14},{13, 8},{2, 1},{12, 6},{0, 2},{12, 0},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z18_CNG10[10][2] = {{1, 12},{4, 13},{14, 4},{12, 4},{12, 1},{8, 13},{15, 2},{14, 7},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z20_CNG10[10][2] = {{17, 18},{4, 15},{0, 2},{0, 6},{8, 12},{0, 17},{0, 13},{15, 6},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z22_CNG10[10][2] = {{17, 9},{21, 20},{19, 20},{1, 17},{11, 17},{15, 1},{17, 17},{1, 17},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z24_CNG10[10][2] = {{3, 16},{12, 7},{0, 17},{20, 0},{7, 3},{19, 22},{17, 4},{3, 18},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z26_CNG10[10][2] = {{18, 10},{1, 22},{3, 18},{24, 7},{3, 21},{17, 3},{14, 9},{24, 4},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z28_CNG10[10][2] = {{25, 17},{16, 6},{3, 8},{16, 12},{3, 12},{4, 26},{1, 16},{3, 21},{0, 1},{0, 0}}; +static const uint16_t circShift_BG2_Z30_CNG10[10][2] = {{22, 29},{24, 21},{3, 5},{27, 20},{6, 9},{23, 19},{9, 13},{17, 15},{0, 1},{0, 0}}; +static const uint16_t circShift_BG2_Z32_CNG10[10][2] = {{7, 8},{6, 26},{29, 30},{29, 8},{2, 17},{28, 22},{0, 18},{28, 0},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z36_CNG10[10][2] = {{19, 12},{22, 13},{32, 22},{30, 4},{12, 1},{26, 13},{33, 20},{14, 7},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z40_CNG10[10][2] = {{17, 38},{4, 15},{0, 22},{0, 26},{8, 12},{0, 17},{0, 13},{15, 6},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z44_CNG10[10][2] = {{17, 9},{21, 42},{19, 42},{1, 39},{11, 17},{37, 1},{39, 17},{1, 17},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z48_CNG10[10][2] = {{27, 40},{36, 31},{0, 17},{44, 24},{31, 27},{43, 22},{41, 28},{3, 42},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z52_CNG10[10][2] = {{18, 36},{27, 48},{3, 18},{50, 7},{29, 47},{17, 3},{14, 35},{24, 4},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z56_CNG10[10][2] = {{53, 17},{44, 6},{3, 36},{44, 12},{3, 12},{32, 54},{29, 44},{31, 21},{0, 1},{0, 0}}; +static const uint16_t circShift_BG2_Z60_CNG10[10][2] = {{22, 59},{54, 51},{3, 35},{27, 50},{36, 39},{23, 19},{9, 43},{47, 15},{0, 1},{0, 0}}; +static const uint16_t circShift_BG2_Z64_CNG10[10][2] = {{39, 8},{38, 58},{61, 30},{61, 40},{34, 17},{28, 54},{32, 18},{60, 0},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z72_CNG10[10][2] = {{19, 48},{22, 49},{32, 22},{66, 4},{12, 1},{26, 49},{69, 56},{50, 7},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z80_CNG10[10][2] = {{57, 38},{44, 15},{0, 22},{0, 66},{8, 12},{0, 57},{0, 53},{55, 46},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z88_CNG10[10][2] = {{17, 53},{65, 86},{63, 86},{1, 39},{55, 17},{37, 1},{83, 17},{45, 17},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z96_CNG10[10][2] = {{27, 40},{36, 79},{48, 17},{92, 72},{31, 27},{91, 22},{89, 28},{3, 90},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z104_CNG10[10][2] = {{18, 36},{27, 48},{3, 18},{102, 7},{81, 99},{17, 3},{14, 87},{76, 56},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z112_CNG10[10][2] = {{53, 73},{44, 6},{3, 36},{44, 12},{3, 12},{88, 110},{29, 44},{31, 21},{0, 1},{0, 0}}; +static const uint16_t circShift_BG2_Z120_CNG10[10][2] = {{22, 119},{54, 51},{63, 95},{27, 110},{96, 39},{23, 79},{9, 43},{47, 75},{0, 1},{0, 0}}; +static const uint16_t circShift_BG2_Z128_CNG10[10][2] = {{39, 8},{38, 58},{125, 30},{125, 104},{98, 81},{28, 54},{96, 18},{124, 0},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z144_CNG10[10][2] = {{19, 120},{94, 121},{104, 22},{66, 4},{84, 73},{98, 49},{69, 128},{50, 79},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z160_CNG10[10][2] = {{137, 38},{124, 15},{0, 102},{0, 146},{88, 12},{0, 57},{0, 53},{55, 46},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z176_CNG10[10][2] = {{17, 53},{65, 174},{63, 174},{1, 127},{55, 17},{37, 89},{171, 17},{133, 105},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z192_CNG10[10][2] = {{27, 136},{36, 175},{48, 113},{92, 72},{31, 123},{187, 118},{185, 28},{3, 186},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z208_CNG10[10][2] = {{18, 36},{27, 48},{3, 18},{102, 111},{185, 203},{17, 3},{14, 191},{180, 160},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z224_CNG10[10][2] = {{53, 185},{156, 6},{115, 36},{156, 124},{115, 124},{200, 110},{29, 156},{31, 133},{0, 1},{0, 0}}; +static const uint16_t circShift_BG2_Z240_CNG10[10][2] = {{142, 239},{174, 171},{183, 95},{27, 110},{96, 159},{23, 199},{9, 43},{167, 75},{0, 1},{0, 0}}; +static const uint16_t circShift_BG2_Z256_CNG10[10][2] = {{167, 8},{166, 58},{253, 158},{125, 104},{226, 209},{156, 54},{224, 18},{252, 128},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z288_CNG10[10][2] = {{19, 120},{94, 121},{104, 22},{66, 4},{84, 73},{98, 49},{69, 128},{50, 79},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z320_CNG10[10][2] = {{137, 38},{124, 15},{0, 102},{0, 146},{88, 12},{0, 57},{0, 53},{55, 46},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z352_CNG10[10][2] = {{17, 53},{65, 174},{63, 174},{1, 127},{55, 17},{37, 89},{171, 17},{133, 105},{0, 0},{0, 0}}; +static const uint16_t circShift_BG2_Z384_CNG10[10][2] = {{27, 136},{36, 175},{48, 113},{92, 72},{31, 123},{187, 118},{185, 28},{3, 186},{0, 0},{0, 0}}; + +// Startaddressses in the BN buffer of the different CN groups +// BG1 +// Group of CNs with 3 BNs (CNG3) / only this group is independent of Z +// R13 +static const uint32_t startAddrBnProcBuf_BG1_R13_CNG3[3][1] = {{111360},{100224},{0}}; +// R23 +static const uint32_t startAddrBnProcBuf_BG1_R23_CNG3[3][1] = {{52224},{47616},{0}}; +// R89 +static const uint32_t startAddrBnProcBuf_BG1_R89_CNG3[3][1] = {{29952},{28032},{0}}; + +// Group of CNs with 4 BNs (CNG4) +static const uint32_t startAddrBnProcBuf_BG1_R13_CNG4[4][5] = {{105984, 107904, 120192, 120576, 109440},{40704, 74496, 43008, 47232, 43008},{41856, 19200, 61440, 23424, 92544},{0, 0, 0, 0, 0}}; +static const uint8_t bnPosBnProcBuf_BG1_R13_CNG4[4][5] = {{0, 0, 0, 0, 0},{0, 1, 1, 0, 0},{1, 0, 3, 0, 1},{23, 33, 36, 38, 41}}; + +// Group of CNs with 5 BNs (CNG5) +// R13 +static const uint32_t startAddrBnProcBuf_BG1_R13_CNG5[5][18] = {{116736, 105216, 105600, 117504, 117888, 106368, 118272, 106752, 118656, 107136, 119040, 107520, 119424, 119808, 108288, 108672, 109056, 120960},{87936, 30336, 39552, 31872, 46848, 58368, 89472, 87936, 89472, 33408, 89472, 41856, 61440, 30336, 73344, 74496, 61440, 92544},{72192, 87936, 86400, 46464, 31872, 97536, 73344, 89472, 59904, 61440, 31872, 91008, 33408, 91008, 91008, 31872, 98688, 33408},{58368, 97152, 56832, 30336, 73344, 21888, 21888, 22656, 22656, 74496, 59904, 91008, 97920, 92544, 33408, 98304, 23424, 92544},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint8_t bnPosBnProcBuf_BG1_R13_CNG5[5][18] = {{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},{2, 0, 0, 0, 0, 1, 1, 0, 2, 0, 0, 0, 1, 1, 0, 0, 2, 0},{1, 1, 0, 0, 3, 0, 1, 3, 1, 0, 2, 2, 2, 1, 0, 1, 0, 1},{3, 0, 1, 2, 2, 1, 0, 1, 0, 2, 3, 3, 0, 2, 3, 0, 1, 3},{18, 19, 21, 22, 24, 25, 26, 27, 28, 29, 30, 31, 32, 34, 35, 37, 39, 40}}; + +// Group of CNs with 6 BNs (CNG6) +// R13 +static const uint32_t startAddrBnProcBuf_BG1_R13_CNG6[6][8] = {{114432, 103680, 115584, 104064, 115968, 116352, 104832, 117120},{68736, 69888, 55296, 86400, 104448, 71040, 17280, 72192},{83328, 56832, 58368, 71040, 84864, 28800, 59904, 46080},{40704, 41856, 56832, 96768, 40704, 58368, 43008, 59904},{18816, 84864, 71040, 30336, 86400, 86400, 72192, 87936},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint8_t bnPosBnProcBuf_BG1_R13_CNG6[6][8] = {{0, 0, 0, 0, 0, 0, 0, 0},{0, 0, 1, 2, 0, 0, 0, 0},{0, 0, 2, 1, 0, 1, 2, 0},{2, 2, 3, 0, 1, 0, 2, 0},{0, 3, 2, 3, 1, 3, 2, 3},{9, 12, 13, 14, 15, 16, 17, 20}}; + +// Group of CNs with 7 BNs (CNG7) +// R13 +static const uint32_t startAddrBnProcBuf_BG1_R13_CNG7[7][5] = {{112512, 102144, 114048, 114816, 115200},{100992, 28800, 102912, 84864, 103296},{45312, 45696, 83328, 28800, 84864},{80256, 81792, 55296, 56832, 69888},{38400, 39552, 68736, 55296, 96384},{52224, 53760, 96000, 69888, 21120},{0, 0, 0, 0, 0}}; +static const uint8_t bnPosBnProcBuf_BG1_R13_CNG7[7][5] = {{0, 0, 0, 0, 0},{0, 0, 0, 2, 0},{0, 0, 1, 2, 1},{0, 0, 0, 2, 1},{1, 1, 1, 3, 0},{1, 1, 0, 2, 1},{3, 6, 8, 10, 11}}; +// R23 +static const uint32_t startAddrBnProcBuf_BG1_R23_CNG7[7][3] = {{53376, 49536, 54912},{48384, 13440, 50304},{22656, 25344, 43392},{22656, 25344, 43392},{22656, 25344, 43392},{22656, 25344, 43392},{0, 0, 0}}; +static const uint8_t bnPosBnProcBuf_BG1_R23_CNG7[7][3] = {{0, 0, 0},{0, 0, 0},{1, 1, 0},{2, 2, 1},{3, 3, 3},{4, 4, 5},{3, 6, 8}}; + +// Group of CNs with 8 BNs (CNG8) +// R13 +static const uint32_t startAddrBnProcBuf_BG1_R13_CNG8[8][2] = {{111744, 113664},{100608, 102528},{66432, 83328},{80256, 55296},{52224, 68736},{66432, 83328},{80256, 18432},{0, 0}}; +static const uint8_t bnPosBnProcBuf_BG1_R13_CNG8[8][2] = {{0, 0},{0, 0},{0, 2},{2, 2},{2, 2},{2, 3},{3, 0},{1, 7}}; +// R23 +static const uint32_t startAddrBnProcBuf_BG1_R23_CNG8[8][2] = {{52608, 54528},{48000, 49920},{22656, 43392},{37248, 43392},{37248, 43392},{37248, 43392},{37248, 8064},{0, 0}}; +static const uint8_t bnPosBnProcBuf_BG1_R23_CNG8[8][2] = {{0, 0},{0, 0},{0, 2},{2, 4},{4, 6},{6, 7},{7, 3},{1, 7}}; + +// Group of CNs with 9 BNs (CNG9) +// R13 +static const uint32_t startAddrBnProcBuf_BG1_R13_CNG9[9][2] = {{112128, 113280},{38400, 101760},{80256, 81792},{52224, 53760},{66432, 67584},{52224, 53760},{95232, 95616},{38400, 39552},{0, 0}}; +static const uint8_t bnPosBnProcBuf_BG1_R13_CNG9[9][2] = {{0, 0},{0, 0},{1, 1},{0, 0},{1, 1},{3, 3},{0, 0},{2, 2},{2, 5}}; +// R23 +static const uint32_t startAddrBnProcBuf_BG1_R23_CNG9[9][2] = {{52992, 54144},{13440, 49152},{37248, 40320},{37248, 40320},{37248, 40320},{22656, 25344},{37248, 40320},{22656, 25344},{0, 0}}; +static const uint8_t bnPosBnProcBuf_BG1_R23_CNG9[9][2] = {{0, 0},{1, 0},{0, 0},{1, 1},{3, 3},{5, 5},{5, 5},{6, 6},{2, 5}}; + +// Group of CNs with 10 BNs (CNG10) +// R13 +static const uint32_t startAddrBnProcBuf_BG1_R13_CNG10[10][1] = {{112896},{101376},{67584},{81792},{53760},{28800},{67584},{81792},{21120},{0}}; +static const uint8_t bnPosBnProcBuf_BG1_R13_CNG10[10][1] = {{0},{0},{0},{2},{2},{3},{2},{3},{0},{4}}; +// R23 +static const uint32_t startAddrBnProcBuf_BG1_R23_CNG10[10][1] = {{53760},{48768},{25344},{40320},{40320},{13440},{40320},{40320},{8064},{0}}; +static const uint8_t bnPosBnProcBuf_BG1_R23_CNG10[10][1] = {{0},{0},{0},{2},{4},{2},{6},{7},{4},{4}}; + +// Group of CNs with 19 BNs (CNG19) +// R13 +static const uint32_t startAddrBnProcBuf_BG1_R13_CNG19[19][4] = {{109824, 110208, 110592, 110976},{99072, 25728, 99456, 99840},{24192, 64128, 27264, 65280},{62976, 44160, 44544, 44928},{16128, 16512, 16896, 37248},{34944, 75648, 36096, 78720},{24192, 34944, 77184, 37248},{75648, 25728, 36096, 78720},{47616, 49152, 27264, 50688},{75648, 77184, 77184, 78720},{62976, 47616, 64128, 65280},{24192, 25728, 49152, 50688},{47616, 49152, 27264, 50688},{94080, 47616, 49152, 50688},{24192, 25728, 94464, 94848},{34944, 64128, 27264, 37248},{62976, 77184, 36096, 65280},{75648, 18048, 20352, 78720},{17664, 19584, 19584, 20352}}; +static const uint8_t bnPosBnProcBuf_BG1_R13_CNG19[19][4] = {{0, 0, 0, 0},{0, 0, 0, 0},{0, 0, 0, 0},{0, 0, 0, 0},{0, 0, 0, 0},{0, 0, 0, 0},{1, 1, 0, 1},{1, 1, 1, 1},{0, 0, 1, 0},{2, 2, 1, 2},{1, 1, 1, 1},{2, 2, 1, 1},{2, 2, 2, 2},{0, 3, 3, 3},{3, 3, 0, 0},{2, 2, 3, 2},{2, 3, 2, 2},{3, 0, 0, 3},{0, 0, 1, 1}}; +// R23 +static const uint32_t startAddrBnProcBuf_BG1_R23_CNG19[19][4] = {{50688, 51072, 51456, 51840},{46464, 11136, 46848, 47232},{9984, 17280, 12288, 19968},{14592, 14592, 17280, 19968},{4224, 6144, 8064, 12288},{9984, 14592, 11136, 19968},{4224, 14592, 17280, 19968},{28032, 6144, 17280, 34176},{28032, 31104, 8064, 34176},{28032, 31104, 31104, 34176},{28032, 14592, 31104, 34176},{4224, 6144, 17280, 19968},{28032, 31104, 8064, 34176},{28032, 14592, 17280, 19968},{9984, 11136, 31104, 34176},{14592, 31104, 12288, 19968},{28032, 31104, 17280, 34176},{28032, 6144, 6144, 34176},{4224, 4224, 3456, 3840}}; +static const uint8_t bnPosBnProcBuf_BG1_R23_CNG19[19][4] = {{0, 0, 0, 0},{0, 0, 0, 0},{0, 0, 0, 0},{0, 1, 1, 1},{0, 0, 0, 1},{1, 2, 1, 2},{1, 3, 2, 3},{0, 1, 3, 0},{1, 1, 1, 1},{2, 2, 0, 2},{3, 4, 3, 3},{2, 2, 4, 4},{4, 4, 2, 4},{5, 5, 5, 5},{2, 2, 5, 5},{6, 6, 2, 6},{6, 7, 6, 6},{7, 3, 4, 7},{3, 4, 0, 0}}; +// R89 +static const uint32_t startAddrBnProcBuf_BG1_R89_CNG19[19][4] = {{28416, 28800, 29184, 29568},{26880, 10752, 27264, 27648},{2688, 10752, 18816, 18816},{2688, 2688, 10752, 18816},{2688, 10752, 18816, 18816},{2688, 2688, 10752, 18816},{2688, 2688, 10752, 18816},{2688, 10752, 10752, 18816},{2688, 10752, 18816, 18816},{2688, 10752, 10752, 18816},{2688, 2688, 10752, 18816},{2688, 10752, 10752, 18816},{2688, 10752, 18816, 18816},{2688, 2688, 10752, 18816},{2688, 10752, 10752, 18816},{2688, 10752, 18816, 18816},{2688, 10752, 10752, 18816},{2688, 1536, 1536, 18816},{384, 384, 384, 1536}}; +static const uint8_t bnPosBnProcBuf_BG1_R89_CNG19[19][4] = {{0, 0, 0, 0},{0, 0, 0, 0},{0, 1, 0, 1},{1, 2, 2, 2},{3, 3, 3, 4},{4, 5, 4, 5},{7, 6, 5, 6},{8, 7, 6, 8},{9, 9, 7, 9},{10, 10, 8, 10},{11, 12, 11, 11},{13, 13, 12, 12},{14, 14, 13, 14},{16, 15, 15, 15},{17, 17, 16, 16},{18, 19, 17, 18},{19, 20, 18, 19},{20, 0, 1, 20},{0, 1, 2, 2}}; + +// BG2 +// CNG3 +static const uint32_t startAddrBnProcBuf_BG2_R15_CNG3[3][6] = {{72960, 64128, 64512, 64896, 74112, 31872},{34176, 49536, 23040, 17664, 40320, 41088},{0, 0, 0, 0, 0, 0}}; +static const uint8_t bnPosBnProcBuf_BG2_R15_CNG3[3][6] = {{0, 0, 0, 0, 0, 0},{0, 0, 0, 1, 0, 0},{18, 21, 23, 25, 27, 33}}; +// CNG4 +// R15 +static const uint32_t startAddrBnProcBuf_BG2_R15_CNG4[4][20] = {{59520, 60672, 70272, 62208, 62592, 62976, 72576, 63360, 63744, 73344, 73728, 65280, 36096, 65664, 74496, 66048, 74880, 66432, 36864, 75264},{67968, 69120, 16896, 29568, 22656, 72192, 16896, 20352, 17664, 34560, 35328, 50688, 45696, 31872, 51072, 36480, 51456, 46464, 32640, 51840},{52992, 28032, 54912, 55680, 44544, 30336, 56832, 39552, 49152, 25728, 49920, 31104, 31104, 40704, 57216, 46080, 57600, 32640, 41472, 57984},{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint8_t bnPosBnProcBuf_BG2_R15_CNG4[4][20] = {{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},{0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0},{0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0},{0, 4, 8, 11, 14, 15, 16, 17, 19, 20, 24, 28, 29, 30, 31, 32, 34, 35, 36, 37}}; +// R13 +static const uint32_t startAddrBnProcBuf_BG2_R13_CNG4[4][8] = {{36096, 37248, 43776, 38784, 39168, 39552, 46080, 39936},{41472, 42624, 9984, 20352, 22272, 45696, 9984, 22272},{30720, 11520, 32640, 33408, 29568, 22272, 34560, 26496},{0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint8_t bnPosBnProcBuf_BG2_R13_CNG4[4][8] = {{0, 0, 0, 0, 0, 0, 0, 0},{0, 0, 0, 4, 1, 0, 1, 2},{0, 0, 0, 0, 0, 4, 0, 0},{0, 4, 8, 11, 14, 15, 16, 17}}; +// R23 +static const uint32_t startAddrBnProcBuf_BG2_R23_CNG4[4][1] = {{18816},{16128},{15360},{0}}; +static const uint8_t bnPosBnProcBuf_BG2_R23_CNG4[4][1] = {{0},{0},{1},{0}}; +// CNG5 +// R15 +static const uint32_t startAddrBnProcBuf_BG2_R15_CNG5[5][9] = {{69504, 61056, 61440, 61824, 71040, 71424, 71808, 34944, 35712},{19584, 69888, 44160, 70656, 22272, 25344, 48768, 44928, 50304},{28800, 21888, 24960, 19968, 55296, 56064, 56448, 30336, 45312},{54528, 43776, 38400, 38784, 39168, 28800, 29568, 39936, 26112},{0, 0, 0, 0, 0, 0, 0, 0, 0}}; +static const uint8_t bnPosBnProcBuf_BG2_R15_CNG5[5][9] = {{0, 0, 0, 0, 0, 0, 0, 0, 0},{0, 0, 0, 0, 0, 0, 0, 0, 0},{0, 0, 0, 0, 0, 0, 0, 1, 0},{0, 0, 0, 0, 0, 1, 1, 0, 0},{5, 6, 7, 9, 10, 12, 13, 22, 26}}; +// R13 +static const uint32_t startAddrBnProcBuf_BG2_R13_CNG5[5][7] = {{43008, 37632, 38016, 38400, 44544, 44928, 45312},{18432, 43392, 29184, 44160, 20352, 22272, 22272},{18432, 18432, 20352, 20352, 33024, 33792, 34176},{32256, 28800, 25344, 25728, 26112, 11904, 12288},{0, 0, 0, 0, 0, 0, 0}}; +static const uint8_t bnPosBnProcBuf_BG2_R13_CNG5[5][7] = {{0, 0, 0, 0, 0, 0, 0},{2, 0, 0, 0, 1, 3, 0},{4, 1, 3, 2, 0, 0, 0},{0, 0, 0, 0, 0, 0, 0},{5, 6, 7, 9, 10, 12, 13}}; +// R23 +// none +// CNG6 +// R15 +static const uint32_t startAddrBnProcBuf_BG2_R15_CNG6[6][3] = {{59904, 60288, 68736},{68352, 48000, 48384},{47616, 43008, 43392},{42624, 24576, 54144},{53376, 53760, 38016},{0, 0, 0}}; +static const uint8_t bnPosBnProcBuf_BG2_R15_CNG6[6][3] = {{0, 0, 0},{0, 0, 0},{0, 0, 0},{0, 0, 0},{0, 0, 0},{1, 2, 3}}; +// R13 +static const uint32_t startAddrBnProcBuf_BG2_R13_CNG6[6][3] = {{36480, 36864, 42240},{41856, 18432, 20352},{16512, 28032, 28416},{27648, 18432, 31872},{31104, 31488, 24960},{0, 0, 0}}; +static const uint8_t bnPosBnProcBuf_BG2_R13_CNG6[6][3] = {{0, 0, 0},{0, 0, 0},{0, 0, 0},{0, 3, 0},{0, 0, 0},{1, 2, 3}}; +// R23 +static const uint32_t startAddrBnProcBuf_BG2_R23_CNG6[6][2] = {{19200, 19584},{16896, 12672},{11520, 12672},{11520, 12672},{16128, 16896},{0, 0}}; +static const uint8_t bnPosBnProcBuf_BG2_R23_CNG6[6][2] = {{0, 0},{0, 0},{0, 1},{1, 2},{1, 1},{1, 2}}; +// CNG8 +// R15 +static const uint32_t startAddrBnProcBuf_BG2_R15_CNG8[8][2] = {{58368, 59136},{66816, 67200},{33408, 16128},{14592, 15360},{20736, 18816},{23424, 27264},{26496, 27264},{52224, 37248}}; +static const uint8_t bnPosBnProcBuf_BG2_R15_CNG8[8][2] = {{0, 0},{0, 0},{0, 0},{0, 1},{0, 0},{0, 0},{0, 1},{0, 0}}; +// R13 +static const uint32_t startAddrBnProcBuf_BG2_R13_CNG8[8][2] = {{34944, 35712},{40320, 40704},{6912, 9216},{7680, 8448},{12672, 14592},{12672, 14592},{12672, 11136},{29952, 24192}}; +static const uint8_t bnPosBnProcBuf_BG2_R13_CNG8[8][2] = {{0, 0},{0, 0},{0, 0},{0, 1},{1, 2},{3, 4},{4, 0},{0, 0}}; +// R23 +static const uint32_t startAddrBnProcBuf_BG2_R23_CNG8[8][2] = {{17664, 18432},{13824, 14592},{1152, 7296},{3456, 5376},{3456, 5376},{9216, 5376},{3456, 2304},{13824, 1152}}; +static const uint8_t bnPosBnProcBuf_BG2_R23_CNG8[8][2] = {{0, 0},{0, 0},{0, 0},{0, 1},{2, 3},{2, 4},{4, 1},{1, 2}}; +// CNG10 +// R15 +static const uint32_t startAddrBnProcBuf_BG2_R15_CNG10[10][2] = {{58752, 67584},{15360, 33792},{14592, 16128},{46848, 47232},{21120, 21504},{41856, 42240},{18432, 19200},{23808, 24192},{52608, 28032},{26496, 37632}}; +static const uint8_t bnPosBnProcBuf_BG2_R15_CNG10[10][2] = {{0, 0},{0, 0},{1, 1},{0, 0},{0, 0},{0, 0},{0, 0},{0, 0},{0, 0},{1, 0}}; +// R13 +static const uint32_t startAddrBnProcBuf_BG2_R13_CNG10[10][2] = {{35328, 41088},{8448, 7296},{7680, 9216},{12672, 14592},{14592, 16512},{26880, 27264},{12672, 16512},{14592, 16512},{30336, 16512},{10752, 24576}}; +static const uint8_t bnPosBnProcBuf_BG2_R13_CNG10[10][2] = {{0, 0},{0, 0},{1, 1},{0, 0},{1, 1},{0, 0},{2, 2},{3, 3},{0, 4},{0, 0}}; +// R23 +static const uint32_t startAddrBnProcBuf_BG2_R23_CNG10[10][2] = {{18048, 15360},{5376, 2304},{3456, 7296},{9216, 10368},{5376, 7296},{9216, 10368},{3456, 7296},{10368, 11520},{14592, 7296},{1152, 2304}}; +static const uint8_t bnPosBnProcBuf_BG2_R23_CNG10[10][2] = {{0, 0},{0, 0},{1, 1},{0, 0},{2, 2},{1, 1},{3, 3},{2, 2},{1, 4},{1, 2}}; + +// LUT for llr2llrProcBuf +// BG1 +// R13 +static const uint16_t llr2llrProcBufAddr_BG1_R13[26] = {25728,25344,17664,22272,20352,16128,19200,23424,19200,17664,23424,20736,23424,22272,20736,17664,20736,20736,24960,17664,19200,22272,23424,16512,16896,16896}; +static const uint8_t llr2llrProcBufBnPos_BG1_R13[26] = {0,0,0,0,0,0,0,0,1,1,1,0,2,1,1,2,2,3,0,3,2,2,3,0,0,1}; +// R23 +static const uint16_t llr2llrProcBufAddr_BG1_R23[26] = {13056,12672,5760,6912,6912,3840,5760,6912,6912,3840,9600,9600,9600,9600,6912,3840,9600,6912,9600,5760,6912,9600,9600,3840,3840,3456}; +static const uint8_t llr2llrProcBufBnPos_BG1_R23[26] = {0,0,0,0,1,0,1,2,3,1,0,1,2,3,4,2,4,5,5,2,6,6,7,3,4,0}; +// R89 +static const uint16_t llr2llrProcBufAddr_BG1_R89[26] = {9984,9600,1536,1536,1536,1536,1536,1536,1536,1536,1536,1536,1536,1536,1536,1536,1536,1536,1536,1536,1536,1536,1536,384,384,384}; +static const uint8_t llr2llrProcBufBnPos_BG1_R89[26] = {0,0,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,0,1,2}; + +// BG2 +// R15 +static const uint16_t llr2llrProcBufAddr_BG2_R15[14] = {19200,19584,17280,14592,14592,18432,15744,18048,15360,16128,16512,18816,16512,17664}; +static const uint8_t llr2llrProcBufBnPos_BG2_R15[14] = {0,0,0,0,1,0,0,0,0,0,0,0,1,0}; +// R13 +static const uint16_t llr2llrProcBufAddr_BG2_R13[14] = {11520,11904,6912,7296,7296,8448,8448,10752,8448,8448,8448,11136,8064,10368}; +static const uint8_t llr2llrProcBufBnPos_BG2_R13[14] = {0,0,0,0,1,0,1,0,2,3,4,0,0,0}; +// R23 +static const uint16_t llr2llrProcBufAddr_BG2_R23[14] = {6144,5376,1152,2304,2304,4224,2304,4224,2304,4224,2304,5376,1152,1152}; +static const uint8_t llr2llrProcBufBnPos_BG2_R23[14] = {0,0,0,0,1,0,2,1,3,2,4,1,1,2}; + +// LUT for llr2cnProcBuf +// LUTs are not rate dependent, however the number of CN in a CNG varies with rate +// CNG3 +// BG1 +static const uint8_t posBnInCnProcBuf_BG1_CNG3[3][1] = {{0},{1},{26}}; +// CNG4 +static const uint8_t posBnInCnProcBuf_BG1_CNG4[4][5] = {{1, 1, 0, 0, 1},{6, 13, 8, 4, 6},{8, 23, 17, 24, 10},{49, 59, 62, 64, 67}}; +// CNG5 +static const uint8_t posBnInCnProcBuf_BG1_CNG5[5][18] = {{0, 1, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0},{12, 2, 6, 2, 4, 14, 10, 7, 12, 2, 7, 6, 14, 9, 3, 3, 16, 7},{13, 10, 7, 4, 19, 18, 13, 22, 14, 11, 15, 12, 15, 10, 7, 9, 18, 9},{17, 18, 14, 15, 21, 25, 24, 25, 24, 21, 17, 22, 18, 12, 19, 18, 25, 22},{44, 45, 47, 48, 50, 51, 52, 53, 54, 55, 56, 57, 58, 60, 61, 63, 65, 66}}; +// CNG6 +static const uint8_t posBnInCnProcBuf_BG1_CNG6[6][8] = {{0, 1, 0, 1, 0, 0, 1, 0},{3, 3, 14, 12, 1, 3, 5, 3},{7, 11, 16, 13, 7, 9, 16, 4},{20, 20, 17, 18, 8, 11, 20, 11},{23, 22, 21, 19, 10, 22, 21, 22},{35, 38, 39, 40, 41, 42, 43, 46}}; +// CNG7 +static const uint8_t posBnInCnProcBuf_BG1_CNG7[7][5] = {{0, 1, 0, 0, 0},{1, 2, 1, 12, 1},{4, 4, 10, 15, 10},{7, 7, 11, 16, 13},{8, 8, 13, 17, 18},{14, 14, 18, 21, 25},{29, 32, 34, 36, 37}}; +// CNG8 +static const uint8_t posBnInCnProcBuf_BG1_CNG8[8][2] = {{0, 0},{1, 1},{3, 12},{12, 16},{16, 21},{21, 22},{22, 23},{27, 33}}; +// CNG9 +static const uint8_t posBnInCnProcBuf_BG1_CNG9[9][2] = {{0, 0},{6, 1},{10, 10},{11, 11},{13, 13},{17, 17},{18, 18},{20, 20},{28, 31}}; +// CNG10 +static const uint8_t posBnInCnProcBuf_BG1_CNG10[10][1] = {{0},{1},{3},{12},{16},{19},{21},{22},{24},{30}}; +// CNG19 +static const uint8_t posBnInCnProcBuf_BG1_CNG19[19][4] = {{0, 0, 0, 0},{1, 2, 1, 1},{2, 3, 2, 3},{3, 4, 4, 4},{5, 5, 5, 6},{6, 7, 6, 7},{9, 8, 7, 8},{10, 9, 8, 10},{11, 11, 9, 11},{12, 12, 10, 12},{13, 14, 13, 13},{15, 15, 14, 14},{16, 16, 15, 16},{18, 17, 17, 17},{19, 19, 18, 18},{20, 21, 19, 20},{21, 22, 20, 21},{22, 23, 24, 22},{23, 24, 25, 25}}; + +// BG2 +static const uint8_t posBnInCnProcBuf_BG2_CNG3[3][6] = {{1, 0, 0, 0, 1, 10},{2, 5, 6, 4, 13, 13},{32, 35, 37, 39, 41, 47}}; +static const uint8_t posBnInCnProcBuf_BG2_CNG4[4][20] = {{0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 2, 0, 1, 0, 1, 0, 2, 1},{1, 1, 3, 10, 6, 1, 4, 8, 3, 2, 2, 5, 7, 12, 5, 2, 5, 7, 10, 5},{11, 12, 11, 11, 7, 10, 11, 13, 5, 9, 5, 12, 10, 13, 11, 7, 11, 12, 13, 11},{14, 18, 22, 25, 28, 29, 30, 31, 33, 34, 38, 42, 43, 44, 45, 46, 48, 49, 50, 51}}; +static const uint8_t posBnInCnProcBuf_BG2_CNG5[5][9] = {{1, 0, 0, 0, 1, 1, 1, 2, 2},{8, 1, 7, 1, 6, 9, 5, 7, 5},{10, 6, 9, 8, 11, 11, 11, 12, 7},{11, 7, 13, 13, 13, 12, 12, 13, 9},{19, 20, 21, 23, 24, 26, 27, 36, 40}}; +static const uint8_t posBnInCnProcBuf_BG2_CNG6[6][3] = {{0, 0, 1},{1, 5, 5},{5, 7, 7},{7, 9, 11},{11, 11, 13},{15, 16, 17}}; +static const uint8_t posBnInCnProcBuf_BG2_CNG8[8][2] = {{0, 0},{1, 1},{2, 3},{3, 4},{6, 8},{9, 10},{10, 12},{11, 13}}; +static const uint8_t posBnInCnProcBuf_BG2_CNG10[10][2] = {{0, 1},{3, 2},{4, 4},{5, 5},{6, 6},{7, 7},{8, 8},{9, 9},{11, 10},{12, 13}}; + +#endif diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/nrLDPC_mPass.h b/openair1/PHY/CODING/nrLDPC_decoder_full/nrLDPC_mPass.h new file mode 100644 index 0000000000..801349fd36 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/nrLDPC_mPass.h @@ -0,0 +1,1199 @@ +/* + * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The OpenAirInterface Software Alliance licenses this file to You under + * the OAI Public License, Version 1.1 (the "License"); you may not use this file + * except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.openairinterface.org/?page_id=698 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *------------------------------------------------------------------------------- + * For more information about the OpenAirInterface (OAI) Software Alliance: + * contact@openairinterface.org + */ + +/*!\file nrLDPC_mPass.h + * \brief Defines the functions for message passing + * \author Sebastian Wagner (TCL Communications) Email: + * \date 30-09-2019 + * \version 2.0 + * \note + * \warning + */ + +#ifndef __NR_LDPC_MPASS__H__ +#define __NR_LDPC_MPASS__H__ + +#include +#include "nrLDPCdecoder_defs.h" +//#include +/** + \brief Circular memcpy + |<- rem->|<- circular shift ->| + (src) str2 = |--------xxxxxxxxxxxxxxxxxxxxx| + \_______________ + \ + (dst) str1 = |xxxxxxxxxxxxxxxxxxxxx---------| + \param str1 Pointer to the start of the destination buffer + \param str2 Pointer to the source buffer + \param Z Lifting size + \param cshift Circular shift +*/ +static inline void *nrLDPC_inv_circ_memcpy(int8_t *str1, const int8_t *str2, uint16_t Z, uint16_t cshift) +{ + uint16_t rem = Z - cshift; + memcpy(str1+cshift, str2 , rem); + memcpy(str1 , str2+rem, cshift); + + return(str1); +} + +/** + \brief Inverse circular memcpy + |<- circular shift ->|<- rem->| + (src) str2 = |xxxxxxxxxxxxxxxxxxxx\--------| + \ + (dst) str1 = |--------xxxxxxxxxxxxxxxxxxxxx| + \param str1 Pointer to the start of the destination buffer + \param str2 Pointer to the source buffer + \param Z Lifting size + \param cshift Circular shift +*/ +static inline void *nrLDPC_circ_memcpy(int8_t *str1, const int8_t *str2, uint16_t Z, uint16_t cshift) +{ + uint16_t rem = Z - cshift; + memcpy(str1 , str2+cshift, rem); + memcpy(str1+rem , str2 , cshift); + + return(str1); +} + +/** + \brief Copies the input LLRs to their corresponding place in the LLR processing buffer. + Example: BG2 + | 0| 0| LLRs --> | + BN Groups |22|23|10| 5| 5|14| 7|13| 6| 8| 9|16| 9|12|1|1|...|1| + ^---------------------------------------/---- / + _________________________/ | / + / ____________________________|___/ + / / \ + LLR Proc Buffer (BNG) | 1| 5| 6| 7| 8| 9|10|12|13|14|16|22|23| + Number BN in BNG(R15) |38| 2| 1| 1| 1| 2| 1| 1| 1| 1| 1| 1| 1| + Idx: 0 ^ ^ ^ + 38*384=14592 _____| ... | | + 50*384=19200 ----------------------------------- | + 51*384=19584 -------------------------------------- + + \param p_lut Pointer to decoder LUTs + \param llr Pointer to input LLRs + \param p_procBuf Pointer the processing buffers + \param Z Lifting size + \param BG Base graph +*/ +static inline void nrLDPC_llr2llrProcBuf(t_nrLDPC_lut* p_lut, int8_t* llr, t_nrLDPC_procBuf* p_procBuf, uint16_t Z, uint8_t BG) +{ + uint32_t i; + const uint8_t numBn2CnG1 = p_lut->numBnInBnGroups[0]; + uint32_t startColParity = (BG ==1 ) ? (NR_LDPC_START_COL_PARITY_BG1) : (NR_LDPC_START_COL_PARITY_BG2); + + uint32_t colG1 = startColParity*Z; + + const uint16_t* lut_llr2llrProcBufAddr = p_lut->llr2llrProcBufAddr; + const uint8_t* lut_llr2llrProcBufBnPos = p_lut->llr2llrProcBufBnPos; + + uint32_t idxBn; + int8_t* llrProcBuf = p_procBuf->llrProcBuf; + + // Copy LLRs connected to 1 CN + if (numBn2CnG1 > 0) + { + memcpy(&llrProcBuf[0], &llr[colG1], numBn2CnG1*Z); + } + + // First 2 columns might be set to zero directly if it's true they always belong to the groups with highest number of connected CNs... + for (i=0; icircShift[0]; + const uint16_t (*lut_circShift_CNG4) [lut_numCnInCnGroups_BG1_R13[1]] = (uint16_t(*)[lut_numCnInCnGroups_BG1_R13[1]]) p_lut->circShift[1]; + const uint16_t (*lut_circShift_CNG5) [lut_numCnInCnGroups_BG1_R13[2]] = (uint16_t(*)[lut_numCnInCnGroups_BG1_R13[2]]) p_lut->circShift[2]; + const uint16_t (*lut_circShift_CNG6) [lut_numCnInCnGroups_BG1_R13[3]] = (uint16_t(*)[lut_numCnInCnGroups_BG1_R13[3]]) p_lut->circShift[3]; + const uint16_t (*lut_circShift_CNG7) [lut_numCnInCnGroups_BG1_R13[4]] = (uint16_t(*)[lut_numCnInCnGroups_BG1_R13[4]]) p_lut->circShift[4]; + const uint16_t (*lut_circShift_CNG8) [lut_numCnInCnGroups_BG1_R13[5]] = (uint16_t(*)[lut_numCnInCnGroups_BG1_R13[5]]) p_lut->circShift[5]; + const uint16_t (*lut_circShift_CNG9) [lut_numCnInCnGroups_BG1_R13[6]] = (uint16_t(*)[lut_numCnInCnGroups_BG1_R13[6]]) p_lut->circShift[6]; + const uint16_t (*lut_circShift_CNG10)[lut_numCnInCnGroups_BG1_R13[7]] = (uint16_t(*)[lut_numCnInCnGroups_BG1_R13[7]]) p_lut->circShift[7]; + const uint16_t (*lut_circShift_CNG19)[lut_numCnInCnGroups_BG1_R13[8]] = (uint16_t(*)[lut_numCnInCnGroups_BG1_R13[8]]) p_lut->circShift[8]; + + const uint8_t (*lut_posBnInCnProcBuf_CNG3) [lut_numCnInCnGroups_BG1_R13[0]] = (uint8_t(*)[lut_numCnInCnGroups_BG1_R13[0]]) p_lut->posBnInCnProcBuf[0]; + const uint8_t (*lut_posBnInCnProcBuf_CNG4) [lut_numCnInCnGroups_BG1_R13[1]] = (uint8_t(*)[lut_numCnInCnGroups_BG1_R13[1]]) p_lut->posBnInCnProcBuf[1]; + const uint8_t (*lut_posBnInCnProcBuf_CNG5) [lut_numCnInCnGroups_BG1_R13[2]] = (uint8_t(*)[lut_numCnInCnGroups_BG1_R13[2]]) p_lut->posBnInCnProcBuf[2]; + const uint8_t (*lut_posBnInCnProcBuf_CNG6) [lut_numCnInCnGroups_BG1_R13[3]] = (uint8_t(*)[lut_numCnInCnGroups_BG1_R13[3]]) p_lut->posBnInCnProcBuf[3]; + const uint8_t (*lut_posBnInCnProcBuf_CNG7) [lut_numCnInCnGroups_BG1_R13[4]] = (uint8_t(*)[lut_numCnInCnGroups_BG1_R13[4]]) p_lut->posBnInCnProcBuf[4]; + const uint8_t (*lut_posBnInCnProcBuf_CNG8) [lut_numCnInCnGroups_BG1_R13[5]] = (uint8_t(*)[lut_numCnInCnGroups_BG1_R13[5]]) p_lut->posBnInCnProcBuf[5]; + const uint8_t (*lut_posBnInCnProcBuf_CNG9) [lut_numCnInCnGroups_BG1_R13[6]] = (uint8_t(*)[lut_numCnInCnGroups_BG1_R13[6]]) p_lut->posBnInCnProcBuf[6]; + const uint8_t (*lut_posBnInCnProcBuf_CNG10)[lut_numCnInCnGroups_BG1_R13[7]] = (uint8_t(*)[lut_numCnInCnGroups_BG1_R13[7]]) p_lut->posBnInCnProcBuf[7]; + const uint8_t (*lut_posBnInCnProcBuf_CNG19)[lut_numCnInCnGroups_BG1_R13[8]] = (uint8_t(*)[lut_numCnInCnGroups_BG1_R13[8]]) p_lut->posBnInCnProcBuf[8]; + + const uint8_t* lut_numCnInCnGroups = p_lut->numCnInCnGroups; + const uint32_t* lut_startAddrCnGroups = p_lut->startAddrCnGroups; + + int8_t* cnProcBuf = p_procBuf->cnProcBuf; + uint32_t i; + uint32_t j; + + uint32_t idxBn = 0; + int8_t* p_cnProcBuf; + uint32_t bitOffsetInGroup; + + // ===================================================================== + // CN group with 3 BNs + + bitOffsetInGroup = lut_numCnInCnGroups_BG1_R13[0]*NR_LDPC_ZMAX; + +// #pragma omp simd +// #pragma omp parallel for schedule(dynamic) + + for (j=0; j<3; j++) + { + p_cnProcBuf = &cnProcBuf[lut_startAddrCnGroups[0] + j*bitOffsetInGroup]; + + idxBn = lut_posBnInCnProcBuf_CNG3[j][0]*Z; + + nrLDPC_circ_memcpy(p_cnProcBuf, &llr[idxBn], Z, lut_circShift_CNG3[j][0]); + } + + // ===================================================================== + // CN group with 4 BNs + + bitOffsetInGroup = lut_numCnInCnGroups_BG1_R13[1]*NR_LDPC_ZMAX; + + for (j=0; j<4; j++) + { + p_cnProcBuf = &cnProcBuf[lut_startAddrCnGroups[1] + j*bitOffsetInGroup]; + + for (i=0; i | + BN Groups |22|23|10| 5| 5|14| 7|13| 6| 8| 9|16| 9|12|1|1|...|1| + + + CN Processing Buffer (CNGs) | 3| 4| 5| 6| 8|10| + Number of CN per CNG (R15) | 6|20| 9| 3| 2| 2| + 0 ^ ^\ \ + 3*6*384=6912 _________| || \_____________ + (3*6+4*20+5*9)*384=54912____|| \ + Bit | 1| 2| 3| 4| 5| 6| + 3*Z CNs>| |< + ^ + 54912 + 3*384______| + + \param p_lut Pointer to decoder LUTs + \param llr Pointer to input LLRs + \param p_procBuf Pointer to the processing buffers + \param Z Lifting size +*/ +static inline void nrLDPC_llr2CnProcBuf_BG2(t_nrLDPC_lut* p_lut, int8_t* llr, t_nrLDPC_procBuf* p_procBuf, uint16_t Z) +{ + const uint16_t (*lut_circShift_CNG3) [lut_numCnInCnGroups_BG2_R15[0]] = (uint16_t(*)[lut_numCnInCnGroups_BG2_R15[0]]) p_lut->circShift[0]; + const uint16_t (*lut_circShift_CNG4) [lut_numCnInCnGroups_BG2_R15[1]] = (uint16_t(*)[lut_numCnInCnGroups_BG2_R15[1]]) p_lut->circShift[1]; + const uint16_t (*lut_circShift_CNG5) [lut_numCnInCnGroups_BG2_R15[2]] = (uint16_t(*)[lut_numCnInCnGroups_BG2_R15[2]]) p_lut->circShift[2]; + const uint16_t (*lut_circShift_CNG6) [lut_numCnInCnGroups_BG2_R15[3]] = (uint16_t(*)[lut_numCnInCnGroups_BG2_R15[3]]) p_lut->circShift[3]; + const uint16_t (*lut_circShift_CNG8) [lut_numCnInCnGroups_BG2_R15[4]] = (uint16_t(*)[lut_numCnInCnGroups_BG2_R15[4]]) p_lut->circShift[4]; + const uint16_t (*lut_circShift_CNG10) [lut_numCnInCnGroups_BG2_R15[5]] = (uint16_t(*)[lut_numCnInCnGroups_BG2_R15[5]]) p_lut->circShift[5]; + + const uint8_t (*lut_posBnInCnProcBuf_CNG3) [lut_numCnInCnGroups_BG2_R15[0]] = (uint8_t(*)[lut_numCnInCnGroups_BG2_R15[0]]) p_lut->posBnInCnProcBuf[0]; + const uint8_t (*lut_posBnInCnProcBuf_CNG4) [lut_numCnInCnGroups_BG2_R15[1]] = (uint8_t(*)[lut_numCnInCnGroups_BG2_R15[1]]) p_lut->posBnInCnProcBuf[1]; + const uint8_t (*lut_posBnInCnProcBuf_CNG5) [lut_numCnInCnGroups_BG2_R15[2]] = (uint8_t(*)[lut_numCnInCnGroups_BG2_R15[2]]) p_lut->posBnInCnProcBuf[2]; + const uint8_t (*lut_posBnInCnProcBuf_CNG6) [lut_numCnInCnGroups_BG2_R15[3]] = (uint8_t(*)[lut_numCnInCnGroups_BG2_R15[3]]) p_lut->posBnInCnProcBuf[3]; + const uint8_t (*lut_posBnInCnProcBuf_CNG8) [lut_numCnInCnGroups_BG2_R15[4]] = (uint8_t(*)[lut_numCnInCnGroups_BG2_R15[4]]) p_lut->posBnInCnProcBuf[4]; + const uint8_t (*lut_posBnInCnProcBuf_CNG10) [lut_numCnInCnGroups_BG2_R15[5]] = (uint8_t(*)[lut_numCnInCnGroups_BG2_R15[5]]) p_lut->posBnInCnProcBuf[5]; + + const uint8_t* lut_numCnInCnGroups = p_lut->numCnInCnGroups; + const uint32_t* lut_startAddrCnGroups = p_lut->startAddrCnGroups; + + int8_t* cnProcBuf = p_procBuf->cnProcBuf; + uint32_t i; + uint32_t j; + + uint32_t idxBn = 0; + int8_t* p_cnProcBuf; + uint32_t bitOffsetInGroup; + + // ===================================================================== + // CN group with 3 BNs + + bitOffsetInGroup = lut_numCnInCnGroups_BG2_R15[0]*NR_LDPC_ZMAX; + + for (j=0; j<3; j++) + { + p_cnProcBuf = &cnProcBuf[lut_startAddrCnGroups[0] + j*bitOffsetInGroup]; + + for (i=0; inumCnInCnGroups; + const uint32_t* lut_startAddrCnGroups = p_lut->startAddrCnGroups; + + const uint16_t (*lut_circShift_CNG3) [lut_numCnInCnGroups_BG2_R15[0]] = (uint16_t(*)[lut_numCnInCnGroups_BG2_R15[0]]) p_lut->circShift[0]; + const uint16_t (*lut_circShift_CNG4) [lut_numCnInCnGroups_BG2_R15[1]] = (uint16_t(*)[lut_numCnInCnGroups_BG2_R15[1]]) p_lut->circShift[1]; + const uint16_t (*lut_circShift_CNG5) [lut_numCnInCnGroups_BG2_R15[2]] = (uint16_t(*)[lut_numCnInCnGroups_BG2_R15[2]]) p_lut->circShift[2]; + const uint16_t (*lut_circShift_CNG6) [lut_numCnInCnGroups_BG2_R15[3]] = (uint16_t(*)[lut_numCnInCnGroups_BG2_R15[3]]) p_lut->circShift[3]; + const uint16_t (*lut_circShift_CNG8) [lut_numCnInCnGroups_BG2_R15[4]] = (uint16_t(*)[lut_numCnInCnGroups_BG2_R15[4]]) p_lut->circShift[4]; + const uint16_t (*lut_circShift_CNG10) [lut_numCnInCnGroups_BG2_R15[5]] = (uint16_t(*)[lut_numCnInCnGroups_BG2_R15[5]]) p_lut->circShift[5]; + + const uint32_t (*lut_startAddrBnProcBuf_CNG3) [lut_numCnInCnGroups[0]] = (uint32_t(*)[lut_numCnInCnGroups[0]]) p_lut->startAddrBnProcBuf[0]; + const uint32_t (*lut_startAddrBnProcBuf_CNG4) [lut_numCnInCnGroups[1]] = (uint32_t(*)[lut_numCnInCnGroups[1]]) p_lut->startAddrBnProcBuf[1]; + const uint32_t (*lut_startAddrBnProcBuf_CNG5) [lut_numCnInCnGroups[2]] = (uint32_t(*)[lut_numCnInCnGroups[2]]) p_lut->startAddrBnProcBuf[2]; + const uint32_t (*lut_startAddrBnProcBuf_CNG6) [lut_numCnInCnGroups[3]] = (uint32_t(*)[lut_numCnInCnGroups[3]]) p_lut->startAddrBnProcBuf[3]; + const uint32_t (*lut_startAddrBnProcBuf_CNG8) [lut_numCnInCnGroups[4]] = (uint32_t(*)[lut_numCnInCnGroups[4]]) p_lut->startAddrBnProcBuf[4]; + const uint32_t (*lut_startAddrBnProcBuf_CNG10) [lut_numCnInCnGroups[5]] = (uint32_t(*)[lut_numCnInCnGroups[5]]) p_lut->startAddrBnProcBuf[5]; + + const uint8_t (*lut_bnPosBnProcBuf_CNG3) [lut_numCnInCnGroups[0]] = (uint8_t(*)[lut_numCnInCnGroups[0]]) p_lut->bnPosBnProcBuf[0]; + const uint8_t (*lut_bnPosBnProcBuf_CNG4) [lut_numCnInCnGroups[1]] = (uint8_t(*)[lut_numCnInCnGroups[1]]) p_lut->bnPosBnProcBuf[1]; + const uint8_t (*lut_bnPosBnProcBuf_CNG5) [lut_numCnInCnGroups[2]] = (uint8_t(*)[lut_numCnInCnGroups[2]]) p_lut->bnPosBnProcBuf[2]; + const uint8_t (*lut_bnPosBnProcBuf_CNG6) [lut_numCnInCnGroups[3]] = (uint8_t(*)[lut_numCnInCnGroups[3]]) p_lut->bnPosBnProcBuf[3]; + const uint8_t (*lut_bnPosBnProcBuf_CNG8) [lut_numCnInCnGroups[4]] = (uint8_t(*)[lut_numCnInCnGroups[4]]) p_lut->bnPosBnProcBuf[4]; + const uint8_t (*lut_bnPosBnProcBuf_CNG10) [lut_numCnInCnGroups[5]] = (uint8_t(*)[lut_numCnInCnGroups[5]]) p_lut->bnPosBnProcBuf[5]; + + int8_t* cnProcBufRes = p_procBuf->cnProcBufRes; + int8_t* bnProcBuf = p_procBuf->bnProcBuf; + + int8_t* p_cnProcBufRes; + uint32_t bitOffsetInGroup; + uint32_t i; + uint32_t j; + uint32_t idxBn = 0; + + // ===================================================================== + // CN group with 3 BNs + + bitOffsetInGroup = lut_numCnInCnGroups_BG2_R15[0]*NR_LDPC_ZMAX; + + for (j=0; j<3; j++) + { + p_cnProcBufRes = &cnProcBufRes[lut_startAddrCnGroups[0] + j*bitOffsetInGroup]; + for (i=0; inumCnInCnGroups; + const uint32_t* lut_startAddrCnGroups = p_lut->startAddrCnGroups; + + const uint16_t (*lut_circShift_CNG3) [lut_numCnInCnGroups_BG1_R13[0]] = (uint16_t(*)[lut_numCnInCnGroups_BG1_R13[0]]) p_lut->circShift[0]; + const uint16_t (*lut_circShift_CNG4) [lut_numCnInCnGroups_BG1_R13[1]] = (uint16_t(*)[lut_numCnInCnGroups_BG1_R13[1]]) p_lut->circShift[1]; + const uint16_t (*lut_circShift_CNG5) [lut_numCnInCnGroups_BG1_R13[2]] = (uint16_t(*)[lut_numCnInCnGroups_BG1_R13[2]]) p_lut->circShift[2]; + const uint16_t (*lut_circShift_CNG6) [lut_numCnInCnGroups_BG1_R13[3]] = (uint16_t(*)[lut_numCnInCnGroups_BG1_R13[3]]) p_lut->circShift[3]; + const uint16_t (*lut_circShift_CNG7) [lut_numCnInCnGroups_BG1_R13[4]] = (uint16_t(*)[lut_numCnInCnGroups_BG1_R13[4]]) p_lut->circShift[4]; + const uint16_t (*lut_circShift_CNG8) [lut_numCnInCnGroups_BG1_R13[5]] = (uint16_t(*)[lut_numCnInCnGroups_BG1_R13[5]]) p_lut->circShift[5]; + const uint16_t (*lut_circShift_CNG9) [lut_numCnInCnGroups_BG1_R13[6]] = (uint16_t(*)[lut_numCnInCnGroups_BG1_R13[6]]) p_lut->circShift[6]; + const uint16_t (*lut_circShift_CNG10)[lut_numCnInCnGroups_BG1_R13[7]] = (uint16_t(*)[lut_numCnInCnGroups_BG1_R13[7]]) p_lut->circShift[7]; + const uint16_t (*lut_circShift_CNG19)[lut_numCnInCnGroups_BG1_R13[8]] = (uint16_t(*)[lut_numCnInCnGroups_BG1_R13[8]]) p_lut->circShift[8]; + + const uint32_t (*lut_startAddrBnProcBuf_CNG3) [lut_numCnInCnGroups[0]] = (uint32_t(*)[lut_numCnInCnGroups[0]]) p_lut->startAddrBnProcBuf[0]; + const uint32_t (*lut_startAddrBnProcBuf_CNG4) [lut_numCnInCnGroups[1]] = (uint32_t(*)[lut_numCnInCnGroups[1]]) p_lut->startAddrBnProcBuf[1]; + const uint32_t (*lut_startAddrBnProcBuf_CNG5) [lut_numCnInCnGroups[2]] = (uint32_t(*)[lut_numCnInCnGroups[2]]) p_lut->startAddrBnProcBuf[2]; + const uint32_t (*lut_startAddrBnProcBuf_CNG6) [lut_numCnInCnGroups[3]] = (uint32_t(*)[lut_numCnInCnGroups[3]]) p_lut->startAddrBnProcBuf[3]; + const uint32_t (*lut_startAddrBnProcBuf_CNG7) [lut_numCnInCnGroups[4]] = (uint32_t(*)[lut_numCnInCnGroups[4]]) p_lut->startAddrBnProcBuf[4]; + const uint32_t (*lut_startAddrBnProcBuf_CNG8) [lut_numCnInCnGroups[5]] = (uint32_t(*)[lut_numCnInCnGroups[5]]) p_lut->startAddrBnProcBuf[5]; + const uint32_t (*lut_startAddrBnProcBuf_CNG9) [lut_numCnInCnGroups[6]] = (uint32_t(*)[lut_numCnInCnGroups[6]]) p_lut->startAddrBnProcBuf[6]; + const uint32_t (*lut_startAddrBnProcBuf_CNG10)[lut_numCnInCnGroups[7]] = (uint32_t(*)[lut_numCnInCnGroups[7]]) p_lut->startAddrBnProcBuf[7]; + const uint32_t (*lut_startAddrBnProcBuf_CNG19)[lut_numCnInCnGroups[8]] = (uint32_t(*)[lut_numCnInCnGroups[8]]) p_lut->startAddrBnProcBuf[8]; + + const uint8_t (*lut_bnPosBnProcBuf_CNG4) [lut_numCnInCnGroups[1]] = (uint8_t(*)[lut_numCnInCnGroups[1]]) p_lut->bnPosBnProcBuf[1]; + const uint8_t (*lut_bnPosBnProcBuf_CNG5) [lut_numCnInCnGroups[2]] = (uint8_t(*)[lut_numCnInCnGroups[2]]) p_lut->bnPosBnProcBuf[2]; + const uint8_t (*lut_bnPosBnProcBuf_CNG6) [lut_numCnInCnGroups[3]] = (uint8_t(*)[lut_numCnInCnGroups[3]]) p_lut->bnPosBnProcBuf[3]; + const uint8_t (*lut_bnPosBnProcBuf_CNG7) [lut_numCnInCnGroups[4]] = (uint8_t(*)[lut_numCnInCnGroups[4]]) p_lut->bnPosBnProcBuf[4]; + const uint8_t (*lut_bnPosBnProcBuf_CNG8) [lut_numCnInCnGroups[5]] = (uint8_t(*)[lut_numCnInCnGroups[5]]) p_lut->bnPosBnProcBuf[5]; + const uint8_t (*lut_bnPosBnProcBuf_CNG9) [lut_numCnInCnGroups[6]] = (uint8_t(*)[lut_numCnInCnGroups[6]]) p_lut->bnPosBnProcBuf[6]; + const uint8_t (*lut_bnPosBnProcBuf_CNG10)[lut_numCnInCnGroups[7]] = (uint8_t(*)[lut_numCnInCnGroups[7]]) p_lut->bnPosBnProcBuf[7]; + const uint8_t (*lut_bnPosBnProcBuf_CNG19)[lut_numCnInCnGroups[8]] = (uint8_t(*)[lut_numCnInCnGroups[8]]) p_lut->bnPosBnProcBuf[8]; + + int8_t* cnProcBufRes = p_procBuf->cnProcBufRes; + int8_t* bnProcBuf = p_procBuf->bnProcBuf; + + int8_t* p_cnProcBufRes; + uint32_t bitOffsetInGroup; + uint32_t i; + uint32_t j; + uint32_t idxBn = 0; + + // ===================================================================== + // CN group with 3 BNs + + bitOffsetInGroup = lut_numCnInCnGroups_BG1_R13[0]*NR_LDPC_ZMAX; + + for (j=0; j<3; j++) + { + p_cnProcBufRes = &cnProcBufRes[lut_startAddrCnGroups[0] + j*bitOffsetInGroup]; + nrLDPC_inv_circ_memcpy(&bnProcBuf[lut_startAddrBnProcBuf_CNG3[j][0]],p_cnProcBufRes,Z,lut_circShift_CNG3[j][0]); + } + + // ===================================================================== + // CN group with 4 BNs + + bitOffsetInGroup = lut_numCnInCnGroups_BG1_R13[1]*NR_LDPC_ZMAX; + + for (j=0; j<4; j++) + { + p_cnProcBufRes = &cnProcBufRes[lut_startAddrCnGroups[1] + j*bitOffsetInGroup]; + + for (i=0; inumCnInCnGroups; + const uint32_t* lut_startAddrCnGroups = p_lut->startAddrCnGroups; + + const uint16_t (*lut_circShift_CNG3) [lut_numCnInCnGroups_BG2_R15[0]] = (uint16_t(*)[lut_numCnInCnGroups_BG2_R15[0]]) p_lut->circShift[0]; + const uint16_t (*lut_circShift_CNG4) [lut_numCnInCnGroups_BG2_R15[1]] = (uint16_t(*)[lut_numCnInCnGroups_BG2_R15[1]]) p_lut->circShift[1]; + const uint16_t (*lut_circShift_CNG5) [lut_numCnInCnGroups_BG2_R15[2]] = (uint16_t(*)[lut_numCnInCnGroups_BG2_R15[2]]) p_lut->circShift[2]; + const uint16_t (*lut_circShift_CNG6) [lut_numCnInCnGroups_BG2_R15[3]] = (uint16_t(*)[lut_numCnInCnGroups_BG2_R15[3]]) p_lut->circShift[3]; + const uint16_t (*lut_circShift_CNG8) [lut_numCnInCnGroups_BG2_R15[4]] = (uint16_t(*)[lut_numCnInCnGroups_BG2_R15[4]]) p_lut->circShift[4]; + const uint16_t (*lut_circShift_CNG10) [lut_numCnInCnGroups_BG2_R15[5]] = (uint16_t(*)[lut_numCnInCnGroups_BG2_R15[5]]) p_lut->circShift[5]; + + const uint32_t (*lut_startAddrBnProcBuf_CNG3) [lut_numCnInCnGroups[0]] = (uint32_t(*)[lut_numCnInCnGroups[0]]) p_lut->startAddrBnProcBuf[0]; + const uint32_t (*lut_startAddrBnProcBuf_CNG4) [lut_numCnInCnGroups[1]] = (uint32_t(*)[lut_numCnInCnGroups[1]]) p_lut->startAddrBnProcBuf[1]; + const uint32_t (*lut_startAddrBnProcBuf_CNG5) [lut_numCnInCnGroups[2]] = (uint32_t(*)[lut_numCnInCnGroups[2]]) p_lut->startAddrBnProcBuf[2]; + const uint32_t (*lut_startAddrBnProcBuf_CNG6) [lut_numCnInCnGroups[3]] = (uint32_t(*)[lut_numCnInCnGroups[3]]) p_lut->startAddrBnProcBuf[3]; + const uint32_t (*lut_startAddrBnProcBuf_CNG8) [lut_numCnInCnGroups[4]] = (uint32_t(*)[lut_numCnInCnGroups[4]]) p_lut->startAddrBnProcBuf[4]; + const uint32_t (*lut_startAddrBnProcBuf_CNG10) [lut_numCnInCnGroups[5]] = (uint32_t(*)[lut_numCnInCnGroups[5]]) p_lut->startAddrBnProcBuf[5]; + + const uint8_t (*lut_bnPosBnProcBuf_CNG3) [lut_numCnInCnGroups[0]] = (uint8_t(*)[lut_numCnInCnGroups[0]]) p_lut->bnPosBnProcBuf[0]; + const uint8_t (*lut_bnPosBnProcBuf_CNG4) [lut_numCnInCnGroups[1]] = (uint8_t(*)[lut_numCnInCnGroups[1]]) p_lut->bnPosBnProcBuf[1]; + const uint8_t (*lut_bnPosBnProcBuf_CNG5) [lut_numCnInCnGroups[2]] = (uint8_t(*)[lut_numCnInCnGroups[2]]) p_lut->bnPosBnProcBuf[2]; + const uint8_t (*lut_bnPosBnProcBuf_CNG6) [lut_numCnInCnGroups[3]] = (uint8_t(*)[lut_numCnInCnGroups[3]]) p_lut->bnPosBnProcBuf[3]; + const uint8_t (*lut_bnPosBnProcBuf_CNG8) [lut_numCnInCnGroups[4]] = (uint8_t(*)[lut_numCnInCnGroups[4]]) p_lut->bnPosBnProcBuf[4]; + const uint8_t (*lut_bnPosBnProcBuf_CNG10) [lut_numCnInCnGroups[5]] = (uint8_t(*)[lut_numCnInCnGroups[5]]) p_lut->bnPosBnProcBuf[5]; + + int8_t* cnProcBuf = p_procBuf->cnProcBuf; + int8_t* bnProcBufRes = p_procBuf->bnProcBufRes; + + int8_t* p_cnProcBuf; + uint32_t bitOffsetInGroup; + uint32_t i; + uint32_t j; + uint32_t idxBn = 0; + + // For CN groups 3 to 6 no need to send the last BN back since it's single edge + // and BN processing does not change the value already in the CN proc buf + + // ===================================================================== + // CN group with 3 BNs + + bitOffsetInGroup = lut_numCnInCnGroups_BG2_R15[0]*NR_LDPC_ZMAX; + + for (j=0; j<2; j++) + { + p_cnProcBuf = &cnProcBuf[lut_startAddrCnGroups[0] + j*bitOffsetInGroup]; + for (i=0; inumCnInCnGroups; + const uint32_t* lut_startAddrCnGroups = p_lut->startAddrCnGroups; + + const uint16_t (*lut_circShift_CNG3) [lut_numCnInCnGroups_BG1_R13[0]] = (uint16_t(*)[lut_numCnInCnGroups_BG1_R13[0]]) p_lut->circShift[0]; + const uint16_t (*lut_circShift_CNG4) [lut_numCnInCnGroups_BG1_R13[1]] = (uint16_t(*)[lut_numCnInCnGroups_BG1_R13[1]]) p_lut->circShift[1]; + const uint16_t (*lut_circShift_CNG5) [lut_numCnInCnGroups_BG1_R13[2]] = (uint16_t(*)[lut_numCnInCnGroups_BG1_R13[2]]) p_lut->circShift[2]; + const uint16_t (*lut_circShift_CNG6) [lut_numCnInCnGroups_BG1_R13[3]] = (uint16_t(*)[lut_numCnInCnGroups_BG1_R13[3]]) p_lut->circShift[3]; + const uint16_t (*lut_circShift_CNG7) [lut_numCnInCnGroups_BG1_R13[4]] = (uint16_t(*)[lut_numCnInCnGroups_BG1_R13[4]]) p_lut->circShift[4]; + const uint16_t (*lut_circShift_CNG8) [lut_numCnInCnGroups_BG1_R13[5]] = (uint16_t(*)[lut_numCnInCnGroups_BG1_R13[5]]) p_lut->circShift[5]; + const uint16_t (*lut_circShift_CNG9) [lut_numCnInCnGroups_BG1_R13[6]] = (uint16_t(*)[lut_numCnInCnGroups_BG1_R13[6]]) p_lut->circShift[6]; + const uint16_t (*lut_circShift_CNG10)[lut_numCnInCnGroups_BG1_R13[7]] = (uint16_t(*)[lut_numCnInCnGroups_BG1_R13[7]]) p_lut->circShift[7]; + const uint16_t (*lut_circShift_CNG19)[lut_numCnInCnGroups_BG1_R13[8]] = (uint16_t(*)[lut_numCnInCnGroups_BG1_R13[8]]) p_lut->circShift[8]; + + const uint32_t (*lut_startAddrBnProcBuf_CNG3) [lut_numCnInCnGroups[0]] = (uint32_t(*)[lut_numCnInCnGroups[0]]) p_lut->startAddrBnProcBuf[0]; + const uint32_t (*lut_startAddrBnProcBuf_CNG4) [lut_numCnInCnGroups[1]] = (uint32_t(*)[lut_numCnInCnGroups[1]]) p_lut->startAddrBnProcBuf[1]; + const uint32_t (*lut_startAddrBnProcBuf_CNG5) [lut_numCnInCnGroups[2]] = (uint32_t(*)[lut_numCnInCnGroups[2]]) p_lut->startAddrBnProcBuf[2]; + const uint32_t (*lut_startAddrBnProcBuf_CNG6) [lut_numCnInCnGroups[3]] = (uint32_t(*)[lut_numCnInCnGroups[3]]) p_lut->startAddrBnProcBuf[3]; + const uint32_t (*lut_startAddrBnProcBuf_CNG7) [lut_numCnInCnGroups[4]] = (uint32_t(*)[lut_numCnInCnGroups[4]]) p_lut->startAddrBnProcBuf[4]; + const uint32_t (*lut_startAddrBnProcBuf_CNG8) [lut_numCnInCnGroups[5]] = (uint32_t(*)[lut_numCnInCnGroups[5]]) p_lut->startAddrBnProcBuf[5]; + const uint32_t (*lut_startAddrBnProcBuf_CNG9) [lut_numCnInCnGroups[6]] = (uint32_t(*)[lut_numCnInCnGroups[6]]) p_lut->startAddrBnProcBuf[6]; + const uint32_t (*lut_startAddrBnProcBuf_CNG10)[lut_numCnInCnGroups[7]] = (uint32_t(*)[lut_numCnInCnGroups[7]]) p_lut->startAddrBnProcBuf[7]; + const uint32_t (*lut_startAddrBnProcBuf_CNG19)[lut_numCnInCnGroups[8]] = (uint32_t(*)[lut_numCnInCnGroups[8]]) p_lut->startAddrBnProcBuf[8]; + + const uint8_t (*lut_bnPosBnProcBuf_CNG4) [lut_numCnInCnGroups[1]] = (uint8_t(*)[lut_numCnInCnGroups[1]]) p_lut->bnPosBnProcBuf[1]; + const uint8_t (*lut_bnPosBnProcBuf_CNG5) [lut_numCnInCnGroups[2]] = (uint8_t(*)[lut_numCnInCnGroups[2]]) p_lut->bnPosBnProcBuf[2]; + const uint8_t (*lut_bnPosBnProcBuf_CNG6) [lut_numCnInCnGroups[3]] = (uint8_t(*)[lut_numCnInCnGroups[3]]) p_lut->bnPosBnProcBuf[3]; + const uint8_t (*lut_bnPosBnProcBuf_CNG7) [lut_numCnInCnGroups[4]] = (uint8_t(*)[lut_numCnInCnGroups[4]]) p_lut->bnPosBnProcBuf[4]; + const uint8_t (*lut_bnPosBnProcBuf_CNG8) [lut_numCnInCnGroups[5]] = (uint8_t(*)[lut_numCnInCnGroups[5]]) p_lut->bnPosBnProcBuf[5]; + const uint8_t (*lut_bnPosBnProcBuf_CNG9) [lut_numCnInCnGroups[6]] = (uint8_t(*)[lut_numCnInCnGroups[6]]) p_lut->bnPosBnProcBuf[6]; + const uint8_t (*lut_bnPosBnProcBuf_CNG10)[lut_numCnInCnGroups[7]] = (uint8_t(*)[lut_numCnInCnGroups[7]]) p_lut->bnPosBnProcBuf[7]; + const uint8_t (*lut_bnPosBnProcBuf_CNG19)[lut_numCnInCnGroups[8]] = (uint8_t(*)[lut_numCnInCnGroups[8]]) p_lut->bnPosBnProcBuf[8]; + + int8_t* cnProcBuf = p_procBuf->cnProcBuf; + int8_t* bnProcBufRes = p_procBuf->bnProcBufRes; + + int8_t* p_cnProcBuf; + uint32_t bitOffsetInGroup; + uint32_t i; + uint32_t j; + uint32_t idxBn = 0; + + // For CN groups 3 to 19 no need to send the last BN back since it's single edge + // and BN processing does not change the value already in the CN proc buf + + // ===================================================================== + // CN group with 3 BNs + + bitOffsetInGroup = lut_numCnInCnGroups_BG1_R13[0]*NR_LDPC_ZMAX; + + for (j=0;j<2; j++) + { + p_cnProcBuf = &cnProcBuf[lut_startAddrCnGroups[0] + j*bitOffsetInGroup]; + nrLDPC_circ_memcpy(p_cnProcBuf, &bnProcBufRes[lut_startAddrBnProcBuf_CNG3[j][0]], Z, lut_circShift_CNG3[j][0]); + } + + // ===================================================================== + // CN group with 4 BNs + + bitOffsetInGroup = lut_numCnInCnGroups_BG1_R13[1]*NR_LDPC_ZMAX; + + for (j=0; j<3; j++) + { + p_cnProcBuf = &cnProcBuf[lut_startAddrCnGroups[1] + j*bitOffsetInGroup]; + for (i=0; inumBnInBnGroups[0]; + uint32_t startColParity = (BG ==1 ) ? (NR_LDPC_START_COL_PARITY_BG1) : (NR_LDPC_START_COL_PARITY_BG2); + + uint32_t colG1 = startColParity*Z; + + const uint16_t* lut_llr2llrProcBufAddr = p_lut->llr2llrProcBufAddr; + const uint8_t* lut_llr2llrProcBufBnPos = p_lut->llr2llrProcBufBnPos; + + int8_t* llrRes = p_procBuf->llrRes; + int8_t* p_llrOut = &llrOut[0]; + uint32_t idxBn; + + // Copy LLRs connected to 1 CN + if (numBn2CnG1 > 0) + { + memcpy(&llrOut[colG1], llrRes, numBn2CnG1*Z); + } + + for (i=0; i + * \date 27-03-2018 + * \version 1.0 + * \note + * \warning + */ + +#ifndef __NR_LDPC_DEBUG__H__ +#define __NR_LDPC_DEBUG__H__ + +#include + +/** + Enum with possible LDPC data buffers + */ +typedef enum nrLDPC_buffers { + nrLDPC_buffers_LLR_PROC, /**< LLR processing buffer */ + nrLDPC_buffers_CN_PROC, /**< CN processing buffer */ + nrLDPC_buffers_CN_PROC_RES, /**< CN processing results buffer */ + nrLDPC_buffers_BN_PROC, /**< BN processing buffer */ + nrLDPC_buffers_BN_PROC_RES, /**< BN processing results buffer */ + nrLDPC_buffers_LLR_RES /**< LLR results buffer */ +} e_nrLDPC_buffers; + +/** + \brief Writes N data samples to a file + \param fileName Name of the file + \param p_data Pointer to the data + \param N Number of values to write +*/ +static inline void nrLDPC_writeFile(const char* fileName, int8_t* p_data, const uint32_t N) +{ + FILE *f; + uint32_t i; + + f = fopen(fileName, "a"); + + // Newline indicating new data + fprintf(f, "\n"); + for (i=0; i < N; i++) + { + fprintf(f, "%d, ", p_data[i]); + } + + fclose(f); +} + +/** + \brief Creates empty new file + \param fileName Name of the file +*/ +static inline void nrLDPC_initFile(const char* fileName) +{ + FILE *f; + + f = fopen(fileName, "w"); + + fprintf(f, " "); + + fclose(f); +} + +/** + \brief Writes data of predefined buffers to file + \param buffer Enum of buffer name to write +*/ +static inline void nrLDPC_debug_writeBuffer2File(e_nrLDPC_buffers buffer, t_nrLDPC_procBuf* p_procBuf) +{ + switch (buffer) + { + case nrLDPC_buffers_LLR_PROC: + { + nrLDPC_writeFile("llrProcBuf.txt", p_procBuf->llrProcBuf, NR_LDPC_MAX_NUM_LLR); + break; + } + case nrLDPC_buffers_CN_PROC: + { + nrLDPC_writeFile("cnProcBuf.txt", p_procBuf->cnProcBuf, NR_LDPC_SIZE_CN_PROC_BUF); + break; + } + case nrLDPC_buffers_CN_PROC_RES: + { + nrLDPC_writeFile("cnProcBufRes.txt", p_procBuf->cnProcBufRes, NR_LDPC_SIZE_CN_PROC_BUF); + break; + } + case nrLDPC_buffers_BN_PROC: + { + nrLDPC_writeFile("bnProcBuf.txt", p_procBuf->bnProcBuf, NR_LDPC_SIZE_BN_PROC_BUF); + break; + } + case nrLDPC_buffers_BN_PROC_RES: + { + nrLDPC_writeFile("bnProcBufRes.txt", p_procBuf->bnProcBufRes, NR_LDPC_SIZE_BN_PROC_BUF); + break; + } + case nrLDPC_buffers_LLR_RES: + { + nrLDPC_writeFile("llrRes.txt", p_procBuf->llrRes, NR_LDPC_MAX_NUM_LLR); + break; + } + } +} + +/** + \brief Initializes file for writing a buffer + \param buffer Enum of buffer name to write +*/ +static inline void nrLDPC_debug_initBuffer2File(e_nrLDPC_buffers buffer) +{ + switch (buffer) + { + case nrLDPC_buffers_LLR_PROC: + { + nrLDPC_initFile("llrProcBuf.txt"); + break; + } + case nrLDPC_buffers_CN_PROC: + { + nrLDPC_initFile("cnProcBuf.txt"); + break; + } + case nrLDPC_buffers_CN_PROC_RES: + { + nrLDPC_initFile("cnProcBufRes.txt"); + break; + } + case nrLDPC_buffers_BN_PROC: + { + nrLDPC_initFile("bnProcBuf.txt"); + break; + } + case nrLDPC_buffers_BN_PROC_RES: + { + nrLDPC_initFile("bnProcBufRes.txt"); + break; + } + case nrLDPC_buffers_LLR_RES: + { + nrLDPC_initFile("llrRes.txt"); + break; + } + } +} + +/** + \brief Prints 256 data type + \param in Input to print +*/ +static inline void nrLDPC_debug_print256i_epi8(__m256i* in) +{ + uint32_t i; + + for (i=0; i<32; i++) + { + mexPrintf("%d ", ((int8_t*)&in)[i]); + } +} + +#endif diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/nrLDPC_types.h b/openair1/PHY/CODING/nrLDPC_decoder_full/nrLDPC_types.h new file mode 100644 index 0000000000..b802e07104 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/nrLDPC_types.h @@ -0,0 +1,106 @@ +/* + * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The OpenAirInterface Software Alliance licenses this file to You under + * the OAI Public License, Version 1.1 (the "License"); you may not use this file + * except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.openairinterface.org/?page_id=698 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *------------------------------------------------------------------------------- + * For more information about the OpenAirInterface (OAI) Software Alliance: + * contact@openairinterface.org + */ + +/*!\file nrLDPC_types.h + * \brief Defines all types for the LDPC decoder + * \author Sebastian Wagner (TCL Communications) Email: + * \date 27-03-2018 + * \version 1.0 + * \note + * \warning + */ + +#ifndef __NR_LDPC_TYPES__H__ +#define __NR_LDPC_TYPES__H__ +#include "../../TOOLS/time_meas.h" +#include "nrLDPCdecoder_defs.h" +// ============================================================================== +// TYPES + +/** + Structure containing the pointers to the LUTs. + */ +typedef struct nrLDPC_lut { + const uint32_t* startAddrCnGroups; /**< Start addresses for CN groups in CN processing buffer */ + const uint8_t* numCnInCnGroups; /**< Number of CNs in every CN group */ + const uint8_t* numBnInBnGroups; /**< Number of CNs in every BN group */ + const uint32_t* startAddrBnGroups; /**< Start addresses for BN groups in BN processing buffer */ + const uint16_t* startAddrBnGroupsLlr; /**< Start addresses for BN groups in LLR processing buffer */ + const uint16_t** circShift[NR_LDPC_NUM_CN_GROUPS_BG1]; /**< LUT for circular shift values for all CN groups and Zs */ + const uint32_t** startAddrBnProcBuf[NR_LDPC_NUM_CN_GROUPS_BG1]; /**< LUT of start addresses of CN groups in BN proc buffer */ + const uint8_t** bnPosBnProcBuf[NR_LDPC_NUM_CN_GROUPS_BG1]; /**< LUT of BN positions in BG for CN groups */ + const uint16_t* llr2llrProcBufAddr; /**< LUT for transferring input LLRs to LLR processing buffer */ + const uint8_t* llr2llrProcBufBnPos; /**< LUT BN position in BG */ + const uint8_t** posBnInCnProcBuf[NR_LDPC_NUM_CN_GROUPS_BG1]; /**< LUT for llr2cnProcBuf */ +} t_nrLDPC_lut; + +/** + Enum with possible LDPC output formats. + */ +typedef enum nrLDPC_outMode { + nrLDPC_outMode_BIT, /**< 32 bits per uint32_t output */ + nrLDPC_outMode_BITINT8, /**< 1 bit per int8_t output */ + nrLDPC_outMode_LLRINT8 /**< Single LLR value per int8_t output */ +} e_nrLDPC_outMode; + +/** + Structure containing LDPC decoder parameters. + */ +typedef struct nrLDPC_dec_params { + uint8_t BG; /**< Base graph */ + uint16_t Z; /**< Lifting size */ + uint8_t R; /**< Decoding rate: Format 15,13,... for code rates 1/5, 1/3,... */ + uint8_t numMaxIter; /**< Maximum number of iterations */ + e_nrLDPC_outMode outMode; /**< Output format */ +} t_nrLDPC_dec_params; + +/** + Structure containing LDPC decoder processing time statistics. + */ +typedef struct nrLDPC_time_stats { + time_stats_t llr2llrProcBuf; /**< Statistics for function llr2llrProcBuf */ + time_stats_t llr2CnProcBuf; /**< Statistics for function llr2CnProcBuf */ + time_stats_t cnProc; /**< Statistics for function cnProc */ + time_stats_t cnProcPc; /**< Statistics for function cnProcPc */ + time_stats_t bnProcPc; /**< Statistics for function bnProcPc */ + time_stats_t bnProc; /**< Statistics for function bnProc */ + time_stats_t cn2bnProcBuf; /**< Statistics for function cn2bnProcBuf */ + time_stats_t bn2cnProcBuf; /**< Statistics for function bn2cnProcBuf */ + time_stats_t llrRes2llrOut; /**< Statistics for function llrRes2llrOut */ + time_stats_t llr2bit; /**< Statistics for function llr2bit */ + time_stats_t total; /**< Statistics for total processing time */ +} t_nrLDPC_time_stats; + +/** + Structure containing the processing buffers + */ +typedef struct nrLDPC_procBuf { + int8_t* cnProcBuf; /**< CN processing buffer */ + int8_t* cnProcBufRes; /**< Buffer for CN processing results */ + int8_t* bnProcBuf; /**< BN processing buffer */ + int8_t* bnProcBufRes; /**< Buffer for BN processing results */ + int8_t* llrRes; /**< Buffer for LLR results */ + int8_t* llrProcBuf; /**< LLR processing buffer */ +} t_nrLDPC_procBuf; + + + +#endif diff --git a/openair1/PHY/CODING/nrLDPC_decoder_full/nrLDPCdecoder_defs.h b/openair1/PHY/CODING/nrLDPC_decoder_full/nrLDPCdecoder_defs.h new file mode 100644 index 0000000000..083fe39bff --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_decoder_full/nrLDPCdecoder_defs.h @@ -0,0 +1,211 @@ +/* + * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The OpenAirInterface Software Alliance licenses this file to You under + * the OAI Public License, Version 1.1 (the "License"); you may not use this file + * except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.openairinterface.org/?page_id=698 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *------------------------------------------------------------------------------- + * For more information about the OpenAirInterface (OAI) Software Alliance: + * contact@openairinterface.org + */ + +/*!\file nrLDPCdecoder_defs.h + * \brief Defines all constants and buffers for the LDPC decoder + * \author Sebastian Wagner (TCL Communications) Email: + * \date 27-03-2018 + * \version 1.0 + * \note + * \warning + */ + +#ifndef __NR_LDPC_DEFS__H__ +#define __NR_LDPC_DEFS__H__ + +// ============================================================================== +// DEFINES + +/** Maximum lifting size */ +#define NR_LDPC_ZMAX 384 + +/** Number of columns in BG1 */ +#define NR_LDPC_NCOL_BG1 68 +/** Number of rows in BG1 */ +#define NR_LDPC_NROW_BG1 46 +/** Number of edges/entries in BG1 */ +#define NR_LDPC_NUM_EDGE_BG1 316 +/** Number of check node (CN) groups in BG1 + A CN group is defined by its number of connected bit nodes. */ +#define NR_LDPC_NUM_CN_GROUPS_BG1 9 +/** First column in BG1 that is connected to only a single CN */ +#define NR_LDPC_START_COL_PARITY_BG1 26 + +/** Number of columns in BG1 for rate 1/3 = 22/(68-2) */ +#define NR_LDPC_NCOL_BG1_R13 NR_LDPC_NCOL_BG1 +/** Number of columns in BG1 for rate 2/3 = 22/(35-2) */ +#define NR_LDPC_NCOL_BG1_R23 35 +/** Number of columns in BG1 for rate 8/9 ~ 22/(27-2) */ +#define NR_LDPC_NCOL_BG1_R89 27 + +/** Number of bit node (BN) groups in BG1 for rate 1/3 + A BN group is defined by its number of connected CNs. */ +#define NR_LDPC_NUM_BN_GROUPS_BG1_R13 30 +/** Number of bit node (BN) groups in BG1 for rate 2/3 */ +#define NR_LDPC_NUM_BN_GROUPS_BG1_R23 8 +/** Number of bit node (BN) groups in BG1 for rate 8/9 */ +#define NR_LDPC_NUM_BN_GROUPS_BG1_R89 5 + +/** Number of columns in BG2 */ +#define NR_LDPC_NCOL_BG2 52 +/** Number of rows in BG2 */ +#define NR_LDPC_NROW_BG2 42 +/** Number of edges/entries in BG2 */ +#define NR_LDPC_NUM_EDGE_BG2 197 +/** Number of check node (CN) groups in BG2 + A CN group is defined by its number of connected bit nodes. */ +#define NR_LDPC_NUM_CN_GROUPS_BG2 6 +/** First column in BG2 that is connected to only a single CN */ +#define NR_LDPC_START_COL_PARITY_BG2 14 + +/** Number of columns in BG2 for rate 1/5 = 10/(52-2) */ +#define NR_LDPC_NCOL_BG2_R15 NR_LDPC_NCOL_BG2 +/** Number of columns in BG2 for rate 1/3 = 10/(32-2) */ +#define NR_LDPC_NCOL_BG2_R13 32 +/** Number of columns in BG2 for rate 2/3 = 10/(17-2) */ +#define NR_LDPC_NCOL_BG2_R23 17 + +/** Number of bit node (BN) groups in BG2 for rate 1/5 + A BN group is defined by its number of connected CNs. */ +#define NR_LDPC_NUM_BN_GROUPS_BG2_R15 23 +/** Number of bit node (BN) groups in BG2 for rate 1/3 */ +#define NR_LDPC_NUM_BN_GROUPS_BG2_R13 10 +/** Number of bit node (BN) groups in BG2 for rate 2/3 */ +#define NR_LDPC_NUM_BN_GROUPS_BG2_R23 6 + +/** Worst case size of the CN processing buffer */ +#define NR_LDPC_SIZE_CN_PROC_BUF NR_LDPC_NUM_EDGE_BG1*NR_LDPC_ZMAX +/** Worst case size of the BN processing buffer */ +#define NR_LDPC_SIZE_BN_PROC_BUF NR_LDPC_NUM_EDGE_BG1*NR_LDPC_ZMAX + +/** Maximum number of possible input LLR = NR_LDPC_NCOL_BG1*NR_LDPC_ZMAX */ +#define NR_LDPC_MAX_NUM_LLR 27000 + +// ============================================================================== +// GLOBAL CONSTANT VARIABLES + +/** Start addresses for the cnProcBuf for each CN group in BG1*/ +static const uint32_t lut_startAddrCnGroups_BG1[NR_LDPC_NUM_CN_GROUPS_BG1] = {0, 1152, 8832, 43392, 61824, 75264, 81408, 88320, 92160}; +/** Start addresses for the cnProcBuf for each CN group in BG2*/ +static const uint32_t lut_startAddrCnGroups_BG2[NR_LDPC_NUM_CN_GROUPS_BG2] = {0, 6912, 37632, 54912, 61824, 67968}; + +/** Number of BNs of CN group for BG1. + E.g. 10 means that there is a CN group where every CN is connected to 10 BNs */ +static const uint8_t lut_numBnInCnGroups_BG1_R13[NR_LDPC_NUM_CN_GROUPS_BG1] = {3, 4, 5, 6, 7, 8, 9, 10, 19}; +/** Number of rows/CNs in every CN group for rate = 1/3 BG1, e.g. 5 rows of CNs connected to 4 BNs */ +static const uint8_t lut_numCnInCnGroups_BG1_R13[NR_LDPC_NUM_CN_GROUPS_BG1] = {1, 5, 18, 8, 5, 2, 2, 1, 4}; +/** Number of rows/CNs in every CN group for rate = 2/3 BG1, e.g. 3 rows of CNs connected to 7 BNs */ +static const uint8_t lut_numCnInCnGroups_BG1_R23[NR_LDPC_NUM_CN_GROUPS_BG1] = {1, 0, 0, 0, 3, 2, 2, 1, 4}; +/** Number of rows/CNs in every CN group for rate = 8/9 BG1, e.g. 4 rows of CNs connected to 19 BNs */ +static const uint8_t lut_numCnInCnGroups_BG1_R89[NR_LDPC_NUM_CN_GROUPS_BG1] = {1, 0, 0, 0, 0, 0, 0, 0, 4}; + +/** Number of connected BNs for every column in BG1 rate = 1/3, e.g. in first column all BNs are connected to 30 CNs */ +static const uint8_t lut_numEdgesPerBn_BG1_R13[NR_LDPC_START_COL_PARITY_BG1] = {30, 28, 7, 11, 9, 4, 8, 12, 8, 7, 12, 10, 12, 11, 10, 7, 10, 10, 13, 7, 8, 11, 12, 5, 6, 6}; +/** Number of connected BNs for every column in BG1 rate = 2/3, e.g. in first column all BNs are connected to 12 CNs */ +static const uint8_t lut_numEdgesPerBn_BG1_R23[NR_LDPC_START_COL_PARITY_BG1] = {12, 11, 4, 5, 5, 3, 4, 5, 5, 3, 6, 6, 6, 6, 5, 3, 6, 5, 6, 4, 5, 6, 6, 3, 3, 2}; +/** Number of connected BNs for every column in BG1 rate = 8/9, e.g. in first column all BNs are connected to 5 CNs */ +static const uint8_t lut_numEdgesPerBn_BG1_R89[NR_LDPC_START_COL_PARITY_BG1] = {5, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2}; + +/** Number of BNs of CN group for BG2. + E.g. 3 means that there is a CN group where every CN is connected to 3 BNs */ +static const uint8_t lut_numBnInCnGroups_BG2_R15[NR_LDPC_NUM_CN_GROUPS_BG2] = {3, 4, 5, 6, 8, 10}; +/** Number of rows/CNs in every CN group for rate = 1/5 BG2, e.g. 6 rows of CNs connected to 3 BNs */ +static const uint8_t lut_numCnInCnGroups_BG2_R15[NR_LDPC_NUM_CN_GROUPS_BG2] = {6, 20, 9, 3, 2, 2}; +/** Number of rows/CNs in every CN group for rate = 1/3 BG2, e.g. 8 rows of CNs connected to 4 BNs */ +static const uint8_t lut_numCnInCnGroups_BG2_R13[NR_LDPC_NUM_CN_GROUPS_BG2] = {0, 8, 7, 3, 2, 2}; +/** Number of rows/CNs in every CN group for rate = 2/3 BG2, e.g. 1 row of CNs connected to 4 BNs */ +static const uint8_t lut_numCnInCnGroups_BG2_R23[NR_LDPC_NUM_CN_GROUPS_BG2] = {0, 1, 0, 2, 2, 2}; + +/** Number of connected BNs for every column in BG2 rate = 1/5, e.g. in first column all BNs are connected to 22 CNs */ +static const uint8_t lut_numEdgesPerBn_BG2_R15[NR_LDPC_START_COL_PARITY_BG2] = {22, 23, 10, 5, 5, 14, 7, 13, 6, 8, 9, 16, 9, 12}; +/** Number of connected BNs for every column in BG2 rate = 1/3, e.g. in first column all BNs are connected to 14 CNs */ +static const uint8_t lut_numEdgesPerBn_BG2_R13[NR_LDPC_START_COL_PARITY_BG2] = {14, 16, 2, 4, 4, 6, 6, 8, 6, 6, 6, 13, 5, 7}; +/** Number of connected BNs for every column in BG2 rate = 2/3, e.g. in first column all BNs are connected to 6 CNs */ +static const uint8_t lut_numEdgesPerBn_BG2_R23[NR_LDPC_START_COL_PARITY_BG2] = { 6, 5, 2, 3, 3, 4, 3, 4, 3, 4, 3, 5, 2, 2}; + +// Number of groups for bit node processing +/** Number of connected CNs for every column/BN in BG1 for rate = 1/3, Worst case is BG1 with up to 30 CNs connected to one BN + E.g. 42 parity BNs connected to single CN */ + // BG1: 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 +static const uint8_t lut_numBnInBnGroups_BG1_R13[NR_LDPC_NUM_BN_GROUPS_BG1_R13] = {42, 0, 0, 1, 1, 2, 4, 3, 1, 4, 3, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1}; +/** Number of connected CNs for every column/BN in BG1 for rate = 2/3 */ +static const uint8_t lut_numBnInBnGroups_BG1_R23[NR_LDPC_NUM_BN_GROUPS_BG1_R13] = { 9, 1, 5, 3, 7, 8, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; +/** Number of connected CNs for every column/BN in BG1 for rate = 8/9 */ +static const uint8_t lut_numBnInBnGroups_BG1_R89[NR_LDPC_NUM_BN_GROUPS_BG1_R13] = { 1, 3,21, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + +/** Number of connected CNs for every column/BN in BG2 for rate = 1/5, Worst case is BG1 with up to 30 CNs connected to one BN + E.g. 38 parity BNs connected to single CN */ + // BG2: 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 +static const uint8_t lut_numBnInBnGroups_BG2_R15[NR_LDPC_NUM_BN_GROUPS_BG1_R13] = {38, 0, 0, 0, 2, 1, 1, 1, 2, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0}; +/** Number of connected CNs for every column/BN in BG2 for rate = 1/3 */ +static const uint8_t lut_numBnInBnGroups_BG2_R13[NR_LDPC_NUM_BN_GROUPS_BG1_R13] = {18, 1, 0, 2, 1, 5, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; +/** Number of connected CNs for every column/BN in BG2 for rate = 2/3 */ +static const uint8_t lut_numBnInBnGroups_BG2_R23[NR_LDPC_NUM_BN_GROUPS_BG1_R13] = { 3, 3, 5, 3, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + +// Start addresses for the bnProcBuf for each BN group +// BG1 +/** Start address for every BN group within the BN processing buffer for BG1 rate = 1/3 */ +static const uint32_t lut_startAddrBnGroups_BG1_R13[NR_LDPC_NUM_BN_GROUPS_BG1_R13] = {0, 16128, 17664, 19584, 24192, 34944, 44160, 47616, 62976, 75648, 94080, 99072, 109824}; +/** Start address for every BN group within the BN processing buffer for BG1 rate = 2/3 */ +static const uint32_t lut_startAddrBnGroups_BG1_R23[NR_LDPC_NUM_BN_GROUPS_BG1_R23] = {0, 3456, 4224, 9984, 14592, 28032, 46464, 50688}; +/** Start address for every BN group within the BN processing buffer for BG1 rate = 8/9 */ +static const uint32_t lut_startAddrBnGroups_BG1_R89[NR_LDPC_NUM_BN_GROUPS_BG1_R89] = {0, 384, 2688, 26880, 28416}; + +/** Start address for every BN group within the LLR processing buffer for BG1 rate = 1/3 */ +static const uint16_t lut_startAddrBnGroupsLlr_BG1_R13[NR_LDPC_NUM_BN_GROUPS_BG1_R13] = {0, 16128, 16512, 16896, 17664, 19200, 20352, 20736, 22272, 23424, 24960, 25344, 25728}; +/** Start address for every BN group within the LLR processing buffer for BG1 rate = 2/3 */ +static const uint16_t lut_startAddrBnGroupsLlr_BG1_R23[NR_LDPC_NUM_BN_GROUPS_BG1_R23] = {0, 3456, 3840, 5760, 6912, 9600, 12672, 13056}; +/** Start address for every BN group within the LLR processing buffer for BG1 rate = 8/9 */ +static const uint16_t lut_startAddrBnGroupsLlr_BG1_R89[NR_LDPC_NUM_BN_GROUPS_BG1_R89] = {0, 384, 1536, 9600, 9984}; + +// BG2 +/** Start address for every BN group within the BN processing buffer for BG2 rate = 1/5 */ +static const uint32_t lut_startAddrBnGroups_BG2_R15[NR_LDPC_NUM_BN_GROUPS_BG2_R15] = {0, 14592, 18432, 20736, 23424, 26496, 33408, 37248, 41856, 46848, 52224, 58368, 66816}; +/** Start address for every BN group within the BN processing buffer for BG2 rate = 1/3 */ +static const uint32_t lut_startAddrBnGroups_BG2_R13[NR_LDPC_NUM_BN_GROUPS_BG2_R13] = {0, 6912, 7680, 10752, 12672, 24192, 26880, 29952, 34944, 40320}; +/** Start address for every BN group within the BN processing buffer for BG2 rate = 2/3 */ +static const uint32_t lut_startAddrBnGroups_BG2_R23[NR_LDPC_NUM_BN_GROUPS_BG2_R23] = {0, 1152, 3456, 9216, 13824, 17664}; + +/** Start address for every BN group within the LLR processing buffer for BG2 rate = 1/5 */ +static const uint16_t lut_startAddrBnGroupsLlr_BG2_R15[NR_LDPC_NUM_BN_GROUPS_BG2_R15] = {0, 14592, 15360, 15744, 16128, 16512, 17280, 17664, 18048, 18432, 18816, 19200, 19584}; +/** Start address for every BN group within the LLR processing buffer for BG2 rate = 1/3 */ +static const uint16_t lut_startAddrBnGroupsLlr_BG2_R13[NR_LDPC_NUM_BN_GROUPS_BG2_R13] = {0, 6912, 7296, 8064, 8448, 10368, 10752, 11136, 11520, 11904}; +/** Start address for every BN group within the LLR processing buffer for BG2 rate = 2/3 */ +static const uint16_t lut_startAddrBnGroupsLlr_BG2_R23[NR_LDPC_NUM_BN_GROUPS_BG2_R23] = {0, 1152, 2304, 4224, 5376, 6144}; + +/** Vector of 32 '1' in int8 for application with AVX2 */ +static const int8_t ones256_epi8[32] __attribute__ ((aligned(32))) = {1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}; +/** Vector of 32 '0' in int8 for application with AVX2 */ +static const int8_t zeros256_epi8[32] __attribute__ ((aligned(32))) = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; +/** Vector of 32 '127' in int8 for application with AVX2 */ +static const int8_t maxLLR256_epi8[32] __attribute__ ((aligned(32))) = {127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127}; + +/** Vector of 64 '1' in int8 for application with AVX512 */ +static const int8_t ones512_epi8[64] __attribute__ ((aligned(64))) = {1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}; +/** Vector of 64 '0' in int8 for application with AVX512 */ +static const int8_t zeros512_epi8[64] __attribute__ ((aligned(64))) = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; +/** Vector of 64 '127' in int8 for application with AVX512 */ +static const int8_t maxLLR512_epi8[64] __attribute__ ((aligned(64))) = {127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127}; + + + + +#endif