diff --git a/CMakeLists.txt b/CMakeLists.txt index feb7e41a4f..491dcd7e0b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -108,6 +108,7 @@ add_list_string_option(CMAKE_BUILD_TYPE "RelWithDebInfo" "Choose the type of bui # -so to avoid AVX512 instructions generation by gcc if(EXISTS "/proc/cpuinfo" AND NOT CROSS_COMPILE) file(STRINGS "/proc/cpuinfo" CPUFLAGS REGEX flags LIMIT_COUNT 1) + file(STRINGS "/proc/cpuinfo" CPUPART REGEX "CPU part" LIMIT_COUNT 1) else() message(WARNING "did not find /proc/cpuinfo -- not setting any x86-specific compilation variables") endif() @@ -146,9 +147,20 @@ elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -DSIMDE_X86_SSSE3_NATIVE") endif() elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "armv7l") - set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -gdwarf-2 -mfloat-abi=hard -mfpu=neon -lgcc -lrt") + set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -gdwarf-2 -mfloat-abi=hard -mfpu=neon -lgcc -lrt") elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64") - set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -gdwarf-2 -lgcc -lrt") +# Neoverse CPU part codes available at https://github.com/util-linux/util-linux/blob/master/sys-utils/lscpu-arm.c + if (CPUPART MATCHES "0xd40") # Neoverse-V1 + set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -gdwarf-2 -mcpu=neoverse-v1 -lgcc -lrt") + elseif (CPUPART MATCHES "0xd4f") # Neoverse-V2 + set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -gdwarf-2 -mcpu=neoverse-v2 -ftree-vectorize -lgcc -lrt") + elseif (CPUPART MATCHES "0xd0c") # Neoverse-N1 + set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -gdwarf-2 -mcpu=neoverse-n1 -lgcc -lrt") + elseif (CPUPART MATCHES "0xd49") # Neoverse-N2 + set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -gdwarf-2 -mcpu=neoverse-n2 -ftree-vectorize -lgcc -lrt") + else () + set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -gdwarf-2 -mcpu=native -lgcc -lrt") + endif() else() message(FATAL_ERROR "compile for CPU architecture ${CPUARCH}, CMAKE_SYSTEM_PROCESSOR ${CMAKE_SYSTEM_PROCESSOR}") endif() diff --git a/ci-scripts/datalog_rt_stats.100.2x2.yaml b/ci-scripts/datalog_rt_stats.100.2x2.yaml index c0a7717346..024dfa8e9e 100644 --- a/ci-scripts/datalog_rt_stats.100.2x2.yaml +++ b/ci-scripts/datalog_rt_stats.100.2x2.yaml @@ -11,8 +11,8 @@ Ref : feptx_prec : 0.0 feptx_ofdm : 60.0 feptx_total : 150.0 - L1 Tx processing : 530.0 - DLSCH encoding : 220.0 + L1 Tx processing : 475.0 + DLSCH encoding : 200.0 L1 Rx processing : 530.0 PUSCH inner-receiver : 360.0 Schedule Response : 1.0 diff --git a/ci-scripts/datalog_rt_stats.1x1.60.yaml b/ci-scripts/datalog_rt_stats.1x1.60.yaml index 47021b502b..85075bac97 100644 --- a/ci-scripts/datalog_rt_stats.1x1.60.yaml +++ b/ci-scripts/datalog_rt_stats.1x1.60.yaml @@ -11,8 +11,8 @@ Ref : feptx_prec : 15.0 feptx_ofdm : 30.0 feptx_total : 45.0 - L1 Tx processing : 205.0 - DLSCH encoding : 140.0 + L1 Tx processing : 175.0 + DLSCH encoding : 110.0 L1 Rx processing : 345.0 PUSCH inner-receiver : 150.0 Schedule Response : 1.0 diff --git a/ci-scripts/datalog_rt_stats.60.2x2.yaml b/ci-scripts/datalog_rt_stats.60.2x2.yaml index 064d6374c1..b371356971 100644 --- a/ci-scripts/datalog_rt_stats.60.2x2.yaml +++ b/ci-scripts/datalog_rt_stats.60.2x2.yaml @@ -11,8 +11,8 @@ Ref : feptx_prec : 14.0 feptx_ofdm : 30.0 feptx_total : 80.0 - L1 Tx processing : 315.0 - DLSCH encoding : 155.0 + L1 Tx processing : 280.0 + DLSCH encoding : 140.0 L1 Rx processing : 345.0 PUSCH inner-receiver : 155.0 Schedule Response : 1.0 diff --git a/ci-scripts/datalog_rt_stats.default.yaml b/ci-scripts/datalog_rt_stats.default.yaml index 17e45cf367..32977944a0 100644 --- a/ci-scripts/datalog_rt_stats.default.yaml +++ b/ci-scripts/datalog_rt_stats.default.yaml @@ -11,8 +11,8 @@ Ref : feptx_prec : 13.0 feptx_ofdm : 30.0 feptx_total : 45.0 - L1 Tx processing : 160.0 - DLSCH encoding : 100.0 + L1 Tx processing : 145.0 + DLSCH encoding : 90.0 L1 Rx processing : 290.0 PUSCH inner-receiver : 115.0 Schedule Response : 1.0 diff --git a/common/utils/nr/nr_common.c b/common/utils/nr/nr_common.c index 30e8e6c32e..078ef027e8 100644 --- a/common/utils/nr/nr_common.c +++ b/common/utils/nr/nr_common.c @@ -37,6 +37,7 @@ #include "nr_common.h" #include #include +#include #define C_SRS_NUMBER (64) #define B_SRS_NUMBER (4) @@ -131,8 +132,24 @@ void reverse_bits_u8(uint8_t const* in, size_t sz, uint8_t* out) DevAssert(in != NULL); DevAssert(out != NULL); +// Bit reversal implementation based on https://wunkolo.github.io/post/2020/11/gf2p8affineqb-bit-reversal/ +#if defined(__GFNI__) && defined(__AVX512F__) + int simde_sz = 64; + int i = 0; + int simde_bound = sz - simde_sz; + for (; i <= simde_bound; i += simde_sz) { + __m512i input = _mm512_loadu_epi8(&in[i]); + __m512i reversed = _mm512_gf2p8affine_epi64_epi8(input, _mm512_set1_epi64(0x8040201008040201), 0); + _mm512_storeu_epi8(&out[i], reversed); + } + + for (; i < sz; ++i) { + out[i] = bit_reverse_table_256[in[i]]; + } +#else for(size_t i = 0; i < sz; ++i) out[i] = bit_reverse_table_256[in[i]]; +#endif } // Reverse bits implementation based on http://graphics.stanford.edu/~seander/bithacks.html diff --git a/doc/Doxyfile b/doc/Doxyfile index a154051692..6b0b390f05 100644 --- a/doc/Doxyfile +++ b/doc/Doxyfile @@ -2133,10 +2133,8 @@ INPUT = \ @CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc_encode_parity_check.c \ @CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc_BG2_Zc176_byte.c \ @CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc_BG2_Zc352_byte.c \ -@CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim.c \ @CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc_BG2_Zc128_byte_128.c \ @CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc224_byte.c \ -@CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8seg.c \ @CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc_generate_coefficient.c \ @CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc_BG2_Zc104_byte.c \ @CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc320_byte_128.c \ diff --git a/openair1/PHY/CODING/CMakeLists.txt b/openair1/PHY/CODING/CMakeLists.txt index 4eee35a2cd..7d8ce40c14 100644 --- a/openair1/PHY/CODING/CMakeLists.txt +++ b/openair1/PHY/CODING/CMakeLists.txt @@ -17,20 +17,6 @@ add_library(ldpc_orig MODULE set_target_properties(ldpc_orig PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) target_link_libraries(ldpc_orig PRIVATE ldpc_segment ldpc_gen_HEADERS) -add_library(ldpc_optim MODULE - nrLDPC_decoder/nrLDPC_decoder.c - nrLDPC_encoder/ldpc_encoder_optim.c -) -set_target_properties(ldpc_optim PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) -target_link_libraries(ldpc_optim PRIVATE ldpc_segment ldpc_gen_HEADERS) - -add_library(ldpc_optim8seg MODULE - nrLDPC_decoder/nrLDPC_decoder.c - nrLDPC_encoder/ldpc_encoder_optim8seg.c -) -set_target_properties(ldpc_optim8seg PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) -target_link_libraries(ldpc_optim8seg PRIVATE ldpc_segment ldpc_gen_HEADERS) - add_library(ldpc MODULE nrLDPC_decoder/nrLDPC_decoder.c nrLDPC_encoder/ldpc_encoder_optim8segmulti.c @@ -38,13 +24,13 @@ add_library(ldpc MODULE set_target_properties(ldpc PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) target_link_libraries(ldpc PRIVATE ldpc_segment ldpc_gen_HEADERS) -add_dependencies(ldpctest ldpc ldpc_orig ldpc_optim ldpc_optim8seg) -add_dependencies(nr-softmodem ldpc ldpc_orig ldpc_optim ldpc_optim8seg) -add_dependencies(nr-uesoftmodem ldpc ldpc_orig ldpc_optim ldpc_optim8seg) -add_dependencies(nr_ulsim ldpc ldpc_orig ldpc_optim ldpc_optim8seg) -add_dependencies(nr_ulschsim ldpc ldpc_orig ldpc_optim ldpc_optim8seg) -add_dependencies(nr_dlsim ldpc ldpc_orig ldpc_optim ldpc_optim8seg) -add_dependencies(nr_dlschsim ldpc ldpc_orig ldpc_optim ldpc_optim8seg) +add_dependencies(ldpctest ldpc ldpc_orig) +add_dependencies(nr-softmodem ldpc ldpc_orig) +add_dependencies(nr-uesoftmodem ldpc ldpc_orig) +add_dependencies(nr_ulsim ldpc ldpc_orig) +add_dependencies(nr_ulschsim ldpc ldpc_orig) +add_dependencies(nr_dlsim ldpc ldpc_orig) +add_dependencies(nr_dlschsim ldpc ldpc_orig) add_custom_target(nrLDPC_decoder_kernels_CL COMMAND gcc nrLDPC_decoder/nrLDPC_decoder_CL.c -dD -DNRLDPC_KERNEL_SOURCE -E -o ${CMAKE_CURRENT_BINARY_DIR}/nrLDPC_decoder_kernels_CL.clc diff --git a/openair1/PHY/CODING/DOC/LDPCImplementation.md b/openair1/PHY/CODING/DOC/LDPCImplementation.md index 2880c45ecf..325b208a64 100644 --- a/openair1/PHY/CODING/DOC/LDPCImplementation.md +++ b/openair1/PHY/CODING/DOC/LDPCImplementation.md @@ -120,16 +120,16 @@ By default the function `int load_nrLDPClib(void)` looks for `libldpc.so`, this #### Examples of ldpc shared lib selection when running nr softmodem's: -loading `libldpc_optim8seg.so` instead of `libldpc.so`: +loading `libldpc_orig.so` instead of `libldpc.so`: ``` -./nr-softmodem -O libconfig:gnb.band78.tm1.106PRB.usrpx300.conf:dbgl5 --loader.ldpc.shlibversion _optim8seg +./nr-softmodem -O libconfig:gnb.band78.tm1.106PRB.usrpx300.conf:dbgl5 --loader.ldpc.shlibversion _orig ....................... [CONFIG] loader.ldpc.shlibversion set to default value "" [LIBCONFIG] loader.ldpc: 2/2 parameters successfully set, (1 to default value) -[CONFIG] shlibversion set to _optim8seg from command line +[CONFIG] shlibversion set to _orig from command line [CONFIG] loader.ldpc 1 options set from command line -[LOADER] library libldpc_optim8seg.so successfully loaded +[LOADER] library libldpc_orig.so successfully loaded ........................ ``` diff --git a/openair1/PHY/CODING/TESTBENCH/ldpctest.c b/openair1/PHY/CODING/TESTBENCH/ldpctest.c index 5912c9d0b4..46534c37e3 100644 --- a/openair1/PHY/CODING/TESTBENCH/ldpctest.c +++ b/openair1/PHY/CODING/TESTBENCH/ldpctest.c @@ -123,7 +123,7 @@ one_measurement_t test_ldpc(short max_iterations, uint8_t estimated_output[MAX_NUM_DLSCH_SEGMENTS][Kprime]; memset(estimated_output, 0, sizeof(estimated_output)); uint8_t *channel_input[MAX_NUM_DLSCH_SEGMENTS]; - uint8_t *channel_input_optim[MAX_NUM_DLSCH_SEGMENTS]; + uint8_t *channel_input_optim; // double channel_output[68 * 384]; double modulated_input[MAX_NUM_DLSCH_SEGMENTS][68 * 384] = {0}; int8_t channel_output_fixed[MAX_NUM_DLSCH_SEGMENTS][68 * 384] = {0}; @@ -265,9 +265,9 @@ one_measurement_t test_ldpc(short max_iterations, memset(test_input[j], 0, ((K + 7) & ~7) / 8); channel_input[j] = malloc16(68 * 384); memset(channel_input[j], 0, 68 * 384); - channel_input_optim[j] = malloc16(68 * 384); - memset(channel_input_optim[j], 0, 68 * 384); } + channel_input_optim = malloc16(68 * 384); + memset(channel_input_optim, 0, 68 * 384); // Fill input segments with random values for (int j = 0; j < MAX_NUM_DLSCH_SEGMENTS; j++) { @@ -282,10 +282,10 @@ one_measurement_t test_ldpc(short max_iterations, } encoder_implemparams_t impp = {.Zc = Zc, .Kb = Kb, .BG = BG, .K = K}; - impp.gen_code = 2; + impp.gen_code = 1; if (ntrials == 0) - ldpc_orig.LDPCencoder(test_input, channel_input, &impp); + ldpc_orig.LDPCencoder(test_input, channel_input[0], &impp); impp.gen_code = 0; decode_abort_t dec_abort; init_abort(&dec_abort); @@ -294,23 +294,21 @@ one_measurement_t test_ldpc(short max_iterations, //// encoder start_meas(&time); for (int j = 0; j < n_segments; j++) { - ldpc_orig.LDPCencoder(&test_input[j], &channel_input[j], &impp); + ldpc_orig.LDPCencoder(&test_input[j], channel_input[j], &impp); } stop_meas(&time); impp.n_segments = n_segments; - for (int j = 0; j < (n_segments / 8 + 1); j++) { - start_meas(&ret.time_optim); - impp.macro_num = j; - ldpc_toCompare.LDPCencoder(test_input, channel_input_optim, &impp); - stop_meas(&ret.time_optim); - } + start_meas(&ret.time_optim); + impp.first_seg = 0; + ldpc_toCompare.LDPCencoder(test_input, channel_input_optim, &impp); + stop_meas(&ret.time_optim); if (ntrials == 1) for (int j = 0; j < n_segments; j++) for (int i = 0; i < K + (nrows - no_punctured_columns) * Zc - removed_bit; i++) { - if (channel_input[j][i] != channel_input_optim[j][i]) { - printf("differ in seg %d pos %d (%u,%u)\n", j, i, channel_input[j][i], channel_input_optim[j][i]); + if (channel_input[j][i] != ((channel_input_optim[i] >> j) & 0x1)) { + printf("differ in seg %d pos %d (%u,%u)\n", j, i, channel_input[j][i], (channel_input_optim[i] >> j) & 0x1); return ret; } } @@ -322,7 +320,7 @@ one_measurement_t test_ldpc(short max_iterations, printf("\ne %d..%d: ", i, i + 15); #endif - if (channel_input_optim[j][i - 2 * Zc] == 0) + if (((channel_input_optim[i - 2 * Zc] >> j) & 0x1) == 0) modulated_input[j][i] = 1.0; /// sqrt(2); //QPSK else modulated_input[j][i] = -1.0; /// sqrt(2); @@ -332,7 +330,7 @@ one_measurement_t test_ldpc(short max_iterations, // Uncoded BER uint8_t channel_output_uncoded = channel_output_fixed[j][i] < 0 ? 1 /* QPSK demod */ : 0; - if (channel_output_uncoded != channel_input_optim[j][i - 2 * Zc]) + if (channel_output_uncoded != ((channel_input_optim[i - 2 * Zc] >> j) & 0x1)) ret.errors_bit_uncoded++; } @@ -396,8 +394,8 @@ one_measurement_t test_ldpc(short max_iterations, for (int j = 0; j < MAX_NUM_DLSCH_SEGMENTS; j++) { free(test_input[j]); free(channel_input[j]); - free(channel_input_optim[j]); } + free(channel_input_optim); print_meas(&time, "ldpc_encoder", NULL, NULL); print_meas(&ret.time_optim, "ldpc_encoder_optim", NULL, NULL); diff --git a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_interface.h b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_interface.h index 7544f8cf55..ca57c9bab3 100644 --- a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_interface.h +++ b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_interface.h @@ -140,9 +140,7 @@ typedef struct nrLDPC_slot_decoding_parameters_s{ * \struct nrLDPC_segment_encoding_parameters_s * \brief encoding parameter of segments * \var E input llr segment size - * \var output input llr segment array * \var c Pointers to code blocks before LDPC encoding (38.212 V15.4.0 section 5.2.2) - * flag indicating that the decoding of the segment was successful * IT MUST BE FILLED BY THE IMPLEMENTATION * \var ts_interleave interleaving time stats * \var ts_rate_match rate matching time stats @@ -150,7 +148,6 @@ typedef struct nrLDPC_slot_decoding_parameters_s{ */ typedef struct nrLDPC_segment_encoding_parameters_s{ int E; - unsigned char *output; uint8_t *c; time_stats_t ts_interleave; time_stats_t ts_rate_match; @@ -179,6 +176,7 @@ typedef struct nrLDPC_segment_encoding_parameters_s{ * \var F filler bits size * \var C number of segments * \var segments array of segments parameters + * \var output input llr TB array */ typedef struct nrLDPC_TB_encoding_parameters_s{ @@ -202,6 +200,7 @@ typedef struct nrLDPC_TB_encoding_parameters_s{ uint32_t C; nrLDPC_segment_encoding_parameters_t *segments; + unsigned char *output; } nrLDPC_TB_encoding_parameters_t; /** diff --git a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c index b51712ed71..5d0e90e46b 100644 --- a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c +++ b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c @@ -41,12 +41,184 @@ #include -// #define DEBUG_LDPC_ENCODING -// #define DEBUG_LDPC_ENCODING_FREE 1 +#define DEBUG_LDPC_ENCODING +//#define DEBUG_LDPC_ENCODING_FREE 1 +/** + * \brief write nrLDPC_coding_segment_encoder output with interleaver output + * the interleaver output has 8 bits from 8 different segments per byte + * nrLDPC_coding_segment_encoder is made of concatenated packed segments + * \param f input interleaved segment, 8 bits from 8 segments per byte + * \param E number of bits per segment in f + * \param f2 input interleaved segment, 8 bits from 8 segments per byte + * interleaved with E2 if there was a E shift in the segment group + * \param E2 number of bits per segment in f2 + * \param Eshift flag indicating if there was a E shift within the segment group + * \param E2_first_segment index of the first segment of size E2 if there is a E shift in the segment group + * \param nb_segments number of segments in the group + * \param output nrLDPC_coding_segment_encoder with concatenated segments and packed bits + * \param Eoffset offset in number of bits of the first segment of the segment group within output + */ +static void write_task_output(uint8_t *f, + uint32_t E, + uint8_t *f2, + uint32_t E2, + bool Eshift, + uint32_t E2_first_segment, + uint32_t nb_segments, + uint8_t *output, + uint32_t Eoffset) +{ + +#if defined(__AVX512VBMI__) + uint64_t *output_p = (uint64_t*)output; + __m512i inc = _mm512_set1_epi8(0x1); + + for (int i=0;i>/<< because when the Eoffset2_bit is 64 or 0, the << + // and >> operations are undefined and in fact don't give "0" which is + // what we want here. The SIMD version do give 0 when the shift is 64 + uint32_t Eoffset2_byte = Eoffset2 >> 6; + uint32_t Eoffset2_bit = Eoffset2 & 63; + __m64 tmp = (__m64)_mm512_bitshuffle_epi64_mask(((__m512i *)f)[i >> 6],bitperm); + *(__m64*)(output_p + Eoffset2_byte) = _mm_or_si64(*(__m64*)(output_p + Eoffset2_byte),_mm_slli_si64(tmp,Eoffset2_bit)); + *(__m64*)(output_p + Eoffset2_byte+1) = _mm_or_si64(*(__m64*)(output_p + Eoffset2_byte+1),_mm_srli_si64(tmp,(64-Eoffset2_bit))); + Eoffset2 += E; + bitperm = _mm512_add_epi8(bitperm ,inc); + } + } else { + for (int j=0; j < E2_first_segment; j++) { + Eoffset2 += E; + bitperm = _mm512_add_epi8(bitperm ,inc); + } + } + for (int j=E2_first_segment; j < nb_segments; j++) { + uint32_t Eoffset2_byte = Eoffset2 >> 6; + uint32_t Eoffset2_bit = Eoffset2 & 63; + __m64 tmp = (__m64)_mm512_bitshuffle_epi64_mask(((__m512i *)f2)[i >> 6],bitperm); + *(__m64*)(output_p + Eoffset2_byte) = _mm_or_si64(*(__m64*)(output_p + Eoffset2_byte),_mm_slli_si64(tmp,Eoffset2_bit)); + *(__m64*)(output_p + Eoffset2_byte+1) = _mm_or_si64(*(__m64*)(output_p + Eoffset2_byte+1),_mm_srli_si64(tmp,(64-Eoffset2_bit))); + Eoffset2 += E2; + bitperm = _mm512_add_epi8(bitperm ,inc); + } + output_p++; + } + +#elif defined(__aarch64__) + uint16_t *output_p = (uint16_t*)output; + const int8_t __attribute__ ((aligned (16))) ucShift[8][16] = { + {0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7}, // segment 0 + {-1,0,1,2,3,4,5,6,-1,0,1,2,3,4,5,6}, // segment 1 + {-2,-1,0,1,2,3,4,5,-2,-1,0,1,2,3,4,5}, // segment 2 + {-3,-2,-1,0,1,2,3,4,-3,-2,-1,0,1,2,3,4}, // segment 3 + {-4,-3,-2,-1,0,1,2,3,-4,-3,-2,-1,0,1,2,3}, // segment 4 + {-5,-4,-3,-2,-1,0,1,2,-5,-4,-3,-2,-1,0,1,2}, // segment 5 + {-6,-5,-4,-3,-2,-1,0,1,-6,-5,-4,-3,-2,-1,0,1}, // segment 6 + {-7,-6,-5,-4,-3,-2,-1,0,-7,-6,-5,-4,-3,-2,-1,0}}; // segment 7 + const uint8_t __attribute__ ((aligned (16))) masks[16] = + {0x1,0x2,0x4,0x8,0x10,0x20,0x40,0x80,0x1,0x2,0x4,0x8,0x10,0x20,0x40,0x80}; + int8x16_t vshift[8]; + for (int n=0;n<8;n++) vshift[n] = vld1q_s8(ucShift[n]); + uint8x16_t vmask = vld1q_u8(masks); + + for (int i=0;i> 4; + uint32_t Eoffset2_bit = Eoffset2 & 15; + uint8x16_t cshift = vandq_u8(vshlq_u8(((uint8x16_t*)f)[i >> 4],vshift[j]),vmask); + int32_t tmp = (int)vaddv_u8(vget_low_u8(cshift)); + tmp += (int)(vaddv_u8(vget_high_u8(cshift))<<8); + *(output_p + Eoffset2_byte) |= (uint16_t)(tmp<>(16-Eoffset2_bit)); + Eoffset2 += E; + } + } else { + for (int j=0; j < E2_first_segment; j++) { + Eoffset2 += E; + } + } + for (int j=E2_first_segment; j < nb_segments; j++) { + uint32_t Eoffset2_byte = Eoffset2 >> 4; + uint32_t Eoffset2_bit = Eoffset2 & 15; + uint8x16_t cshift = vandq_u8(vshlq_u8(((uint8x16_t*)f2)[i >> 4],vshift[j]),vmask); + int32_t tmp = (int)vaddv_u8(vget_low_u8(cshift)); + tmp += (int)(vaddv_u8(vget_high_u8(cshift))<<8); + *(output_p + Eoffset2_byte) |= (uint16_t)(tmp<>(16-Eoffset2_bit)); + Eoffset2 += E2; + } + output_p++; + } + +#else + uint32_t *output_p = (uint32_t*)output; + + for (int i=0; i < E2; i += 32) { + uint32_t Eoffset2 = Eoffset; + if (i < E) { + for (int j = 0; j < E2_first_segment; j++) { + // Note: Here and below, we are using the 64-bit SIMD instruction + // instead of C >>/<< because when the Eoffset2_bit is 64 or 0, the << + // and >> operations are undefined and in fact don't give "0" which is + // what we want here. The SIMD version do give 0 when the shift is 64 + uint32_t Eoffset2_byte = Eoffset2 >> 5; + uint32_t Eoffset2_bit = Eoffset2 & 31; + int tmp = _mm256_movemask_epi8(_mm256_slli_epi16(((__m256i *)f)[i >> 5], 7 - j)); + __m64 tmp64 = _mm_set1_pi32(tmp); + __m64 out64 = _mm_set_pi32(*(output_p + Eoffset2_byte + 1), *(output_p + Eoffset2_byte)); + __m64 tmp64b = _mm_or_si64(out64, _mm_slli_pi32(tmp64, Eoffset2_bit)); + __m64 tmp64c = _mm_or_si64(out64, _mm_srli_pi32(tmp64, (32 - Eoffset2_bit))); + *(output_p + Eoffset2_byte) = _m_to_int(tmp64b); + *(output_p + Eoffset2_byte + 1) = _m_to_int(_mm_srli_si64(tmp64c, 32)); + Eoffset2 += E; + } + } else { + for (int j = 0; j < E2_first_segment; j++) { + Eoffset2 += E; + } + } + for (int j = E2_first_segment; j < nb_segments; j++) { + uint32_t Eoffset2_byte = Eoffset2 >> 5; + uint32_t Eoffset2_bit = Eoffset2 & 31; + int tmp = _mm256_movemask_epi8(_mm256_slli_epi16(((__m256i *)f2)[i >> 5], 7 - j)); + __m64 tmp64 = _mm_set1_pi32(tmp); + __m64 out64 = _mm_set_pi32(*(output_p + Eoffset2_byte + 1), *(output_p + Eoffset2_byte)); + __m64 tmp64b = _mm_or_si64(out64, _mm_slli_pi32(tmp64, Eoffset2_bit)); + __m64 tmp64c = _mm_or_si64(out64, _mm_srli_pi32(tmp64, (32 - Eoffset2_bit))); + *(output_p + Eoffset2_byte) = _m_to_int(tmp64b); + *(output_p + Eoffset2_byte + 1) = _m_to_int(_mm_srli_si64(tmp64c, 32)); + Eoffset2 += E2; + } + output_p++; + } + + +#endif +} + +/** + * \typedef ldpc8blocks_args_t + * \struct ldpc8blocks_args_s + * \brief Arguments of an encoding task + * encode up to 8 code blocks + * \var nrLDPC_TB_encoding_parameters TB encoding parameters as defined in the coding library interface + * \var impp encoder implementation specific parameters for the task + * \var f first interleaver output to be filled by the task + * \var f2 second interleaver output to be filled by the task + * in case of a shift of E in the code blocks group processed by the task + */ typedef struct ldpc8blocks_args_s { nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters; encoder_implemparams_t impp; + uint8_t *f; + uint8_t *f2; } ldpc8blocks_args_t; static void ldpc8blocks(void *p) @@ -64,156 +236,214 @@ static void ldpc8blocks(void *p) // nrLDPC_encoder output is in "d" // let's make this interface happy! - uint8_t tmp[8][68 * 384] __attribute__((aligned(32))); - uint8_t *d[impp->n_segments]; - for (int rr = impp->macro_num * 8, i = 0; rr < impp->n_segments && rr < (impp->macro_num + 1) * 8; rr++, i++) - d[rr] = tmp[i]; - + uint8_t d[68 * 384] __attribute__((aligned(64))); uint8_t *c[nrLDPC_TB_encoding_parameters->C]; + unsigned int macro_segment, macro_segment_end; + + + macro_segment = impp->first_seg; + macro_segment_end = (impp->n_segments > impp->first_seg + 8) ? impp->first_seg + 8 : impp->n_segments; for (int r = 0; r < nrLDPC_TB_encoding_parameters->C; r++) c[r] = nrLDPC_TB_encoding_parameters->segments[r].c; - start_meas(&nrLDPC_TB_encoding_parameters->segments[impp->macro_num * 8].ts_ldpc_encode); + start_meas(&nrLDPC_TB_encoding_parameters->segments[impp->first_seg].ts_ldpc_encode); LDPCencoder(c, d, impp); - stop_meas(&nrLDPC_TB_encoding_parameters->segments[impp->macro_num * 8].ts_ldpc_encode); + stop_meas(&nrLDPC_TB_encoding_parameters->segments[impp->first_seg].ts_ldpc_encode); // Compute where to place in output buffer that is concatenation of all segments - uint32_t r_offset = 0; - for (int i = 0; i < impp->macro_num * 8; i++) - r_offset += nrLDPC_TB_encoding_parameters->segments[i].E; - for (int rr = impp->macro_num * 8; rr < impp->n_segments && rr < (impp->macro_num + 1) * 8; rr++) { - if (impp->F > 0) { - // writing into positions d[r][k-2Zc] as in clause 5.3.2 step 2) in 38.212 - memset(&d[rr][impp->K - impp->F - 2 * impp->Zc], NR_NULL, impp->F); - } - #ifdef DEBUG_LDPC_ENCODING - LOG_D(PHY, "rvidx in encoding = %d\n", rel15->rvIndex[0]); + LOG_D(PHY, "rvidx in encoding = %d\n", nrLDPC_TB_encoding_parameters->rv_index); #endif - uint32_t E = nrLDPC_TB_encoding_parameters->segments[rr].E; - LOG_D(NR_PHY, - "Rate Matching, Code segment %d/%d (coded bits (G) %u, E %d, Filler bits %d, Filler offset %d mod_order %d, nb_rb " + const uint32_t E = nrLDPC_TB_encoding_parameters->segments[macro_segment].E; + uint32_t E2=E; + bool Eshift=false; + uint32_t Emax = E; + for (int s=macro_segment;ssegments[s].E != E) { + E2=nrLDPC_TB_encoding_parameters->segments[s].E; + Eshift=true; + if(E2 > Emax) + Emax = E2; + break; + } + + + LOG_D(NR_PHY, + "Rate Matching, Code segment %d...%d/%d (coded bits (G) %u, E %d, E2 %d Filler bits %d, Filler offset %d mod_order %d, nb_rb " "%d,nrOfLayer %d)...\n", - rr, + macro_segment, + macro_segment_end, + impp->n_segments, + G, + E,E2, + impp->F, + impp->K - impp->F - 2 * impp->Zc, + mod_order, + nb_rb, + nrLDPC_TB_encoding_parameters->nb_layers); +/* + printf("Rate Matching, Code segment %d...%d/%d (coded bits (G) %u, E %d, E2 %d Filler bits %d, Filler offset %d mod_order %d, nb_rb " + "%d,nrOfLayer %d)...\n", + macro_segment, + macro_segment_end, + impp->n_segments, + G, + E,E2, + impp->F, + impp->K - impp->F - 2 * impp->Zc, + mod_order, + nb_rb, + nrLDPC_TB_encoding_parameters->nb_layers); +*/ + uint32_t Tbslbrm = nrLDPC_TB_encoding_parameters->tbslbrm; + + uint8_t e[Emax] __attribute__((aligned(64))); + // Interleaver outputs are stored in the output arrays + uint8_t *f = args->f; + uint8_t *f2 = args->f2; + bzero(e, Emax); + bzero(f, ceil_mod(E, 64)); + if (Eshift) { + bzero(f2, ceil_mod(E2, 64)); + } + start_meas(&nrLDPC_TB_encoding_parameters->segments[macro_segment].ts_rate_match); + nr_rate_matching_ldpc(Tbslbrm, + impp->BG, + impp->Zc, + d, + e, + impp->n_segments, + impp->F, + impp->K - impp->F - 2 * impp->Zc, + nrLDPC_TB_encoding_parameters->rv_index, + Emax); + + stop_meas(&nrLDPC_TB_encoding_parameters->segments[macro_segment].ts_rate_match); + if (impp->K - impp->F - 2 * impp->Zc > E) { + LOG_E(PHY, + "dlsch coding A %d Kr %d G %d (nb_rb %d, mod_order %d)\n", + A, + impp->K, + G, + nb_rb, + (int)mod_order); + + LOG_E(NR_PHY, + "Rate Matching, Code segments %d...%d/%d (coded bits (G) %u, E %d, Kr %d, Filler bits %d, Filler offset %d mod_order %d, " + "nb_rb %d)...\n", + macro_segment, + macro_segment_end, impp->n_segments, G, E, + impp->K, impp->F, impp->K - impp->F - 2 * impp->Zc, mod_order, - nb_rb, - nrLDPC_TB_encoding_parameters->nb_layers); - - uint32_t Tbslbrm = nrLDPC_TB_encoding_parameters->tbslbrm; - - uint8_t e[E]; - bzero(e, E); - start_meas(&nrLDPC_TB_encoding_parameters->segments[rr].ts_rate_match); - nr_rate_matching_ldpc(Tbslbrm, - impp->BG, - impp->Zc, - d[rr], - e, - impp->n_segments, - impp->F, - impp->K - impp->F - 2 * impp->Zc, - nrLDPC_TB_encoding_parameters->rv_index, - E); - stop_meas(&nrLDPC_TB_encoding_parameters->segments[rr].ts_rate_match); - if (impp->K - impp->F - 2 * impp->Zc > E) { - LOG_E(PHY, "dlsch coding A %d K %d G %d (nb_rb %d, mod_order %d)\n", A, impp->K, G, nb_rb, (int)mod_order); - - LOG_E(NR_PHY, - "Rate Matching, Code segment %d/%d (coded bits (G) %u, E %d, K %d, Filler bits %d, Filler offset %d mod_order %d, " - "nb_rb %d)...\n", - rr, - impp->n_segments, - G, - E, - impp->K, - impp->F, - impp->K - impp->F - 2 * impp->Zc, - mod_order, - nb_rb); - } -#ifdef DEBUG_LDPC_ENCODING - - for (int i = 0; i < 16; i++) - printf("output ratematching e[%d]= %d r_offset %u\n", i, e[i], r_offset); - -#endif - start_meas(&nrLDPC_TB_encoding_parameters->segments[rr].ts_interleave); - nr_interleaving_ldpc(E, mod_order, e, impp->output + r_offset); - stop_meas(&nrLDPC_TB_encoding_parameters->segments[rr].ts_interleave); -#ifdef DEBUG_LDPC_ENCODING - - for (int i = 0; i < 16; i++) - printf("output interleaving f[%d]= %d r_offset %u\n", i, impp->output[i + r_offset], r_offset); - - if (r == impp->n_segments - 1) - write_output("enc_output.m", "enc", impp->output, G, 1, 4); - -#endif - r_offset += E; + nb_rb); } + start_meas(&nrLDPC_TB_encoding_parameters->segments[macro_segment].ts_interleave); + nr_interleaving_ldpc(E, + mod_order, + e, + f); + if (Eshift) + nr_interleaving_ldpc(E2, + mod_order, + e, + f2); + stop_meas(&nrLDPC_TB_encoding_parameters->segments[macro_segment].ts_interleave); - // Task running in // completed completed_task_ans(impp->ans); } -static int nrLDPC_prepare_TB_encoding(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters, +static int nrLDPC_launch_TB_encoding(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters, int dlsch_id, - thread_info_tm_t *t_info) + thread_info_tm_t *t_info, + uint8_t **f, + uint8_t **f2) { nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id]; - encoder_implemparams_t impp = {0}; + encoder_implemparams_t common_segment_params = { + .n_segments = nrLDPC_TB_encoding_parameters->C, + .tinput = nrLDPC_slot_encoding_parameters->tinput, + .tprep = nrLDPC_slot_encoding_parameters->tprep, + .tparity = nrLDPC_slot_encoding_parameters->tparity, + .toutput = nrLDPC_slot_encoding_parameters->toutput, + .Kb = nrLDPC_TB_encoding_parameters->Kb, + .Zc = nrLDPC_TB_encoding_parameters->Z, + .BG = nrLDPC_TB_encoding_parameters->BG, + .output = nrLDPC_TB_encoding_parameters->output, + .K = nrLDPC_TB_encoding_parameters->K, + .F = nrLDPC_TB_encoding_parameters->F, + }; - impp.n_segments = nrLDPC_TB_encoding_parameters->C; - impp.tinput = nrLDPC_slot_encoding_parameters->tinput; - impp.tprep = nrLDPC_slot_encoding_parameters->tprep; - impp.tparity = nrLDPC_slot_encoding_parameters->tparity; - impp.toutput = nrLDPC_slot_encoding_parameters->toutput; - impp.Kb = nrLDPC_TB_encoding_parameters->Kb; - impp.Zc = nrLDPC_TB_encoding_parameters->Z; - impp.BG = nrLDPC_TB_encoding_parameters->BG; - impp.output = nrLDPC_TB_encoding_parameters->segments->output; - impp.K = nrLDPC_TB_encoding_parameters->K; - impp.F = nrLDPC_TB_encoding_parameters->F; + const size_t n_tasks = (common_segment_params.n_segments + 7) >> 3; - size_t const n_seg = (impp.n_segments / 8 + ((impp.n_segments & 7) == 0 ? 0 : 1)); - - for (int j = 0; j < n_seg; j++) { + for (int j = 0; j < n_tasks; j++) { ldpc8blocks_args_t *perJobImpp = &((ldpc8blocks_args_t *)t_info->buf)[t_info->len]; DevAssert(t_info->len < t_info->cap); - impp.ans = t_info->ans; t_info->len += 1; - impp.macro_num = j; - perJobImpp->impp = impp; + perJobImpp->impp = common_segment_params; + perJobImpp->impp.first_seg = j * 8; + perJobImpp->impp.ans = t_info->ans; perJobImpp->nrLDPC_TB_encoding_parameters = nrLDPC_TB_encoding_parameters; + perJobImpp->f = f[j]; + perJobImpp->f2 = f2[j]; task_t t = {.func = ldpc8blocks, .args = perJobImpp}; pushTpool(nrLDPC_slot_encoding_parameters->threadPool, t); } - return n_seg; + return n_tasks; } int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters) { int nbTasks = 0; + + uint32_t Emax = 0; for (int dlsch_id = 0; dlsch_id < nrLDPC_slot_encoding_parameters->nb_TBs; dlsch_id++) { + // Compute number of tasks to encode TB nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id]; size_t n_seg = (nrLDPC_TB_encoding_parameters->C / 8 + ((nrLDPC_TB_encoding_parameters->C & 7) == 0 ? 0 : 1)); nbTasks += n_seg; + + // Search for maximum E for sizing encoder output f and f2 + for (int seg_id = 0; seg_id < nrLDPC_TB_encoding_parameters->C; seg_id++) { + uint32_t E = nrLDPC_TB_encoding_parameters->segments[seg_id].E; + Emax = E > Emax ? E : Emax; + } } + + // Create f and f2 to old encoding tasks outputs + uint32_t Emax_ceil_mod = ceil_mod(Emax, 64); + uint8_t f[nbTasks][Emax_ceil_mod] __attribute__((aligned(64))); + uint8_t f2[nbTasks][Emax_ceil_mod] __attribute__((aligned(64))); + ldpc8blocks_args_t arr[nbTasks]; task_ans_t ans; init_task_ans(&ans, nbTasks); thread_info_tm_t t_info = {.buf = (uint8_t *)arr, .len = 0, .cap = nbTasks, .ans = &ans}; int nbEncode = 0; + nbTasks = 0; for (int dlsch_id = 0; dlsch_id < nrLDPC_slot_encoding_parameters->nb_TBs; dlsch_id++) { - nbEncode += nrLDPC_prepare_TB_encoding(nrLDPC_slot_encoding_parameters, dlsch_id, &t_info); + // For easier indexing we store the pointers to sub arrays of f and f2 in pointer arrays + // Then a function to which we pass the pointer arrays can directly use f_2d[j] ans f2_2d[j] + nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id]; + size_t n_seg = (nrLDPC_TB_encoding_parameters->C / 8 + ((nrLDPC_TB_encoding_parameters->C & 7) == 0 ? 0 : 1)); + uint8_t *f_2d[n_seg]; + uint8_t *f2_2d[n_seg]; + for (int j = 0; j < n_seg; j++) { + f_2d[j] = &f[nbTasks + j][0]; + f2_2d[j] = &f2[nbTasks + j][0]; + } + + // Launch tasks encoding TB + nbEncode += nrLDPC_launch_TB_encoding(nrLDPC_slot_encoding_parameters, dlsch_id, &t_info, f_2d, f2_2d); + + nbTasks += n_seg; } if (nbEncode < nbTasks) { completed_many_task_ans(&ans, nbTasks - nbEncode); @@ -221,5 +451,53 @@ int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encodin // Execute thread pool tasks join_task_ans(&ans); + // Write output + nbTasks = 0; + for (int dlsch_id = 0; dlsch_id < nrLDPC_slot_encoding_parameters->nb_TBs; dlsch_id++) { + nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id]; + uint32_t C = nrLDPC_TB_encoding_parameters->C; + size_t n_seg = (C / 8 + ((C & 7) == 0 ? 0 : 1)); + + time_stats_t *toutput = nrLDPC_slot_encoding_parameters->toutput; + + for (int j = 0; j < n_seg; j++) { + unsigned int macro_segment = j * 8; + unsigned int macro_segment_end = (C > macro_segment + 8) ? macro_segment + 8 : C; + const uint32_t E = nrLDPC_TB_encoding_parameters->segments[macro_segment].E; + uint32_t E2 = E, E2_first_segment = macro_segment_end - macro_segment; + bool Eshift = false; + uint32_t Emax = E; + for (int s = macro_segment; s < macro_segment_end; s++) { + if (nrLDPC_TB_encoding_parameters->segments[s].E != E) { + E2 = nrLDPC_TB_encoding_parameters->segments[s].E; + Eshift = true; + E2_first_segment = s-macro_segment; + if(E2 > Emax) + Emax = E2; + break; + } + } + + if(toutput != NULL) start_meas(toutput); + + uint32_t Eoffset=0; + for (int s=0; ssegments[s].E); + + write_task_output(&f[nbTasks + j][0], + E, + &f2[nbTasks + j][0], + E2, + Eshift, + E2_first_segment, + macro_segment_end - macro_segment, + nrLDPC_TB_encoding_parameters->output, + Eoffset); + + if(toutput != NULL) stop_meas(toutput); + } + nbTasks += n_seg; + } + return 0; } diff --git a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.c b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.c index ebd1ddf2dd..4a507fa381 100644 --- a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.c +++ b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.c @@ -29,274 +29,419 @@ static const uint8_t index_k0[2][4] = {{0, 17, 33, 56}, {0, 13, 25, 43}}; void nr_interleaving_ldpc(uint32_t E, uint8_t Qm, uint8_t *e, uint8_t *f) { - uint32_t EQm; - - EQm = E / Qm; + const uint32_t EQm = E / Qm; memset(f, 0, E * sizeof(uint8_t)); - uint8_t *e0, *e1, *e2, *e3, *e4, *e5, *e6, *e7; - uint8_t *fp; -#if 0 // def __WASAVX2__ - simde__m256i tmp0,tmp1,tmp2,tmp0b,tmp1b,tmp3,tmp4,tmp5; - simde__m256i *e0_256,*e1_256,*e2_256,*e3_256,*e4_256,*e5_256,*e6_256,*e7_256; - - simde__m256i *f_256=(simde__m256i *)f; - - uint8_t *fp2; switch(Qm) { - case 2: - e0=e; - e1=e0+EQm; - e0_256=(simde__m256i *)e0; - e1_256=(simde__m256i *)e1; - for (int k=0,j=0;j>5;j++,k+=2) { - f_256[k] = simde_mm256_unpacklo_epi8(e0_256[j],e1_256[j]); - f_256[k+1] = simde_mm256_unpackhi_epi8(e0_256[j],e1_256[j]); - } - break; - case 4: - e0=e; - e1=e0+EQm; - e2=e1+EQm; - e3=e2+EQm; - e0_256=(simde__m256i *)e0; - e1_256=(simde__m256i *)e1; - e2_256=(simde__m256i *)e2; - e3_256=(simde__m256i *)e3; - for (int k=0,j=0;j>5;j++,k+=4) { - tmp0 = simde_mm256_unpacklo_epi8(e0_256[j],e1_256[j]); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15) - tmp1 = simde_mm256_unpacklo_epi8(e2_256[j],e3_256[j]); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15) - f_256[k] = simde_mm256_unpacklo_epi8(tmp0,tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) - f_256[k+1] = simde_mm256_unpackhi_epi8(tmp0,tmp1); // e0(i+8) e1(i+8) e2(i+8) e3(i+8) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) - tmp0 = simde_mm256_unpackhi_epi8(e0_256[j],e1_256[j]); // e0(i+16) e1(i+16) e0(i+17) e1(i+17) .... e0(i+31) e1(i+31) - tmp1 = simde_mm256_unpackhi_epi8(e2_256[j],e3_256[j]); // e2(i+16) e3(i+16) e2(i+17) e3(i+17) .... e2(i+31) e3(i+31) - f_256[k+2] = simde_mm256_unpacklo_epi8(tmp0,tmp1); - f_256[k+3] = simde_mm256_unpackhi_epi8(tmp0,tmp1); - } - break; - case 6: - e0=e; - e1=e0+EQm; - e2=e1+EQm; - e3=e2+EQm; - e4=e3+EQm; - e5=e4+EQm; - e0_256=(simde__m256i *)e0; - e1_256=(simde__m256i *)e1; - e2_256=(simde__m256i *)e2; - e3_256=(simde__m256i *)e3; - e4_256=(simde__m256i *)e4; - e5_256=(simde__m256i *)e5; - - for (int j=0,k=0;j>5;j++,k+=192) { - fp = f+k; - fp2 = fp+96; - - tmp0 = simde_mm256_unpacklo_epi8(e0_256[j],e1_256[j]); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15) - tmp1 = simde_mm256_unpacklo_epi8(e2_256[j],e3_256[j]); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15) - tmp0b = simde_mm256_unpacklo_epi16(tmp0,tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) - tmp1b = simde_mm256_unpackhi_epi16(tmp0,tmp1); // e0(i+8) e1(i+8) e2(i+8) e3(i+8) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) - tmp0 = simde_mm256_unpacklo_epi8(e4_256[j],e5_256[j]); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+15) e5(i+15) - *((uint32_t*)fp) = simde_mm256_extract_epi32(tmp0b,0); - *((uint16_t*)(fp+4)) = simde_mm256_extract_epi16(tmp0,0); - *((uint32_t*)(fp+6)) = simde_mm256_extract_epi32(tmp0b,1); - *((uint16_t*)(fp+10)) = simde_mm256_extract_epi16(tmp0,1); - *((uint32_t*)(fp+12)) = simde_mm256_extract_epi32(tmp0b,2); - *((uint16_t*)(fp+16)) = simde_mm256_extract_epi16(tmp0,2); - *((uint32_t*)(fp+18)) = simde_mm256_extract_epi32(tmp0b,3); - *((uint16_t*)(fp+22)) = simde_mm256_extract_epi16(tmp0,3); - *((uint32_t*)(fp+24)) = simde_mm256_extract_epi32(tmp0b,4); - *((uint16_t*)(fp+26)) = simde_mm256_extract_epi16(tmp0,4); - *((uint32_t*)(fp+30)) = simde_mm256_extract_epi32(tmp0b,5); - *((uint16_t*)(fp+34)) = simde_mm256_extract_epi16(tmp0,5); - *((uint32_t*)(fp+36)) = simde_mm256_extract_epi32(tmp0,6); - *((uint16_t*)(fp+40)) = simde_mm256_extract_epi16(tmp0,6); - *((uint32_t*)(fp+42)) = simde_mm256_extract_epi32(tmp0b,7); - *((uint16_t*)(fp+46)) = simde_mm256_extract_epi16(tmp0,7); - - *((uint32_t*)(fp+48)) = simde_mm256_extract_epi32(tmp1b,0); - *((uint16_t*)(fp+52)) = simde_mm256_extract_epi16(tmp0,8); - *((uint32_t*)(fp+56)) = simde_mm256_extract_epi32(tmp1b,1); - *((uint16_t*)(fp+60)) = simde_mm256_extract_epi16(tmp0,9); - *((uint32_t*)(fp+62)) = simde_mm256_extract_epi32(tmp1b,2); - *((uint16_t*)(fp+66)) = simde_mm256_extract_epi16(tmp0,10); - *((uint32_t*)(fp+68)) = simde_mm256_extract_epi32(tmp1b,3); - *((uint16_t*)(fp+72)) = simde_mm256_extract_epi16(tmp0,11); - *((uint32_t*)(fp+74)) = simde_mm256_extract_epi32(tmp1b,4); - *((uint16_t*)(fp+76)) = simde_mm256_extract_epi16(tmp0,12); - *((uint32_t*)(fp+80)) = simde_mm256_extract_epi32(tmp1b,5); - *((uint16_t*)(fp+82)) = simde_mm256_extract_epi16(tmp0,13); - *((uint32_t*)(fp+86)) = simde_mm256_extract_epi32(tmp1b,6); - *((uint16_t*)(fp+90)) = simde_mm256_extract_epi16(tmp0,14); - *((uint32_t*)(fp+92)) = simde_mm256_extract_epi32(tmp1b,7); - *((uint16_t*)(fp+94)) = simde_mm256_extract_epi16(tmp0,15); - - tmp0 = simde_mm256_unpackhi_epi8(e0_256[j],e1_256[j]); // e0(i+16) e1(i+16) e0(i+17) e1(i+17) .... e0(i+31) e1(i+31) - tmp1 = simde_mm256_unpackhi_epi8(e2_256[j],e3_256[j]); // e2(i+16) e3(i+16) e2(i+17) e3(i+17) .... e2(i+31) e3(i+31) - tmp0b = simde_mm256_unpacklo_epi16(tmp0,tmp1); // e0(i+16) e1(i+16) e2(i+16) e3(i+16) ... e0(i+23) e1(i+23) e2(i+23) e3(i+23) - tmp1b = simde_mm256_unpackhi_epi16(tmp0,tmp1); // e0(i+24) e1(i+24) e2(i+24) e3(i+24) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31) - tmp0 = simde_mm256_unpackhi_epi8(e4_256[j],e5_256[j]); // e4(i+16) e5(i+16) e4(i+17) e5(i+17) .... e4(i+31) e5(i+31) - *((uint32_t*)fp2) = simde_mm256_extract_epi32(tmp0b,0); - *((uint16_t*)(fp2+4)) = simde_mm256_extract_epi16(tmp0,0); - *((uint32_t*)(fp2+6)) = simde_mm256_extract_epi32(tmp0b,1); - *((uint16_t*)(fp2+10)) = simde_mm256_extract_epi16(tmp0,1); - *((uint32_t*)(fp2+12)) = simde_mm256_extract_epi32(tmp0b,2); - *((uint16_t*)(fp2+16)) = simde_mm256_extract_epi16(tmp0,2); - *((uint32_t*)(fp2+18)) = simde_mm256_extract_epi32(tmp0b,3); - *((uint16_t*)(fp2+22)) = simde_mm256_extract_epi16(tmp0,3); - *((uint32_t*)(fp2+24)) = simde_mm256_extract_epi32(tmp0b,4); - *((uint16_t*)(fp2+26)) = simde_mm256_extract_epi16(tmp0,4); - *((uint32_t*)(fp2+30)) = simde_mm256_extract_epi32(tmp0b,5); - *((uint16_t*)(fp2+34)) = simde_mm256_extract_epi16(tmp0,5); - *((uint32_t*)(fp2+36)) = simde_mm256_extract_epi32(tmp0,6); - *((uint16_t*)(fp2+40)) = simde_mm256_extract_epi16(tmp0,6); - *((uint32_t*)(fp2+42)) = simde_mm256_extract_epi32(tmp0b,7); - *((uint16_t*)(fp2+46)) = simde_mm256_extract_epi16(tmp0,7); - - *((uint32_t*)(fp2+48)) = simde_mm256_extract_epi32(tmp1b,0); - *((uint16_t*)(fp2+52)) = simde_mm256_extract_epi16(tmp0,8); - *((uint32_t*)(fp2+56)) = simde_mm256_extract_epi32(tmp1b,1); - *((uint16_t*)(fp2+60)) = simde_mm256_extract_epi16(tmp0,9); - *((uint32_t*)(fp2+62)) = simde_mm256_extract_epi32(tmp1b,2); - *((uint16_t*)(fp2+66)) = simde_mm256_extract_epi16(tmp0,10); - *((uint32_t*)(fp2+68)) = simde_mm256_extract_epi32(tmp1b,3); - *((uint16_t*)(fp2+72)) = simde_mm256_extract_epi16(tmp0,11); - *((uint32_t*)(fp2+74)) = simde_mm256_extract_epi32(tmp1b,4); - *((uint16_t*)(fp2+76)) = simde_mm256_extract_epi16(tmp0,12); - *((uint32_t*)(fp2+80)) = simde_mm256_extract_epi32(tmp1b,5); - *((uint16_t*)(fp2+82)) = simde_mm256_extract_epi16(tmp0,13); - *((uint32_t*)(fp2+86)) = simde_mm256_extract_epi32(tmp1b,6); - *((uint16_t*)(fp2+90)) = simde_mm256_extract_epi16(tmp0,14); - *((uint32_t*)(fp2+92)) = simde_mm256_extract_epi32(tmp1b,7); - *((uint16_t*)(fp2+94)) = simde_mm256_extract_epi16(tmp0,15); - } - break; - case 8: - e0=e; - e1=e0+EQm; - e2=e1+EQm; - e3=e2+EQm; - e4=e3+EQm; - e5=e4+EQm; - e6=e5+EQm; - e7=e6+EQm; - - e0_256=(simde__m256i *)e0; - e1_256=(simde__m256i *)e1; - e2_256=(simde__m256i *)e2; - e3_256=(simde__m256i *)e3; - e4_256=(simde__m256i *)e4; - e5_256=(simde__m256i *)e5; - e6_256=(simde__m256i *)e6; - e7_256=(simde__m256i *)e7; - for (int k=0,j=0;j>5;j++,k+=8) { - tmp0 = simde_mm256_unpacklo_epi8(e0_256[j],e1_256[j]); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15) - tmp1 = simde_mm256_unpacklo_epi8(e2_256[j],e3_256[j]); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15) - tmp2 = simde_mm256_unpacklo_epi8(e4_256[j],e5_256[j]); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+15) e5(i+15) - tmp3 = simde_mm256_unpacklo_epi8(e6_256[j],e7_256[j]); // e6(i) e7(i) e6(i+1) e7(i+1) .... e6(i+15) e7(i+15) - tmp4 = simde_mm256_unpacklo_epi16(tmp0,tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) - tmp5 = simde_mm256_unpacklo_epi16(tmp2,tmp3); // e4(i) e5(i) e6(i) e7(i) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7) - f_256[k] = simde_mm256_unpacklo_epi16(tmp4,tmp5); // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i)... e0(i+3) e1(i+3) e2(i+3) e3(i+3) e4(i+3) e5(i+3) e6(i+3) e7(i+3)) - f_256[k+1] = simde_mm256_unpackhi_epi16(tmp4,tmp5); // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7(i+4)... e0(i+7) e1(i+7) e2(i+7) e3(i+7) e4(i+7) e5(i+7) e6(i+7) e7(i+7)) - - tmp4 = simde_mm256_unpackhi_epi16(tmp0,tmp1); // e0(i+8) e1(i+8) e2(i+8) e3(i+8) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) - tmp5 = simde_mm256_unpackhi_epi16(tmp2,tmp3); // e4(i+8) e5(i+8) e6(i+8) e7(i+8) ... e4(i+15) e5(i+15) e6(i+15) e7(i+15) - f_256[k+2] = simde_mm256_unpacklo_epi16(tmp4,tmp5); // e0(i+8) e1(i+8) e2(i+8) e3(i+8) e4(i+8) e5(i+8) e6(i+8) e7(i+8)... e0(i+11) e1(i+11) e2(i+11) e3(i+11) e4(i+11) e5(i+11) e6(i+11) e7(i+11)) - f_256[k+3] = simde_mm256_unpackhi_epi16(tmp4,tmp5); // e0(i+12) e1(i+12) e2(i+12) e3(i+12) e4(i+12) e5(i+12) e6(i+12) e7(i+12)... e0(i+15) e1(i+15) e2(i+15) e3(i+15) e4(i+15) e5(i+15) e6(i+15) e7(i+15)) - - tmp0 = simde_mm256_unpackhi_epi8(e0_256[j],e1_256[j]); // e0(i+16) e1(i+16) e0(i+17) e1(i+17) .... e0(i+31) e1(i+31) - tmp1 = simde_mm256_unpackhi_epi8(e2_256[j],e3_256[j]); // e2(i+16) e3(i+16) e2(i+17) e3(i+17) .... e2(i+31) e3(i+31) - tmp2 = simde_mm256_unpackhi_epi8(e4_256[j],e5_256[j]); // e4(i+16) e5(i+16) e4(i+17) e5(i+17) .... e4(i+31) e5(i+31) - tmp3 = simde_mm256_unpackhi_epi8(e6_256[j],e7_256[j]); // e6(i+16) e7(i+16) e6(i+17) e7(i+17) .... e6(i+31) e7(i+31) - tmp4 = simde_mm256_unpacklo_epi16(tmp0,tmp1); // e0(i+!6) e1(i+16) e2(i+16) e3(i+16) ... e0(i+23) e1(i+23) e2(i+23) e3(i+23) - tmp5 = simde_mm256_unpacklo_epi16(tmp2,tmp3); // e4(i+16) e5(i+16) e6(i+16) e7(i+16) ... e4(i+23) e5(i+23) e6(i+23) e7(i+23) - f_256[k+4] = simde_mm256_unpacklo_epi16(tmp4,tmp5); // e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16) e6(i+16) e7(i+16)... e0(i+19) e1(i+19) e2(i+19) e3(i+19) e4(i+19) e5(i+19) e6(i+19) e7(i+19)) - f_256[k+5] = simde_mm256_unpackhi_epi16(tmp4,tmp5); // e0(i+20) e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e6(i+20) e7(i+20)... e0(i+23) e1(i+23) e2(i+23) e3(i+23) e4(i+23) e5(i+23) e6(i+23) e7(i+23)) - - tmp4 = simde_mm256_unpackhi_epi16(tmp0,tmp1); // e0(i+24) e1(i+24) e2(i+24) e3(i+24) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31) - tmp5 = simde_mm256_unpackhi_epi16(tmp2,tmp3); // e4(i+24) e5(i+24) e6(i+24) e7(i+24) ... e4(i+31) e5(i+31) e6(i+31) e7(i+31) - f_256[k+6] = simde_mm256_unpacklo_epi16(tmp4,tmp5); // e0(i+24) e1(i+24) e2(i+24) e3(i+24) e4(i+24) e5(i+24) e6(i+24) e7(i+24)... e0(i+27) e1(i+27) e2(i+27) e3(i+27) e4(i+27) e5(i+27) e6(i+27) e7(i+27)) - f_256[k+7] = simde_mm256_unpackhi_epi16(tmp4,tmp5); // e0(i+28) e1(i+28) e2(i+28) e3(i+28) e4(i+28) e5(i+28) e6(i+28) e7(i+28)... e0(i+31) e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31) e6(i+31) e7(i+31)) - } - break; - default: AssertFatal(1==0,"Should be here!\n"); - } - -#else - // original unoptimized loops - /* - for (int j = 0; j< EQm; j++,j2+=2){ - for (int i = 0; i< Qm; i++){ - f[(i+j*Qm)] = e[(i*EQm + j)]; - } - } - */ - - fp = f; - switch (Qm) { - case 2: - e0 = e; - e1 = e0 + EQm; - for (int j = 0, j2 = 0; j < EQm; j++, j2 += 2) { - fp = &f[j2]; - fp[0] = e0[j]; - fp[1] = e1[j]; + case 2: { + uint8_t *e0 = e; + uint8_t *e1 = e0 + EQm; + int i = 0; +#ifdef __AVX512VBMI__ + // clang-format off + const __m512i p8a = _mm512_set_epi8(95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, + 87, 23, 86, 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, + 79, 15, 78, 14, 77, 13, 76, 12, 75, 11, 74, 10, 73, 9, 72, 8, + 71, 7, 70, 6, 69, 5, 68, 4, 67, 3, 66, 2, 65, 1, 64, 0); + const __m512i p8b = _mm512_set_epi8(127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, + 56, 119, 55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, + 48, 111, 47, 110, 46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, + 40, 103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32); + // clang-format on + __m512i *f_512 = (__m512i *)f; + __m512i *e0_512 = (__m512i *)e0; + __m512i *e1_512 = (__m512i *)e1; + for (; i < (EQm & ~63); i += 64) { + __m512i e0j = _mm512_loadu_si512(e0_512++); + __m512i e1j = _mm512_loadu_si512(e1_512++); + // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi8(e0j, p8a, e1j)); + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi8(e0j, p8b, e1j)); } - break; - case 4: - e0 = e; - e1 = e0 + EQm; - e2 = e1 + EQm; - e3 = e2 + EQm; - for (int j = 0, j2 = 0; j < EQm; j++, j2 += 4) { - fp = &f[j2]; - fp[0] = e0[j]; - fp[1] = e1[j]; - fp[2] = e2[j]; - fp[3] = e3[j]; - } - break; - case 6: - e0 = e; - e1 = e0 + EQm; - e2 = e1 + EQm; - e3 = e2 + EQm; - e4 = e3 + EQm; - e5 = e4 + EQm; - fp = f; - for (int j = 0; j < EQm; j++) { - *fp++ = e0[j]; - *fp++ = e1[j]; - *fp++ = e2[j]; - *fp++ = e3[j]; - *fp++ = e4[j]; - *fp++ = e5[j]; - } - break; - case 8: - e0 = e; - e1 = e0 + EQm; - e2 = e1 + EQm; - e3 = e2 + EQm; - e4 = e3 + EQm; - e5 = e4 + EQm; - e6 = e5 + EQm; - e7 = e6 + EQm; - for (int j = 0, j2 = 0; j < EQm; j++, j2 += 8) { - fp = &f[j2]; - fp[0] = e0[j]; - fp[1] = e1[j]; - fp[2] = e2[j]; - fp[3] = e3[j]; - fp[4] = e4[j]; - fp[5] = e5[j]; - fp[6] = e6[j]; - fp[7] = e7[j]; - } - break; - default: - AssertFatal(1 == 0, "Should never be here!\n"); - } + e0 = (uint8_t *)e0_512; + e1 = (uint8_t *)e1_512; + f = (uint8_t *)f_512; #endif +#ifdef USE128BIT + simde__m128i *f_128 = (simde__m128i *)f; + simde__m128i *e0_128 = (simde__m128i *)e0; + simde__m128i *e1_128 = (simde__m128i *)e1; + for (; i < (EQm & ~15); i += 64) { + simde__m128i e0j = simde_mm_loadu_si128(e0_128++); + simde__m128i e1j = simde_mm_loadu_si128(e1_128++); + simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi8(e0j, e1j)); + simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi8(e0j, e1j)); + } + e0 = (uint8_t *)e0_128; + e1 = (uint8_t *)e1_128; + f = (uint8_t *)f_128; +#endif + for (; i < EQm; i++) { + *f++ = *e0++; + *f++ = *e1++; + } + } break; + case 4: { + uint8_t *e0 = e; + uint8_t *e1 = e0 + EQm; + uint8_t *e2 = e1 + EQm; + uint8_t *e3 = e2 + EQm; + int i = 0; +#ifdef __AVX512VBMI__ + // clang-format off + const __m512i p8a = _mm512_set_epi8(95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, 87, 23, 86, + 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, 79, 15, 78, 14, 77, 13, + 76, 12, 75, 11, 74, 10, 73, 9, 72, 8, 71, 7, 70, 6, 69, 5, 68, 4, 67, 3, 66, 2, 65, 1, 64, 0); + const __m512i p8b = _mm512_set_epi8(127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56, 119, + 55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48, 111, 47, + 110, 46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40, 103, 39, 102, + 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32); + const __m512i p16a = _mm512_set_epi16(47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7, 38, 6, + 37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0); + const __m512i p16b = _mm512_set_epi16(63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23, 54, + 22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16); + // clang-format on + __m512i *e0_512 = (__m512i *)e0; + __m512i *e1_512 = (__m512i *)e1; + __m512i *e2_512 = (__m512i *)e2; + __m512i *e3_512 = (__m512i *)e3; + __m512i *f_512 = (__m512i *)f; + for (; i < (EQm & ~63); i += 64) { + __m512i e0j = _mm512_loadu_si512(e0_512++); + __m512i e1j = _mm512_loadu_si512(e1_512++); + __m512i e2j = _mm512_loadu_si512(e2_512++); + __m512i e3j = _mm512_loadu_si512(e3_512++); + __m512i tmp0 = _mm512_permutex2var_epi8(e0j, p8a, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15) + __m512i tmp1 = _mm512_permutex2var_epi8(e2j, p8a, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15) + // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp0, p16a, tmp1)); + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp0, p16b, tmp1)); + tmp0 = _mm512_permutex2var_epi8(e0j, p8b, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15) + tmp1 = _mm512_permutex2var_epi8(e2j, p8b, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15) + // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp0, p16a, tmp1)); + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp0, p16b, tmp1)); + } + e0 = (uint8_t *)e0_512; + e1 = (uint8_t *)e1_512; + e2 = (uint8_t *)e2_512; + e3 = (uint8_t *)e3_512; + f = (uint8_t *)f_512; +#endif +#ifdef USE128BIT + simde__m128i *e0_128 = (simde__m128i *)e0; + simde__m128i *e1_128 = (simde__m128i *)e1; + simde__m128i *e2_128 = (simde__m128i *)e2; + simde__m128i *e3_128 = (simde__m128i *)e3; + simde__m128i *f_128 = (simde__m128i *)f; + for (; i < (EQm & ~15); i += 16) { + simde__m128i e0j = simde_mm_loadu_si128(e0_128++); + simde__m128i e1j = simde_mm_loadu_si128(e1_128++); + simde__m128i e2j = simde_mm_loadu_si128(e2_128++); + simde__m128i e3j = simde_mm_loadu_si128(e3_128++); + simde__m128i tmp0 = simde_mm_unpacklo_epi8(e0j, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+7) e1(i+7) + simde__m128i tmp1 = simde_mm_unpacklo_epi8(e2j, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+7) e3(i+7) + simde_mm_storeu_si128(f_128++, + simde_mm_unpacklo_epi16(tmp0, tmp1)); // e0(i) e1(i) e2(i) e3(i) ... e0(i+3) e1(i+3) e2(i+3) e3(i+3) + simde_mm_storeu_si128( + f_128++, + simde_mm_unpackhi_epi16(tmp0, tmp1)); // e0(i+4) e1(i+4) e2(i+4) e3(i+4) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) + tmp0 = simde_mm_unpackhi_epi8(e0j, e1j); // e0(i+8) e1(i+8) e0(i+9) e1(i+9) .... e0(i+15) e1(i+15) + tmp1 = simde_mm_unpackhi_epi8(e2j, e3j); // e2(i+8) e3(i+9) e2(i+10) e3(i+10) .... e2(i+31) e3(i+31) + simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi16(tmp0, tmp1)); + simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi16(tmp0, tmp1)); + } + e0 = (uint8_t *)e0_128; + e1 = (uint8_t *)e1_128; + e2 = (uint8_t *)e2_128; + e3 = (uint8_t *)e3_128; + f = (uint8_t *)f_128; +#endif + for (; i < EQm; i++) { + *f++ = *e0++; + *f++ = *e1++; + *f++ = *e2++; + *f++ = *e3++; + } + } break; + case 6: { + uint8_t *e0 = e; + uint8_t *e1 = e0 + EQm; + uint8_t *e2 = e1 + EQm; + uint8_t *e3 = e2 + EQm; + uint8_t *e4 = e3 + EQm; + uint8_t *e5 = e4 + EQm; + int i = 0; +#ifdef __AVX512VBMI__ + // clang-format off + const __m512i p8a = _mm512_set_epi8(95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, 87, 23, + 86, 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, 79, 15, 78, 14, + 77, 13, 76, 12, 75, 11, 74, 10, 73, 9, 72, 8, 71, 7, 70, 6, 69, 5, 68, 4, 67, 3, 66, 2, 65, 1, 64, 0); + const __m512i p8b = _mm512_set_epi8(127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56, 119, + 55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48, 111, 47, 110, + 46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40, 103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32); + const __m512i p16a = _mm512_set_epi16(47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7, 38, 6, 37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0); + const __m512i p16b = _mm512_set_epi16(63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23, 54, 22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16); + const __m512i p16c = _mm512_set_epi16(21, 20, 41, 19, 18, 40, 17, 16, 39, 15, 14, 38, 13, 12, 37, 11, 10, 36, 9, 8, 35, 7, 6, 34, 5, 4, 33, 3, 2, 32, 1, 0); + const __m512i p16d = _mm512_set_epi16(10, 52, 9, 8, 51, 7, 6, 50, 5, 4, 49, 3, 2, 48, 1, 0, 47, 31, 30, 46, 29, 28, 45, 27, 26, 44, 25, 24, 43, 23, 22, 42); + const __m512i p16d2 = _mm512_set_epi16(32 + 10, 30, 32 + 9, 32 + 8, 27, 32 + 7, 32 + 6, 24, 32 + 5, 32 + 4, 21, 32 + 3, 32 + 2, 18, 32 + 1, 32 + 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + const __m512i p16e = _mm512_set_epi16(63, 31, 30, 62, 29, 28, 61, 27, 26, 60, 25, 24, 59, 23, 22, 58, 21, 20, 57, 19, 18, 56, 17, 16, 55, 15, 14, 54, 13, 12, 53, 11); + // clang-format on + __m512i *e0_512 = (__m512i *)e0; + __m512i *e1_512 = (__m512i *)e1; + __m512i *e2_512 = (__m512i *)e2; + __m512i *e3_512 = (__m512i *)e3; + __m512i *e4_512 = (__m512i *)e4; + __m512i *e5_512 = (__m512i *)e5; + __m512i *f_512 = (__m512i *)f; + for (; i < (EQm & ~63); i += 64) { + __m512i e0j = _mm512_loadu_si512(e0_512++); + __m512i e1j = _mm512_loadu_si512(e1_512++); + __m512i e2j = _mm512_loadu_si512(e2_512++); + __m512i e3j = _mm512_loadu_si512(e3_512++); + __m512i e4j = _mm512_loadu_si512(e4_512++); + __m512i e5j = _mm512_loadu_si512(e5_512++); + __m512i tmp0 = _mm512_permutex2var_epi8(e0j, p8a, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+31) e1(i+31) + __m512i tmp1 = _mm512_permutex2var_epi8(e2j, p8a, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+31) e3(i+31) + __m512i tmp2 = _mm512_permutex2var_epi8(e4j, p8a, e5j); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+31) e5(i+31) + // e0(i) e1(i) e2(i) e3(i) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) + __m512i tmp3 = _mm512_permutex2var_epi16(tmp0, p16a, tmp1); + // e0(i+16) e1(i+16) e2(i+16) e3(i+16) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31) + __m512i tmp4 = _mm512_permutex2var_epi16(tmp0, p16b, tmp1); + // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) ... e0(i+9) e1(i+9) e2(i+9) e3(i+9) e4(i+9) e5(i+9) e0(i+10) e1(i+10) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp3, p16c, tmp2)); + // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) e4(i+15) + // e5(i+15) x x x x e4(i+16) e5(i+16) x x x x .... e4(i+20) e5(i+20) x x + __m512i tmp5 = _mm512_permutex2var_epi16(tmp3, p16d, tmp2); + // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) + // e4(i+15) e5(i+15) e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16) e0(i+20) + // e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e0(i+21) e1(i+21) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp5, p16d2, tmp4)); + // e2(i+21) e3(i+21) e4(i+21) e5(i+21) .... e0(i+31) e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp4, p16e, tmp2)); + + tmp0 = _mm512_permutex2var_epi8(e0j, p8b, e1j); // e0(i+32) e1(i+32) e0(i+32) e1(i+32) .... e0(i+63) e1(i+63) + tmp1 = _mm512_permutex2var_epi8(e2j, p8b, e3j); // e2(i+32) e3(i+32) e2(i+32) e3(i+32) .... e2(i+63) e3(i+63) + tmp2 = _mm512_permutex2var_epi8(e4j, p8b, e5j); // e4(i+32) e5(i+32) e4(i+32) e5(i+32) .... e4(i+63) e5(i+63) + tmp3 = _mm512_permutex2var_epi16(tmp0, p16a, tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) + // e0(i+16) e1(i+16) e2(i+16) e3(i+16) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31) + tmp4 = _mm512_permutex2var_epi16(tmp0, p16b, tmp1); + // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) ... e0(i+9) e1(i+9) e2(i+9) e3(i+9) e4(i+9) e5(i+9) e0(i+10) e1(i+10) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp3, p16c, tmp2)); + // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) + // e4(i+15) e5(i+15) x x x x e4(i+16) e5(i+16) x x x x .... e4(i+20) e5(i+20) x x + tmp5 = _mm512_permutex2var_epi16(tmp3, p16d, tmp2); + // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) + // e4(i+15) e5(i+15) e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16) e0(i+20) + // e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e0(i+21) e1(i+21) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp5, p16d2, tmp4)); + // e2(i+21) e3(i+21) e4(i+21) e5(i+21) .... e0(i+31) e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp4, p16e, tmp2)); + } + e0 = (uint8_t *)e0_512; + e1 = (uint8_t *)e1_512; + e2 = (uint8_t *)e2_512; + e3 = (uint8_t *)e3_512; + e4 = (uint8_t *)e4_512; + e5 = (uint8_t *)e5_512; + f = (uint8_t *)f_512; +#endif + for (; i < EQm; i++) { + *f++ = *e0++; + *f++ = *e1++; + *f++ = *e2++; + *f++ = *e3++; + *f++ = *e4++; + *f++ = *e5++; + } + } break; + case 8: { + uint8_t *e0 = e; + uint8_t *e1 = e0 + EQm; + uint8_t *e2 = e1 + EQm; + uint8_t *e3 = e2 + EQm; + uint8_t *e4 = e3 + EQm; + uint8_t *e5 = e4 + EQm; + uint8_t *e6 = e5 + EQm; + uint8_t *e7 = e6 + EQm; + + int i = 0; +#ifdef __AVX512VBMI__ + // clang-format off + const __m512i p8a = _mm512_set_epi8(95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, 87, 23, + 86, 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, 79, 15, 78, 14, 77, 13, + 76, 12, 75, 11, 74, 10, 73, 9, 72, 8, 71, 7, 70, 6, 69, 5, 68, 4, 67, 3, 66, 2, 65, 1, 64, 0); + const __m512i p8b = _mm512_set_epi8(127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56, 119, 55, + 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48, 111, 47, 110, 46, + 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40, 103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32); + + const __m512i p16a = _mm512_set_epi16(47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7, 38, 6, 37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0); + const __m512i p16b = _mm512_set_epi16(63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23, 54, 22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16); + + const __m512i p32a = _mm512_set_epi32(23, 7, 22, 6, 21, 5, 20, 4, 19, 3, 18, 2, 17, 1, 16, 0); + const __m512i p32b = _mm512_set_epi32(31, 15, 30, 14, 29, 13, 28, 12, 27, 11, 26, 10, 25, 9, 24, 8); + // clang-format on + __m512i *e0_512 = (__m512i *)e0; + __m512i *e1_512 = (__m512i *)e1; + __m512i *e2_512 = (__m512i *)e2; + __m512i *e3_512 = (__m512i *)e3; + __m512i *e4_512 = (__m512i *)e4; + __m512i *e5_512 = (__m512i *)e5; + __m512i *e6_512 = (__m512i *)e6; + __m512i *e7_512 = (__m512i *)e7; + __m512i *f_512 = (__m512i *)f; + for (; i < (EQm & ~63); i += 64) { + __m512i e0j = _mm512_loadu_si512(e0_512++); + __m512i e1j = _mm512_loadu_si512(e1_512++); + __m512i e2j = _mm512_loadu_si512(e2_512++); + __m512i e3j = _mm512_loadu_si512(e3_512++); + __m512i e4j = _mm512_loadu_si512(e4_512++); + __m512i e5j = _mm512_loadu_si512(e5_512++); + __m512i e6j = _mm512_loadu_si512(e6_512++); + __m512i e7j = _mm512_loadu_si512(e7_512++); + __m512i tmp0 = _mm512_permutex2var_epi8(e0j, p8a, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15) + __m512i tmp1 = _mm512_permutex2var_epi8(e2j, p8a, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15) + __m512i tmp2 = _mm512_permutex2var_epi8(e4j, p8a, e5j); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+15) e5(i+15) + __m512i tmp3 = _mm512_permutex2var_epi8(e6j, p8a, e7j); // e6(i) e7(i) e6(i+1) e7(i+1) .... e6(i+15) e7(i+15) + __m512i tmp4 = _mm512_permutex2var_epi16(tmp0, p16a, tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) + // e4(i) e5(i) e6(i) e7(i) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7) + // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i)... e0(i+3) e1(i+3) + __m512i tmp5 = _mm512_permutex2var_epi16(tmp2, p16a, tmp3); + // e2(i+3) e3(i+3) e4(i+3) e5(i+3) e6(i+3) e7(i+3)) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32a, tmp5)); + // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7(i+4)... + // e0(i+7) e1(i+7) e2(i+7) e3(i+7) e4(i+7) e5(i+7) e6(i+7) e7(i+7)) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32b, tmp5)); + // e0(i+8) e1(i+8) e2(i+8) e3(i+8) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) + tmp4 = _mm512_permutex2var_epi16(tmp0, p16b, tmp1); + // e4(i+8) e5(i+8) e6(i+8) e7(i+8) ... e4(i+15) e5(i+15) e6(i+15) e7(i+15) + tmp5 = _mm512_permutex2var_epi16(tmp2, p16b, tmp3); + // e0(i+8) e1(i+8) e2(i+8) e3(i+8) e4(i+8) e5(i+8) e6(i+8) e7(i+8)... e0(i+11) + // e1(i+11) e2(i+11) e3(i+11) e4(i+11) e5(i+11) e6(i+11) e7(i+11)) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32a, tmp5)); + // e0(i+12) e1(i+12) e2(i+12) e3(i+12) e4(i+12) e5(i+12) e6(i+12) e7(i+12)... e0(i+15) + // e1(i+15) e2(i+15) e3(i+15) e4(i+15) e5(i+15) e6(i+15) e7(i+15)) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32b, tmp5)); + + tmp0 = _mm512_permutex2var_epi8(e0j, p8b, e1j); // e0(i+16) e1(i+16) e0(i+17) e1(i+17) .... e0(i+31) e1(i+31) + tmp1 = _mm512_permutex2var_epi8(e2j, p8b, e3j); // e2(i+16) e3(i+16) e2(i+17) e3(i+17) .... e2(i+31) e3(i+31) + tmp2 = _mm512_permutex2var_epi8(e4j, p8b, e5j); // e4(i+16) e5(i+16) e4(i+17) e5(i+17) .... e4(i+31) e5(i+31) + tmp3 = _mm512_permutex2var_epi8(e6j, p8b, e7j); // e6(i+16) e7(i+16) e6(i+17) e7(i+17) .... e6(i+31) e7(i+31) + // e0(i+!6) e1(i+16) e2(i+16) e3(i+16) ... e0(i+23) e1(i+23) e2(i+23) e3(i+23) + tmp4 = _mm512_permutex2var_epi16(tmp0, p16a, tmp1); + // e4(i+16) e5(i+16) e6(i+16) e7(i+16) ... e4(i+23) e5(i+23) e6(i+23) e7(i+23) + tmp5 = _mm512_permutex2var_epi16(tmp2, p16a, tmp3); + // e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16) e6(i+16) e7(i+16)... e0(i+19) + // e1(i+19) e2(i+19) e3(i+19) e4(i+19) e5(i+19) e6(i+19) e7(i+19)) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32a, tmp5)); + // e0(i+20) e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e6(i+20) e7(i+20)... e0(i+23) + // e1(i+23) e2(i+23) e3(i+23) e4(i+23) e5(i+23) e6(i+23) e7(i+23)) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32b, tmp5)); + // e0(i+24) e1(i+24) e2(i+24) e3(i+24) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31) + tmp4 = _mm512_permutex2var_epi16(tmp0, p16b, tmp1); + // e4(i+24) e5(i+24) e6(i+24) e7(i+24) ... e4(i+31) e5(i+31) e6(i+31) e7(i+31) + tmp5 = _mm512_permutex2var_epi16(tmp2, p16b, tmp3); + // e0(i+24) e1(i+24) e2(i+24) e3(i+24) e4(i+24) e5(i+24) e6(i+24) e7(i+24)... e0(i+27) + // e1(i+27) e2(i+27) e3(i+27) e4(i+27) e5(i+27) e6(i+27) e7(i+27)) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32a, tmp5)); + // e0(i+28) e1(i+28) e2(i+28) e3(i+28) e4(i+28) e5(i+28) e6(i+28) e7(i+28)... e0(i+31) + // e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31) e6(i+31) e7(i+31)) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32b, tmp5)); + } + e0 = (uint8_t *)e0_512; + e1 = (uint8_t *)e1_512; + e2 = (uint8_t *)e2_512; + e3 = (uint8_t *)e3_512; + e4 = (uint8_t *)e4_512; + e5 = (uint8_t *)e5_512; + e6 = (uint8_t *)e6_512; + e7 = (uint8_t *)e7_512; + f = (uint8_t *)f_512; +#endif +#ifdef USE128BIT + e0_128 = (simde__m128i *)e0; + e1_128 = (simde__m128i *)e1; + e2_128 = (simde__m128i *)e2; + e3_128 = (simde__m128i *)e3; + e4_128 = (simde__m128i *)e4; + e5_128 = (simde__m128i *)e5; + e6_128 = (simde__m128i *)e6; + e7_128 = (simde__m128i *)e7; + for (; i < (EQm & ~15); i += 16) { + simde__m128i e0j = simde_mm_loadu_si128(e0_128++); + simde__m128i e1j = simde_mm_loadu_si128(e1_128++); + simde__m128i e2j = simde_mm_loadu_si128(e2_128++); + simde__m128i e3j = simde_mm_loadu_si128(e3_128++); + simde__m128i e4j = simde_mm_loadu_si128(e4_128++); + simde__m128i e5j = simde_mm_loadu_si128(e5_128++); + simde__m128i e6j = simde_mm_loadu_si128(e6_128++); + simde__m128i e7j = simde_mm_loadu_si128(e7_128++); + simde__m128i tmp0 = simde_mm_unpacklo_epi8(e0j, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+7) e1(i+7) + simde__m128i tmp1 = simde_mm_unpacklo_epi8(e2j, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+7) e3(i+7) + // e0(i) e1(i) e2(i) e3(i) e0(i+1) e1(i+1) e2(i+1) e3(i+1) ... e0(i+3) e1(i+3) e2(i+3) e3(i+3) + simde__m128i tmp2 = simde_mm_unpacklo_epi16(tmp0, tmp1); + // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e0(i+5) e1(i+5) e2(i+5) e3(i+5) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) + simde__m128i tmp3 = simde_mm_unpackhi_epi16(tmp0, tmp1); + simde__m128i tmp4 = simde_mm_unpacklo_epi8(e4j, e5j); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+7) e5(i+7) + simde__m128i tmp5 = simde_mm_unpacklo_epi8(e6j, e7j); // e6(i) e7(i) e6(i+1) e7(i+1) .... e6(i+7) e7(i+7) + // e4(i) e5(i) e6(i) e7(i) e4(i+1) e5(i+1) e6(i+1) e7(i+1) ... e4(i+3) e5(i+3) e6(i+) e7(i+3) + simde__m128i tmp6 = simde_mm_unpacklo_epi16(tmp4, tmp5); + // e4(i+4) e5(i+4) e6(i+4) e7(i+4) e4(i+5) e5(i+5) e6(i+5) e7(i+5) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7) + simde__m128i tmp7 = simde_mm_unpackhi_epi16(tmp4, tmp5); + // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i) e0(i+1) ... e7(i+1) + simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi32(tmp2, tmp6)); + // e0(i+2) e1(i+2) e2(i+2) e3(i+2) e4(i+2) e5(i+2) e6(i+2) e7(i+2) e0(i+3) e1(i+3) ... e7(i+3) + simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi32(tmp2, tmp6)); + // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7+4(i) e0(i+5) ... e7(i+5) + simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi32(tmp3, tmp7)); + // e0(i+6) e1(i+6) e2(i+6) e3(i+6) e4(i+6) e5(i+6) e6(i+6) e7(i+6) e0(i+7) e0(i+7) ... e7(i+7) + simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi32(tmp3, tmp7)); + + tmp0 = simde_mm_unpackhi_epi8(e0j, e1j); // e0(i+8) e1(i+8) e0(i+9) e1(i+9) .... e0(i+15) e1(i+15) + tmp1 = simde_mm_unpackhi_epi8(e2j, e3j); // e2(i+8) e3(i+8) e2(i+9) e3(i+9) .... e2(i+15) e3(i+15) + // e0(i) e1(i) e2(i) e3(i) e0(i+1) e1(i+1) e2(i+1) e3(i+1) ... e0(i+3) e1(i+3) e2(i+3) e3(i+3) + tmp2 = simde_mm_unpacklo_epi16(tmp0, tmp1); + // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e0(i+5) e1(i+5) e2(i+5) e3(i+5) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) + tmp3 = simde_mm_unpackhi_epi16(tmp0, tmp1); + tmp4 = simde_mm_unpackhi_epi8(e4j, e5j); // e4(i+8) e5(i+8) e4(i+9) e5(i+9) .... e4(i+15) e515i+7) + tmp5 = simde_mm_unpackhi_epi8(e6j, e7j); // e6(i+8) e7(i+8) e6(i+9) e7(i+9) .... e6(i+15) e7(i+15) + // e4(i) e5(i) e6(i) e7(i) e4(i+1) e5(i+1) e6(i+1) e7(i+1) ... e4(i+3) e5(i+3) e6(i+) e7(i+3) + tmp6 = simde_mm_unpacklo_epi16(tmp4, tmp5); + // e4(i+4) e5(i+4) e6(i+4) e7(i+4) e4(i+5) e5(i+5) e6(i+5) e7(i+5) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7) + tmp7 = simde_mm_unpackhi_epi16(tmp4, tmp5); + // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i) e0(i+1) ... e7(i+1) + simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi32(tmp2, tmp6)); + // e0(i+2) e1(i+2) e2(i+2) e3(i+2) e4(i+2) e5(i+2) e6(i+2) e7(i+2) e0(i+3) e1(i+3) ... e7(i+3) + simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi32(tmp2, tmp6)); + // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7+4(i) e0(i+5) ... e7(i+5) + simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi32(tmp3, tmp7)); + // e0(i+6) e1(i+6) e2(i+6) e3(i+6) e4(i+6) e5(i+6) e6(i+6) e7(i+6) e0(i+7) e0(i+7) ... e7(i+7) + simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi32(tmp3, tmp7)); + } + e0 = (uint8_t *)e0_128; + e1 = (uint8_t *)e1_128; + e2 = (uint8_t *)e2_128; + e3 = (uint8_t *)e3_128; + e4 = (uint8_t *)e4_128; + e5 = (uint8_t *)e5_128; + e6 = (uint8_t *)e6_128; + e7 = (uint8_t *)e7_128; + f = (uint8_t *)f_128; +#endif + for (; i < EQm; i++) { + *f++ = *e0++; + *f++ = *e1++; + *f++ = *e2++; + *f++ = *e3++; + *f++ = *e4++; + *f++ = *e5++; + *f++ = *e6++; + *f++ = *e7++; + } + } break; + default: + AssertFatal(false, "Should be here!\n"); + } } void nr_deinterleaving_ldpc(uint32_t E, uint8_t Qm, int16_t *e, int16_t *f) @@ -454,8 +599,11 @@ int nr_rate_matching_ldpc(uint32_t Tbslbrm, printf("RM_TX k%u Ind: %u (%d)\n", k, ind, d[ind]); #endif - if (d[ind] != NR_NULL) - e[k++] = d[ind]; + if (ind == Foffset) + ind = F + Foffset; // skip filler bits + + e[k++] = d[ind]; + } } diff --git a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.h b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.h index ac6a496801..597a53070a 100644 --- a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.h +++ b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.h @@ -24,8 +24,6 @@ #include -#define NR_NULL 2 - /** * \brief interleave a code segment after encoding and rate matching * \param E size of the code segment in bits diff --git a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_t2/nrLDPC_coding_t2.c b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_t2/nrLDPC_coding_t2.c index c16b21bfdc..6eb8f0dacc 100644 --- a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_t2/nrLDPC_coding_t2.c +++ b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_t2/nrLDPC_coding_t2.c @@ -650,21 +650,44 @@ static int retrieve_ldpc_enc_op(struct rte_bbdev_enc_op **ops, nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters) { + uint8_t *p_out = NULL; unsigned int j = 0; for (unsigned int h = 0; h < nrLDPC_slot_encoding_parameters->nb_TBs; ++h){ - for (unsigned int i = 0; i < nrLDPC_slot_encoding_parameters->TBs[h].C; ++i) { + int E_sum = 0; + int bit_offset = 0; + int byte_offset = 0; + p_out = nrLDPC_slot_encoding_parameters->TBs[h].output; + for (unsigned int r = 0; r < nrLDPC_slot_encoding_parameters->TBs[h].C; ++r) { struct rte_bbdev_op_data *output = &ops[j]->ldpc_enc.output; struct rte_mbuf *m = output->data; uint16_t data_len = rte_pktmbuf_data_len(m) - output->offset; - uint8_t *out = nrLDPC_slot_encoding_parameters->TBs[h].segments[i].output; - const char *data = m->buf_addr + m->data_off; - const char *end = data + data_len; - while (data < end) { - uint8_t byte = *data++; // get the current byte - for (int bit = 7; bit >= 0; --bit) { - *out++ = (byte >> bit) & 1; // extract each bit + uint8_t *data = m->buf_addr + m->data_off; + reverse_bits_u8(data, data_len, data); + if (bit_offset == 0) { + memcpy(&p_out[byte_offset], data, data_len); + } else { + size_t i = 0; + for (; i < (data_len & ~0x7); i += 8) { + uint8_t carry = *data << bit_offset; + p_out[byte_offset + i - 1] |= carry; + + simde__m64 current = *((simde__m64 *)data); + data += 8; + current = simde_mm_srli_si64(current, 8 - bit_offset); + *(simde__m64 *)&p_out[byte_offset + i] = current; + } + for (; i < data_len; i++) { + uint8_t current = *data++; + + uint8_t carry = current << bit_offset; + p_out[byte_offset + i - 1] |= carry; + + p_out[byte_offset + i] = (current >> (8 - bit_offset)); } } + E_sum += nrLDPC_slot_encoding_parameters->TBs[h].segments[r].E; + byte_offset = (E_sum + 7) / 8; + bit_offset = E_sum % 8; rte_pktmbuf_free(m); rte_pktmbuf_free(ops[j]->ldpc_enc.input.data); ++j; @@ -815,6 +838,10 @@ static int pmd_lcore_ldpc_enc(void *arg) AssertFatal(ret == 0, "Allocation failed for %d ops", num_segments); set_ldpc_enc_op(ops_enq, 0, bufs->inputs, bufs->hard_outputs, nrLDPC_slot_encoding_parameters); + if (nrLDPC_slot_encoding_parameters->tprep != NULL) + stop_meas(nrLDPC_slot_encoding_parameters->tprep); + if (nrLDPC_slot_encoding_parameters->tparity != NULL) + start_meas(nrLDPC_slot_encoding_parameters->tparity); for (enq = 0, deq = 0; enq < num_segments;) { num_to_enq = num_segments; if (unlikely(num_segments - enq < num_to_enq)) @@ -828,10 +855,14 @@ static int pmd_lcore_ldpc_enc(void *arg) time_out++; DevAssert(time_out <= TIME_OUT_POLL); } - + if (nrLDPC_slot_encoding_parameters->tparity != NULL) + stop_meas(nrLDPC_slot_encoding_parameters->tparity); + if (nrLDPC_slot_encoding_parameters->toutput != NULL) + start_meas(nrLDPC_slot_encoding_parameters->toutput); ret = retrieve_ldpc_enc_op(ops_deq, nrLDPC_slot_encoding_parameters); AssertFatal(ret == 0, "Failed to retrieve LDPC encoding op!"); - + if (nrLDPC_slot_encoding_parameters->toutput != NULL) + stop_meas(nrLDPC_slot_encoding_parameters->toutput); rte_bbdev_enc_op_free_bulk(ops_enq, num_segments); rte_free(ops_enq); rte_free(ops_deq); @@ -1074,6 +1105,8 @@ int32_t nrLDPC_coding_decoder(nrLDPC_slot_decoding_parameters_t *nrLDPC_slot_dec int32_t nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters) { pthread_mutex_lock(&encode_mutex); + if (nrLDPC_slot_encoding_parameters->tprep != NULL) + start_meas(nrLDPC_slot_encoding_parameters->tprep); // hardcoded to use the first found board struct active_device *ad = active_devs; int ret; diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/generator_cnProc/cnProc_gen_BG1_128.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/generator_cnProc/cnProc_gen_BG1_128.c index e4d971472f..9e3d212697 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/generator_cnProc/cnProc_gen_BG1_128.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/generator_cnProc/cnProc_gen_BG1_128.c @@ -24,7 +24,9 @@ #include #include "../../nrLDPCdecoder_defs.h" -#define AVOID_MM256_SIGN 1 +#ifdef __aarch64__ +#define AVOID_SIGN 1 +#endif #define DROP_MAXLLR 1 void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) { @@ -126,7 +128,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) // ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i]; fprintf(fd," ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[0]>>4)+lut_idxCnProcG3[j][0]*2); // sgn = simde_mm_sign_epi8(ones, ymm0); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(ones, ymm0);\n"); +#else fprintf(fd," sgn = simde_mm_sign_epi8(ones, ymm0);\n"); +#endif // min = simde_mm_abs_epi8(ymm0); fprintf(fd," min = simde_mm_abs_epi8(ymm0);\n"); @@ -138,7 +144,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) fprintf(fd," min = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n"); // sgn = simde_mm_sign_epi8(sgn, ymm0); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(sgn, ymm0);\n"); +#else fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#endif // Store result // min = simde_mm_min_epu8(min, maxLLR); // 128 in epi8 is -127 #ifndef DROP_MAXLLR @@ -185,7 +195,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) // ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i]; fprintf(fd," ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[1]>>4)+lut_idxCnProcG4[j][0]*2); // sgn = simde_mm_sign_epi8(ones, ymm0); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(ones, ymm0);\n"); +#else fprintf(fd," sgn = simde_mm_sign_epi8(ones, ymm0);\n"); +#endif // min = simde_mm_abs_epi8(ymm0); fprintf(fd," min = simde_mm_abs_epi8(ymm0);\n"); @@ -199,7 +213,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) fprintf(fd," min = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n"); // sgn = simde_mm_sign_epi8(sgn, ymm0); - fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(sgn, ymm0);\n"); +#else + fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#endif } // Store result @@ -252,7 +270,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) // ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i]; fprintf(fd," ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[2]>>4)+lut_idxCnProcG5[j][0]*2); // sgn = simde_mm_sign_epi8(ones, ymm0); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(ones, ymm0);\n"); +#else fprintf(fd," sgn = simde_mm_sign_epi8(ones, ymm0);\n"); +#endif // min = simde_mm_abs_epi8(ymm0); fprintf(fd," min = simde_mm_abs_epi8(ymm0);\n"); @@ -266,7 +288,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) fprintf(fd," min = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n"); // sgn = simde_mm_sign_epi8(sgn, ymm0); - fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(sgn, ymm0);\n"); +#else + fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#endif } // Store result @@ -319,7 +345,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) // ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i]; fprintf(fd," ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[3]>>4)+lut_idxCnProcG6[j][0]*2); // sgn = simde_mm_sign_epi8(ones, ymm0); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(ones, ymm0);\n"); +#else fprintf(fd," sgn = simde_mm_sign_epi8(ones, ymm0);\n"); +#endif // min = simde_mm_abs_epi8(ymm0); fprintf(fd," min = simde_mm_abs_epi8(ymm0);\n"); @@ -333,7 +363,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) fprintf(fd," min = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n"); // sgn = simde_mm_sign_epi8(sgn, ymm0); - fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(sgn, ymm0);\n"); +#else + fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#endif } // Store result @@ -389,7 +423,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) // ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i]; fprintf(fd," ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[4]>>4)+lut_idxCnProcG7[j][0]*2); // sgn = simde_mm_sign_epi8(ones, ymm0); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(ones, ymm0);\n"); +#else fprintf(fd," sgn = simde_mm_sign_epi8(ones, ymm0);\n"); +#endif // min = simde_mm_abs_epi8(ymm0); fprintf(fd," min = simde_mm_abs_epi8(ymm0);\n"); @@ -403,7 +441,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) fprintf(fd," min = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n"); // sgn = simde_mm_sign_epi8(sgn, ymm0); - fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(sgn, ymm0);\n"); +#else + fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#endif } // Store result @@ -460,7 +502,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) // ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i]; fprintf(fd," ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[5]>>4)+lut_idxCnProcG8[j][0]*2); // sgn = simde_mm_sign_epi8(ones, ymm0); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(ones, ymm0);\n"); +#else fprintf(fd," sgn = simde_mm_sign_epi8(ones, ymm0);\n"); +#endif // min = simde_mm_abs_epi8(ymm0); fprintf(fd," min = simde_mm_abs_epi8(ymm0);\n"); @@ -474,7 +520,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) fprintf(fd," min = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n"); // sgn = simde_mm_sign_epi8(sgn, ymm0); - fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(sgn, ymm0);\n"); +#else + fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#endif } // Store result @@ -531,7 +581,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) // ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i]; fprintf(fd," ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[6]>>4)+lut_idxCnProcG9[j][0]*2); // sgn = simde_mm_sign_epi8(ones, ymm0); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(ones, ymm0);\n"); +#else fprintf(fd," sgn = simde_mm_sign_epi8(ones, ymm0);\n"); +#endif // min = simde_mm_abs_epi8(ymm0); fprintf(fd," min = simde_mm_abs_epi8(ymm0);\n"); @@ -545,7 +599,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) fprintf(fd," min = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n"); // sgn = simde_mm_sign_epi8(sgn, ymm0); - fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(sgn, ymm0);\n"); +#else + fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#endif } // Store result @@ -603,7 +661,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) // ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i]; fprintf(fd," ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[7]>>4)+lut_idxCnProcG10[j][0]*2); // sgn = simde_mm_sign_epi8(ones, ymm0); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(ones, ymm0);\n"); +#else fprintf(fd," sgn = simde_mm_sign_epi8(ones, ymm0);\n"); +#endif // min = simde_mm_abs_epi8(ymm0); fprintf(fd," min = simde_mm_abs_epi8(ymm0);\n"); @@ -617,7 +679,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) fprintf(fd," min = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n"); // sgn = simde_mm_sign_epi8(sgn, ymm0); - fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(sgn, ymm0);\n"); +#else + fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#endif } // Store result @@ -678,7 +744,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) // ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i]; fprintf(fd," ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[8]>>4)+lut_idxCnProcG19[j][0]*2); // sgn = simde_mm_sign_epi8(ones, ymm0); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(ones, ymm0);\n"); +#else fprintf(fd," sgn = simde_mm_sign_epi8(ones, ymm0);\n"); +#endif // min = simde_mm_abs_epi8(ymm0); fprintf(fd," min = simde_mm_abs_epi8(ymm0);\n"); @@ -692,7 +762,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) fprintf(fd," min = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n"); // sgn = simde_mm_sign_epi8(sgn, ymm0); - fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(sgn, ymm0);\n"); +#else + fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#endif } // Store result diff --git a/openair1/PHY/CODING/nrLDPC_defs.h b/openair1/PHY/CODING/nrLDPC_defs.h index 3db41a0e29..2298400078 100644 --- a/openair1/PHY/CODING/nrLDPC_defs.h +++ b/openair1/PHY/CODING/nrLDPC_defs.h @@ -28,7 +28,7 @@ /** \brief LDPC encoder parameter structure \var n_segments number of segments in the transport block - \var macro_num index of the subset of segments to encode + \var first_seg index of the first segment of the subset to encode within the transport block \var gen_code flag to generate parity check code 0 -> encoding @@ -52,7 +52,7 @@ */ typedef struct { unsigned int n_segments; // optim8seg - unsigned int macro_num; // optim8segmulti + unsigned int first_seg; // optim8segmulti unsigned char gen_code; //orig time_stats_t *tinput; time_stats_t *tprep; @@ -92,6 +92,6 @@ typedef int32_t(LDPC_decoderfunc_t)(t_nrLDPC_dec_params *p_decParams, int8_t *p_out, t_nrLDPC_time_stats *, decode_abort_t *ab); -typedef int32_t(LDPC_encoderfunc_t)(uint8_t **, uint8_t **, encoder_implemparams_t *); +typedef int32_t(LDPC_encoderfunc_t)(uint8_t **, uint8_t *, encoder_implemparams_t *); #endif diff --git a/openair1/PHY/CODING/nrLDPC_encoder/ldpc384_simd512_byte.c b/openair1/PHY/CODING/nrLDPC_encoder/ldpc384_simd512_byte.c new file mode 100644 index 0000000000..16ddbd3e48 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_encoder/ldpc384_simd512_byte.c @@ -0,0 +1,151 @@ +#include "PHY/sse_intrin.h" +// generated code for Zc=384, byte encoding +static inline void ldpc384_byte(uint8_t *c,uint8_t *d) { + __m512i *csimd=(__m512i *)c,*dsimd=(__m512i *)d; + + __m512i *c2,*d2; + + int i2; + for (i2=0; i2<6; i2++) { + c2=&csimd[i2]; + d2=&dsimd[i2]; + +//row: 0 + d2[0]=_mm512_xor_si512(c2[13468],_mm512_xor_si512(c2[3169],_mm512_xor_si512(c2[3435],_mm512_xor_si512(c2[5284],_mm512_xor_si512(c2[5028],_mm512_xor_si512(c2[15327],_mm512_xor_si512(c2[6085],_mm512_xor_si512(c2[13224],_mm512_xor_si512(c2[3193],_mm512_xor_si512(c2[2141],_mm512_xor_si512(c2[12977],_mm512_xor_si512(c2[2413],_mm512_xor_si512(c2[36],_mm512_xor_si512(c2[8500],_mm512_xor_si512(c2[3221],_mm512_xor_si512(c2[5068],_mm512_xor_si512(c2[14054],_mm512_xor_si512(c2[4286],_mm512_xor_si512(c2[64],_mm512_xor_si512(c2[6411],_mm512_xor_si512(c2[8786],_mm512_xor_si512(c2[1923],_mm512_xor_si512(c2[2993],_mm512_xor_si512(c2[2992],_mm512_xor_si512(c2[6686],_mm512_xor_si512(c2[3005],_mm512_xor_si512(c2[8546],_mm512_xor_si512(c2[14880],_mm512_xor_si512(c2[16216],_mm512_xor_si512(c2[13310],_mm512_xor_si512(c2[16740],_mm512_xor_si512(c2[8572],_mm512_xor_si512(c2[386],_mm512_xor_si512(c2[1178],_mm512_xor_si512(c2[12013],_mm512_xor_si512(c2[10432],_mm512_xor_si512(c2[13072],_mm512_xor_si512(c2[4632],_mm512_xor_si512(c2[5957],_mm512_xor_si512(c2[10443],_mm512_xor_si512(c2[9929],_mm512_xor_si512(c2[2271],_mm512_xor_si512(c2[5705],_mm512_xor_si512(c2[6771],_mm512_xor_si512(c2[6505],_mm512_xor_si512(c2[5451],_mm512_xor_si512(c2[6255],_mm512_xor_si512(c2[9421],_mm512_xor_si512(c2[14172],_mm512_xor_si512(c2[11281],_mm512_xor_si512(c2[9173],_mm512_xor_si512(c2[12868],_mm512_xor_si512(c2[13405],_mm512_xor_si512(c2[998],_mm512_xor_si512(c2[11824],_mm512_xor_si512(c2[13419],_mm512_xor_si512(c2[12891],_mm512_xor_si512(c2[4180],_mm512_xor_si512(c2[13958],_mm512_xor_si512(c2[3137],_mm512_xor_si512(c2[3663],_mm512_xor_si512(c2[2885],_mm512_xor_si512(c2[3672],_mm512_xor_si512(c2[10536],_mm512_xor_si512(c2[7121],_mm512_xor_si512(c2[12925],c2[10025])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 1 + d2[6]=_mm512_xor_si512(c2[13468],_mm512_xor_si512(c2[13732],_mm512_xor_si512(c2[3433],_mm512_xor_si512(c2[3699],_mm512_xor_si512(c2[5548],_mm512_xor_si512(c2[5028],_mm512_xor_si512(c2[5292],_mm512_xor_si512(c2[15591],_mm512_xor_si512(c2[6349],_mm512_xor_si512(c2[13224],_mm512_xor_si512(c2[13488],_mm512_xor_si512(c2[3457],_mm512_xor_si512(c2[2405],_mm512_xor_si512(c2[12977],_mm512_xor_si512(c2[13241],_mm512_xor_si512(c2[2677],_mm512_xor_si512(c2[300],_mm512_xor_si512(c2[8764],_mm512_xor_si512(c2[3485],_mm512_xor_si512(c2[5332],_mm512_xor_si512(c2[14054],_mm512_xor_si512(c2[14318],_mm512_xor_si512(c2[4550],_mm512_xor_si512(c2[328],_mm512_xor_si512(c2[6411],_mm512_xor_si512(c2[6675],_mm512_xor_si512(c2[9050],_mm512_xor_si512(c2[2187],_mm512_xor_si512(c2[3257],_mm512_xor_si512(c2[3256],_mm512_xor_si512(c2[6950],_mm512_xor_si512(c2[3269],_mm512_xor_si512(c2[8810],_mm512_xor_si512(c2[15144],_mm512_xor_si512(c2[16216],_mm512_xor_si512(c2[16480],_mm512_xor_si512(c2[13574],_mm512_xor_si512(c2[109],_mm512_xor_si512(c2[8572],_mm512_xor_si512(c2[8836],_mm512_xor_si512(c2[650],_mm512_xor_si512(c2[1442],_mm512_xor_si512(c2[12013],_mm512_xor_si512(c2[12277],_mm512_xor_si512(c2[10696],_mm512_xor_si512(c2[13336],_mm512_xor_si512(c2[4632],_mm512_xor_si512(c2[4896],_mm512_xor_si512(c2[6221],_mm512_xor_si512(c2[10707],_mm512_xor_si512(c2[9929],_mm512_xor_si512(c2[10193],_mm512_xor_si512(c2[2535],_mm512_xor_si512(c2[5969],_mm512_xor_si512(c2[7035],_mm512_xor_si512(c2[6769],_mm512_xor_si512(c2[5715],_mm512_xor_si512(c2[6255],_mm512_xor_si512(c2[6519],_mm512_xor_si512(c2[9685],_mm512_xor_si512(c2[14436],_mm512_xor_si512(c2[11281],_mm512_xor_si512(c2[11545],_mm512_xor_si512(c2[9437],_mm512_xor_si512(c2[13132],_mm512_xor_si512(c2[13669],_mm512_xor_si512(c2[1262],_mm512_xor_si512(c2[12088],_mm512_xor_si512(c2[13419],_mm512_xor_si512(c2[13683],_mm512_xor_si512(c2[13155],_mm512_xor_si512(c2[4444],_mm512_xor_si512(c2[13958],_mm512_xor_si512(c2[14222],_mm512_xor_si512(c2[3401],_mm512_xor_si512(c2[3927],_mm512_xor_si512(c2[2885],_mm512_xor_si512(c2[3149],_mm512_xor_si512(c2[3936],_mm512_xor_si512(c2[10800],_mm512_xor_si512(c2[7121],_mm512_xor_si512(c2[7385],_mm512_xor_si512(c2[13189],c2[10289]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 2 + d2[12]=_mm512_xor_si512(c2[13732],_mm512_xor_si512(c2[3433],_mm512_xor_si512(c2[3435],_mm512_xor_si512(c2[3699],_mm512_xor_si512(c2[5284],_mm512_xor_si512(c2[5548],_mm512_xor_si512(c2[5292],_mm512_xor_si512(c2[15327],_mm512_xor_si512(c2[15591],_mm512_xor_si512(c2[6085],_mm512_xor_si512(c2[6349],_mm512_xor_si512(c2[13488],_mm512_xor_si512(c2[3457],_mm512_xor_si512(c2[2141],_mm512_xor_si512(c2[2405],_mm512_xor_si512(c2[13241],_mm512_xor_si512(c2[2677],_mm512_xor_si512(c2[36],_mm512_xor_si512(c2[300],_mm512_xor_si512(c2[8764],_mm512_xor_si512(c2[3221],_mm512_xor_si512(c2[3485],_mm512_xor_si512(c2[5068],_mm512_xor_si512(c2[5332],_mm512_xor_si512(c2[14318],_mm512_xor_si512(c2[4550],_mm512_xor_si512(c2[64],_mm512_xor_si512(c2[328],_mm512_xor_si512(c2[6675],_mm512_xor_si512(c2[8786],_mm512_xor_si512(c2[9050],_mm512_xor_si512(c2[1923],_mm512_xor_si512(c2[2187],_mm512_xor_si512(c2[3257],_mm512_xor_si512(c2[2992],_mm512_xor_si512(c2[3256],_mm512_xor_si512(c2[6686],_mm512_xor_si512(c2[6950],_mm512_xor_si512(c2[3269],_mm512_xor_si512(c2[8546],_mm512_xor_si512(c2[8810],_mm512_xor_si512(c2[14880],_mm512_xor_si512(c2[15144],_mm512_xor_si512(c2[16480],_mm512_xor_si512(c2[13574],_mm512_xor_si512(c2[16740],_mm512_xor_si512(c2[109],_mm512_xor_si512(c2[8836],_mm512_xor_si512(c2[386],_mm512_xor_si512(c2[650],_mm512_xor_si512(c2[1178],_mm512_xor_si512(c2[1442],_mm512_xor_si512(c2[12277],_mm512_xor_si512(c2[10696],_mm512_xor_si512(c2[13072],_mm512_xor_si512(c2[13336],_mm512_xor_si512(c2[4896],_mm512_xor_si512(c2[6221],_mm512_xor_si512(c2[10443],_mm512_xor_si512(c2[10707],_mm512_xor_si512(c2[10193],_mm512_xor_si512(c2[2271],_mm512_xor_si512(c2[2535],_mm512_xor_si512(c2[5705],_mm512_xor_si512(c2[5969],_mm512_xor_si512(c2[7035],_mm512_xor_si512(c2[6505],_mm512_xor_si512(c2[6769],_mm512_xor_si512(c2[5451],_mm512_xor_si512(c2[5715],_mm512_xor_si512(c2[6519],_mm512_xor_si512(c2[9685],_mm512_xor_si512(c2[14172],_mm512_xor_si512(c2[14436],_mm512_xor_si512(c2[11545],_mm512_xor_si512(c2[9437],_mm512_xor_si512(c2[12868],_mm512_xor_si512(c2[13132],_mm512_xor_si512(c2[13669],_mm512_xor_si512(c2[998],_mm512_xor_si512(c2[1262],_mm512_xor_si512(c2[11824],_mm512_xor_si512(c2[12088],_mm512_xor_si512(c2[13683],_mm512_xor_si512(c2[12891],_mm512_xor_si512(c2[13155],_mm512_xor_si512(c2[4180],_mm512_xor_si512(c2[4444],_mm512_xor_si512(c2[14222],_mm512_xor_si512(c2[3401],_mm512_xor_si512(c2[3663],_mm512_xor_si512(c2[3927],_mm512_xor_si512(c2[3149],_mm512_xor_si512(c2[3672],_mm512_xor_si512(c2[3936],_mm512_xor_si512(c2[10536],_mm512_xor_si512(c2[10800],_mm512_xor_si512(c2[7385],_mm512_xor_si512(c2[13189],_mm512_xor_si512(c2[10025],c2[10289])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 3 + d2[18]=_mm512_xor_si512(c2[13732],_mm512_xor_si512(c2[3433],_mm512_xor_si512(c2[3699],_mm512_xor_si512(c2[5284],_mm512_xor_si512(c2[5548],_mm512_xor_si512(c2[5292],_mm512_xor_si512(c2[15591],_mm512_xor_si512(c2[6085],_mm512_xor_si512(c2[6349],_mm512_xor_si512(c2[13488],_mm512_xor_si512(c2[3457],_mm512_xor_si512(c2[2405],_mm512_xor_si512(c2[13241],_mm512_xor_si512(c2[2677],_mm512_xor_si512(c2[36],_mm512_xor_si512(c2[300],_mm512_xor_si512(c2[8764],_mm512_xor_si512(c2[3485],_mm512_xor_si512(c2[5068],_mm512_xor_si512(c2[5332],_mm512_xor_si512(c2[14318],_mm512_xor_si512(c2[4550],_mm512_xor_si512(c2[328],_mm512_xor_si512(c2[6675],_mm512_xor_si512(c2[9050],_mm512_xor_si512(c2[1923],_mm512_xor_si512(c2[2187],_mm512_xor_si512(c2[3257],_mm512_xor_si512(c2[3256],_mm512_xor_si512(c2[6686],_mm512_xor_si512(c2[6950],_mm512_xor_si512(c2[3269],_mm512_xor_si512(c2[8810],_mm512_xor_si512(c2[14880],_mm512_xor_si512(c2[15144],_mm512_xor_si512(c2[16480],_mm512_xor_si512(c2[13574],_mm512_xor_si512(c2[109],_mm512_xor_si512(c2[8836],_mm512_xor_si512(c2[650],_mm512_xor_si512(c2[1178],_mm512_xor_si512(c2[1442],_mm512_xor_si512(c2[12277],_mm512_xor_si512(c2[10696],_mm512_xor_si512(c2[13072],_mm512_xor_si512(c2[13336],_mm512_xor_si512(c2[4896],_mm512_xor_si512(c2[6221],_mm512_xor_si512(c2[10443],_mm512_xor_si512(c2[10707],_mm512_xor_si512(c2[10193],_mm512_xor_si512(c2[2535],_mm512_xor_si512(c2[5705],_mm512_xor_si512(c2[5969],_mm512_xor_si512(c2[7035],_mm512_xor_si512(c2[6769],_mm512_xor_si512(c2[5451],_mm512_xor_si512(c2[5715],_mm512_xor_si512(c2[6519],_mm512_xor_si512(c2[9685],_mm512_xor_si512(c2[14436],_mm512_xor_si512(c2[11545],_mm512_xor_si512(c2[9437],_mm512_xor_si512(c2[12868],_mm512_xor_si512(c2[13132],_mm512_xor_si512(c2[13669],_mm512_xor_si512(c2[1262],_mm512_xor_si512(c2[11824],_mm512_xor_si512(c2[12088],_mm512_xor_si512(c2[13683],_mm512_xor_si512(c2[13155],_mm512_xor_si512(c2[4180],_mm512_xor_si512(c2[4444],_mm512_xor_si512(c2[14222],_mm512_xor_si512(c2[3401],_mm512_xor_si512(c2[3927],_mm512_xor_si512(c2[3149],_mm512_xor_si512(c2[3936],_mm512_xor_si512(c2[10536],_mm512_xor_si512(c2[10800],_mm512_xor_si512(c2[7385],_mm512_xor_si512(c2[13189],_mm512_xor_si512(c2[10025],c2[10289]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 4 + d2[24]=_mm512_xor_si512(c2[3173],c2[14006]); + +//row: 5 + d2[30]=_mm512_xor_si512(c2[4225],_mm512_xor_si512(c2[10827],_mm512_xor_si512(c2[11093],_mm512_xor_si512(c2[12936],_mm512_xor_si512(c2[795],_mm512_xor_si512(c2[12686],_mm512_xor_si512(c2[6084],_mm512_xor_si512(c2[13743],_mm512_xor_si512(c2[3708],_mm512_xor_si512(c2[3987],_mm512_xor_si512(c2[10851],_mm512_xor_si512(c2[9793],_mm512_xor_si512(c2[3734],_mm512_xor_si512(c2[10071],_mm512_xor_si512(c2[7694],_mm512_xor_si512(c2[13501],_mm512_xor_si512(c2[16152],_mm512_xor_si512(c2[10873],_mm512_xor_si512(c2[12720],_mm512_xor_si512(c2[4817],_mm512_xor_si512(c2[11944],_mm512_xor_si512(c2[7716],_mm512_xor_si512(c2[14069],_mm512_xor_si512(c2[16444],_mm512_xor_si512(c2[9581],_mm512_xor_si512(c2[10645],_mm512_xor_si512(c2[10644],_mm512_xor_si512(c2[14344],_mm512_xor_si512(c2[10657],_mm512_xor_si512(c2[16204],_mm512_xor_si512(c2[5643],_mm512_xor_si512(c2[6973],_mm512_xor_si512(c2[4073],_mm512_xor_si512(c2[7503],_mm512_xor_si512(c2[16224],_mm512_xor_si512(c2[8044],_mm512_xor_si512(c2[8836],_mm512_xor_si512(c2[2776],_mm512_xor_si512(c2[1189],_mm512_xor_si512(c2[3829],_mm512_xor_si512(c2[12290],_mm512_xor_si512(c2[13609],_mm512_xor_si512(c2[1200],_mm512_xor_si512(c2[10178],_mm512_xor_si512(c2[686],_mm512_xor_si512(c2[9929],_mm512_xor_si512(c2[13357],_mm512_xor_si512(c2[14429],_mm512_xor_si512(c2[14163],_mm512_xor_si512(c2[13109],_mm512_xor_si512(c2[13913],_mm512_xor_si512(c2[184],_mm512_xor_si512(c2[4935],_mm512_xor_si512(c2[2044],_mm512_xor_si512(c2[16825],_mm512_xor_si512(c2[3625],_mm512_xor_si512(c2[13131],_mm512_xor_si512(c2[4168],_mm512_xor_si512(c2[8656],_mm512_xor_si512(c2[2581],_mm512_xor_si512(c2[4176],_mm512_xor_si512(c2[3648],_mm512_xor_si512(c2[11832],_mm512_xor_si512(c2[4721],_mm512_xor_si512(c2[10789],_mm512_xor_si512(c2[11321],_mm512_xor_si512(c2[10537],_mm512_xor_si512(c2[11330],_mm512_xor_si512(c2[1299],_mm512_xor_si512(c2[14773],_mm512_xor_si512(c2[3688],_mm512_xor_si512(c2[782],c2[13716])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 6 + d2[36]=_mm512_xor_si512(c2[5812],_mm512_xor_si512(c2[340],_mm512_xor_si512(c2[384],_mm512_xor_si512(c2[8321],_mm512_xor_si512(c2[7549],_mm512_xor_si512(c2[16311],_mm512_xor_si512(c2[4968],c2[8955]))))))); + +//row: 7 + d2[42]=_mm512_xor_si512(c2[2376],_mm512_xor_si512(c2[16380],_mm512_xor_si512(c2[15892],_mm512_xor_si512(c2[3521],_mm512_xor_si512(c2[9076],c2[13369]))))); + +//row: 8 + d2[48]=_mm512_xor_si512(c2[14261],_mm512_xor_si512(c2[7921],_mm512_xor_si512(c2[3962],_mm512_xor_si512(c2[14523],_mm512_xor_si512(c2[4228],_mm512_xor_si512(c2[14525],_mm512_xor_si512(c2[14789],_mm512_xor_si512(c2[6077],_mm512_xor_si512(c2[16368],_mm512_xor_si512(c2[16632],_mm512_xor_si512(c2[13468],_mm512_xor_si512(c2[5821],_mm512_xor_si512(c2[16382],_mm512_xor_si512(c2[16120],_mm512_xor_si512(c2[9516],_mm512_xor_si512(c2[9780],_mm512_xor_si512(c2[6878],_mm512_xor_si512(c2[280],_mm512_xor_si512(c2[544],_mm512_xor_si512(c2[13478],_mm512_xor_si512(c2[14017],_mm512_xor_si512(c2[7683],_mm512_xor_si512(c2[3986],_mm512_xor_si512(c2[14547],_mm512_xor_si512(c2[2928],_mm512_xor_si512(c2[13225],_mm512_xor_si512(c2[13489],_mm512_xor_si512(c2[13764],_mm512_xor_si512(c2[7430],_mm512_xor_si512(c2[3206],_mm512_xor_si512(c2[13767],_mm512_xor_si512(c2[829],_mm512_xor_si512(c2[11126],_mm512_xor_si512(c2[11390],_mm512_xor_si512(c2[9806],_mm512_xor_si512(c2[9293],_mm512_xor_si512(c2[2953],_mm512_xor_si512(c2[4008],_mm512_xor_si512(c2[14305],_mm512_xor_si512(c2[14569],_mm512_xor_si512(c2[5861],_mm512_xor_si512(c2[16152],_mm512_xor_si512(c2[16416],_mm512_xor_si512(c2[14847],_mm512_xor_si512(c2[8513],_mm512_xor_si512(c2[5079],_mm512_xor_si512(c2[15640],_mm512_xor_si512(c2[857],_mm512_xor_si512(c2[11148],_mm512_xor_si512(c2[11412],_mm512_xor_si512(c2[7204],_mm512_xor_si512(c2[864],_mm512_xor_si512(c2[9579],_mm512_xor_si512(c2[2981],_mm512_xor_si512(c2[3245],_mm512_xor_si512(c2[2716],_mm512_xor_si512(c2[13013],_mm512_xor_si512(c2[13277],_mm512_xor_si512(c2[3780],_mm512_xor_si512(c2[14341],_mm512_xor_si512(c2[3785],_mm512_xor_si512(c2[14076],_mm512_xor_si512(c2[14340],_mm512_xor_si512(c2[7479],_mm512_xor_si512(c2[881],_mm512_xor_si512(c2[1145],_mm512_xor_si512(c2[3792],_mm512_xor_si512(c2[14353],_mm512_xor_si512(c2[9339],_mm512_xor_si512(c2[2741],_mm512_xor_si512(c2[3005],_mm512_xor_si512(c2[15673],_mm512_xor_si512(c2[9075],_mm512_xor_si512(c2[9339],_mm512_xor_si512(c2[108],_mm512_xor_si512(c2[10669],_mm512_xor_si512(c2[14103],_mm512_xor_si512(c2[7769],_mm512_xor_si512(c2[638],_mm512_xor_si512(c2[10935],_mm512_xor_si512(c2[11199],_mm512_xor_si512(c2[9365],_mm512_xor_si512(c2[3025],_mm512_xor_si512(c2[1179],_mm512_xor_si512(c2[11476],_mm512_xor_si512(c2[11740],_mm512_xor_si512(c2[1971],_mm512_xor_si512(c2[12268],_mm512_xor_si512(c2[12532],_mm512_xor_si512(c2[12806],_mm512_xor_si512(c2[6472],_mm512_xor_si512(c2[11225],_mm512_xor_si512(c2[4885],_mm512_xor_si512(c2[13865],_mm512_xor_si512(c2[7261],_mm512_xor_si512(c2[7525],_mm512_xor_si512(c2[5425],_mm512_xor_si512(c2[15986],_mm512_xor_si512(c2[6744],_mm512_xor_si512(c2[410],_mm512_xor_si512(c2[11236],_mm512_xor_si512(c2[4632],_mm512_xor_si512(c2[4896],_mm512_xor_si512(c2[4896],_mm512_xor_si512(c2[10716],_mm512_xor_si512(c2[4382],_mm512_xor_si512(c2[3064],_mm512_xor_si512(c2[13361],_mm512_xor_si512(c2[13625],_mm512_xor_si512(c2[6492],_mm512_xor_si512(c2[16789],_mm512_xor_si512(c2[158],_mm512_xor_si512(c2[7564],_mm512_xor_si512(c2[1224],_mm512_xor_si512(c2[7298],_mm512_xor_si512(c2[700],_mm512_xor_si512(c2[964],_mm512_xor_si512(c2[6244],_mm512_xor_si512(c2[16541],_mm512_xor_si512(c2[16805],_mm512_xor_si512(c2[7048],_mm512_xor_si512(c2[708],_mm512_xor_si512(c2[10214],_mm512_xor_si512(c2[3880],_mm512_xor_si512(c2[14965],_mm512_xor_si512(c2[8367],_mm512_xor_si512(c2[8631],_mm512_xor_si512(c2[12074],_mm512_xor_si512(c2[5740],_mm512_xor_si512(c2[9960],_mm512_xor_si512(c2[3626],_mm512_xor_si512(c2[13661],_mm512_xor_si512(c2[7057],_mm512_xor_si512(c2[7321],_mm512_xor_si512(c2[10488],_mm512_xor_si512(c2[14198],_mm512_xor_si512(c2[7864],_mm512_xor_si512(c2[1791],_mm512_xor_si512(c2[12088],_mm512_xor_si512(c2[12352],_mm512_xor_si512(c2[12617],_mm512_xor_si512(c2[6013],_mm512_xor_si512(c2[6277],_mm512_xor_si512(c2[14212],_mm512_xor_si512(c2[7872],_mm512_xor_si512(c2[13684],_mm512_xor_si512(c2[7080],_mm512_xor_si512(c2[7344],_mm512_xor_si512(c2[4973],_mm512_xor_si512(c2[15264],_mm512_xor_si512(c2[15528],_mm512_xor_si512(c2[14751],_mm512_xor_si512(c2[8417],_mm512_xor_si512(c2[3924],_mm512_xor_si512(c2[14485],_mm512_xor_si512(c2[4456],_mm512_xor_si512(c2[14753],_mm512_xor_si512(c2[15017],_mm512_xor_si512(c2[8679],_mm512_xor_si512(c2[3672],_mm512_xor_si512(c2[14233],_mm512_xor_si512(c2[4465],_mm512_xor_si512(c2[14762],_mm512_xor_si512(c2[15026],_mm512_xor_si512(c2[11329],_mm512_xor_si512(c2[4731],_mm512_xor_si512(c2[4995],_mm512_xor_si512(c2[7908],_mm512_xor_si512(c2[1574],_mm512_xor_si512(c2[13718],_mm512_xor_si512(c2[7384],_mm512_xor_si512(c2[10812],_mm512_xor_si512(c2[4214],_mm512_xor_si512(c2[4478],c2[12929])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 9 + d2[54]=_mm512_xor_si512(c2[12149],_mm512_xor_si512(c2[10575],_mm512_xor_si512(c2[389],_mm512_xor_si512(c2[1454],_mm512_xor_si512(c2[15204],_mm512_xor_si512(c2[12616],_mm512_xor_si512(c2[16848],c2[4993]))))))); + +//row: 10 + d2[60]=_mm512_xor_si512(c2[9781],_mm512_xor_si512(c2[5045],_mm512_xor_si512(c2[4804],_mm512_xor_si512(c2[12493],_mm512_xor_si512(c2[16733],c2[9149]))))); + +//row: 11 + d2[66]=_mm512_xor_si512(c2[268],_mm512_xor_si512(c2[10032],_mm512_xor_si512(c2[10296],_mm512_xor_si512(c2[6864],_mm512_xor_si512(c2[3],_mm512_xor_si512(c2[7130],_mm512_xor_si512(c2[269],_mm512_xor_si512(c2[8979],_mm512_xor_si512(c2[2112],_mm512_xor_si512(c2[12672],_mm512_xor_si512(c2[8729],_mm512_xor_si512(c2[1598],_mm512_xor_si512(c2[1862],_mm512_xor_si512(c2[2127],_mm512_xor_si512(c2[12161],_mm512_xor_si512(c2[9780],_mm512_xor_si512(c2[2919],_mm512_xor_si512(c2[10045],_mm512_xor_si512(c2[24],_mm512_xor_si512(c2[9794],_mm512_xor_si512(c2[10058],_mm512_xor_si512(c2[6888],_mm512_xor_si512(c2[27],_mm512_xor_si512(c2[5836],_mm512_xor_si512(c2[15864],_mm512_xor_si512(c2[16672],_mm512_xor_si512(c2[9541],_mm512_xor_si512(c2[9805],_mm512_xor_si512(c2[6108],_mm512_xor_si512(c2[16142],_mm512_xor_si512(c2[3737],_mm512_xor_si512(c2[13765],_mm512_xor_si512(c2[12195],_mm512_xor_si512(c2[5328],_mm512_xor_si512(c2[6916],_mm512_xor_si512(c2[49],_mm512_xor_si512(c2[8763],_mm512_xor_si512(c2[1896],_mm512_xor_si512(c2[854],_mm512_xor_si512(c2[10624],_mm512_xor_si512(c2[10888],_mm512_xor_si512(c2[7981],_mm512_xor_si512(c2[1120],_mm512_xor_si512(c2[3759],_mm512_xor_si512(c2[13793],_mm512_xor_si512(c2[10106],_mm512_xor_si512(c2[2981],_mm512_xor_si512(c2[3245],_mm512_xor_si512(c2[12481],_mm512_xor_si512(c2[5620],_mm512_xor_si512(c2[5618],_mm512_xor_si512(c2[15652],_mm512_xor_si512(c2[6688],_mm512_xor_si512(c2[16716],_mm512_xor_si512(c2[6687],_mm512_xor_si512(c2[16721],_mm512_xor_si512(c2[10381],_mm512_xor_si512(c2[3520],_mm512_xor_si512(c2[6700],_mm512_xor_si512(c2[16728],_mm512_xor_si512(c2[12241],_mm512_xor_si512(c2[5380],_mm512_xor_si512(c2[1680],_mm512_xor_si512(c2[11714],_mm512_xor_si512(c2[3016],_mm512_xor_si512(c2[12780],_mm512_xor_si512(c2[13044],_mm512_xor_si512(c2[110],_mm512_xor_si512(c2[10144],_mm512_xor_si512(c2[3540],_mm512_xor_si512(c2[13574],_mm512_xor_si512(c2[12267],_mm512_xor_si512(c2[5136],_mm512_xor_si512(c2[5400],_mm512_xor_si512(c2[4081],_mm512_xor_si512(c2[14115],_mm512_xor_si512(c2[4873],_mm512_xor_si512(c2[14907],_mm512_xor_si512(c2[15708],_mm512_xor_si512(c2[8583],_mm512_xor_si512(c2[8847],_mm512_xor_si512(c2[14127],_mm512_xor_si512(c2[7260],_mm512_xor_si512(c2[16767],_mm512_xor_si512(c2[9900],_mm512_xor_si512(c2[8333],_mm512_xor_si512(c2[1202],_mm512_xor_si512(c2[1466],_mm512_xor_si512(c2[9652],_mm512_xor_si512(c2[2785],_mm512_xor_si512(c2[14138],_mm512_xor_si512(c2[7277],_mm512_xor_si512(c2[2256],_mm512_xor_si512(c2[13624],_mm512_xor_si512(c2[6493],_mm512_xor_si512(c2[6757],_mm512_xor_si512(c2[5966],_mm512_xor_si512(c2[16000],_mm512_xor_si512(c2[9400],_mm512_xor_si512(c2[2533],_mm512_xor_si512(c2[10466],_mm512_xor_si512(c2[3605],_mm512_xor_si512(c2[10200],_mm512_xor_si512(c2[3339],_mm512_xor_si512(c2[9146],_mm512_xor_si512(c2[2285],_mm512_xor_si512(c2[9950],_mm512_xor_si512(c2[2825],_mm512_xor_si512(c2[3089],_mm512_xor_si512(c2[13116],_mm512_xor_si512(c2[6255],_mm512_xor_si512(c2[972],_mm512_xor_si512(c2[11006],_mm512_xor_si512(c2[14976],_mm512_xor_si512(c2[7851],_mm512_xor_si512(c2[8115],_mm512_xor_si512(c2[12868],_mm512_xor_si512(c2[6001],_mm512_xor_si512(c2[16563],_mm512_xor_si512(c2[9696],_mm512_xor_si512(c2[12600],_mm512_xor_si512(c2[205],_mm512_xor_si512(c2[10239],_mm512_xor_si512(c2[4693],_mm512_xor_si512(c2[14727],_mm512_xor_si512(c2[15519],_mm512_xor_si512(c2[8652],_mm512_xor_si512(c2[219],_mm512_xor_si512(c2[9989],_mm512_xor_si512(c2[10253],_mm512_xor_si512(c2[16586],_mm512_xor_si512(c2[9725],_mm512_xor_si512(c2[7875],_mm512_xor_si512(c2[1008],_mm512_xor_si512(c2[758],_mm512_xor_si512(c2[10528],_mm512_xor_si512(c2[10792],_mm512_xor_si512(c2[6832],_mm512_xor_si512(c2[16860],_mm512_xor_si512(c2[7358],_mm512_xor_si512(c2[497],_mm512_xor_si512(c2[6580],_mm512_xor_si512(c2[16344],_mm512_xor_si512(c2[16608],_mm512_xor_si512(c2[7373],_mm512_xor_si512(c2[506],_mm512_xor_si512(c2[14237],_mm512_xor_si512(c2[7370],_mm512_xor_si512(c2[10816],_mm512_xor_si512(c2[3685],_mm512_xor_si512(c2[3949],_mm512_xor_si512(c2[16620],_mm512_xor_si512(c2[9759],_mm512_xor_si512(c2[13720],_mm512_xor_si512(c2[6853],c2[16094]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 12 + d2[72]=_mm512_xor_si512(c2[3433],_mm512_xor_si512(c2[15326],_mm512_xor_si512(c2[12266],_mm512_xor_si512(c2[10695],_mm512_xor_si512(c2[13356],c2[2857]))))); + +//row: 13 + d2[78]=_mm512_xor_si512(c2[9772],_mm512_xor_si512(c2[10036],_mm512_xor_si512(c2[16632],_mm512_xor_si512(c2[3],_mm512_xor_si512(c2[1852],_mm512_xor_si512(c2[15052],_mm512_xor_si512(c2[1332],_mm512_xor_si512(c2[1596],_mm512_xor_si512(c2[11895],_mm512_xor_si512(c2[2653],_mm512_xor_si512(c2[9528],_mm512_xor_si512(c2[9792],_mm512_xor_si512(c2[16656],_mm512_xor_si512(c2[15604],_mm512_xor_si512(c2[9281],_mm512_xor_si512(c2[9545],_mm512_xor_si512(c2[15876],_mm512_xor_si512(c2[13505],_mm512_xor_si512(c2[12974],_mm512_xor_si512(c2[5068],_mm512_xor_si512(c2[16684],_mm512_xor_si512(c2[1636],_mm512_xor_si512(c2[10358],_mm512_xor_si512(c2[10622],_mm512_xor_si512(c2[854],_mm512_xor_si512(c2[13527],_mm512_xor_si512(c2[2715],_mm512_xor_si512(c2[2979],_mm512_xor_si512(c2[5354],_mm512_xor_si512(c2[15386],_mm512_xor_si512(c2[16456],_mm512_xor_si512(c2[16455],_mm512_xor_si512(c2[3254],_mm512_xor_si512(c2[2728],_mm512_xor_si512(c2[16468],_mm512_xor_si512(c2[5114],_mm512_xor_si512(c2[11448],_mm512_xor_si512(c2[12520],_mm512_xor_si512(c2[12784],_mm512_xor_si512(c2[9878],_mm512_xor_si512(c2[13308],_mm512_xor_si512(c2[4876],_mm512_xor_si512(c2[5140],_mm512_xor_si512(c2[13849],_mm512_xor_si512(c2[14641],_mm512_xor_si512(c2[8317],_mm512_xor_si512(c2[8581],_mm512_xor_si512(c2[7000],_mm512_xor_si512(c2[9640],_mm512_xor_si512(c2[936],_mm512_xor_si512(c2[1200],_mm512_xor_si512(c2[2525],_mm512_xor_si512(c2[7011],_mm512_xor_si512(c2[6233],_mm512_xor_si512(c2[6497],_mm512_xor_si512(c2[15734],_mm512_xor_si512(c2[2273],_mm512_xor_si512(c2[3339],_mm512_xor_si512(c2[3073],_mm512_xor_si512(c2[2019],_mm512_xor_si512(c2[2559],_mm512_xor_si512(c2[2823],_mm512_xor_si512(c2[5989],_mm512_xor_si512(c2[10740],_mm512_xor_si512(c2[7585],_mm512_xor_si512(c2[7849],_mm512_xor_si512(c2[5741],_mm512_xor_si512(c2[9436],_mm512_xor_si512(c2[9973],_mm512_xor_si512(c2[14461],_mm512_xor_si512(c2[8392],_mm512_xor_si512(c2[9723],_mm512_xor_si512(c2[9987],_mm512_xor_si512(c2[9459],_mm512_xor_si512(c2[748],_mm512_xor_si512(c2[10262],_mm512_xor_si512(c2[10526],_mm512_xor_si512(c2[16600],_mm512_xor_si512(c2[231],_mm512_xor_si512(c2[16084],_mm512_xor_si512(c2[16348],_mm512_xor_si512(c2[240],_mm512_xor_si512(c2[7104],_mm512_xor_si512(c2[13705],_mm512_xor_si512(c2[3425],_mm512_xor_si512(c2[3689],_mm512_xor_si512(c2[9493],c2[6593]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 14 + d2[84]=_mm512_xor_si512(c2[3698],_mm512_xor_si512(c2[14931],_mm512_xor_si512(c2[2558],_mm512_xor_si512(c2[6793],_mm512_xor_si512(c2[7337],c2[3420]))))); + +//row: 15 + d2[90]=_mm512_xor_si512(c2[267],_mm512_xor_si512(c2[6869],_mm512_xor_si512(c2[7129],_mm512_xor_si512(c2[8714],_mm512_xor_si512(c2[8978],_mm512_xor_si512(c2[12939],_mm512_xor_si512(c2[8728],_mm512_xor_si512(c2[2126],_mm512_xor_si512(c2[9521],_mm512_xor_si512(c2[9785],_mm512_xor_si512(c2[540],_mm512_xor_si512(c2[29],_mm512_xor_si512(c2[6893],_mm512_xor_si512(c2[5835],_mm512_xor_si512(c2[16671],_mm512_xor_si512(c2[6113],_mm512_xor_si512(c2[3472],_mm512_xor_si512(c2[3736],_mm512_xor_si512(c2[12194],_mm512_xor_si512(c2[6915],_mm512_xor_si512(c2[8498],_mm512_xor_si512(c2[8762],_mm512_xor_si512(c2[853],_mm512_xor_si512(c2[7980],_mm512_xor_si512(c2[3758],_mm512_xor_si512(c2[10105],_mm512_xor_si512(c2[12480],_mm512_xor_si512(c2[5353],_mm512_xor_si512(c2[5617],_mm512_xor_si512(c2[6687],_mm512_xor_si512(c2[6686],_mm512_xor_si512(c2[10116],_mm512_xor_si512(c2[10380],_mm512_xor_si512(c2[6699],_mm512_xor_si512(c2[12240],_mm512_xor_si512(c2[1421],_mm512_xor_si512(c2[1685],_mm512_xor_si512(c2[3015],_mm512_xor_si512(c2[109],_mm512_xor_si512(c2[3545],_mm512_xor_si512(c2[12266],_mm512_xor_si512(c2[4080],_mm512_xor_si512(c2[4608],_mm512_xor_si512(c2[4872],_mm512_xor_si512(c2[4875],_mm512_xor_si512(c2[15713],_mm512_xor_si512(c2[14126],_mm512_xor_si512(c2[16502],_mm512_xor_si512(c2[16766],_mm512_xor_si512(c2[8332],_mm512_xor_si512(c2[9651],_mm512_xor_si512(c2[13873],_mm512_xor_si512(c2[14137],_mm512_xor_si512(c2[13623],_mm512_xor_si512(c2[5965],_mm512_xor_si512(c2[9135],_mm512_xor_si512(c2[9399],_mm512_xor_si512(c2[16528],_mm512_xor_si512(c2[10465],_mm512_xor_si512(c2[10205],_mm512_xor_si512(c2[8881],_mm512_xor_si512(c2[9145],_mm512_xor_si512(c2[9949],_mm512_xor_si512(c2[13121],_mm512_xor_si512(c2[977],_mm512_xor_si512(c2[14981],_mm512_xor_si512(c2[12867],_mm512_xor_si512(c2[16298],_mm512_xor_si512(c2[16562],_mm512_xor_si512(c2[204],_mm512_xor_si512(c2[4692],_mm512_xor_si512(c2[15254],_mm512_xor_si512(c2[15518],_mm512_xor_si512(c2[218],_mm512_xor_si512(c2[16585],_mm512_xor_si512(c2[7610],_mm512_xor_si512(c2[7874],_mm512_xor_si512(c2[14736],_mm512_xor_si512(c2[757],_mm512_xor_si512(c2[6831],_mm512_xor_si512(c2[7357],_mm512_xor_si512(c2[6579],_mm512_xor_si512(c2[7372],_mm512_xor_si512(c2[13972],_mm512_xor_si512(c2[14236],_mm512_xor_si512(c2[10815],_mm512_xor_si512(c2[16625],_mm512_xor_si512(c2[13455],c2[13719])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 16 + d2[96]=_mm512_xor_si512(c2[11621],_mm512_xor_si512(c2[1322],_mm512_xor_si512(c2[1588],_mm512_xor_si512(c2[3437],_mm512_xor_si512(c2[3181],_mm512_xor_si512(c2[13480],_mm512_xor_si512(c2[4238],_mm512_xor_si512(c2[3444],_mm512_xor_si512(c2[11377],_mm512_xor_si512(c2[1346],_mm512_xor_si512(c2[288],_mm512_xor_si512(c2[11124],_mm512_xor_si512(c2[566],_mm512_xor_si512(c2[15084],_mm512_xor_si512(c2[4793],_mm512_xor_si512(c2[6653],_mm512_xor_si512(c2[1368],_mm512_xor_si512(c2[3221],_mm512_xor_si512(c2[12207],_mm512_xor_si512(c2[2439],_mm512_xor_si512(c2[15112],_mm512_xor_si512(c2[4564],_mm512_xor_si512(c2[6939],_mm512_xor_si512(c2[76],_mm512_xor_si512(c2[1140],_mm512_xor_si512(c2[1145],_mm512_xor_si512(c2[4839],_mm512_xor_si512(c2[1152],_mm512_xor_si512(c2[6699],_mm512_xor_si512(c2[13033],_mm512_xor_si512(c2[14369],_mm512_xor_si512(c2[11463],_mm512_xor_si512(c2[14893],_mm512_xor_si512(c2[6725],_mm512_xor_si512(c2[15434],_mm512_xor_si512(c2[16226],_mm512_xor_si512(c2[10166],_mm512_xor_si512(c2[8585],_mm512_xor_si512(c2[11225],_mm512_xor_si512(c2[15180],_mm512_xor_si512(c2[2785],_mm512_xor_si512(c2[4104],_mm512_xor_si512(c2[8596],_mm512_xor_si512(c2[8076],_mm512_xor_si512(c2[424],_mm512_xor_si512(c2[3852],_mm512_xor_si512(c2[4924],_mm512_xor_si512(c2[4658],_mm512_xor_si512(c2[3604],_mm512_xor_si512(c2[4408],_mm512_xor_si512(c2[7574],_mm512_xor_si512(c2[12325],_mm512_xor_si512(c2[9434],_mm512_xor_si512(c2[7320],_mm512_xor_si512(c2[11021],_mm512_xor_si512(c2[11558],_mm512_xor_si512(c2[16046],_mm512_xor_si512(c2[9977],_mm512_xor_si512(c2[11572],_mm512_xor_si512(c2[11044],_mm512_xor_si512(c2[2333],_mm512_xor_si512(c2[12111],_mm512_xor_si512(c2[1284],_mm512_xor_si512(c2[1816],_mm512_xor_si512(c2[1032],_mm512_xor_si512(c2[1825],_mm512_xor_si512(c2[8689],_mm512_xor_si512(c2[8956],_mm512_xor_si512(c2[5268],_mm512_xor_si512(c2[11078],c2[8172])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 17 + d2[102]=_mm512_xor_si512(c2[1060],_mm512_xor_si512(c2[12580],_mm512_xor_si512(c2[4681],_mm512_xor_si512(c2[10241],c2[14777])))); + +//row: 18 + d2[108]=_mm512_xor_si512(c2[542],_mm512_xor_si512(c2[9386],_mm512_xor_si512(c2[6496],_mm512_xor_si512(c2[1274],c2[1284])))); + +//row: 19 + d2[114]=_mm512_xor_si512(c2[4490],_mm512_xor_si512(c2[5559],_mm512_xor_si512(c2[6425],_mm512_xor_si512(c2[13299],c2[1443])))); + +//row: 20 + d2[120]=_mm512_xor_si512(c2[12149],_mm512_xor_si512(c2[1850],_mm512_xor_si512(c2[2116],_mm512_xor_si512(c2[3965],_mm512_xor_si512(c2[15578],_mm512_xor_si512(c2[3709],_mm512_xor_si512(c2[14008],_mm512_xor_si512(c2[4766],_mm512_xor_si512(c2[11905],_mm512_xor_si512(c2[1874],_mm512_xor_si512(c2[816],_mm512_xor_si512(c2[11652],_mm512_xor_si512(c2[1094],_mm512_xor_si512(c2[15612],_mm512_xor_si512(c2[3735],_mm512_xor_si512(c2[7181],_mm512_xor_si512(c2[1896],_mm512_xor_si512(c2[3749],_mm512_xor_si512(c2[12735],_mm512_xor_si512(c2[2967],_mm512_xor_si512(c2[15640],_mm512_xor_si512(c2[5092],_mm512_xor_si512(c2[7467],_mm512_xor_si512(c2[604],_mm512_xor_si512(c2[1668],_mm512_xor_si512(c2[1673],_mm512_xor_si512(c2[5367],_mm512_xor_si512(c2[1680],_mm512_xor_si512(c2[7227],_mm512_xor_si512(c2[13561],_mm512_xor_si512(c2[14897],_mm512_xor_si512(c2[11991],_mm512_xor_si512(c2[15421],_mm512_xor_si512(c2[2224],_mm512_xor_si512(c2[7253],_mm512_xor_si512(c2[15962],_mm512_xor_si512(c2[16754],_mm512_xor_si512(c2[10694],_mm512_xor_si512(c2[9113],_mm512_xor_si512(c2[11753],_mm512_xor_si512(c2[5681],_mm512_xor_si512(c2[3313],_mm512_xor_si512(c2[4632],_mm512_xor_si512(c2[9124],_mm512_xor_si512(c2[8604],_mm512_xor_si512(c2[952],_mm512_xor_si512(c2[4380],_mm512_xor_si512(c2[5452],_mm512_xor_si512(c2[5186],_mm512_xor_si512(c2[4132],_mm512_xor_si512(c2[4936],_mm512_xor_si512(c2[8102],_mm512_xor_si512(c2[12853],_mm512_xor_si512(c2[9962],_mm512_xor_si512(c2[7848],_mm512_xor_si512(c2[11549],_mm512_xor_si512(c2[12086],_mm512_xor_si512(c2[16574],_mm512_xor_si512(c2[10505],_mm512_xor_si512(c2[12100],_mm512_xor_si512(c2[11572],_mm512_xor_si512(c2[2861],_mm512_xor_si512(c2[12639],_mm512_xor_si512(c2[1812],_mm512_xor_si512(c2[2344],_mm512_xor_si512(c2[1560],_mm512_xor_si512(c2[2353],_mm512_xor_si512(c2[9217],_mm512_xor_si512(c2[5796],_mm512_xor_si512(c2[11606],c2[8700])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 21 + d2[126]=_mm512_xor_si512(c2[3447],_mm512_xor_si512(c2[10093],_mm512_xor_si512(c2[2309],_mm512_xor_si512(c2[5787],c2[8965])))); + +//row: 22 + d2[132]=_mm512_xor_si512(c2[7920],_mm512_xor_si512(c2[3048],_mm512_xor_si512(c2[10983],c2[6012]))); + +//row: 23 + d2[138]=_mm512_xor_si512(c2[6348],_mm512_xor_si512(c2[6625],_mm512_xor_si512(c2[16224],c2[7344]))); + +//row: 24 + d2[144]=_mm512_xor_si512(c2[7658],_mm512_xor_si512(c2[14260],_mm512_xor_si512(c2[14520],_mm512_xor_si512(c2[16369],_mm512_xor_si512(c2[11092],_mm512_xor_si512(c2[16119],_mm512_xor_si512(c2[9517],_mm512_xor_si512(c2[281],_mm512_xor_si512(c2[7420],_mm512_xor_si512(c2[14284],_mm512_xor_si512(c2[13226],_mm512_xor_si512(c2[7167],_mm512_xor_si512(c2[13504],_mm512_xor_si512(c2[11127],_mm512_xor_si512(c2[7958],_mm512_xor_si512(c2[2690],_mm512_xor_si512(c2[14306],_mm512_xor_si512(c2[16153],_mm512_xor_si512(c2[11403],_mm512_xor_si512(c2[8244],_mm512_xor_si512(c2[15377],_mm512_xor_si512(c2[11149],_mm512_xor_si512(c2[601],_mm512_xor_si512(c2[2976],_mm512_xor_si512(c2[13008],_mm512_xor_si512(c2[14078],_mm512_xor_si512(c2[14077],_mm512_xor_si512(c2[876],_mm512_xor_si512(c2[14090],_mm512_xor_si512(c2[2736],_mm512_xor_si512(c2[9076],_mm512_xor_si512(c2[10406],_mm512_xor_si512(c2[7500],_mm512_xor_si512(c2[10936],_mm512_xor_si512(c2[2762],_mm512_xor_si512(c2[11477],_mm512_xor_si512(c2[12269],_mm512_xor_si512(c2[6209],_mm512_xor_si512(c2[4622],_mm512_xor_si512(c2[7262],_mm512_xor_si512(c2[5153],_mm512_xor_si512(c2[15723],_mm512_xor_si512(c2[147],_mm512_xor_si512(c2[4633],_mm512_xor_si512(c2[4119],_mm512_xor_si512(c2[13356],_mm512_xor_si512(c2[16790],_mm512_xor_si512(c2[961],_mm512_xor_si512(c2[701],_mm512_xor_si512(c2[16536],_mm512_xor_si512(c2[445],_mm512_xor_si512(c2[3617],_mm512_xor_si512(c2[8368],_mm512_xor_si512(c2[5477],_mm512_xor_si512(c2[3363],_mm512_xor_si512(c2[7058],_mm512_xor_si512(c2[7601],_mm512_xor_si512(c2[12089],_mm512_xor_si512(c2[6014],_mm512_xor_si512(c2[7609],_mm512_xor_si512(c2[7081],_mm512_xor_si512(c2[15265],_mm512_xor_si512(c2[8148],_mm512_xor_si512(c2[14222],_mm512_xor_si512(c2[14748],_mm512_xor_si512(c2[13970],_mm512_xor_si512(c2[14763],_mm512_xor_si512(c2[4732],_mm512_xor_si512(c2[1311],_mm512_xor_si512(c2[7121],c2[4215])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 25 + d2[150]=_mm512_xor_si512(c2[2125],_mm512_xor_si512(c2[4560],_mm512_xor_si512(c2[16721],c2[14956]))); + +//row: 26 + d2[156]=_mm512_xor_si512(c2[1849],_mm512_xor_si512(c2[4513],_mm512_xor_si512(c2[3217],c2[2294]))); + +//row: 27 + d2[162]=_mm512_xor_si512(c2[543],_mm512_xor_si512(c2[603],c2[9865])); + +//row: 28 + d2[168]=_mm512_xor_si512(c2[7923],_mm512_xor_si512(c2[5064],_mm512_xor_si512(c2[13959],c2[5008]))); + +//row: 29 + d2[174]=_mm512_xor_si512(c2[528],_mm512_xor_si512(c2[7130],_mm512_xor_si512(c2[7396],_mm512_xor_si512(c2[8981],_mm512_xor_si512(c2[9245],_mm512_xor_si512(c2[8989],_mm512_xor_si512(c2[2393],_mm512_xor_si512(c2[9782],_mm512_xor_si512(c2[10046],_mm512_xor_si512(c2[15855],_mm512_xor_si512(c2[290],_mm512_xor_si512(c2[7154],_mm512_xor_si512(c2[6096],_mm512_xor_si512(c2[37],_mm512_xor_si512(c2[6374],_mm512_xor_si512(c2[3733],_mm512_xor_si512(c2[3997],_mm512_xor_si512(c2[12461],_mm512_xor_si512(c2[7176],_mm512_xor_si512(c2[8765],_mm512_xor_si512(c2[9029],_mm512_xor_si512(c2[1120],_mm512_xor_si512(c2[8247],_mm512_xor_si512(c2[4025],_mm512_xor_si512(c2[10372],_mm512_xor_si512(c2[12747],_mm512_xor_si512(c2[5620],_mm512_xor_si512(c2[5884],_mm512_xor_si512(c2[6948],_mm512_xor_si512(c2[6953],_mm512_xor_si512(c2[10383],_mm512_xor_si512(c2[10647],_mm512_xor_si512(c2[6960],_mm512_xor_si512(c2[12507],_mm512_xor_si512(c2[1682],_mm512_xor_si512(c2[1946],_mm512_xor_si512(c2[3276],_mm512_xor_si512(c2[376],_mm512_xor_si512(c2[3806],_mm512_xor_si512(c2[12533],_mm512_xor_si512(c2[4347],_mm512_xor_si512(c2[4875],_mm512_xor_si512(c2[5139],_mm512_xor_si512(c2[15974],_mm512_xor_si512(c2[14393],_mm512_xor_si512(c2[16769],_mm512_xor_si512(c2[132],_mm512_xor_si512(c2[8593],_mm512_xor_si512(c2[9912],_mm512_xor_si512(c2[14140],_mm512_xor_si512(c2[14404],_mm512_xor_si512(c2[13884],_mm512_xor_si512(c2[6232],_mm512_xor_si512(c2[9396],_mm512_xor_si512(c2[9660],_mm512_xor_si512(c2[10732],_mm512_xor_si512(c2[10466],_mm512_xor_si512(c2[9148],_mm512_xor_si512(c2[9412],_mm512_xor_si512(c2[1488],_mm512_xor_si512(c2[10216],_mm512_xor_si512(c2[13382],_mm512_xor_si512(c2[1238],_mm512_xor_si512(c2[15242],_mm512_xor_si512(c2[13128],_mm512_xor_si512(c2[16565],_mm512_xor_si512(c2[16829],_mm512_xor_si512(c2[471],_mm512_xor_si512(c2[4959],_mm512_xor_si512(c2[15521],_mm512_xor_si512(c2[15785],_mm512_xor_si512(c2[485],_mm512_xor_si512(c2[16852],_mm512_xor_si512(c2[7877],_mm512_xor_si512(c2[8141],_mm512_xor_si512(c2[5234],_mm512_xor_si512(c2[1024],_mm512_xor_si512(c2[7092],_mm512_xor_si512(c2[7624],_mm512_xor_si512(c2[6840],_mm512_xor_si512(c2[7633],_mm512_xor_si512(c2[14233],_mm512_xor_si512(c2[14497],_mm512_xor_si512(c2[11076],_mm512_xor_si512(c2[16886],_mm512_xor_si512(c2[13716],c2[13980])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 30 + d2[180]=_mm512_xor_si512(c2[3696],_mm512_xor_si512(c2[10298],_mm512_xor_si512(c2[10300],_mm512_xor_si512(c2[10564],_mm512_xor_si512(c2[12149],_mm512_xor_si512(c2[12413],_mm512_xor_si512(c2[8186],_mm512_xor_si512(c2[12157],_mm512_xor_si512(c2[5297],_mm512_xor_si512(c2[5561],_mm512_xor_si512(c2[12950],_mm512_xor_si512(c2[13214],_mm512_xor_si512(c2[3458],_mm512_xor_si512(c2[10322],_mm512_xor_si512(c2[9000],_mm512_xor_si512(c2[9264],_mm512_xor_si512(c2[3205],_mm512_xor_si512(c2[9542],_mm512_xor_si512(c2[6901],_mm512_xor_si512(c2[7165],_mm512_xor_si512(c2[15629],_mm512_xor_si512(c2[10080],_mm512_xor_si512(c2[10344],_mm512_xor_si512(c2[11933],_mm512_xor_si512(c2[12197],_mm512_xor_si512(c2[4288],_mm512_xor_si512(c2[11415],_mm512_xor_si512(c2[6929],_mm512_xor_si512(c2[7193],_mm512_xor_si512(c2[13540],_mm512_xor_si512(c2[15651],_mm512_xor_si512(c2[15915],_mm512_xor_si512(c2[8788],_mm512_xor_si512(c2[9052],_mm512_xor_si512(c2[10116],_mm512_xor_si512(c2[9857],_mm512_xor_si512(c2[10121],_mm512_xor_si512(c2[13551],_mm512_xor_si512(c2[13815],_mm512_xor_si512(c2[10128],_mm512_xor_si512(c2[15411],_mm512_xor_si512(c2[15675],_mm512_xor_si512(c2[4850],_mm512_xor_si512(c2[5114],_mm512_xor_si512(c2[6444],_mm512_xor_si512(c2[3544],_mm512_xor_si512(c2[6710],_mm512_xor_si512(c2[6974],_mm512_xor_si512(c2[15701],_mm512_xor_si512(c2[7251],_mm512_xor_si512(c2[7515],_mm512_xor_si512(c2[8043],_mm512_xor_si512(c2[8307],_mm512_xor_si512(c2[9891],_mm512_xor_si512(c2[2247],_mm512_xor_si512(c2[660],_mm512_xor_si512(c2[3036],_mm512_xor_si512(c2[3300],_mm512_xor_si512(c2[11761],_mm512_xor_si512(c2[13080],_mm512_xor_si512(c2[413],_mm512_xor_si512(c2[677],_mm512_xor_si512(c2[157],_mm512_xor_si512(c2[9136],_mm512_xor_si512(c2[9400],_mm512_xor_si512(c2[12564],_mm512_xor_si512(c2[12828],_mm512_xor_si512(c2[1216],_mm512_xor_si512(c2[13900],_mm512_xor_si512(c2[13370],_mm512_xor_si512(c2[13634],_mm512_xor_si512(c2[12316],_mm512_xor_si512(c2[12580],_mm512_xor_si512(c2[13384],_mm512_xor_si512(c2[16550],_mm512_xor_si512(c2[4142],_mm512_xor_si512(c2[4406],_mm512_xor_si512(c2[1515],_mm512_xor_si512(c2[16296],_mm512_xor_si512(c2[2832],_mm512_xor_si512(c2[3096],_mm512_xor_si512(c2[3639],_mm512_xor_si512(c2[7863],_mm512_xor_si512(c2[8127],_mm512_xor_si512(c2[1788],_mm512_xor_si512(c2[2052],_mm512_xor_si512(c2[3653],_mm512_xor_si512(c2[2861],_mm512_xor_si512(c2[3125],_mm512_xor_si512(c2[11045],_mm512_xor_si512(c2[11309],_mm512_xor_si512(c2[4192],_mm512_xor_si512(c2[10260],_mm512_xor_si512(c2[10528],_mm512_xor_si512(c2[10792],_mm512_xor_si512(c2[10008],_mm512_xor_si512(c2[10537],_mm512_xor_si512(c2[10801],_mm512_xor_si512(c2[506],_mm512_xor_si512(c2[770],_mm512_xor_si512(c2[14244],_mm512_xor_si512(c2[3159],_mm512_xor_si512(c2[16884],c2[253]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 31 + d2[186]=_mm512_xor_si512(c2[13994],_mm512_xor_si512(c2[13730],_mm512_xor_si512(c2[3701],_mm512_xor_si512(c2[3437],_mm512_xor_si512(c2[3961],_mm512_xor_si512(c2[3697],_mm512_xor_si512(c2[5810],_mm512_xor_si512(c2[5282],_mm512_xor_si512(c2[5546],_mm512_xor_si512(c2[5560],_mm512_xor_si512(c2[5296],_mm512_xor_si512(c2[15853],_mm512_xor_si512(c2[15589],_mm512_xor_si512(c2[6617],_mm512_xor_si512(c2[6089],_mm512_xor_si512(c2[6353],_mm512_xor_si512(c2[9517],_mm512_xor_si512(c2[13756],_mm512_xor_si512(c2[13492],_mm512_xor_si512(c2[3725],_mm512_xor_si512(c2[3461],_mm512_xor_si512(c2[2667],_mm512_xor_si512(c2[2403],_mm512_xor_si512(c2[13503],_mm512_xor_si512(c2[13239],_mm512_xor_si512(c2[2945],_mm512_xor_si512(c2[2681],_mm512_xor_si512(c2[568],_mm512_xor_si512(c2[40],_mm512_xor_si512(c2[304],_mm512_xor_si512(c2[9026],_mm512_xor_si512(c2[8762],_mm512_xor_si512(c2[3747],_mm512_xor_si512(c2[3483],_mm512_xor_si512(c2[5594],_mm512_xor_si512(c2[5066],_mm512_xor_si512(c2[5330],_mm512_xor_si512(c2[14580],_mm512_xor_si512(c2[14316],_mm512_xor_si512(c2[4812],_mm512_xor_si512(c2[4548],_mm512_xor_si512(c2[590],_mm512_xor_si512(c2[326],_mm512_xor_si512(c2[6937],_mm512_xor_si512(c2[6673],_mm512_xor_si512(c2[9312],_mm512_xor_si512(c2[9048],_mm512_xor_si512(c2[2449],_mm512_xor_si512(c2[1921],_mm512_xor_si512(c2[2185],_mm512_xor_si512(c2[3519],_mm512_xor_si512(c2[3255],_mm512_xor_si512(c2[3518],_mm512_xor_si512(c2[3254],_mm512_xor_si512(c2[7212],_mm512_xor_si512(c2[6684],_mm512_xor_si512(c2[6948],_mm512_xor_si512(c2[6159],_mm512_xor_si512(c2[3531],_mm512_xor_si512(c2[3267],_mm512_xor_si512(c2[9072],_mm512_xor_si512(c2[8808],_mm512_xor_si512(c2[15412],_mm512_xor_si512(c2[14884],_mm512_xor_si512(c2[15148],_mm512_xor_si512(c2[16742],_mm512_xor_si512(c2[16478],_mm512_xor_si512(c2[13836],_mm512_xor_si512(c2[13572],_mm512_xor_si512(c2[377],_mm512_xor_si512(c2[113],_mm512_xor_si512(c2[9098],_mm512_xor_si512(c2[8834],_mm512_xor_si512(c2[912],_mm512_xor_si512(c2[648],_mm512_xor_si512(c2[1704],_mm512_xor_si512(c2[1176],_mm512_xor_si512(c2[1440],_mm512_xor_si512(c2[12545],_mm512_xor_si512(c2[12281],_mm512_xor_si512(c2[10958],_mm512_xor_si512(c2[10694],_mm512_xor_si512(c2[13598],_mm512_xor_si512(c2[13070],_mm512_xor_si512(c2[13334],_mm512_xor_si512(c2[5164],_mm512_xor_si512(c2[4900],_mm512_xor_si512(c2[6483],_mm512_xor_si512(c2[6219],_mm512_xor_si512(c2[10969],_mm512_xor_si512(c2[10441],_mm512_xor_si512(c2[10705],_mm512_xor_si512(c2[10455],_mm512_xor_si512(c2[10191],_mm512_xor_si512(c2[2797],_mm512_xor_si512(c2[2533],_mm512_xor_si512(c2[6231],_mm512_xor_si512(c2[5703],_mm512_xor_si512(c2[5967],_mm512_xor_si512(c2[7297],_mm512_xor_si512(c2[7033],_mm512_xor_si512(c2[7037],_mm512_xor_si512(c2[6773],_mm512_xor_si512(c2[5977],_mm512_xor_si512(c2[5449],_mm512_xor_si512(c2[5713],_mm512_xor_si512(c2[6781],_mm512_xor_si512(c2[6517],_mm512_xor_si512(c2[9953],_mm512_xor_si512(c2[9689],_mm512_xor_si512(c2[14704],_mm512_xor_si512(c2[14440],_mm512_xor_si512(c2[11813],_mm512_xor_si512(c2[11549],_mm512_xor_si512(c2[9699],_mm512_xor_si512(c2[9435],_mm512_xor_si512(c2[13394],_mm512_xor_si512(c2[12866],_mm512_xor_si512(c2[13130],_mm512_xor_si512(c2[13937],_mm512_xor_si512(c2[13673],_mm512_xor_si512(c2[1524],_mm512_xor_si512(c2[1260],_mm512_xor_si512(c2[12350],_mm512_xor_si512(c2[11822],_mm512_xor_si512(c2[12086],_mm512_xor_si512(c2[13945],_mm512_xor_si512(c2[13681],_mm512_xor_si512(c2[13417],_mm512_xor_si512(c2[13153],_mm512_xor_si512(c2[4706],_mm512_xor_si512(c2[4178],_mm512_xor_si512(c2[4442],_mm512_xor_si512(c2[14484],_mm512_xor_si512(c2[14220],_mm512_xor_si512(c2[3663],_mm512_xor_si512(c2[3399],_mm512_xor_si512(c2[4189],_mm512_xor_si512(c2[3925],_mm512_xor_si512(c2[3411],_mm512_xor_si512(c2[3147],_mm512_xor_si512(c2[4204],_mm512_xor_si512(c2[3940],_mm512_xor_si512(c2[11068],_mm512_xor_si512(c2[10540],_mm512_xor_si512(c2[10804],_mm512_xor_si512(c2[7647],_mm512_xor_si512(c2[7383],_mm512_xor_si512(c2[13457],_mm512_xor_si512(c2[13193],_mm512_xor_si512(c2[10551],_mm512_xor_si512(c2[10023],c2[10287])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 32 + d2[192]=_mm512_xor_si512(c2[5019],_mm512_xor_si512(c2[11621],_mm512_xor_si512(c2[11617],_mm512_xor_si512(c2[11881],_mm512_xor_si512(c2[13466],_mm512_xor_si512(c2[13730],_mm512_xor_si512(c2[10033],_mm512_xor_si512(c2[13480],_mm512_xor_si512(c2[6614],_mm512_xor_si512(c2[6878],_mm512_xor_si512(c2[14273],_mm512_xor_si512(c2[14537],_mm512_xor_si512(c2[4781],_mm512_xor_si512(c2[11645],_mm512_xor_si512(c2[10323],_mm512_xor_si512(c2[10587],_mm512_xor_si512(c2[4528],_mm512_xor_si512(c2[10865],_mm512_xor_si512(c2[8224],_mm512_xor_si512(c2[8488],_mm512_xor_si512(c2[51],_mm512_xor_si512(c2[11403],_mm512_xor_si512(c2[11667],_mm512_xor_si512(c2[13250],_mm512_xor_si512(c2[13514],_mm512_xor_si512(c2[5605],_mm512_xor_si512(c2[12732],_mm512_xor_si512(c2[8246],_mm512_xor_si512(c2[8510],_mm512_xor_si512(c2[14857],_mm512_xor_si512(c2[73],_mm512_xor_si512(c2[337],_mm512_xor_si512(c2[10105],_mm512_xor_si512(c2[10369],_mm512_xor_si512(c2[11439],_mm512_xor_si512(c2[11174],_mm512_xor_si512(c2[11438],_mm512_xor_si512(c2[14868],_mm512_xor_si512(c2[15132],_mm512_xor_si512(c2[11451],_mm512_xor_si512(c2[16728],_mm512_xor_si512(c2[97],_mm512_xor_si512(c2[6173],_mm512_xor_si512(c2[6437],_mm512_xor_si512(c2[7767],_mm512_xor_si512(c2[4861],_mm512_xor_si512(c2[8033],_mm512_xor_si512(c2[8297],_mm512_xor_si512(c2[123],_mm512_xor_si512(c2[8568],_mm512_xor_si512(c2[8832],_mm512_xor_si512(c2[9360],_mm512_xor_si512(c2[9624],_mm512_xor_si512(c2[3564],_mm512_xor_si512(c2[1983],_mm512_xor_si512(c2[4359],_mm512_xor_si512(c2[4623],_mm512_xor_si512(c2[13084],_mm512_xor_si512(c2[14403],_mm512_xor_si512(c2[1730],_mm512_xor_si512(c2[1994],_mm512_xor_si512(c2[2523],_mm512_xor_si512(c2[1480],_mm512_xor_si512(c2[10453],_mm512_xor_si512(c2[10717],_mm512_xor_si512(c2[13887],_mm512_xor_si512(c2[14151],_mm512_xor_si512(c2[15217],_mm512_xor_si512(c2[14693],_mm512_xor_si512(c2[14957],_mm512_xor_si512(c2[13633],_mm512_xor_si512(c2[13897],_mm512_xor_si512(c2[12578],_mm512_xor_si512(c2[14701],_mm512_xor_si512(c2[972],_mm512_xor_si512(c2[5465],_mm512_xor_si512(c2[5729],_mm512_xor_si512(c2[2832],_mm512_xor_si512(c2[724],_mm512_xor_si512(c2[4155],_mm512_xor_si512(c2[4419],_mm512_xor_si512(c2[4956],_mm512_xor_si512(c2[9180],_mm512_xor_si512(c2[9444],_mm512_xor_si512(c2[3111],_mm512_xor_si512(c2[3375],_mm512_xor_si512(c2[4970],_mm512_xor_si512(c2[4178],_mm512_xor_si512(c2[4442],_mm512_xor_si512(c2[12362],_mm512_xor_si512(c2[12626],_mm512_xor_si512(c2[5509],_mm512_xor_si512(c2[11583],_mm512_xor_si512(c2[11845],_mm512_xor_si512(c2[12109],_mm512_xor_si512(c2[11331],_mm512_xor_si512(c2[11860],_mm512_xor_si512(c2[12124],_mm512_xor_si512(c2[1829],_mm512_xor_si512(c2[2093],_mm512_xor_si512(c2[15567],_mm512_xor_si512(c2[4476],_mm512_xor_si512(c2[1312],c2[1576]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 33 + d2[198]=_mm512_xor_si512(c2[809],_mm512_xor_si512(c2[2136],_mm512_xor_si512(c2[10961],c2[11077]))); + +//row: 34 + d2[204]=_mm512_xor_si512(c2[10035],_mm512_xor_si512(c2[5366],_mm512_xor_si512(c2[2823],c2[14992]))); + +//row: 35 + d2[210]=_mm512_xor_si512(c2[16106],_mm512_xor_si512(c2[5813],_mm512_xor_si512(c2[6073],_mm512_xor_si512(c2[7922],_mm512_xor_si512(c2[7672],_mm512_xor_si512(c2[1070],_mm512_xor_si512(c2[8729],_mm512_xor_si512(c2[17],_mm512_xor_si512(c2[15868],_mm512_xor_si512(c2[5837],_mm512_xor_si512(c2[4779],_mm512_xor_si512(c2[15615],_mm512_xor_si512(c2[5057],_mm512_xor_si512(c2[2680],_mm512_xor_si512(c2[11138],_mm512_xor_si512(c2[5859],_mm512_xor_si512(c2[7706],_mm512_xor_si512(c2[16692],_mm512_xor_si512(c2[6924],_mm512_xor_si512(c2[2702],_mm512_xor_si512(c2[9049],_mm512_xor_si512(c2[11424],_mm512_xor_si512(c2[4561],_mm512_xor_si512(c2[4037],_mm512_xor_si512(c2[5631],_mm512_xor_si512(c2[5630],_mm512_xor_si512(c2[9324],_mm512_xor_si512(c2[5643],_mm512_xor_si512(c2[11184],_mm512_xor_si512(c2[629],_mm512_xor_si512(c2[1959],_mm512_xor_si512(c2[15948],_mm512_xor_si512(c2[2489],_mm512_xor_si512(c2[11210],_mm512_xor_si512(c2[3024],_mm512_xor_si512(c2[3816],_mm512_xor_si512(c2[14657],_mm512_xor_si512(c2[13070],_mm512_xor_si512(c2[15710],_mm512_xor_si512(c2[7276],_mm512_xor_si512(c2[8595],_mm512_xor_si512(c2[13081],_mm512_xor_si512(c2[672],_mm512_xor_si512(c2[12567],_mm512_xor_si512(c2[4909],_mm512_xor_si512(c2[8343],_mm512_xor_si512(c2[9409],_mm512_xor_si512(c2[9149],_mm512_xor_si512(c2[8089],_mm512_xor_si512(c2[8893],_mm512_xor_si512(c2[12065],_mm512_xor_si512(c2[16816],_mm512_xor_si512(c2[13925],_mm512_xor_si512(c2[11811],_mm512_xor_si512(c2[15506],_mm512_xor_si512(c2[16049],_mm512_xor_si512(c2[3636],_mm512_xor_si512(c2[14462],_mm512_xor_si512(c2[16057],_mm512_xor_si512(c2[15529],_mm512_xor_si512(c2[6818],_mm512_xor_si512(c2[16596],_mm512_xor_si512(c2[5775],_mm512_xor_si512(c2[6301],_mm512_xor_si512(c2[5523],_mm512_xor_si512(c2[6316],_mm512_xor_si512(c2[13180],_mm512_xor_si512(c2[9759],_mm512_xor_si512(c2[15569],c2[12663]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 36 + d2[216]=_mm512_xor_si512(c2[4755],_mm512_xor_si512(c2[15220],_mm512_xor_si512(c2[11008],c2[5760]))); + +//row: 37 + d2[222]=_mm512_xor_si512(c2[10032],_mm512_xor_si512(c2[10296],_mm512_xor_si512(c2[3],_mm512_xor_si512(c2[269],_mm512_xor_si512(c2[2112],_mm512_xor_si512(c2[1598],_mm512_xor_si512(c2[1862],_mm512_xor_si512(c2[12161],_mm512_xor_si512(c2[2919],_mm512_xor_si512(c2[3448],_mm512_xor_si512(c2[9794],_mm512_xor_si512(c2[10058],_mm512_xor_si512(c2[27],_mm512_xor_si512(c2[15864],_mm512_xor_si512(c2[9541],_mm512_xor_si512(c2[9805],_mm512_xor_si512(c2[16142],_mm512_xor_si512(c2[13765],_mm512_xor_si512(c2[5328],_mm512_xor_si512(c2[49],_mm512_xor_si512(c2[1896],_mm512_xor_si512(c2[10624],_mm512_xor_si512(c2[10888],_mm512_xor_si512(c2[1120],_mm512_xor_si512(c2[13793],_mm512_xor_si512(c2[2981],_mm512_xor_si512(c2[3245],_mm512_xor_si512(c2[5620],_mm512_xor_si512(c2[15652],_mm512_xor_si512(c2[16716],_mm512_xor_si512(c2[16721],_mm512_xor_si512(c2[3520],_mm512_xor_si512(c2[16728],_mm512_xor_si512(c2[5380],_mm512_xor_si512(c2[11714],_mm512_xor_si512(c2[12780],_mm512_xor_si512(c2[13044],_mm512_xor_si512(c2[10144],_mm512_xor_si512(c2[13574],_mm512_xor_si512(c2[5136],_mm512_xor_si512(c2[5400],_mm512_xor_si512(c2[14115],_mm512_xor_si512(c2[14907],_mm512_xor_si512(c2[8583],_mm512_xor_si512(c2[8847],_mm512_xor_si512(c2[7260],_mm512_xor_si512(c2[9900],_mm512_xor_si512(c2[1202],_mm512_xor_si512(c2[1466],_mm512_xor_si512(c2[2785],_mm512_xor_si512(c2[7277],_mm512_xor_si512(c2[6493],_mm512_xor_si512(c2[6757],_mm512_xor_si512(c2[16000],_mm512_xor_si512(c2[2533],_mm512_xor_si512(c2[4909],_mm512_xor_si512(c2[3605],_mm512_xor_si512(c2[3339],_mm512_xor_si512(c2[2285],_mm512_xor_si512(c2[2825],_mm512_xor_si512(c2[3089],_mm512_xor_si512(c2[6255],_mm512_xor_si512(c2[11006],_mm512_xor_si512(c2[7851],_mm512_xor_si512(c2[8115],_mm512_xor_si512(c2[6001],_mm512_xor_si512(c2[9696],_mm512_xor_si512(c2[10239],_mm512_xor_si512(c2[14727],_mm512_xor_si512(c2[8652],_mm512_xor_si512(c2[9989],_mm512_xor_si512(c2[10253],_mm512_xor_si512(c2[9725],_mm512_xor_si512(c2[1008],_mm512_xor_si512(c2[10528],_mm512_xor_si512(c2[10792],_mm512_xor_si512(c2[16860],_mm512_xor_si512(c2[497],_mm512_xor_si512(c2[16344],_mm512_xor_si512(c2[16608],_mm512_xor_si512(c2[506],_mm512_xor_si512(c2[7370],_mm512_xor_si512(c2[3685],_mm512_xor_si512(c2[3949],_mm512_xor_si512(c2[9759],c2[6853]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 38 + d2[228]=_mm512_xor_si512(c2[15050],_mm512_xor_si512(c2[13046],_mm512_xor_si512(c2[8836],c2[5955]))); + +//row: 39 + d2[234]=_mm512_xor_si512(c2[544],_mm512_xor_si512(c2[7693],_mm512_xor_si512(c2[6953],c2[11056]))); + +//row: 40 + d2[240]=_mm512_xor_si512(c2[12410],_mm512_xor_si512(c2[9864],c2[14992])); + +//row: 41 + d2[246]=_mm512_xor_si512(c2[13740],_mm512_xor_si512(c2[15352],_mm512_xor_si512(c2[3014],c2[8668]))); + +//row: 42 + d2[252]=_mm512_xor_si512(c2[3698],_mm512_xor_si512(c2[10300],_mm512_xor_si512(c2[10296],_mm512_xor_si512(c2[10560],_mm512_xor_si512(c2[12145],_mm512_xor_si512(c2[12409],_mm512_xor_si512(c2[12937],_mm512_xor_si512(c2[12159],_mm512_xor_si512(c2[5293],_mm512_xor_si512(c2[5557],_mm512_xor_si512(c2[12952],_mm512_xor_si512(c2[13216],_mm512_xor_si512(c2[3460],_mm512_xor_si512(c2[10324],_mm512_xor_si512(c2[9002],_mm512_xor_si512(c2[9266],_mm512_xor_si512(c2[3207],_mm512_xor_si512(c2[9544],_mm512_xor_si512(c2[6903],_mm512_xor_si512(c2[7167],_mm512_xor_si512(c2[15625],_mm512_xor_si512(c2[10082],_mm512_xor_si512(c2[10346],_mm512_xor_si512(c2[11929],_mm512_xor_si512(c2[12193],_mm512_xor_si512(c2[3744],_mm512_xor_si512(c2[4284],_mm512_xor_si512(c2[11417],_mm512_xor_si512(c2[6925],_mm512_xor_si512(c2[7189],_mm512_xor_si512(c2[13536],_mm512_xor_si512(c2[15653],_mm512_xor_si512(c2[15917],_mm512_xor_si512(c2[8784],_mm512_xor_si512(c2[9048],_mm512_xor_si512(c2[10118],_mm512_xor_si512(c2[9853],_mm512_xor_si512(c2[10117],_mm512_xor_si512(c2[13553],_mm512_xor_si512(c2[13817],_mm512_xor_si512(c2[10130],_mm512_xor_si512(c2[15413],_mm512_xor_si512(c2[15677],_mm512_xor_si512(c2[4852],_mm512_xor_si512(c2[5116],_mm512_xor_si512(c2[6446],_mm512_xor_si512(c2[3540],_mm512_xor_si512(c2[6712],_mm512_xor_si512(c2[6976],_mm512_xor_si512(c2[15697],_mm512_xor_si512(c2[7253],_mm512_xor_si512(c2[7517],_mm512_xor_si512(c2[8045],_mm512_xor_si512(c2[8309],_mm512_xor_si512(c2[2249],_mm512_xor_si512(c2[662],_mm512_xor_si512(c2[3038],_mm512_xor_si512(c2[3302],_mm512_xor_si512(c2[11763],_mm512_xor_si512(c2[13082],_mm512_xor_si512(c2[409],_mm512_xor_si512(c2[673],_mm512_xor_si512(c2[159],_mm512_xor_si512(c2[9132],_mm512_xor_si512(c2[9396],_mm512_xor_si512(c2[12566],_mm512_xor_si512(c2[12830],_mm512_xor_si512(c2[13896],_mm512_xor_si512(c2[13372],_mm512_xor_si512(c2[13636],_mm512_xor_si512(c2[12312],_mm512_xor_si512(c2[12576],_mm512_xor_si512(c2[13380],_mm512_xor_si512(c2[16552],_mm512_xor_si512(c2[4144],_mm512_xor_si512(c2[4408],_mm512_xor_si512(c2[1517],_mm512_xor_si512(c2[16298],_mm512_xor_si512(c2[2834],_mm512_xor_si512(c2[3098],_mm512_xor_si512(c2[3641],_mm512_xor_si512(c2[7865],_mm512_xor_si512(c2[8129],_mm512_xor_si512(c2[1790],_mm512_xor_si512(c2[2054],_mm512_xor_si512(c2[3649],_mm512_xor_si512(c2[2857],_mm512_xor_si512(c2[3121],_mm512_xor_si512(c2[11041],_mm512_xor_si512(c2[11305],_mm512_xor_si512(c2[4188],_mm512_xor_si512(c2[10262],_mm512_xor_si512(c2[10524],_mm512_xor_si512(c2[10788],_mm512_xor_si512(c2[10010],_mm512_xor_si512(c2[10539],_mm512_xor_si512(c2[10803],_mm512_xor_si512(c2[508],_mm512_xor_si512(c2[772],_mm512_xor_si512(c2[14246],_mm512_xor_si512(c2[3161],_mm512_xor_si512(c2[16886],c2[255])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 43 + d2[258]=_mm512_xor_si512(c2[7393],_mm512_xor_si512(c2[13995],_mm512_xor_si512(c2[14261],_mm512_xor_si512(c2[15840],_mm512_xor_si512(c2[16104],_mm512_xor_si512(c2[15854],_mm512_xor_si512(c2[9252],_mm512_xor_si512(c2[16647],_mm512_xor_si512(c2[16],_mm512_xor_si512(c2[12949],_mm512_xor_si512(c2[7155],_mm512_xor_si512(c2[14019],_mm512_xor_si512(c2[12961],_mm512_xor_si512(c2[6902],_mm512_xor_si512(c2[13239],_mm512_xor_si512(c2[10598],_mm512_xor_si512(c2[10862],_mm512_xor_si512(c2[2425],_mm512_xor_si512(c2[14041],_mm512_xor_si512(c2[15624],_mm512_xor_si512(c2[15888],_mm512_xor_si512(c2[7985],_mm512_xor_si512(c2[15112],_mm512_xor_si512(c2[10884],_mm512_xor_si512(c2[336],_mm512_xor_si512(c2[2717],_mm512_xor_si512(c2[12485],_mm512_xor_si512(c2[12749],_mm512_xor_si512(c2[13813],_mm512_xor_si512(c2[13812],_mm512_xor_si512(c2[353],_mm512_xor_si512(c2[617],_mm512_xor_si512(c2[13825],_mm512_xor_si512(c2[2477],_mm512_xor_si512(c2[8547],_mm512_xor_si512(c2[8811],_mm512_xor_si512(c2[10141],_mm512_xor_si512(c2[7241],_mm512_xor_si512(c2[10671],_mm512_xor_si512(c2[2497],_mm512_xor_si512(c2[11212],_mm512_xor_si512(c2[11740],_mm512_xor_si512(c2[12004],_mm512_xor_si512(c2[5944],_mm512_xor_si512(c2[4357],_mm512_xor_si512(c2[6733],_mm512_xor_si512(c2[6997],_mm512_xor_si512(c2[15458],_mm512_xor_si512(c2[16777],_mm512_xor_si512(c2[4104],_mm512_xor_si512(c2[4368],_mm512_xor_si512(c2[3854],_mm512_xor_si512(c2[13097],_mm512_xor_si512(c2[16261],_mm512_xor_si512(c2[16525],_mm512_xor_si512(c2[696],_mm512_xor_si512(c2[436],_mm512_xor_si512(c2[16013],_mm512_xor_si512(c2[16277],_mm512_xor_si512(c2[180],_mm512_xor_si512(c2[3352],_mm512_xor_si512(c2[8103],_mm512_xor_si512(c2[5212],_mm512_xor_si512(c2[3098],_mm512_xor_si512(c2[6529],_mm512_xor_si512(c2[6793],_mm512_xor_si512(c2[1250],_mm512_xor_si512(c2[7336],_mm512_xor_si512(c2[11824],_mm512_xor_si512(c2[5485],_mm512_xor_si512(c2[5749],_mm512_xor_si512(c2[7344],_mm512_xor_si512(c2[6816],_mm512_xor_si512(c2[14736],_mm512_xor_si512(c2[15000],_mm512_xor_si512(c2[13417],_mm512_xor_si512(c2[7889],_mm512_xor_si512(c2[13957],_mm512_xor_si512(c2[14489],_mm512_xor_si512(c2[13705],_mm512_xor_si512(c2[14498],_mm512_xor_si512(c2[4203],_mm512_xor_si512(c2[4467],_mm512_xor_si512(c2[1046],_mm512_xor_si512(c2[6856],_mm512_xor_si512(c2[3686],c2[3950])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 44 + d2[264]=_mm512_xor_si512(c2[1323],_mm512_xor_si512(c2[7925],_mm512_xor_si512(c2[8185],_mm512_xor_si512(c2[10034],_mm512_xor_si512(c2[4225],_mm512_xor_si512(c2[9784],_mm512_xor_si512(c2[3182],_mm512_xor_si512(c2[10841],_mm512_xor_si512(c2[1085],_mm512_xor_si512(c2[7949],_mm512_xor_si512(c2[6891],_mm512_xor_si512(c2[832],_mm512_xor_si512(c2[7169],_mm512_xor_si512(c2[4792],_mm512_xor_si512(c2[13250],_mm512_xor_si512(c2[7971],_mm512_xor_si512(c2[9818],_mm512_xor_si512(c2[1909],_mm512_xor_si512(c2[9036],_mm512_xor_si512(c2[4814],_mm512_xor_si512(c2[11161],_mm512_xor_si512(c2[13536],_mm512_xor_si512(c2[6673],_mm512_xor_si512(c2[7743],_mm512_xor_si512(c2[7742],_mm512_xor_si512(c2[11436],_mm512_xor_si512(c2[3781],_mm512_xor_si512(c2[7755],_mm512_xor_si512(c2[13296],_mm512_xor_si512(c2[2741],_mm512_xor_si512(c2[4071],_mm512_xor_si512(c2[1165],_mm512_xor_si512(c2[4601],_mm512_xor_si512(c2[9350],_mm512_xor_si512(c2[13322],_mm512_xor_si512(c2[5136],_mm512_xor_si512(c2[5928],_mm512_xor_si512(c2[16769],_mm512_xor_si512(c2[15182],_mm512_xor_si512(c2[927],_mm512_xor_si512(c2[9388],_mm512_xor_si512(c2[10707],_mm512_xor_si512(c2[15193],_mm512_xor_si512(c2[14679],_mm512_xor_si512(c2[7021],_mm512_xor_si512(c2[10455],_mm512_xor_si512(c2[11521],_mm512_xor_si512(c2[11261],_mm512_xor_si512(c2[10201],_mm512_xor_si512(c2[11005],_mm512_xor_si512(c2[14177],_mm512_xor_si512(c2[2033],_mm512_xor_si512(c2[16037],_mm512_xor_si512(c2[13923],_mm512_xor_si512(c2[723],_mm512_xor_si512(c2[1260],_mm512_xor_si512(c2[5748],_mm512_xor_si512(c2[16574],_mm512_xor_si512(c2[1274],_mm512_xor_si512(c2[746],_mm512_xor_si512(c2[8930],_mm512_xor_si512(c2[1813],_mm512_xor_si512(c2[7887],_mm512_xor_si512(c2[8413],_mm512_xor_si512(c2[7635],_mm512_xor_si512(c2[8428],_mm512_xor_si512(c2[15292],_mm512_xor_si512(c2[11871],_mm512_xor_si512(c2[780],c2[14775]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 45 + d2[270]=_mm512_xor_si512(c2[1862],_mm512_xor_si512(c2[5618],c2[4080])); + } +} diff --git a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encode_parity_check.c b/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encode_parity_check.c index 55e02eea6a..94eb0a4d92 100644 --- a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encode_parity_check.c +++ b/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encode_parity_check.c @@ -37,7 +37,11 @@ #include "common/utils/LOG/log.h" +#ifdef __AVX512F__ +#include "ldpc384_simd512_byte.c" +#else #include "ldpc384_byte.c" +#endif #include "ldpc384_byte_128.c" #include "ldpc352_byte.c" #include "ldpc352_byte_128.c" @@ -87,7 +91,7 @@ static void encode_parity_check_part_optim(uint8_t *cc, uint8_t *d, short BG,short Zc,short Kb, int simd_size, int ncols) { - unsigned char c[2 * 22 * Zc * simd_size] __attribute__((aligned(32))); //double size matrix of c + unsigned char c[2 * 22 * Zc * simd_size] __attribute__((aligned(64))); //double size matrix of c for (int i1 = 0; i1 < ncols; i1++) { memcpy(&c[2 * i1 * Zc], &cc[i1 * Zc], Zc * sizeof(unsigned char)); memcpy(&c[(2 * i1 + 1) * Zc], &cc[i1 * Zc], Zc * sizeof(unsigned char)); @@ -95,7 +99,6 @@ static void encode_parity_check_part_optim(uint8_t *cc, uint8_t *d, short BG,sho for (int i1 = 1; i1 < simd_size; i1++) { memcpy(&c[(2 * ncols * Zc * i1)], &c[i1], (2 * ncols * Zc * sizeof(unsigned char)) - i1); } - if (BG == 1) { switch (Zc) { case 176: diff --git a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder.c b/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder.c index ebcb6bf4f5..055c0a7eb6 100644 --- a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder.c +++ b/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder.c @@ -41,11 +41,11 @@ #include "openair1/PHY/CODING/nrLDPC_extern.h" #include "ldpc_generate_coefficient.c" -int LDPCencoder(unsigned char **inputArray, unsigned char **outputArray, encoder_implemparams_t *impp) +int LDPCencoder(unsigned char **inputArray, unsigned char *outputArray, encoder_implemparams_t *impp) { const unsigned char *input = inputArray[0]; // channel input is the output of this function! - unsigned char *output = outputArray[0]; + unsigned char *output = outputArray; const int Zc = impp->Zc; const int Kb = impp->Kb; const short block_length = impp->K; @@ -114,7 +114,13 @@ int LDPCencoder(unsigned char **inputArray, unsigned char **outputArray, encoder fprintf(fd,"#include \"PHY/sse_intrin.h\"\n"); fprintf(fd2,"#include \"PHY/sse_intrin.h\"\n"); - if (gen_code == 1 && (Zc&31)==0) { + if (gen_code == 1 && (Zc&63)==0) { + shift=6; + mask=63; + strcpy(data_type,"__m512i"); + strcpy(xor_command,"_mm512_xor_si512"); + } + else if (gen_code == 1 && (Zc&31)==0) { shift=5; // AVX2 - 256-bit SIMD mask=31; strcpy(data_type,"simde__m256i"); diff --git a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim.c b/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim.c deleted file mode 100644 index deaba29e0d..0000000000 --- a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim.c +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The OpenAirInterface Software Alliance licenses this file to You under - * the OAI Public License, Version 1.1 (the "License"); you may not use this file - * except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.openairinterface.org/?page_id=698 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *------------------------------------------------------------------------------- - * For more information about the OpenAirInterface (OAI) Software Alliance: - * contact@openairinterface.org - */ - -/*! \file ldpc_encoder_optim.c - * \brief Defines the optimized LDPC encoder - * \author Florian Kaltenberger, Raymond Knopp, Kien le Trung (Eurecom) - * \email openair_tech@eurecom.fr - * \date 27-03-2018 - * \version 1.0 - * \note - * \warning - */ - -#include -#include -#include -#include -#include "assertions.h" -#include "common/utils/LOG/log.h" -#include "time_meas.h" -#include "openair1/PHY/CODING/nrLDPC_defs.h" -#include "openair1/PHY/CODING/nrLDPC_extern.h" -#include "ldpc_encode_parity_check.c" -#include "ldpc_generate_coefficient.c" - -int LDPCencoder(uint8_t **test_input, uint8_t **channel_input, encoder_implemparams_t *impp) -{ - int Zc = impp->Zc; - int Kb = impp->Kb; - int block_length = impp->K; - int BG = impp->BG; - int nrows=0,ncols=0; - int rate=3; - int no_punctured_columns,removed_bit; - - int simd_size; - - //determine number of bits in codeword - //if (block_length>3840) - if (BG==1) - { - //BG=1; - nrows=46; //parity check bits - ncols=22; //info bits - rate=3; - } - //else if (block_length<=3840) - else if (BG==2) - { - //BG=2; - nrows=42; //parity check bits - ncols=10; // info bits - rate=5; - - } - - -#ifdef DEBUG_LDPC - LOG_D(PHY,"ldpc_encoder_optim_8seg: BG %d, Zc %d, Kb %d, block_length %d\n",BG,Zc,Kb,block_length); - LOG_D(PHY,"ldpc_encoder_optim_8seg: PDU %x %x %x %x\n",test_input[0][0],test_input[0][1],test_input[0][2],test_input[0][3]); -#endif - - if ((Zc&31) > 0) simd_size = 16; - else simd_size = 32; - - uint8_t c[22*Zc] __attribute__((aligned(32))); //padded input, unpacked, max size - uint8_t d[46*Zc] __attribute__((aligned(32))); //coded parity part output, unpacked, max size - - // calculate number of punctured bits - no_punctured_columns=(int)((nrows-2)*Zc+block_length-block_length*rate)/Zc; - removed_bit = (nrows - no_punctured_columns - 2) * Zc + block_length - block_length * rate; - // printf("%d\n",no_punctured_columns); - // printf("%d\n",removed_bit); - // unpack input - memset(c, 0, sizeof(c)); - memset(d, 0, sizeof(d)); - - if(impp->tinput != NULL) start_meas(impp->tinput); - for (int i=0; i>(i&7)))>>(7-(i&7)); - //printf("c(%d,%d)=%d\n",j,i,temp); - } - - if(impp->tinput != NULL) stop_meas(impp->tinput); - - if ((BG==1 && Zc>=176) || (BG==2 && Zc>=72)) { - // extend matrix - if(impp->tprep != NULL) start_meas(impp->tprep); - if(impp->tprep != NULL) stop_meas(impp->tprep); - //parity check part - if(impp->tparity != NULL) start_meas(impp->tparity); - encode_parity_check_part_optim(c, d, BG, Zc, Kb,simd_size, ncols); - if(impp->tparity != NULL) stop_meas(impp->tparity); - } - else { - if (encode_parity_check_part_orig(c, d, BG, Zc, Kb, block_length)!=0) { - printf("Problem with encoder\n"); - return(-1); - } - } - if(impp->toutput != NULL) start_meas(impp->toutput); - // information part and puncture columns - memcpy(&channel_input[0][0], &c[2*Zc], (block_length-2*Zc)*sizeof(unsigned char)); - memcpy(&channel_input[0][block_length-2*Zc], &d[0], ((nrows-no_punctured_columns) * Zc-removed_bit)*sizeof(unsigned char)); - - if(impp->toutput != NULL) stop_meas(impp->toutput); - return 0; -} - - diff --git a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8seg.c b/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8seg.c deleted file mode 100644 index a45aac2474..0000000000 --- a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8seg.c +++ /dev/null @@ -1,201 +0,0 @@ -/* - * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The OpenAirInterface Software Alliance licenses this file to You under - * the OAI Public License, Version 1.1 (the "License"); you may not use this file - * except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.openairinterface.org/?page_id=698 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *------------------------------------------------------------------------------- - * For more information about the OpenAirInterface (OAI) Software Alliance: - * contact@openairinterface.org - */ - -/*! \file ldpc_encoder_optim8seg.c - * \brief Defines the optimized LDPC encoder - * \author Florian Kaltenberger, Raymond Knopp, Kien le Trung (Eurecom) - * \email openair_tech@eurecom.fr - * \date 27-03-2018 - * \version 1.0 - * \note - * \warning - */ - -#include -#include -#include -#include -#include "assertions.h" -#include "common/utils/LOG/log.h" -#include "time_meas.h" -#include "openair1/PHY/CODING/nrLDPC_defs.h" -#include "openair1/PHY/CODING/nrLDPC_extern.h" -#include "ldpc_encode_parity_check.c" -#include "ldpc_generate_coefficient.c" -#include "PHY/sse_intrin.h" - -int LDPCencoder(uint8_t **test_input, uint8_t **channel_input, encoder_implemparams_t *impp) -{ - int Zc = impp->Zc; - int Kb = impp->Kb; - short block_length = impp->K; - short BG = impp->BG; - - short nrows=0,ncols=0; - int i,i1,j,rate=3; - int no_punctured_columns,removed_bit; - char temp; - int simd_size; - - simde__m256i shufmask = simde_mm256_set_epi64x(0x0303030303030303, 0x0202020202020202,0x0101010101010101, 0x0000000000000000); - simde__m256i andmask = simde_mm256_set1_epi64x(0x0102040810204080); // every 8 bits -> 8 bytes, pattern repeats. - simde__m256i zero256 = simde_mm256_setzero_si256(); - simde__m256i masks[8]; - register simde__m256i c256; - masks[0] = simde_mm256_set1_epi8(0x1); - masks[1] = simde_mm256_set1_epi8(0x2); - masks[2] = simde_mm256_set1_epi8(0x4); - masks[3] = simde_mm256_set1_epi8(0x8); - masks[4] = simde_mm256_set1_epi8(0x10); - masks[5] = simde_mm256_set1_epi8(0x20); - masks[6] = simde_mm256_set1_epi8(0x40); - masks[7] = simde_mm256_set1_epi8(0x80); - - AssertFatal((impp->n_segments>0&&impp->n_segments<=8),"0 < n_segments %d <= 8\n",impp->n_segments); - - //determine number of bits in codeword - //if (block_length>3840) - if (BG==1) - { - nrows=46; //parity check bits - ncols=22; //info bits - rate=3; - } - //else if (block_length<=3840) - else if (BG==2) - { - //BG=2; - nrows=42; //parity check bits - ncols=10; // info bits - rate=5; - - } - -#ifdef DEBUG_LDPC - LOG_D(PHY,"ldpc_encoder_optim_8seg: BG %d, Zc %d, Kb %d, block_length %d, segments %d\n",BG,Zc,Kb,block_length,n_segments); - LOG_D(PHY,"ldpc_encoder_optim_8seg: PDU (seg 0) %x %x %x %x\n",test_input[0][0],test_input[0][1],test_input[0][2],test_input[0][3]); -#endif - - AssertFatal(Zc>0,"no valid Zc found for block length %d\n",block_length); - - if ((Zc&31) > 0) simd_size = 16; - else simd_size = 32; - - unsigned char c[22*Zc] __attribute__((aligned(32))); //padded input, unpacked, max size - unsigned char d[46*Zc] __attribute__((aligned(32))); //coded parity part output, unpacked, max size - - - // calculate number of punctured bits - no_punctured_columns=(int)((nrows-2)*Zc+block_length-block_length*rate)/Zc; - removed_bit=(nrows-no_punctured_columns-2) * Zc+block_length-(int)(block_length*rate); - // printf("%d\n",no_punctured_columns); - // printf("%d\n",removed_bit); - // unpack input - memset(c,0,sizeof(unsigned char) * ncols * Zc); - memset(d,0,sizeof(unsigned char) * nrows * Zc); - - if(impp->tinput != NULL) start_meas(impp->tinput); -#if 0 - for (i=0; i>(i&7)))>>(7-(i&7)); - //printf("c(%d,%d)=%d\n",j,i,temp); - c[i] |= (temp << j); - } - } -#else - for (i=0; i>5; i++) { - c256 = simde_mm256_and_si256(simde_mm256_cmpeq_epi8(simde_mm256_andnot_si256(simde_mm256_shuffle_epi8(simde_mm256_set1_epi32(((uint32_t*)test_input[0])[i]), shufmask),andmask),zero256),masks[0]); - for (j=1; jn_segments; j++) { - c256 = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_cmpeq_epi8(simde_mm256_andnot_si256(simde_mm256_shuffle_epi8(simde_mm256_set1_epi32(((uint32_t*)test_input[j])[i]), shufmask),andmask),zero256),masks[j]),c256); - } - ((simde__m256i *)c)[i] = c256; - } - - for (i=(block_length>>5)<<5;in_segments; j++) { - - temp = (test_input[j][i/8]&(128>>(i&7)))>>(7-(i&7)); - //printf("c(%d,%d)=%d\n",j,i,temp); - c[i] |= (temp << j); - } - } -#endif - - if(impp->tinput != NULL) stop_meas(impp->tinput); - - if ((BG==1 && Zc>=176) || (BG==2 && Zc>=72)) { - // extend matrix - if(impp->tprep != NULL) start_meas(impp->tprep); - if(impp->tprep != NULL) stop_meas(impp->tprep); - //parity check part - if(impp->tparity != NULL) start_meas(impp->tparity); - encode_parity_check_part_optim(c, d, BG, Zc, Kb, simd_size, ncols); - if(impp->tparity != NULL) stop_meas(impp->tparity); - } - else { - if (encode_parity_check_part_orig(c, d, BG, Zc, Kb, block_length)!=0) { - printf("Problem with encoder\n"); - return(-1); - } - } - if(impp->toutput != NULL) start_meas(impp->toutput); - // information part and puncture columns - /* - memcpy(&channel_input[0], &c[2*Zc], (block_length-2*Zc)*sizeof(unsigned char)); - memcpy(&channel_input[block_length-2*Zc], &d[0], ((nrows-no_punctured_columns) * Zc-removed_bit)*sizeof(unsigned char)); - */ - if ((((2*Zc)&31) == 0) && (((block_length-(2*Zc))&31) == 0)) { - //AssertFatal(((2*Zc)&31) == 0,"2*Zc needs to be a multiple of 32 for now\n"); - //AssertFatal(((block_length-(2*Zc))&31) == 0,"block_length-(2*Zc) needs to be a multiple of 32 for now\n"); - uint32_t l1 = (block_length-(2*Zc))>>5; - uint32_t l2 = ((nrows-no_punctured_columns) * Zc-removed_bit)>>5; - simde__m256i *c256p = (simde__m256i *)&c[2*Zc]; - simde__m256i *d256p = (simde__m256i *)&d[0]; - // if (((block_length-(2*Zc))&31)>0) l1++; - - for (i=0;in_segments;j++) ((simde__m256i *)channel_input[j])[i] = simde_mm256_and_si256(simde_mm256_srai_epi16(c256p[i],j),masks[0]); - - // if ((((nrows-no_punctured_columns) * Zc-removed_bit)&31)>0) l2++; - - for (i1=0;i1n_segments;j++) ((simde__m256i *)channel_input[j])[i] = simde_mm256_and_si256(simde_mm256_srai_epi16(d256p[i1],j),masks[0]); - } - else { -#ifdef DEBUG_LDPC - LOG_W(PHY,"using non-optimized version\n"); -#endif - // do non-SIMD version - for (i=0;i<(block_length-2*Zc);i++) - for (j=0; jn_segments; j++) - channel_input[j][i] = (c[2*Zc+i]>>j)&1; - for (i=0;i<((nrows-no_punctured_columns) * Zc-removed_bit);i++) - for (j=0; jn_segments; j++) - channel_input[j][block_length-2*Zc+i] = (d[i]>>j)&1; - } - - if(impp->toutput != NULL) stop_meas(impp->toutput); - return 0; -} - - diff --git a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8segmulti.c b/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8segmulti.c index b91d1f5825..b8f46969ec 100644 --- a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8segmulti.c +++ b/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8segmulti.c @@ -43,7 +43,7 @@ #include "ldpc_encode_parity_check.c" #include "ldpc_generate_coefficient.c" -int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp) +int LDPCencoder(uint8_t **input, uint8_t *output, encoder_implemparams_t *impp) { //set_log(PHY, 4); @@ -61,27 +61,11 @@ int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp) unsigned int macro_segment, macro_segment_end; - macro_segment = 8*impp->macro_num; - macro_segment_end = (impp->n_segments > 8*(impp->macro_num+1)) ? 8*(impp->macro_num+1) : impp->n_segments; + macro_segment = impp->first_seg; + macro_segment_end = (impp->n_segments > impp->first_seg + 8) ? impp->first_seg + 8 : impp->n_segments; ///printf("macro_segment: %d\n", macro_segment); ///printf("macro_segment_end: %d\n", macro_segment_end ); - simde__m256i shufmask = simde_mm256_set_epi64x(0x0303030303030303, 0x0202020202020202,0x0101010101010101, 0x0000000000000000); - simde__m256i andmask = simde_mm256_set1_epi64x(0x0102040810204080); // every 8 bits -> 8 bytes, pattern repeats. - simde__m256i zero256 = simde_mm256_setzero_si256(); - simde__m256i masks[8]; - register simde__m256i c256; - masks[0] = simde_mm256_set1_epi8(0x1); - masks[1] = simde_mm256_set1_epi8(0x2); - masks[2] = simde_mm256_set1_epi8(0x4); - masks[3] = simde_mm256_set1_epi8(0x8); - masks[4] = simde_mm256_set1_epi8(0x10); - masks[5] = simde_mm256_set1_epi8(0x20); - masks[6] = simde_mm256_set1_epi8(0x40); - masks[7] = simde_mm256_set1_epi8(0x80); - - - //determine number of bits in codeword if (BG==1) { @@ -103,10 +87,12 @@ int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp) AssertFatal(Zc > 0, "no valid Zc found for block length %d\n", block_length); if ((Zc&31) > 0) simd_size = 16; - else - simd_size = 32; - unsigned char cc[22*Zc] __attribute__((aligned(32))); //padded input, unpacked, max size - unsigned char dd[46*Zc] __attribute__((aligned(32))); //coded parity part output, unpacked, max size +#ifdef __AVX512F__ + else if ((BG==1) && Zc==384) simd_size =64; +#endif + else simd_size=32; + unsigned char cc[22*Zc] __attribute__((aligned(64))); //padded input, unpacked, max size + unsigned char dd[46*Zc] __attribute__((aligned(64))); //coded parity part output, unpacked, max size // calculate number of punctured bits no_punctured_columns=(int)((nrows-2)*Zc+block_length-block_length*rate)/Zc; @@ -114,40 +100,95 @@ int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp) //printf("%d\n",no_punctured_columns); //printf("%d\n",removed_bit); // unpack input - memset(cc,0,sizeof(unsigned char) * ncols * Zc); - memset(dd,0,sizeof(unsigned char) * nrows * Zc); + memset(cc,0,sizeof(cc)); + memset(dd,0,sizeof(dd)); if(impp->tinput != NULL) start_meas(impp->tinput); -#if 0 - for (i=0; i>(i&7); - //printf("c(%d,%d)=%d\n",j,i,temp); - c[i] |= (temp << (j-macro_segment)); + unsigned int i_byte = 0; + +#if defined(__AVX512F__) && defined(__AVX512BW__) && defined(__AVX512VBMI__) + const __m512i masks5[8] = { _mm512_set1_epi8(0x1), _mm512_set1_epi8(0x2), + _mm512_set1_epi8(0x4), _mm512_set1_epi8(0x8), + _mm512_set1_epi8(0x10), _mm512_set1_epi8(0x20), + _mm512_set1_epi8(0x40), _mm512_set1_epi8(0x80)}; + const __m512i zero512 = _mm512_setzero_si512(); + const uint8_t perm[64]__attribute__((aligned(64))) = {7,6,5,4,3,2,1,0, 15,14,13,12,11,10,9,8, + 23,22,21,20,19,18,17,16, 31,30,29,28,27,26,25,24, + 39,38,37,36,35,34,33,32, 47,46,45,44,43,42,41,40, + 55,54,53,52,51,50,49,48, 63,62,61,60,59,58,57,56}; + register __m512i c512; + + for (; i_byte < ((block_length >> 6) << 6); i_byte += 64) { + unsigned int i = i_byte >> 6; + c512 = _mm512_mask_blend_epi8(((uint64_t *)&input[macro_segment][0])[i], zero512, masks5[0]); + for (int j = macro_segment + 1; j < macro_segment_end; j++) { + c512 = _mm512_or_si512(c512, _mm512_mask_blend_epi8(((uint64_t *)&input[j][0])[i], zero512, masks5[j - macro_segment])); } + c512 = _mm512_permutexvar_epi8(*(__m512i*)perm, c512); + ((__m512i *)cc)[i] = c512; } -#else - for (int i=0; i>5; i++) { +#endif + +#ifndef __aarch64__ + simde__m256i shufmask = simde_mm256_set_epi64x(0x0303030303030303, 0x0202020202020202,0x0101010101010101, 0x0000000000000000); + simde__m256i andmask = simde_mm256_set1_epi64x(0x0102040810204080); // every 8 bits -> 8 bytes, pattern repeats. + simde__m256i zero256 = simde_mm256_setzero_si256(); + simde__m256i masks[8]; + register simde__m256i c256; + masks[0] = simde_mm256_set1_epi8(0x1); + masks[1] = simde_mm256_set1_epi8(0x2); + masks[2] = simde_mm256_set1_epi8(0x4); + masks[3] = simde_mm256_set1_epi8(0x8); + masks[4] = simde_mm256_set1_epi8(0x10); + masks[5] = simde_mm256_set1_epi8(0x20); + masks[6] = simde_mm256_set1_epi8(0x40); + masks[7] = simde_mm256_set1_epi8(0x80); + + for (; i_byte < ((block_length >> 5 ) << 5); i_byte += 32) { + unsigned int i = i_byte >> 5; c256 = simde_mm256_and_si256(simde_mm256_cmpeq_epi8(simde_mm256_andnot_si256(simde_mm256_shuffle_epi8(simde_mm256_set1_epi32(((uint32_t*)input[macro_segment])[i]), shufmask),andmask),zero256),masks[0]); - //for (j=1; j>5)<<5;i 8 bytes, pattern repeats. + simde__m128i zero128 = simde_mm_setzero_si128(); + simde__m128i masks[8]; + register simde__m128i c128; + masks[0] = simde_mm_set1_epi8(0x1); + masks[1] = simde_mm_set1_epi8(0x2); + masks[2] = simde_mm_set1_epi8(0x4); + masks[3] = simde_mm_set1_epi8(0x8); + masks[4] = simde_mm_set1_epi8(0x10); + masks[5] = simde_mm_set1_epi8(0x20); + masks[6] = simde_mm_set1_epi8(0x40); + masks[7] = simde_mm_set1_epi8(0x80); - temp = (input[j][i/8]&(128>>(i&7)))>>(7-(i&7)); - //printf("c(%d,%d)=%d\n",j,i,temp); + for (; i_byte < ((block_length >> 4 ) << 4); i_byte += 16) { + unsigned int i = i_byte >> 4; + c128 = simde_mm_and_si128(simde_mm_cmpeq_epi8(simde_mm_andnot_si128(simde_mm_shuffle_epi8(simde_mm_set1_epi16(((uint16_t*)input[macro_segment])[i]), shufmask),andmask),zero128),masks[0]); + for (int j=macro_segment+1; j < macro_segment_end; j++) { + c128 = simde_mm_or_si128(simde_mm_and_si128(simde_mm_cmpeq_epi8(simde_mm_andnot_si128(simde_mm_shuffle_epi8(simde_mm_set1_epi32(((uint16_t*)input[j])[i]), shufmask),andmask),zero128),masks[j-macro_segment]),c128); + } + ((simde__m128i *)cc)[i] = c128; + } +#endif + + for (; i_byte < block_length; i_byte++) { + unsigned int i = i_byte; + for (int j = macro_segment; j < macro_segment_end; j++) { + + temp = (input[j][i/8]&(128>>(i&7)))>>(7-(i&7)); cc[i] |= (temp << (j-macro_segment)); } } -#endif if(impp->tinput != NULL) stop_meas(impp->tinput); @@ -166,48 +207,8 @@ int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp) return(-1); } } - if(impp->toutput != NULL) start_meas(impp->toutput); - // information part and puncture columns - /* - memcpy(&output[0], &c[2*Zc], (block_length-2*Zc)*sizeof(unsigned char)); - memcpy(&output[block_length-2*Zc], &d[0], ((nrows-no_punctured_columns) * Zc-removed_bit)*sizeof(unsigned char)); - */ - if ((((2*Zc)&31) == 0) && (((block_length-(2*Zc))&31) == 0)) { - //AssertFatal(((2*Zc)&31) == 0,"2*Zc needs to be a multiple of 32 for now\n"); - //AssertFatal(((block_length-(2*Zc))&31) == 0,"block_length-(2*Zc) needs to be a multiple of 32 for now\n"); - uint32_t l1 = (block_length-(2*Zc))>>5; - uint32_t l2 = ((nrows-no_punctured_columns) * Zc-removed_bit)>>5; - simde__m256i *c256p = (simde__m256i *)&cc[2*Zc]; - simde__m256i *d256p = (simde__m256i *)&dd[0]; - // if (((block_length-(2*Zc))&31)>0) l1++; - - for (int i=0;i0) l2++; - - for (int i1=0, i=l1;i1>(j-macro_segment))&1; - for (int i=0;i<((nrows-no_punctured_columns) * Zc-removed_bit);i++) - //for (j=0; j>(j-macro_segment))&1; - } - - if(impp->toutput != NULL) stop_meas(impp->toutput); + memcpy(output,&cc[2*Zc],(block_length-(2*Zc))); + memcpy(output+block_length-(2*Zc),dd,((nrows-no_punctured_columns) * Zc-removed_bit)); return 0; } diff --git a/openair1/PHY/CODING/nr_segmentation.c b/openair1/PHY/CODING/nr_segmentation.c index 8db12fd3c0..57c21a5980 100644 --- a/openair1/PHY/CODING/nr_segmentation.c +++ b/openair1/PHY/CODING/nr_segmentation.c @@ -151,13 +151,8 @@ else for (r=0; r<*C; r++) { k = 0; - - while (k<((Kprime - L)>>3)) { - output_buffers[r][k] = input_buffer[s]; - //printf("encoding segment %d : byte %d (%d) => %d\n",r,k,(Kprime-L)>>3,input_buffer[s]); - k++; - s++; - } + memcpy(output_buffers[r],input_buffer+s,(Kprime-L)>>3); + s+=(Kprime-L)>>3; if (*C > 1) { // add CRC crc = crc24b(output_buffers[r],Kprime-L)>>8; diff --git a/openair1/PHY/NR_TRANSPORT/nr_dlsch.c b/openair1/PHY/NR_TRANSPORT/nr_dlsch.c index 517d0ab974..6e68a459c9 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_dlsch.c +++ b/openair1/PHY/NR_TRANSPORT/nr_dlsch.c @@ -600,14 +600,14 @@ static int do_one_dlsch(unsigned char *input_ptr, PHY_VARS_gNB *gNB, NR_gNB_DLSC printf("\nEncoded payload:\n"); for (int i = 0; i < encoded_length; i += 8) { for (int j = 0; j < 8; j++) - printf("%d", input_ptr[i + j]); + printf("%d", (input_ptr[i >> 3] >> j) & 1); printf("\t"); } printf("\n"); #endif if (IS_SOFTMODEM_DLSIM) - memcpy(harq->f, input_ptr, encoded_length); + memcpy(harq->f, input_ptr, (encoded_length + 7) >> 3); c16_t mod_symbs[rel15->NrOfCodewords][encoded_length] __attribute__((aligned(64))); for (int codeWord = 0; codeWord < rel15->NrOfCodewords; codeWord++) { @@ -763,11 +763,11 @@ static int do_one_dlsch(unsigned char *input_ptr, PHY_VARS_gNB *gNB, NR_gNB_DLSC } stop_meas(&gNB->dlsch_precoding_stats); - /* output and its parts for each dlsch should be aligned on 64 bytes - * should remain a multiple of 64 with enough offset to fit each dlsch + /* output and its parts for each dlsch should be aligned on 64 bytes (or 8 * 64 bits) + * should remain a multiple of 8 * 64 with enough offset to fit each dlsch */ uint32_t size_output_tb = rel15->rbSize * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * Qm * rel15->nrOfLayers; - return ceil_mod(size_output_tb,64); + return ((size_output_tb + 511) >> 9) << 6; } void nr_generate_pdsch(processingData_L1tx_t *msgTx, int frame, int slot) @@ -817,15 +817,15 @@ void nr_generate_pdsch(processingData_L1tx_t *msgTx, int frame, int slot) /// CRC, coding, interleaving and rate matching AssertFatal(harq->pdu != NULL, "%4d.%2d no HARQ PDU for PDSCH generation\n", msgTx->frame, msgTx->slot); - /* output and its parts for each dlsch should be aligned on 64 bytes - * => size_output is a sum of parts sizes rounded up to a multiple of 64 + /* output and its parts for each dlsch should be aligned on 64 bytes (or 8 * 64 bits) + * => size_output is a sum of parts sizes rounded up to a multiple of 8 * 64 */ size_t size_output_tb = rel15->rbSize * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * Qm * rel15->nrOfLayers; - size_output += ceil_mod(size_output_tb, 64); + size_output += ceil_mod(size_output_tb, 8 * 64); } - unsigned char output[size_output] __attribute__((aligned(64))); - bzero(output, size_output); + unsigned char output[size_output >> 3] __attribute__((aligned(64))); + bzero(output, sizeof(output)); start_meas(dlsch_encoding_stats); if (nr_dlsch_encoding(gNB, diff --git a/openair1/PHY/NR_TRANSPORT/nr_dlsch_coding.c b/openair1/PHY/NR_TRANSPORT/nr_dlsch_coding.c index 29ea38232c..56a849a4af 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_dlsch_coding.c +++ b/openair1/PHY/NR_TRANSPORT/nr_dlsch_coding.c @@ -265,15 +265,13 @@ int nr_dlsch_encoding(PHY_VARS_gNB *gNB, TB_parameters->tbslbrm = rel15->maintenance_parms_v3.tbSizeLbrmBytes; + TB_parameters->output = &output[dlsch_offset >> 3]; TB_parameters->segments = &segments[segments_offset]; - size_t r_offset = 0; for (int r = 0; r < TB_parameters->C; r++) { nrLDPC_segment_encoding_parameters_t *segment_parameters = &TB_parameters->segments[r]; segment_parameters->c = harq->c[r]; segment_parameters->E = nr_get_E(TB_parameters->G, TB_parameters->C, TB_parameters->Qm, rel15->nrOfLayers, r); - segment_parameters->output = &output[dlsch_offset + r_offset]; - r_offset += segment_parameters->E; reset_meas(&segment_parameters->ts_interleave); reset_meas(&segment_parameters->ts_rate_match); @@ -282,11 +280,11 @@ int nr_dlsch_encoding(PHY_VARS_gNB *gNB, segments_offset += TB_parameters->C; - /* output and its parts for each dlsch should be aligned on 64 bytes - * => dlsch_offset should remain a multiple of 64 with enough offset to fit each dlsch + /* output and its parts for each dlsch should be aligned on 64 bytes (or 8 * 64 bits) + * => dlsch_offset should remain a multiple of 8 * 64 with enough offset to fit each dlsch */ const size_t dlsch_size = rel15->rbSize * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * rel15->qamModOrder[0] * rel15->nrOfLayers; - dlsch_offset += ceil_mod(dlsch_size, 64); + dlsch_offset += ceil_mod(dlsch_size, 8 * 64); } gNB->nrLDPC_coding_interface.nrLDPC_coding_encoder(&slot_parameters); diff --git a/openair1/PHY/NR_TRANSPORT/nr_scrambling.c b/openair1/PHY/NR_TRANSPORT/nr_scrambling.c index b71ee0cce0..9577f943e4 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_scrambling.c +++ b/openair1/PHY/NR_TRANSPORT/nr_scrambling.c @@ -33,11 +33,26 @@ void nr_codeword_scrambling(uint8_t *in, { const int roundedSz = (size + 31) / 32; uint32_t *seq = gold_cache((n_RNTI << 15) + (q << 14) + Nid, roundedSz); - for (int i = 0; i < roundedSz; i++) { - simde__m256i c = ((simde__m256i*)in)[i]; - uint32_t in32 = simde_mm256_movemask_epi8(simde_mm256_slli_epi16(c, 7)); - out[i] = in32 ^ seq[i]; - DEBUG_SCRAMBLING(LOG_D(PHY, "in[%d] %x => %x\n", i, in32, out[i])); + unsigned int i_32 = 0; +#if defined(__AVX512F__) && defined(__AVX512VL__) + for (; i_32 < ((roundedSz >> 3) << 3); i_32 += 8) { + __m256i in_256 = _mm256_load_epi32(&((uint32_t *)in)[i_32]); + __m256i seq_256 = _mm256_load_epi32(&seq[i_32]); + _mm256_storeu_epi32(&out[i_32], _mm256_xor_si256(in_256, seq_256)); + } +#endif +#if defined(__aarch64__) + for (; i_32 < ((roundedSz >> 2) << 2); i_32 += 4) { + unsigned int i = i_32 >> 2; + uint32x4_t in_32x4 = vld1q_u32(&((uint32_t *)in)[i_32]); + uint32x4_t seq_32x4 = vld1q_u32(&seq[i_32]); + ((uint32x4_t *)out)[i] = veorq_u32(in_32x4, seq_32x4); + } +#endif + for (; i_32 < roundedSz; i_32++) { + unsigned int i = i_32; + out[i] = ((uint32_t *)in)[i] ^ seq[i]; + DEBUG_SCRAMBLING(LOG_D(PHY, "in[%d] %x => %x\n", i, ((uint32_t*)in)[i], out[i])); } } diff --git a/openair1/PHY/NR_UE_TRANSPORT/nr_ulsch_coding.c b/openair1/PHY/NR_UE_TRANSPORT/nr_ulsch_coding.c index bed86bf852..2eeb27d457 100644 --- a/openair1/PHY/NR_UE_TRANSPORT/nr_ulsch_coding.c +++ b/openair1/PHY/NR_UE_TRANSPORT/nr_ulsch_coding.c @@ -156,7 +156,11 @@ int nr_ulsch_encoding(PHY_VARS_NR_UE *ue, NR_UL_UE_HARQ_t *harq_process = &ue->ul_harq_processes[harq_pid]; TB_parameters->segments = segments[pusch_id]; - int r_offset = 0; + const nfapi_nr_ue_pusch_pdu_t *pusch_pdu = &ulsch->pusch_pdu; + uint16_t nb_rb = pusch_pdu->rb_size; + memset(harq_process->f, 0, 14 * nb_rb * 12 * 16); + TB_parameters->output = harq_process->f; + for (int r = 0; r < TB_parameters->C; r++) { nrLDPC_segment_encoding_parameters_t *segment_parameters = &TB_parameters->segments[r]; segment_parameters->c = harq_process->c[r]; @@ -165,8 +169,6 @@ int nr_ulsch_encoding(PHY_VARS_NR_UE *ue, TB_parameters->Qm, TB_parameters->nb_layers, r); - segment_parameters->output = harq_process->f + r_offset; - r_offset += segment_parameters->E; reset_meas(&segment_parameters->ts_interleave); reset_meas(&segment_parameters->ts_rate_match); diff --git a/openair1/SIMULATION/NR_PHY/dlschsim.c b/openair1/SIMULATION/NR_PHY/dlschsim.c index 36fe9daa50..c475bfc039 100644 --- a/openair1/SIMULATION/NR_PHY/dlschsim.c +++ b/openair1/SIMULATION/NR_PHY/dlschsim.c @@ -512,8 +512,8 @@ int main(int argc, char **argv) //printf("crc32: [0]->0x%08x\n",crc24c(test_input, 32)); // generate signal - unsigned char output[rel15->rbSize * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * 8 * NR_MAX_NB_LAYERS] __attribute__((aligned(32))); - bzero(output,rel15->rbSize * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * 8 * NR_MAX_NB_LAYERS); + unsigned char output[rel15->rbSize * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * NR_MAX_NB_LAYERS] __attribute__((aligned(64))); + bzero(output, sizeof(output)); if (input_fd == NULL) { msgDataTx.num_pdsch_slot = 1; nr_dlsch_encoding(gNB, &msgDataTx, frame, slot, frame_parms, output, NULL, NULL, NULL, NULL, NULL, NULL, NULL); @@ -533,12 +533,12 @@ int main(int argc, char **argv) //if (i<16) // printf("encoder output f[%d] = %d\n",i,dlsch->harq_processes[0]->f[i]); - if (output[i] == 0) + if ((output[i >> 3] & (1 << (i & 7))) == 0) modulated_input[i] = 1.0; ///sqrt(2); //QPSK else modulated_input[i] = -1.0; ///sqrt(2); - //if (i<16) printf("modulated_input[%d] = %d\n",i,modulated_input[i]); + //if (i<16) printf("modulated_input[%d] = %f (%d)\n",i,modulated_input[i],output[i]); //SNR =10; SNR_lin = pow(10, SNR / 10.0); sigma = 1.0 / sqrt(2 * SNR_lin); diff --git a/openair1/SIMULATION/NR_PHY/ulschsim.c b/openair1/SIMULATION/NR_PHY/ulschsim.c index f68f6885a6..ef6fd34ee4 100644 --- a/openair1/SIMULATION/NR_PHY/ulschsim.c +++ b/openair1/SIMULATION/NR_PHY/ulschsim.c @@ -542,7 +542,7 @@ int main(int argc, char **argv) } */ - if (harq_process_ul_ue->f[i] == 0) + if ((harq_process_ul_ue->f[i >> 3] & (1 << (i & 7))) == 0) modulated_input[i] = 1.0; ///sqrt(2); //QPSK else modulated_input[i] = -1.0; ///sqrt(2); diff --git a/openair1/SIMULATION/NR_PHY/ulsim.c b/openair1/SIMULATION/NR_PHY/ulsim.c index 2ca025865f..da85ef0de4 100644 --- a/openair1/SIMULATION/NR_PHY/ulsim.c +++ b/openair1/SIMULATION/NR_PHY/ulsim.c @@ -248,7 +248,9 @@ int main(int argc, char *argv[]) InitSinLUT(); int c; - while ((c = getopt(argc, argv, "--:O:a:b:c:d:ef:g:h:i:jk:m:n:p:q:r:s:t:u:v:w:y:z:A:C:F:G:H:I:M:N:PR:S:T:U:L:ZW:E:X:")) != -1) { + bool setAffinity=false; + char gNBthreads[128]="n"; + while ((c = getopt(argc, argv, "--:O:a:b:c:d:ef:g:h:i:jk:m:n:p:q:r:s:t:u:v:w:y:z:A:C:F:G:H:I:M:N:PR:S:T:U:L:ZW:E:X:Y:")) != -1) { /* ignore long options starting with '--', option '-O' and their arguments that are handled by configmodule */ /* with this opstring getopt returns 1 for non-option arguments, refer to 'man 3 getopt' */ if (c == 1 || c == '-' || c == 'O') @@ -493,6 +495,13 @@ int main(int argc, char *argv[]) AssertFatal(filename_csv != NULL, "strdup() error: errno %d\n", errno); break; + case 'Y': + threadCnt = sizeof(gNBthreads)-1; + strncpy(gNBthreads, optarg, threadCnt); + gNBthreads[threadCnt]=0; + setAffinity=true; + break; + case 'Z': transform_precoding = transformPrecoder_enabled; num_dmrs_cdm_grps_no_data = 2; @@ -591,7 +600,11 @@ int main(int argc, char *argv[]) gNB->RU_list[0] = calloc(1, sizeof(**gNB->RU_list)); gNB->RU_list[0]->rfdevice.openair0_cfg = openair0_cfg; - initFloatingCoresTpool(threadCnt, &gNB->threadPool, false, "gNB-tpool"); + if (setAffinity == false) + initFloatingCoresTpool(threadCnt, &gNB->threadPool, false, "gNB-tpool"); + else + initNamedTpool(gNBthreads, &gNB->threadPool, true, "gNB-tpool"); + initNotifiedFIFO(&gNB->respDecode); initNotifiedFIFO(&gNB->respPuschSymb);