diff --git a/openair1/PHY/CODING/TESTBENCH/ldpctest.c b/openair1/PHY/CODING/TESTBENCH/ldpctest.c index ee33c7f91a..46534c37e3 100644 --- a/openair1/PHY/CODING/TESTBENCH/ldpctest.c +++ b/openair1/PHY/CODING/TESTBENCH/ldpctest.c @@ -123,7 +123,7 @@ one_measurement_t test_ldpc(short max_iterations, uint8_t estimated_output[MAX_NUM_DLSCH_SEGMENTS][Kprime]; memset(estimated_output, 0, sizeof(estimated_output)); uint8_t *channel_input[MAX_NUM_DLSCH_SEGMENTS]; - uint8_t *channel_input_optim[MAX_NUM_DLSCH_SEGMENTS]; + uint8_t *channel_input_optim; // double channel_output[68 * 384]; double modulated_input[MAX_NUM_DLSCH_SEGMENTS][68 * 384] = {0}; int8_t channel_output_fixed[MAX_NUM_DLSCH_SEGMENTS][68 * 384] = {0}; @@ -265,9 +265,9 @@ one_measurement_t test_ldpc(short max_iterations, memset(test_input[j], 0, ((K + 7) & ~7) / 8); channel_input[j] = malloc16(68 * 384); memset(channel_input[j], 0, 68 * 384); - channel_input_optim[j] = malloc16(68 * 384); - memset(channel_input_optim[j], 0, 68 * 384); } + channel_input_optim = malloc16(68 * 384); + memset(channel_input_optim, 0, 68 * 384); // Fill input segments with random values for (int j = 0; j < MAX_NUM_DLSCH_SEGMENTS; j++) { @@ -285,7 +285,7 @@ one_measurement_t test_ldpc(short max_iterations, impp.gen_code = 1; if (ntrials == 0) - ldpc_orig.LDPCencoder(test_input, channel_input, &impp); + ldpc_orig.LDPCencoder(test_input, channel_input[0], &impp); impp.gen_code = 0; decode_abort_t dec_abort; init_abort(&dec_abort); @@ -294,23 +294,21 @@ one_measurement_t test_ldpc(short max_iterations, //// encoder start_meas(&time); for (int j = 0; j < n_segments; j++) { - ldpc_orig.LDPCencoder(&test_input[j], &channel_input[j], &impp); + ldpc_orig.LDPCencoder(&test_input[j], channel_input[j], &impp); } stop_meas(&time); impp.n_segments = n_segments; - for (int j = 0; j < (n_segments / 8 + 1); j++) { - start_meas(&ret.time_optim); - impp.macro_num = j; - ldpc_toCompare.LDPCencoder(test_input, channel_input_optim, &impp); - stop_meas(&ret.time_optim); - } + start_meas(&ret.time_optim); + impp.first_seg = 0; + ldpc_toCompare.LDPCencoder(test_input, channel_input_optim, &impp); + stop_meas(&ret.time_optim); if (ntrials == 1) for (int j = 0; j < n_segments; j++) for (int i = 0; i < K + (nrows - no_punctured_columns) * Zc - removed_bit; i++) { - if (channel_input[j][i] != channel_input_optim[j][i]) { - printf("differ in seg %d pos %d (%u,%u)\n", j, i, channel_input[j][i], channel_input_optim[j][i]); + if (channel_input[j][i] != ((channel_input_optim[i] >> j) & 0x1)) { + printf("differ in seg %d pos %d (%u,%u)\n", j, i, channel_input[j][i], (channel_input_optim[i] >> j) & 0x1); return ret; } } @@ -322,7 +320,7 @@ one_measurement_t test_ldpc(short max_iterations, printf("\ne %d..%d: ", i, i + 15); #endif - if (channel_input_optim[j][i - 2 * Zc] == 0) + if (((channel_input_optim[i - 2 * Zc] >> j) & 0x1) == 0) modulated_input[j][i] = 1.0; /// sqrt(2); //QPSK else modulated_input[j][i] = -1.0; /// sqrt(2); @@ -332,7 +330,7 @@ one_measurement_t test_ldpc(short max_iterations, // Uncoded BER uint8_t channel_output_uncoded = channel_output_fixed[j][i] < 0 ? 1 /* QPSK demod */ : 0; - if (channel_output_uncoded != channel_input_optim[j][i - 2 * Zc]) + if (channel_output_uncoded != ((channel_input_optim[i - 2 * Zc] >> j) & 0x1)) ret.errors_bit_uncoded++; } @@ -396,8 +394,8 @@ one_measurement_t test_ldpc(short max_iterations, for (int j = 0; j < MAX_NUM_DLSCH_SEGMENTS; j++) { free(test_input[j]); free(channel_input[j]); - free(channel_input_optim[j]); } + free(channel_input_optim); print_meas(&time, "ldpc_encoder", NULL, NULL); print_meas(&ret.time_optim, "ldpc_encoder_optim", NULL, NULL); diff --git a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c index 5e44eaf2ae..554b399f4b 100644 --- a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c +++ b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c @@ -65,19 +65,17 @@ static void ldpc8blocks(void *p) // nrLDPC_encoder output is in "d" // let's make this interface happy! uint8_t d[68 * 384] __attribute__((aligned(64))); - uint8_t *dp[2]; - dp[0] = &d[0]; uint8_t *c[nrLDPC_TB_encoding_parameters->C]; unsigned int macro_segment, macro_segment_end; - macro_segment = 8*impp->macro_num; - macro_segment_end = (impp->n_segments > 8*(impp->macro_num+1)) ? 8*(impp->macro_num+1) : impp->n_segments; + macro_segment = impp->first_seg; + macro_segment_end = (impp->n_segments > impp->first_seg + 8) ? impp->first_seg + 8 : impp->n_segments; for (int r = 0; r < nrLDPC_TB_encoding_parameters->C; r++) c[r] = nrLDPC_TB_encoding_parameters->segments[r].c; - start_meas(&nrLDPC_TB_encoding_parameters->segments[impp->macro_num * 8].ts_ldpc_encode); - LDPCencoder(c, dp, impp); - stop_meas(&nrLDPC_TB_encoding_parameters->segments[impp->macro_num * 8].ts_ldpc_encode); + start_meas(&nrLDPC_TB_encoding_parameters->segments[impp->first_seg].ts_ldpc_encode); + LDPCencoder(c, d, impp); + stop_meas(&nrLDPC_TB_encoding_parameters->segments[impp->first_seg].ts_ldpc_encode); // Compute where to place in output buffer that is concatenation of all segments #ifdef DEBUG_LDPC_ENCODING @@ -86,11 +84,14 @@ static void ldpc8blocks(void *p) const uint32_t E = nrLDPC_TB_encoding_parameters->segments[macro_segment].E; uint32_t E2=E,E2_first_segment=macro_segment_end-macro_segment; bool Eshift=false; + uint32_t Emax = E; for (int s=macro_segment;ssegments[s].E != E) { E2=nrLDPC_TB_encoding_parameters->segments[s].E; Eshift=true; E2_first_segment = s-macro_segment; + if(E2 > Emax) + Emax = E2; break; } @@ -124,14 +125,12 @@ static void ldpc8blocks(void *p) */ uint32_t Tbslbrm = nrLDPC_TB_encoding_parameters->tbslbrm; - uint8_t e[E] __attribute__((aligned(64))); + uint8_t e[Emax] __attribute__((aligned(64))); uint8_t f[E+64] __attribute__((aligned(64))); - uint8_t e2[E2] __attribute__((aligned(64))); uint8_t f2[E2+64] __attribute__((aligned(64))); - bzero(e, E); + bzero(e, Emax); bzero(f, E + 64); if (Eshift) { - bzero(e2, E2); bzero(f2, E2 + 64); } start_meas(&nrLDPC_TB_encoding_parameters->segments[macro_segment].ts_rate_match); @@ -144,18 +143,7 @@ static void ldpc8blocks(void *p) impp->F, impp->K - impp->F - 2 * impp->Zc, nrLDPC_TB_encoding_parameters->rv_index, - E); - if (Eshift) - nr_rate_matching_ldpc(Tbslbrm, - impp->BG, - impp->Zc, - d, - e2, - impp->n_segments, - impp->F, - impp->K - impp->F - 2 * impp->Zc, - nrLDPC_TB_encoding_parameters->rv_index, - E2); + Emax); stop_meas(&nrLDPC_TB_encoding_parameters->segments[macro_segment].ts_rate_match); if (impp->K - impp->F - 2 * impp->Zc > E) { @@ -189,7 +177,7 @@ static void ldpc8blocks(void *p) if (Eshift) nr_interleaving_ldpc(E2, mod_order, - e2, + e, f2); stop_meas(&nrLDPC_TB_encoding_parameters->segments[macro_segment].ts_interleave); if(impp->toutput != NULL) start_meas(impp->toutput); @@ -504,42 +492,42 @@ static void ldpc8blocks(void *p) completed_task_ans(impp->ans); } -static int nrLDPC_prepare_TB_encoding(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters, +static int nrLDPC_launch_TB_encoding(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters, int dlsch_id, thread_info_tm_t *t_info) { nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id]; - encoder_implemparams_t impp = {0}; + encoder_implemparams_t common_segment_params = { + .n_segments = nrLDPC_TB_encoding_parameters->C, + .tinput = nrLDPC_slot_encoding_parameters->tinput, + .tprep = nrLDPC_slot_encoding_parameters->tprep, + .tparity = nrLDPC_slot_encoding_parameters->tparity, + .toutput = nrLDPC_slot_encoding_parameters->toutput, + .Kb = nrLDPC_TB_encoding_parameters->Kb, + .Zc = nrLDPC_TB_encoding_parameters->Z, + .BG = nrLDPC_TB_encoding_parameters->BG, + .output = nrLDPC_TB_encoding_parameters->output, + .K = nrLDPC_TB_encoding_parameters->K, + .F = nrLDPC_TB_encoding_parameters->F, + }; - impp.n_segments = nrLDPC_TB_encoding_parameters->C; - impp.tinput = nrLDPC_slot_encoding_parameters->tinput; - impp.tprep = nrLDPC_slot_encoding_parameters->tprep; - impp.tparity = nrLDPC_slot_encoding_parameters->tparity; - impp.toutput = nrLDPC_slot_encoding_parameters->toutput; - impp.Kb = nrLDPC_TB_encoding_parameters->Kb; - impp.Zc = nrLDPC_TB_encoding_parameters->Z; - impp.BG = nrLDPC_TB_encoding_parameters->BG; - impp.output = nrLDPC_TB_encoding_parameters->output; - impp.K = nrLDPC_TB_encoding_parameters->K; - impp.F = nrLDPC_TB_encoding_parameters->F; + const size_t n_tasks = (common_segment_params.n_segments + 7) >> 3; - size_t const n_seg = (impp.n_segments / 8 + ((impp.n_segments & 7) == 0 ? 0 : 1)); - - for (int j = 0; j < n_seg; j++) { + for (int j = 0; j < n_tasks; j++) { ldpc8blocks_args_t *perJobImpp = &((ldpc8blocks_args_t *)t_info->buf)[t_info->len]; DevAssert(t_info->len < t_info->cap); - impp.ans = t_info->ans; t_info->len += 1; - impp.macro_num = j; - perJobImpp->impp = impp; + perJobImpp->impp = common_segment_params; + perJobImpp->impp.first_seg = j * 8; + perJobImpp->impp.ans = t_info->ans; perJobImpp->nrLDPC_TB_encoding_parameters = nrLDPC_TB_encoding_parameters; task_t t = {.func = ldpc8blocks, .args = perJobImpp}; pushTpool(nrLDPC_slot_encoding_parameters->threadPool, t); } - return n_seg; + return n_tasks; } int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters) @@ -558,7 +546,7 @@ int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encodin int nbEncode = 0; for (int dlsch_id = 0; dlsch_id < nrLDPC_slot_encoding_parameters->nb_TBs; dlsch_id++) { - nbEncode += nrLDPC_prepare_TB_encoding(nrLDPC_slot_encoding_parameters, dlsch_id, &t_info); + nbEncode += nrLDPC_launch_TB_encoding(nrLDPC_slot_encoding_parameters, dlsch_id, &t_info); } if (nbEncode < nbTasks) { completed_many_task_ans(&ans, nbTasks - nbEncode); diff --git a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.c b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.c index 4e5a31f31f..4a507fa381 100644 --- a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.c +++ b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.c @@ -29,446 +29,419 @@ static const uint8_t index_k0[2][4] = {{0, 17, 33, 56}, {0, 13, 25, 43}}; void nr_interleaving_ldpc(uint32_t E, uint8_t Qm, uint8_t *e, uint8_t *f) { - uint32_t EQm; - - EQm = E / Qm; + const uint32_t EQm = E / Qm; memset(f, 0, E * sizeof(uint8_t)); - uint8_t *e0, *e1, *e2, *e3, *e4, *e5, *e6, *e7; - uint8_t *fp; - int j,k; -#if 1 //def __WASAVX2__ -#ifdef __AVX512VBMI__ - __m512i tmp0,tmp1,tmp2,tmp3,tmp4,tmp5; - __m512i *e0_512,*e1_512,*e2_512,*e3_512,*e4_512,*e5_512,*e6_512,*e7_512; - - __m512i *f_512=(__m512i *)f; -#else - simde__m128i tmp0,tmp1,tmp2,tmp3,tmp4,tmp5,tmp6,tmp7; - simde__m128i *e0_128,*e1_128,*e2_128,*e3_128,*e4_128,*e5_128,*e6_128,*e7_128; - - simde__m128i *f_128=(simde__m128i *)f; -#endif switch(Qm) { - case 2: - e0=e; - e1=e0+EQm; + case 2: { + uint8_t *e0 = e; + uint8_t *e1 = e0 + EQm; + int i = 0; #ifdef __AVX512VBMI__ - e0_512=(__m512i *)e0; - e1_512=(__m512i *)e1; - __m512i p8a = _mm512_set_epi8(95,31,94,30,93,29,92,28,91,27,90,26,89,25,88,24,87,23,86,22,85,21,84,20,83,19,82,18,81,17,80,16,79,15,78,14,77,13,76,12,75,11,74,10,73,9,72,8,71,7,70,6,69,5,68,4,67,3,66,2,65,1,64,0); - __m512i p8b = _mm512_set_epi8(127,63,126,62,125,61,124,60,123,59,122,58,121,57,120,56,119,55,118,54,117,53,116,52,115,51,114,50,113,49,112,48,111,47,110,46,109,45,108,44,107,43,106,42,105,41,104,40,103,39,102,38,101,37,100,36,99,35,98,34,97,33,96,32); - - __m512i e0j,e1j; - for (k=0,j=0;j>6;j++,k+=2) { - e0j = _mm512_loadu_si512(e0_512+j); - e1j = _mm512_loadu_si512(e1_512+j); - _mm512_storeu_si512(f_512+k,_mm512_permutex2var_epi8(e0j,p8a,e1j)); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15) - _mm512_storeu_si512(f_512+k+1,_mm512_permutex2var_epi8(e0j,p8b,e1j)); - } - if ((j<<6) != EQm) { - int k2=k<<6; - fp = &f[k2]; - for (int j2=(j<<6) ; j2< EQm ; j2++) { - *fp++ = e0[j2]; - *fp++ = e1[j2]; + // clang-format off + const __m512i p8a = _mm512_set_epi8(95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, + 87, 23, 86, 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, + 79, 15, 78, 14, 77, 13, 76, 12, 75, 11, 74, 10, 73, 9, 72, 8, + 71, 7, 70, 6, 69, 5, 68, 4, 67, 3, 66, 2, 65, 1, 64, 0); + const __m512i p8b = _mm512_set_epi8(127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, + 56, 119, 55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, + 48, 111, 47, 110, 46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, + 40, 103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32); + // clang-format on + __m512i *f_512 = (__m512i *)f; + __m512i *e0_512 = (__m512i *)e0; + __m512i *e1_512 = (__m512i *)e1; + for (; i < (EQm & ~63); i += 64) { + __m512i e0j = _mm512_loadu_si512(e0_512++); + __m512i e1j = _mm512_loadu_si512(e1_512++); + // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi8(e0j, p8a, e1j)); + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi8(e0j, p8b, e1j)); } - } -#else - e0_128=(simde__m128i *)e0; - e1_128=(simde__m128i *)e1; - simde__m128i e0j,e1j; - for (k=0,j=0;j>4;j++,k+=2) { - e0j=simde_mm_loadu_si128(e0_128+j); - e1j=simde_mm_loadu_si128(e1_128+j); - simde_mm_storeu_si128(f_128+k,simde_mm_unpacklo_epi8(e0j,e1j)); - simde_mm_storeu_si128(f_128+k+1,simde_mm_unpackhi_epi8(e0j,e1j)); - } - if ((j<<4) != EQm) { - int k2=k<<4; - fp = &f[k2]; - for (int j2=(j<<4) ; j2< EQm ; j2++) { - *fp++ = e0[j2]; - *fp++ = e1[j2]; - } - } + e0 = (uint8_t *)e0_512; + e1 = (uint8_t *)e1_512; + f = (uint8_t *)f_512; #endif - break; - case 4: - e0=e; - e1=e0+EQm; - e2=e1+EQm; - e3=e2+EQm; +#ifdef USE128BIT + simde__m128i *f_128 = (simde__m128i *)f; + simde__m128i *e0_128 = (simde__m128i *)e0; + simde__m128i *e1_128 = (simde__m128i *)e1; + for (; i < (EQm & ~15); i += 64) { + simde__m128i e0j = simde_mm_loadu_si128(e0_128++); + simde__m128i e1j = simde_mm_loadu_si128(e1_128++); + simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi8(e0j, e1j)); + simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi8(e0j, e1j)); + } + e0 = (uint8_t *)e0_128; + e1 = (uint8_t *)e1_128; + f = (uint8_t *)f_128; +#endif + for (; i < EQm; i++) { + *f++ = *e0++; + *f++ = *e1++; + } + } break; + case 4: { + uint8_t *e0 = e; + uint8_t *e1 = e0 + EQm; + uint8_t *e2 = e1 + EQm; + uint8_t *e3 = e2 + EQm; + int i = 0; #ifdef __AVX512VBMI__ - e0_512=(__m512i *)e0; - e1_512=(__m512i *)e1; - e2_512=(__m512i *)e2; - e3_512=(__m512i *)e3; - p8a = _mm512_set_epi8(95,31,94,30,93,29,92,28,91,27,90,26,89,25,88,24,87,23,86,22,85,21,84,20,83,19,82,18,81,17,80,16,79,15,78,14,77,13,76,12,75,11,74,10,73,9,72,8,71,7,70,6,69,5,68,4,67,3,66,2,65,1,64,0); - p8b = _mm512_set_epi8(127,63,126,62,125,61,124,60,123,59,122,58,121,57,120,56,119,55,118,54,117,53,116,52,115,51,114,50,113,49,112,48,111,47,110,46,109,45,108,44,107,43,106,42,105,41,104,40,103,39,102,38,101,37,100,36,99,35,98,34,97,33,96,32); - - __m512i p16a = _mm512_set_epi16(47,15,46,14,45,13,44,12,43,11,42,10,41,9,40,8,39,7,38,6,37,5,36,4,35,3,34,2,33,1,32,0); - __m512i p16b = _mm512_set_epi16(63,31,62,30,61,29,60,28,59,27,58,26,57,25,56,24,55,23,54,22,53,21,52,20,51,19,50,18,49,17,48,16); - - __m512i p32a = _mm512_set_epi32(23,7,22,6,21,5,20,4,19,3,18,2,17,1,16,0); - __m512i p32b = _mm512_set_epi32(31,15,30,14,29,13,28,12,27,11,26,10,25,9,24,8); - __m512i e2j,e3j; - for (k=0,j=0;j>6;j++,k+=4) { - e0j = _mm512_loadu_si512(e0_512+j); - e1j = _mm512_loadu_si512(e1_512+j); - e2j = _mm512_loadu_si512(e2_512+j); - e3j = _mm512_loadu_si512(e3_512+j); - tmp0 = _mm512_permutex2var_epi8(e0j,p8a,e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15) - tmp1 = _mm512_permutex2var_epi8(e2j,p8a,e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15) - _mm512_storeu_si512(f_512+k,_mm512_permutex2var_epi16(tmp0,p16a,tmp1)); // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) - _mm512_storeu_si512(f_512+k+1,_mm512_permutex2var_epi16(tmp0,p16b,tmp1)); - - tmp0 = _mm512_permutex2var_epi8(e0j,p8b,e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15) - tmp1 = _mm512_permutex2var_epi8(e2j,p8b,e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15) - _mm512_storeu_si512(f_512+k+2,_mm512_permutex2var_epi16(tmp0,p16a,tmp1)); // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) - _mm512_storeu_si512(f_512+k+3,_mm512_permutex2var_epi16(tmp0,p16b,tmp1)); - } - if ((j<<6) != EQm) { - int k2=k<<6; - fp = &f[k2]; - for (int j2=(j<<6) ; j2< EQm ; j2++) { - *fp++ = e0[j2]; - *fp++ = e1[j2]; - *fp++ = e2[j2]; - *fp++ = e3[j2]; + // clang-format off + const __m512i p8a = _mm512_set_epi8(95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, 87, 23, 86, + 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, 79, 15, 78, 14, 77, 13, + 76, 12, 75, 11, 74, 10, 73, 9, 72, 8, 71, 7, 70, 6, 69, 5, 68, 4, 67, 3, 66, 2, 65, 1, 64, 0); + const __m512i p8b = _mm512_set_epi8(127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56, 119, + 55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48, 111, 47, + 110, 46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40, 103, 39, 102, + 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32); + const __m512i p16a = _mm512_set_epi16(47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7, 38, 6, + 37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0); + const __m512i p16b = _mm512_set_epi16(63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23, 54, + 22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16); + // clang-format on + __m512i *e0_512 = (__m512i *)e0; + __m512i *e1_512 = (__m512i *)e1; + __m512i *e2_512 = (__m512i *)e2; + __m512i *e3_512 = (__m512i *)e3; + __m512i *f_512 = (__m512i *)f; + for (; i < (EQm & ~63); i += 64) { + __m512i e0j = _mm512_loadu_si512(e0_512++); + __m512i e1j = _mm512_loadu_si512(e1_512++); + __m512i e2j = _mm512_loadu_si512(e2_512++); + __m512i e3j = _mm512_loadu_si512(e3_512++); + __m512i tmp0 = _mm512_permutex2var_epi8(e0j, p8a, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15) + __m512i tmp1 = _mm512_permutex2var_epi8(e2j, p8a, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15) + // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp0, p16a, tmp1)); + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp0, p16b, tmp1)); + tmp0 = _mm512_permutex2var_epi8(e0j, p8b, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15) + tmp1 = _mm512_permutex2var_epi8(e2j, p8b, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15) + // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp0, p16a, tmp1)); + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp0, p16b, tmp1)); } - } -#else - e0_128=(simde__m128i *)e0; - e1_128=(simde__m128i *)e1; - e2_128=(simde__m128i *)e2; - e3_128=(simde__m128i *)e3; - simde__m128i e2j,e3j; - for (k=0,j=0;j>4;j++,k+=4) { - e0j = simde_mm_loadu_si128(e0_128+j); - e1j = simde_mm_loadu_si128(e1_128+j); - e2j = simde_mm_loadu_si128(e2_128+j); - e3j = simde_mm_loadu_si128(e3_128+j); - tmp0 = simde_mm_unpacklo_epi8(e0j,e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+7) e1(i+7) - tmp1 = simde_mm_unpacklo_epi8(e2j,e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+7) e3(i+7) - simde_mm_storeu_si128(f_128+k,simde_mm_unpacklo_epi16(tmp0,tmp1)); // e0(i) e1(i) e2(i) e3(i) ... e0(i+3) e1(i+3) e2(i+3) e3(i+3) - simde_mm_storeu_si128(f_128+k+1,simde_mm_unpackhi_epi16(tmp0,tmp1)); // e0(i+4) e1(i+4) e2(i+4) e3(i+4) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) - tmp0 = simde_mm_unpackhi_epi8(e0j,e1j); // e0(i+8) e1(i+8) e0(i+9) e1(i+9) .... e0(i+15) e1(i+15) - tmp1 = simde_mm_unpackhi_epi8(e2j,e3j); // e2(i+8) e3(i+9) e2(i+10) e3(i+10) .... e2(i+31) e3(i+31) - simde_mm_storeu_si128(f_128+k+2,simde_mm_unpacklo_epi16(tmp0,tmp1)); - simde_mm_storeu_si128(f_128+k+3,simde_mm_unpackhi_epi16(tmp0,tmp1)); - } - if ((j<<4) != EQm) { - int k2=k<<4; - fp = &f[k2]; - for (int j2=(j<<4) ; j2< EQm ; j2++) { - *fp++ = e0[j2]; - *fp++ = e1[j2]; - *fp++ = e2[j2]; - *fp++ = e3[j2]; - } - } + e0 = (uint8_t *)e0_512; + e1 = (uint8_t *)e1_512; + e2 = (uint8_t *)e2_512; + e3 = (uint8_t *)e3_512; + f = (uint8_t *)f_512; #endif - break; - case 6: - e0=e; - e1=e0+EQm; - e2=e1+EQm; - e3=e2+EQm; - e4=e3+EQm; - e5=e4+EQm; +#ifdef USE128BIT + simde__m128i *e0_128 = (simde__m128i *)e0; + simde__m128i *e1_128 = (simde__m128i *)e1; + simde__m128i *e2_128 = (simde__m128i *)e2; + simde__m128i *e3_128 = (simde__m128i *)e3; + simde__m128i *f_128 = (simde__m128i *)f; + for (; i < (EQm & ~15); i += 16) { + simde__m128i e0j = simde_mm_loadu_si128(e0_128++); + simde__m128i e1j = simde_mm_loadu_si128(e1_128++); + simde__m128i e2j = simde_mm_loadu_si128(e2_128++); + simde__m128i e3j = simde_mm_loadu_si128(e3_128++); + simde__m128i tmp0 = simde_mm_unpacklo_epi8(e0j, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+7) e1(i+7) + simde__m128i tmp1 = simde_mm_unpacklo_epi8(e2j, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+7) e3(i+7) + simde_mm_storeu_si128(f_128++, + simde_mm_unpacklo_epi16(tmp0, tmp1)); // e0(i) e1(i) e2(i) e3(i) ... e0(i+3) e1(i+3) e2(i+3) e3(i+3) + simde_mm_storeu_si128( + f_128++, + simde_mm_unpackhi_epi16(tmp0, tmp1)); // e0(i+4) e1(i+4) e2(i+4) e3(i+4) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) + tmp0 = simde_mm_unpackhi_epi8(e0j, e1j); // e0(i+8) e1(i+8) e0(i+9) e1(i+9) .... e0(i+15) e1(i+15) + tmp1 = simde_mm_unpackhi_epi8(e2j, e3j); // e2(i+8) e3(i+9) e2(i+10) e3(i+10) .... e2(i+31) e3(i+31) + simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi16(tmp0, tmp1)); + simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi16(tmp0, tmp1)); + } + e0 = (uint8_t *)e0_128; + e1 = (uint8_t *)e1_128; + e2 = (uint8_t *)e2_128; + e3 = (uint8_t *)e3_128; + f = (uint8_t *)f_128; +#endif + for (; i < EQm; i++) { + *f++ = *e0++; + *f++ = *e1++; + *f++ = *e2++; + *f++ = *e3++; + } + } break; + case 6: { + uint8_t *e0 = e; + uint8_t *e1 = e0 + EQm; + uint8_t *e2 = e1 + EQm; + uint8_t *e3 = e2 + EQm; + uint8_t *e4 = e3 + EQm; + uint8_t *e5 = e4 + EQm; + int i = 0; #ifdef __AVX512VBMI__ - e0_512=(__m512i *)e0; - e1_512=(__m512i *)e1; - e2_512=(__m512i *)e2; - e3_512=(__m512i *)e3; - e4_512=(__m512i *)e4; - e5_512=(__m512i *)e5; - __m512i e4j,e5j; - p8a = _mm512_set_epi8(95,31,94,30,93,29,92,28,91,27,90,26,89,25,88,24,87,23,86,22,85,21,84,20,83,19,82,18,81,17,80,16,79,15,78,14,77,13,76,12,75,11,74,10,73,9,72,8,71,7,70,6,69,5,68,4,67,3,66,2,65,1,64,0); - p8b = _mm512_set_epi8(127,63,126,62,125,61,124,60,123,59,122,58,121,57,120,56,119,55,118,54,117,53,116,52,115,51,114,50,113,49,112,48,111,47,110,46,109,45,108,44,107,43,106,42,105,41,104,40,103,39,102,38,101,37,100,36,99,35,98,34,97,33,96,32); + // clang-format off + const __m512i p8a = _mm512_set_epi8(95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, 87, 23, + 86, 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, 79, 15, 78, 14, + 77, 13, 76, 12, 75, 11, 74, 10, 73, 9, 72, 8, 71, 7, 70, 6, 69, 5, 68, 4, 67, 3, 66, 2, 65, 1, 64, 0); + const __m512i p8b = _mm512_set_epi8(127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56, 119, + 55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48, 111, 47, 110, + 46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40, 103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32); + const __m512i p16a = _mm512_set_epi16(47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7, 38, 6, 37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0); + const __m512i p16b = _mm512_set_epi16(63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23, 54, 22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16); + const __m512i p16c = _mm512_set_epi16(21, 20, 41, 19, 18, 40, 17, 16, 39, 15, 14, 38, 13, 12, 37, 11, 10, 36, 9, 8, 35, 7, 6, 34, 5, 4, 33, 3, 2, 32, 1, 0); + const __m512i p16d = _mm512_set_epi16(10, 52, 9, 8, 51, 7, 6, 50, 5, 4, 49, 3, 2, 48, 1, 0, 47, 31, 30, 46, 29, 28, 45, 27, 26, 44, 25, 24, 43, 23, 22, 42); + const __m512i p16d2 = _mm512_set_epi16(32 + 10, 30, 32 + 9, 32 + 8, 27, 32 + 7, 32 + 6, 24, 32 + 5, 32 + 4, 21, 32 + 3, 32 + 2, 18, 32 + 1, 32 + 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + const __m512i p16e = _mm512_set_epi16(63, 31, 30, 62, 29, 28, 61, 27, 26, 60, 25, 24, 59, 23, 22, 58, 21, 20, 57, 19, 18, 56, 17, 16, 55, 15, 14, 54, 13, 12, 53, 11); + // clang-format on + __m512i *e0_512 = (__m512i *)e0; + __m512i *e1_512 = (__m512i *)e1; + __m512i *e2_512 = (__m512i *)e2; + __m512i *e3_512 = (__m512i *)e3; + __m512i *e4_512 = (__m512i *)e4; + __m512i *e5_512 = (__m512i *)e5; + __m512i *f_512 = (__m512i *)f; + for (; i < (EQm & ~63); i += 64) { + __m512i e0j = _mm512_loadu_si512(e0_512++); + __m512i e1j = _mm512_loadu_si512(e1_512++); + __m512i e2j = _mm512_loadu_si512(e2_512++); + __m512i e3j = _mm512_loadu_si512(e3_512++); + __m512i e4j = _mm512_loadu_si512(e4_512++); + __m512i e5j = _mm512_loadu_si512(e5_512++); + __m512i tmp0 = _mm512_permutex2var_epi8(e0j, p8a, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+31) e1(i+31) + __m512i tmp1 = _mm512_permutex2var_epi8(e2j, p8a, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+31) e3(i+31) + __m512i tmp2 = _mm512_permutex2var_epi8(e4j, p8a, e5j); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+31) e5(i+31) + // e0(i) e1(i) e2(i) e3(i) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) + __m512i tmp3 = _mm512_permutex2var_epi16(tmp0, p16a, tmp1); + // e0(i+16) e1(i+16) e2(i+16) e3(i+16) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31) + __m512i tmp4 = _mm512_permutex2var_epi16(tmp0, p16b, tmp1); + // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) ... e0(i+9) e1(i+9) e2(i+9) e3(i+9) e4(i+9) e5(i+9) e0(i+10) e1(i+10) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp3, p16c, tmp2)); + // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) e4(i+15) + // e5(i+15) x x x x e4(i+16) e5(i+16) x x x x .... e4(i+20) e5(i+20) x x + __m512i tmp5 = _mm512_permutex2var_epi16(tmp3, p16d, tmp2); + // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) + // e4(i+15) e5(i+15) e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16) e0(i+20) + // e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e0(i+21) e1(i+21) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp5, p16d2, tmp4)); + // e2(i+21) e3(i+21) e4(i+21) e5(i+21) .... e0(i+31) e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp4, p16e, tmp2)); - p16a = _mm512_set_epi16(47,15,46,14,45,13,44,12,43,11,42,10,41,9,40,8,39,7,38,6,37,5,36,4,35,3,34,2,33,1,32,0); - p16b = _mm512_set_epi16(63,31,62,30,61,29,60,28,59,27,58,26,57,25,56,24,55,23,54,22,53,21,52,20,51,19,50,18,49,17,48,16); - - __m512i p16c = _mm512_set_epi16(21,20,41,19,18,40,17,16,39,15,14,38,13,12,37,11,10,36,9,8,35,7,6,34,5,4,33,3,2,32,1,0); - __m512i p16d = _mm512_set_epi16(10,52,9,8,51,7,6,50,5,4,49,3,2,48,1,0,47,31,30,46,29,28,45,27,26,44,25,24,43,23,22,42); - __m512i p16d2 = _mm512_set_epi16(32+10,30,32+9,32+8,27,32+7,32+6,24,32+5,32+4,21,32+3,32+2,18,32+1,32+0,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0); - __m512i p16e = _mm512_set_epi16(63,31,30,62,29,28,61,27,26,60,25,24,59,23,22,58,21,20,57,19,18,56,17,16,55,15,14,54,13,12,53,11); - - for (j=0,k=0;j>6;j++,k+=6) { - e0j = _mm512_loadu_si512(e0_512+j); - e1j = _mm512_loadu_si512(e1_512+j); - e2j = _mm512_loadu_si512(e2_512+j); - e3j = _mm512_loadu_si512(e3_512+j); - e4j = _mm512_loadu_si512(e4_512+j); - e5j = _mm512_loadu_si512(e5_512+j); - tmp0 = _mm512_permutex2var_epi8(e0j,p8a,e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+31) e1(i+31) - tmp1 = _mm512_permutex2var_epi8(e2j,p8a,e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+31) e3(i+31) - tmp2 = _mm512_permutex2var_epi8(e4j,p8a,e5j); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+31) e5(i+31) - tmp3 = _mm512_permutex2var_epi16(tmp0,p16a,tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) - tmp4 = _mm512_permutex2var_epi16(tmp0,p16b,tmp1); // e0(i+16) e1(i+16) e2(i+16) e3(i+16) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31) - _mm512_storeu_si512(f_512+k,_mm512_permutex2var_epi16(tmp3,p16c,tmp2)); // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) ... e0(i+9) e1(i+9) e2(i+9) e3(i+9) e4(i+9) e5(i+9) e0(i+10) e1(i+10) - tmp5 = _mm512_permutex2var_epi16(tmp3,p16d,tmp2); // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) e4(i+15) e5(i+15) x x x x e4(i+16) e5(i+16) x x x x .... e4(i+20) e5(i+20) x x - _mm512_storeu_si512(f_512+k+1,_mm512_permutex2var_epi16(tmp5,p16d2,tmp4)); // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) e4(i+15) e5(i+15) e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16) e0(i+20) e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e0(i+21) e1(i+21) - _mm512_storeu_si512(f_512+k+2,_mm512_permutex2var_epi16(tmp4,p16e,tmp2)); // e2(i+21) e3(i+21) e4(i+21) e5(i+21) .... e0(i+31) e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31) - - tmp0 = _mm512_permutex2var_epi8(e0j,p8b,e1j); // e0(i+32) e1(i+32) e0(i+32) e1(i+32) .... e0(i+63) e1(i+63) - tmp1 = _mm512_permutex2var_epi8(e2j,p8b,e3j); // e2(i+32) e3(i+32) e2(i+32) e3(i+32) .... e2(i+63) e3(i+63) - tmp2 = _mm512_permutex2var_epi8(e4j,p8b,e5j); // e4(i+32) e5(i+32) e4(i+32) e5(i+32) .... e4(i+63) e5(i+63) - tmp3 = _mm512_permutex2var_epi16(tmp0,p16a,tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) - tmp4 = _mm512_permutex2var_epi16(tmp0,p16b,tmp1); // e0(i+16) e1(i+16) e2(i+16) e3(i+16) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31) - - _mm512_storeu_si512(f_512+k+3,_mm512_permutex2var_epi16(tmp3,p16c,tmp2)); // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) ... e0(i+9) e1(i+9) e2(i+9) e3(i+9) e4(i+9) e5(i+9) e0(i+10) e1(i+10) - tmp5 = _mm512_permutex2var_epi16(tmp3,p16d,tmp2); // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) e4(i+15) e5(i+15) x x x x e4(i+16) e5(i+16) x x x x .... e4(i+20) e5(i+20) x x - _mm512_storeu_si512(f_512+k+4,_mm512_permutex2var_epi16(tmp5,p16d2,tmp4)); // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) e4(i+15) e5(i+15) e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16) e0(i+20) e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e0(i+21) e1(i+21) - _mm512_storeu_si512(f_512+k+5,_mm512_permutex2var_epi16(tmp4,p16e,tmp2)); // e2(i+21) e3(i+21) e4(i+21) e5(i+21) .... e0(i+31) e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31) - } - if ((j<<6) != EQm) { - int k2=k<<6; - fp = &f[k2]; - for (int j2=(j<<6) ; j2< EQm ; j2++) { - *fp++ = e0[j2]; - *fp++ = e1[j2]; - *fp++ = e2[j2]; - *fp++ = e3[j2]; - *fp++ = e4[j2]; - *fp++ = e5[j2]; + tmp0 = _mm512_permutex2var_epi8(e0j, p8b, e1j); // e0(i+32) e1(i+32) e0(i+32) e1(i+32) .... e0(i+63) e1(i+63) + tmp1 = _mm512_permutex2var_epi8(e2j, p8b, e3j); // e2(i+32) e3(i+32) e2(i+32) e3(i+32) .... e2(i+63) e3(i+63) + tmp2 = _mm512_permutex2var_epi8(e4j, p8b, e5j); // e4(i+32) e5(i+32) e4(i+32) e5(i+32) .... e4(i+63) e5(i+63) + tmp3 = _mm512_permutex2var_epi16(tmp0, p16a, tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) + // e0(i+16) e1(i+16) e2(i+16) e3(i+16) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31) + tmp4 = _mm512_permutex2var_epi16(tmp0, p16b, tmp1); + // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) ... e0(i+9) e1(i+9) e2(i+9) e3(i+9) e4(i+9) e5(i+9) e0(i+10) e1(i+10) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp3, p16c, tmp2)); + // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) + // e4(i+15) e5(i+15) x x x x e4(i+16) e5(i+16) x x x x .... e4(i+20) e5(i+20) x x + tmp5 = _mm512_permutex2var_epi16(tmp3, p16d, tmp2); + // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) + // e4(i+15) e5(i+15) e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16) e0(i+20) + // e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e0(i+21) e1(i+21) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp5, p16d2, tmp4)); + // e2(i+21) e3(i+21) e4(i+21) e5(i+21) .... e0(i+31) e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp4, p16e, tmp2)); } - } -#else - fp = f; - for (int j = 0; j< EQm; j++){ - *fp++ = e0[j]; - *fp++ = e1[j]; - *fp++ = e2[j]; - *fp++ = e3[j]; - *fp++ = e4[j]; - *fp++ = e5[j]; - } - + e0 = (uint8_t *)e0_512; + e1 = (uint8_t *)e1_512; + e2 = (uint8_t *)e2_512; + e3 = (uint8_t *)e3_512; + e4 = (uint8_t *)e4_512; + e5 = (uint8_t *)e5_512; + f = (uint8_t *)f_512; #endif - break; - case 8: - e0=e; - e1=e0+EQm; - e2=e1+EQm; - e3=e2+EQm; - e4=e3+EQm; - e5=e4+EQm; - e6=e5+EQm; - e7=e6+EQm; - + for (; i < EQm; i++) { + *f++ = *e0++; + *f++ = *e1++; + *f++ = *e2++; + *f++ = *e3++; + *f++ = *e4++; + *f++ = *e5++; + } + } break; + case 8: { + uint8_t *e0 = e; + uint8_t *e1 = e0 + EQm; + uint8_t *e2 = e1 + EQm; + uint8_t *e3 = e2 + EQm; + uint8_t *e4 = e3 + EQm; + uint8_t *e5 = e4 + EQm; + uint8_t *e6 = e5 + EQm; + uint8_t *e7 = e6 + EQm; + + int i = 0; #ifdef __AVX512VBMI__ - e0_512=(__m512i *)e0; - e1_512=(__m512i *)e1; - e2_512=(__m512i *)e2; - e3_512=(__m512i *)e3; - e4_512=(__m512i *)e4; - e5_512=(__m512i *)e5; - e6_512=(__m512i *)e6; - e7_512=(__m512i *)e7; - __m512i e6j,e7j; - p8a = _mm512_set_epi8(95,31,94,30,93,29,92,28,91,27,90,26,89,25,88,24,87,23,86,22,85,21,84,20,83,19,82,18,81,17,80,16,79,15,78,14,77,13,76,12,75,11,74,10,73,9,72,8,71,7,70,6,69,5,68,4,67,3,66,2,65,1,64,0); - p8b = _mm512_set_epi8(127,63,126,62,125,61,124,60,123,59,122,58,121,57,120,56,119,55,118,54,117,53,116,52,115,51,114,50,113,49,112,48,111,47,110,46,109,45,108,44,107,43,106,42,105,41,104,40,103,39,102,38,101,37,100,36,99,35,98,34,97,33,96,32); + // clang-format off + const __m512i p8a = _mm512_set_epi8(95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, 87, 23, + 86, 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, 79, 15, 78, 14, 77, 13, + 76, 12, 75, 11, 74, 10, 73, 9, 72, 8, 71, 7, 70, 6, 69, 5, 68, 4, 67, 3, 66, 2, 65, 1, 64, 0); + const __m512i p8b = _mm512_set_epi8(127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56, 119, 55, + 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48, 111, 47, 110, 46, + 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40, 103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32); - p16a = _mm512_set_epi16(47,15,46,14,45,13,44,12,43,11,42,10,41,9,40,8,39,7,38,6,37,5,36,4,35,3,34,2,33,1,32,0); - p16b = _mm512_set_epi16(63,31,62,30,61,29,60,28,59,27,58,26,57,25,56,24,55,23,54,22,53,21,52,20,51,19,50,18,49,17,48,16); + const __m512i p16a = _mm512_set_epi16(47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7, 38, 6, 37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0); + const __m512i p16b = _mm512_set_epi16(63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23, 54, 22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16); - p32a = _mm512_set_epi32(23,7,22,6,21,5,20,4,19,3,18,2,17,1,16,0); - p32b = _mm512_set_epi32(31,15,30,14,29,13,28,12,27,11,26,10,25,9,24,8); - for (k=0,j=0;j>6;j++,k+=8) { - e0j = _mm512_loadu_si512(e0_512+j); - e1j = _mm512_loadu_si512(e1_512+j); - e2j = _mm512_loadu_si512(e2_512+j); - e3j = _mm512_loadu_si512(e3_512+j); - e4j = _mm512_loadu_si512(e4_512+j); - e5j = _mm512_loadu_si512(e5_512+j); - e6j = _mm512_loadu_si512(e6_512+j); - e7j = _mm512_loadu_si512(e7_512+j); - tmp0 = _mm512_permutex2var_epi8(e0j,p8a,e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15) - tmp1 = _mm512_permutex2var_epi8(e2j,p8a,e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15) - tmp2 = _mm512_permutex2var_epi8(e4j,p8a,e5j); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+15) e5(i+15) - tmp3 = _mm512_permutex2var_epi8(e6j,p8a,e7j); // e6(i) e7(i) e6(i+1) e7(i+1) .... e6(i+15) e7(i+15) - tmp4 = _mm512_permutex2var_epi16(tmp0,p16a,tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) - tmp5 = _mm512_permutex2var_epi16(tmp2,p16a,tmp3); // e4(i) e5(i) e6(i) e7(i) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7) - _mm512_storeu_si512(f_512+k,_mm512_permutex2var_epi32(tmp4,p32a,tmp5)); // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i)... e0(i+3) e1(i+3) e2(i+3) e3(i+3) e4(i+3) e5(i+3) e6(i+3) e7(i+3)) - _mm512_storeu_si512(f_512+k+1,_mm512_permutex2var_epi32(tmp4,p32b,tmp5)); // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7(i+4)... e0(i+7) e1(i+7) e2(i+7) e3(i+7) e4(i+7) e5(i+7) e6(i+7) e7(i+7)) + const __m512i p32a = _mm512_set_epi32(23, 7, 22, 6, 21, 5, 20, 4, 19, 3, 18, 2, 17, 1, 16, 0); + const __m512i p32b = _mm512_set_epi32(31, 15, 30, 14, 29, 13, 28, 12, 27, 11, 26, 10, 25, 9, 24, 8); + // clang-format on + __m512i *e0_512 = (__m512i *)e0; + __m512i *e1_512 = (__m512i *)e1; + __m512i *e2_512 = (__m512i *)e2; + __m512i *e3_512 = (__m512i *)e3; + __m512i *e4_512 = (__m512i *)e4; + __m512i *e5_512 = (__m512i *)e5; + __m512i *e6_512 = (__m512i *)e6; + __m512i *e7_512 = (__m512i *)e7; + __m512i *f_512 = (__m512i *)f; + for (; i < (EQm & ~63); i += 64) { + __m512i e0j = _mm512_loadu_si512(e0_512++); + __m512i e1j = _mm512_loadu_si512(e1_512++); + __m512i e2j = _mm512_loadu_si512(e2_512++); + __m512i e3j = _mm512_loadu_si512(e3_512++); + __m512i e4j = _mm512_loadu_si512(e4_512++); + __m512i e5j = _mm512_loadu_si512(e5_512++); + __m512i e6j = _mm512_loadu_si512(e6_512++); + __m512i e7j = _mm512_loadu_si512(e7_512++); + __m512i tmp0 = _mm512_permutex2var_epi8(e0j, p8a, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15) + __m512i tmp1 = _mm512_permutex2var_epi8(e2j, p8a, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15) + __m512i tmp2 = _mm512_permutex2var_epi8(e4j, p8a, e5j); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+15) e5(i+15) + __m512i tmp3 = _mm512_permutex2var_epi8(e6j, p8a, e7j); // e6(i) e7(i) e6(i+1) e7(i+1) .... e6(i+15) e7(i+15) + __m512i tmp4 = _mm512_permutex2var_epi16(tmp0, p16a, tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) + // e4(i) e5(i) e6(i) e7(i) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7) + // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i)... e0(i+3) e1(i+3) + __m512i tmp5 = _mm512_permutex2var_epi16(tmp2, p16a, tmp3); + // e2(i+3) e3(i+3) e4(i+3) e5(i+3) e6(i+3) e7(i+3)) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32a, tmp5)); + // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7(i+4)... + // e0(i+7) e1(i+7) e2(i+7) e3(i+7) e4(i+7) e5(i+7) e6(i+7) e7(i+7)) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32b, tmp5)); + // e0(i+8) e1(i+8) e2(i+8) e3(i+8) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) + tmp4 = _mm512_permutex2var_epi16(tmp0, p16b, tmp1); + // e4(i+8) e5(i+8) e6(i+8) e7(i+8) ... e4(i+15) e5(i+15) e6(i+15) e7(i+15) + tmp5 = _mm512_permutex2var_epi16(tmp2, p16b, tmp3); + // e0(i+8) e1(i+8) e2(i+8) e3(i+8) e4(i+8) e5(i+8) e6(i+8) e7(i+8)... e0(i+11) + // e1(i+11) e2(i+11) e3(i+11) e4(i+11) e5(i+11) e6(i+11) e7(i+11)) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32a, tmp5)); + // e0(i+12) e1(i+12) e2(i+12) e3(i+12) e4(i+12) e5(i+12) e6(i+12) e7(i+12)... e0(i+15) + // e1(i+15) e2(i+15) e3(i+15) e4(i+15) e5(i+15) e6(i+15) e7(i+15)) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32b, tmp5)); - tmp4 = _mm512_permutex2var_epi16(tmp0,p16b,tmp1); // e0(i+8) e1(i+8) e2(i+8) e3(i+8) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) - tmp5 = _mm512_permutex2var_epi16(tmp2,p16b,tmp3); // e4(i+8) e5(i+8) e6(i+8) e7(i+8) ... e4(i+15) e5(i+15) e6(i+15) e7(i+15) - _mm512_storeu_si512(f_512+k+2,_mm512_permutex2var_epi32(tmp4,p32a,tmp5)); // e0(i+8) e1(i+8) e2(i+8) e3(i+8) e4(i+8) e5(i+8) e6(i+8) e7(i+8)... e0(i+11) e1(i+11) e2(i+11) e3(i+11) e4(i+11) e5(i+11) e6(i+11) e7(i+11)) - _mm512_storeu_si512(f_512+k+3,_mm512_permutex2var_epi32(tmp4,p32b,tmp5)); // e0(i+12) e1(i+12) e2(i+12) e3(i+12) e4(i+12) e5(i+12) e6(i+12) e7(i+12)... e0(i+15) e1(i+15) e2(i+15) e3(i+15) e4(i+15) e5(i+15) e6(i+15) e7(i+15)) - - tmp0 = _mm512_permutex2var_epi8(e0j,p8b,e1j); // e0(i+16) e1(i+16) e0(i+17) e1(i+17) .... e0(i+31) e1(i+31) - tmp1 = _mm512_permutex2var_epi8(e2j,p8b,e3j); // e2(i+16) e3(i+16) e2(i+17) e3(i+17) .... e2(i+31) e3(i+31) - tmp2 = _mm512_permutex2var_epi8(e4j,p8b,e5j); // e4(i+16) e5(i+16) e4(i+17) e5(i+17) .... e4(i+31) e5(i+31) - tmp3 = _mm512_permutex2var_epi8(e6j,p8b,e7j); // e6(i+16) e7(i+16) e6(i+17) e7(i+17) .... e6(i+31) e7(i+31) - tmp4 = _mm512_permutex2var_epi16(tmp0,p16a,tmp1); // e0(i+!6) e1(i+16) e2(i+16) e3(i+16) ... e0(i+23) e1(i+23) e2(i+23) e3(i+23) - tmp5 = _mm512_permutex2var_epi16(tmp2,p16a,tmp3); // e4(i+16) e5(i+16) e6(i+16) e7(i+16) ... e4(i+23) e5(i+23) e6(i+23) e7(i+23) - _mm512_storeu_si512(f_512+k+4,_mm512_permutex2var_epi32(tmp4,p32a,tmp5)); // e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16) e6(i+16) e7(i+16)... e0(i+19) e1(i+19) e2(i+19) e3(i+19) e4(i+19) e5(i+19) e6(i+19) e7(i+19)) - _mm512_storeu_si512(f_512+k+5,_mm512_permutex2var_epi32(tmp4,p32b,tmp5)); // e0(i+20) e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e6(i+20) e7(i+20)... e0(i+23) e1(i+23) e2(i+23) e3(i+23) e4(i+23) e5(i+23) e6(i+23) e7(i+23)) - - tmp4 = _mm512_permutex2var_epi16(tmp0,p16b,tmp1); // e0(i+24) e1(i+24) e2(i+24) e3(i+24) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31) - tmp5 = _mm512_permutex2var_epi16(tmp2,p16b,tmp3); // e4(i+24) e5(i+24) e6(i+24) e7(i+24) ... e4(i+31) e5(i+31) e6(i+31) e7(i+31) - _mm512_storeu_si512(f_512+k+6,_mm512_permutex2var_epi32(tmp4,p32a,tmp5)); // e0(i+24) e1(i+24) e2(i+24) e3(i+24) e4(i+24) e5(i+24) e6(i+24) e7(i+24)... e0(i+27) e1(i+27) e2(i+27) e3(i+27) e4(i+27) e5(i+27) e6(i+27) e7(i+27)) - _mm512_storeu_si512(f_512+k+7,_mm512_permutex2var_epi32(tmp4,p32b,tmp5)); // e0(i+28) e1(i+28) e2(i+28) e3(i+28) e4(i+28) e5(i+28) e6(i+28) e7(i+28)... e0(i+31) e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31) e6(i+31) e7(i+31)) - } - if ((j<<6) != EQm) { - int k2=k<<6; - fp=&f[k2]; - for (int j2=(j<<6) ; j2< EQm ; j2++) { - *fp++ = e0[j2]; - *fp++ = e1[j2]; - *fp++ = e2[j2]; - *fp++ = e3[j2]; - *fp++ = e4[j2]; - *fp++ = e5[j2]; - *fp++ = e6[j2]; - *fp++ = e7[j2]; + tmp0 = _mm512_permutex2var_epi8(e0j, p8b, e1j); // e0(i+16) e1(i+16) e0(i+17) e1(i+17) .... e0(i+31) e1(i+31) + tmp1 = _mm512_permutex2var_epi8(e2j, p8b, e3j); // e2(i+16) e3(i+16) e2(i+17) e3(i+17) .... e2(i+31) e3(i+31) + tmp2 = _mm512_permutex2var_epi8(e4j, p8b, e5j); // e4(i+16) e5(i+16) e4(i+17) e5(i+17) .... e4(i+31) e5(i+31) + tmp3 = _mm512_permutex2var_epi8(e6j, p8b, e7j); // e6(i+16) e7(i+16) e6(i+17) e7(i+17) .... e6(i+31) e7(i+31) + // e0(i+!6) e1(i+16) e2(i+16) e3(i+16) ... e0(i+23) e1(i+23) e2(i+23) e3(i+23) + tmp4 = _mm512_permutex2var_epi16(tmp0, p16a, tmp1); + // e4(i+16) e5(i+16) e6(i+16) e7(i+16) ... e4(i+23) e5(i+23) e6(i+23) e7(i+23) + tmp5 = _mm512_permutex2var_epi16(tmp2, p16a, tmp3); + // e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16) e6(i+16) e7(i+16)... e0(i+19) + // e1(i+19) e2(i+19) e3(i+19) e4(i+19) e5(i+19) e6(i+19) e7(i+19)) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32a, tmp5)); + // e0(i+20) e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e6(i+20) e7(i+20)... e0(i+23) + // e1(i+23) e2(i+23) e3(i+23) e4(i+23) e5(i+23) e6(i+23) e7(i+23)) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32b, tmp5)); + // e0(i+24) e1(i+24) e2(i+24) e3(i+24) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31) + tmp4 = _mm512_permutex2var_epi16(tmp0, p16b, tmp1); + // e4(i+24) e5(i+24) e6(i+24) e7(i+24) ... e4(i+31) e5(i+31) e6(i+31) e7(i+31) + tmp5 = _mm512_permutex2var_epi16(tmp2, p16b, tmp3); + // e0(i+24) e1(i+24) e2(i+24) e3(i+24) e4(i+24) e5(i+24) e6(i+24) e7(i+24)... e0(i+27) + // e1(i+27) e2(i+27) e3(i+27) e4(i+27) e5(i+27) e6(i+27) e7(i+27)) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32a, tmp5)); + // e0(i+28) e1(i+28) e2(i+28) e3(i+28) e4(i+28) e5(i+28) e6(i+28) e7(i+28)... e0(i+31) + // e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31) e6(i+31) e7(i+31)) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32b, tmp5)); } - } -#else - e0_128=(simde__m128i *)e0; - e1_128=(simde__m128i *)e1; - e2_128=(simde__m128i *)e2; - e3_128=(simde__m128i *)e3; - e4_128=(simde__m128i *)e4; - e5_128=(simde__m128i *)e5; - e6_128=(simde__m128i *)e6; - e7_128=(simde__m128i *)e7; - simde__m128i e4j,e5j,e6j,e7j; - - for (k=0,j=0;j>4;j++,k+=8) { - e0j = simde_mm_loadu_si128(e0_128+j); - e1j = simde_mm_loadu_si128(e1_128+j); - e2j = simde_mm_loadu_si128(e2_128+j); - e3j = simde_mm_loadu_si128(e3_128+j); - e4j = simde_mm_loadu_si128(e4_128+j); - e5j = simde_mm_loadu_si128(e5_128+j); - e6j = simde_mm_loadu_si128(e6_128+j); - e7j = simde_mm_loadu_si128(e7_128+j); - tmp0 = simde_mm_unpacklo_epi8(e0j,e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+7) e1(i+7) - tmp1 = simde_mm_unpacklo_epi8(e2j,e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+7) e3(i+7) - tmp2 = simde_mm_unpacklo_epi16(tmp0,tmp1);// e0(i) e1(i) e2(i) e3(i) e0(i+1) e1(i+1) e2(i+1) e3(i+1) ... e0(i+3) e1(i+3) e2(i+3) e3(i+3) - tmp3 = simde_mm_unpackhi_epi16(tmp0,tmp1);// e0(i+4) e1(i+4) e2(i+4) e3(i+4) e0(i+5) e1(i+5) e2(i+5) e3(i+5) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) - tmp4 = simde_mm_unpacklo_epi8(e4j,e5j); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+7) e5(i+7) - tmp5 = simde_mm_unpacklo_epi8(e6j,e7j); // e6(i) e7(i) e6(i+1) e7(i+1) .... e6(i+7) e7(i+7) - tmp6 = simde_mm_unpacklo_epi16(tmp4,tmp5);// e4(i) e5(i) e6(i) e7(i) e4(i+1) e5(i+1) e6(i+1) e7(i+1) ... e4(i+3) e5(i+3) e6(i+) e7(i+3) - tmp7 = simde_mm_unpackhi_epi16(tmp4,tmp5);// e4(i+4) e5(i+4) e6(i+4) e7(i+4) e4(i+5) e5(i+5) e6(i+5) e7(i+5) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7) - simde_mm_storeu_si128(f_128+k,simde_mm_unpacklo_epi32(tmp2,tmp6)); // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i) e0(i+1) ... e7(i+1) - simde_mm_storeu_si128(f_128+k+1,simde_mm_unpackhi_epi32(tmp2,tmp6)); // e0(i+2) e1(i+2) e2(i+2) e3(i+2) e4(i+2) e5(i+2) e6(i+2) e7(i+2) e0(i+3) e1(i+3) ... e7(i+3) - simde_mm_storeu_si128(f_128+k+2,simde_mm_unpacklo_epi32(tmp3,tmp7)); // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7+4(i) e0(i+5) ... e7(i+5) - simde_mm_storeu_si128(f_128+k+3,simde_mm_unpackhi_epi32(tmp3,tmp7)); // e0(i+6) e1(i+6) e2(i+6) e3(i+6) e4(i+6) e5(i+6) e6(i+6) e7(i+6) e0(i+7) e0(i+7) ... e7(i+7) - - - tmp0 = simde_mm_unpackhi_epi8(e0j,e1j); // e0(i+8) e1(i+8) e0(i+9) e1(i+9) .... e0(i+15) e1(i+15) - tmp1 = simde_mm_unpackhi_epi8(e2j,e3j); // e2(i+8) e3(i+8) e2(i+9) e3(i+9) .... e2(i+15) e3(i+15) - tmp2 = simde_mm_unpacklo_epi16(tmp0,tmp1);// e0(i) e1(i) e2(i) e3(i) e0(i+1) e1(i+1) e2(i+1) e3(i+1) ... e0(i+3) e1(i+3) e2(i+3) e3(i+3) - tmp3 = simde_mm_unpackhi_epi16(tmp0,tmp1);// e0(i+4) e1(i+4) e2(i+4) e3(i+4) e0(i+5) e1(i+5) e2(i+5) e3(i+5) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) - tmp4 = simde_mm_unpackhi_epi8(e4j,e5j); // e4(i+8) e5(i+8) e4(i+9) e5(i+9) .... e4(i+15) e515i+7) - tmp5 = simde_mm_unpackhi_epi8(e6j,e7j); // e6(i+8) e7(i+8) e6(i+9) e7(i+9) .... e6(i+15) e7(i+15) - tmp6 = simde_mm_unpacklo_epi16(tmp4,tmp5);// e4(i) e5(i) e6(i) e7(i) e4(i+1) e5(i+1) e6(i+1) e7(i+1) ... e4(i+3) e5(i+3) e6(i+) e7(i+3) - tmp7 = simde_mm_unpackhi_epi16(tmp4,tmp5);// e4(i+4) e5(i+4) e6(i+4) e7(i+4) e4(i+5) e5(i+5) e6(i+5) e7(i+5) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7) - simde_mm_storeu_si128(f_128+k+4,simde_mm_unpacklo_epi32(tmp2,tmp6)); // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i) e0(i+1) ... e7(i+1) - simde_mm_storeu_si128(f_128+k+5,simde_mm_unpackhi_epi32(tmp2,tmp6)); // e0(i+2) e1(i+2) e2(i+2) e3(i+2) e4(i+2) e5(i+2) e6(i+2) e7(i+2) e0(i+3) e1(i+3) ... e7(i+3) - simde_mm_storeu_si128(f_128+k+6,simde_mm_unpacklo_epi32(tmp3,tmp7)); // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7+4(i) e0(i+5) ... e7(i+5) - simde_mm_storeu_si128(f_128+k+7,simde_mm_unpackhi_epi32(tmp3,tmp7)); // e0(i+6) e1(i+6) e2(i+6) e3(i+6) e4(i+6) e5(i+6) e6(i+6) e7(i+6) e0(i+7) e0(i+7) ... e7(i+7) - } - if ((j<<4) != EQm) { - int k2=k<<4; - fp=&f[k2]; - for (int j2=(j<<4) ; j2< EQm ; j2++) { - *fp++ = e0[j2]; - *fp++ = e1[j2]; - *fp++ = e2[j2]; - *fp++ = e3[j2]; - *fp++ = e4[j2]; - *fp++ = e5[j2]; - *fp++ = e6[j2]; - *fp++ = e7[j2]; - } - } + e0 = (uint8_t *)e0_512; + e1 = (uint8_t *)e1_512; + e2 = (uint8_t *)e2_512; + e3 = (uint8_t *)e3_512; + e4 = (uint8_t *)e4_512; + e5 = (uint8_t *)e5_512; + e6 = (uint8_t *)e6_512; + e7 = (uint8_t *)e7_512; + f = (uint8_t *)f_512; #endif - break; - default: AssertFatal(1==0,"Should be here!\n"); - } +#ifdef USE128BIT + e0_128 = (simde__m128i *)e0; + e1_128 = (simde__m128i *)e1; + e2_128 = (simde__m128i *)e2; + e3_128 = (simde__m128i *)e3; + e4_128 = (simde__m128i *)e4; + e5_128 = (simde__m128i *)e5; + e6_128 = (simde__m128i *)e6; + e7_128 = (simde__m128i *)e7; + for (; i < (EQm & ~15); i += 16) { + simde__m128i e0j = simde_mm_loadu_si128(e0_128++); + simde__m128i e1j = simde_mm_loadu_si128(e1_128++); + simde__m128i e2j = simde_mm_loadu_si128(e2_128++); + simde__m128i e3j = simde_mm_loadu_si128(e3_128++); + simde__m128i e4j = simde_mm_loadu_si128(e4_128++); + simde__m128i e5j = simde_mm_loadu_si128(e5_128++); + simde__m128i e6j = simde_mm_loadu_si128(e6_128++); + simde__m128i e7j = simde_mm_loadu_si128(e7_128++); + simde__m128i tmp0 = simde_mm_unpacklo_epi8(e0j, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+7) e1(i+7) + simde__m128i tmp1 = simde_mm_unpacklo_epi8(e2j, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+7) e3(i+7) + // e0(i) e1(i) e2(i) e3(i) e0(i+1) e1(i+1) e2(i+1) e3(i+1) ... e0(i+3) e1(i+3) e2(i+3) e3(i+3) + simde__m128i tmp2 = simde_mm_unpacklo_epi16(tmp0, tmp1); + // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e0(i+5) e1(i+5) e2(i+5) e3(i+5) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) + simde__m128i tmp3 = simde_mm_unpackhi_epi16(tmp0, tmp1); + simde__m128i tmp4 = simde_mm_unpacklo_epi8(e4j, e5j); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+7) e5(i+7) + simde__m128i tmp5 = simde_mm_unpacklo_epi8(e6j, e7j); // e6(i) e7(i) e6(i+1) e7(i+1) .... e6(i+7) e7(i+7) + // e4(i) e5(i) e6(i) e7(i) e4(i+1) e5(i+1) e6(i+1) e7(i+1) ... e4(i+3) e5(i+3) e6(i+) e7(i+3) + simde__m128i tmp6 = simde_mm_unpacklo_epi16(tmp4, tmp5); + // e4(i+4) e5(i+4) e6(i+4) e7(i+4) e4(i+5) e5(i+5) e6(i+5) e7(i+5) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7) + simde__m128i tmp7 = simde_mm_unpackhi_epi16(tmp4, tmp5); + // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i) e0(i+1) ... e7(i+1) + simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi32(tmp2, tmp6)); + // e0(i+2) e1(i+2) e2(i+2) e3(i+2) e4(i+2) e5(i+2) e6(i+2) e7(i+2) e0(i+3) e1(i+3) ... e7(i+3) + simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi32(tmp2, tmp6)); + // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7+4(i) e0(i+5) ... e7(i+5) + simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi32(tmp3, tmp7)); + // e0(i+6) e1(i+6) e2(i+6) e3(i+6) e4(i+6) e5(i+6) e6(i+6) e7(i+6) e0(i+7) e0(i+7) ... e7(i+7) + simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi32(tmp3, tmp7)); -#else - // original unoptimized loops - /* - for (int j = 0; j< EQm; j++,j2+=2){ - for (int i = 0; i< Qm; i++){ - f[(i+j*Qm)] = e[(i*EQm + j)]; - } - } - */ - - fp = f; - switch (Qm) { - case 2: - e0 = e; - e1 = e0 + EQm; - for (int j = 0, j2 = 0; j < EQm; j++, j2 += 2) { - fp = &f[j2]; - fp[0] = e0[j]; - fp[1] = e1[j]; + tmp0 = simde_mm_unpackhi_epi8(e0j, e1j); // e0(i+8) e1(i+8) e0(i+9) e1(i+9) .... e0(i+15) e1(i+15) + tmp1 = simde_mm_unpackhi_epi8(e2j, e3j); // e2(i+8) e3(i+8) e2(i+9) e3(i+9) .... e2(i+15) e3(i+15) + // e0(i) e1(i) e2(i) e3(i) e0(i+1) e1(i+1) e2(i+1) e3(i+1) ... e0(i+3) e1(i+3) e2(i+3) e3(i+3) + tmp2 = simde_mm_unpacklo_epi16(tmp0, tmp1); + // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e0(i+5) e1(i+5) e2(i+5) e3(i+5) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) + tmp3 = simde_mm_unpackhi_epi16(tmp0, tmp1); + tmp4 = simde_mm_unpackhi_epi8(e4j, e5j); // e4(i+8) e5(i+8) e4(i+9) e5(i+9) .... e4(i+15) e515i+7) + tmp5 = simde_mm_unpackhi_epi8(e6j, e7j); // e6(i+8) e7(i+8) e6(i+9) e7(i+9) .... e6(i+15) e7(i+15) + // e4(i) e5(i) e6(i) e7(i) e4(i+1) e5(i+1) e6(i+1) e7(i+1) ... e4(i+3) e5(i+3) e6(i+) e7(i+3) + tmp6 = simde_mm_unpacklo_epi16(tmp4, tmp5); + // e4(i+4) e5(i+4) e6(i+4) e7(i+4) e4(i+5) e5(i+5) e6(i+5) e7(i+5) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7) + tmp7 = simde_mm_unpackhi_epi16(tmp4, tmp5); + // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i) e0(i+1) ... e7(i+1) + simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi32(tmp2, tmp6)); + // e0(i+2) e1(i+2) e2(i+2) e3(i+2) e4(i+2) e5(i+2) e6(i+2) e7(i+2) e0(i+3) e1(i+3) ... e7(i+3) + simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi32(tmp2, tmp6)); + // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7+4(i) e0(i+5) ... e7(i+5) + simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi32(tmp3, tmp7)); + // e0(i+6) e1(i+6) e2(i+6) e3(i+6) e4(i+6) e5(i+6) e6(i+6) e7(i+6) e0(i+7) e0(i+7) ... e7(i+7) + simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi32(tmp3, tmp7)); } - break; - case 4: - e0 = e; - e1 = e0 + EQm; - e2 = e1 + EQm; - e3 = e2 + EQm; - for (int j = 0, j2 = 0; j < EQm; j++, j2 += 4) { - fp = &f[j2]; - fp[0] = e0[j]; - fp[1] = e1[j]; - fp[2] = e2[j]; - fp[3] = e3[j]; + e0 = (uint8_t *)e0_128; + e1 = (uint8_t *)e1_128; + e2 = (uint8_t *)e2_128; + e3 = (uint8_t *)e3_128; + e4 = (uint8_t *)e4_128; + e5 = (uint8_t *)e5_128; + e6 = (uint8_t *)e6_128; + e7 = (uint8_t *)e7_128; + f = (uint8_t *)f_128; +#endif + for (; i < EQm; i++) { + *f++ = *e0++; + *f++ = *e1++; + *f++ = *e2++; + *f++ = *e3++; + *f++ = *e4++; + *f++ = *e5++; + *f++ = *e6++; + *f++ = *e7++; } - break; - case 6: - e0 = e; - e1 = e0 + EQm; - e2 = e1 + EQm; - e3 = e2 + EQm; - e4 = e3 + EQm; - e5 = e4 + EQm; - fp = f; - for (int j = 0; j < EQm; j++) { - *fp++ = e0[j]; - *fp++ = e1[j]; - *fp++ = e2[j]; - *fp++ = e3[j]; - *fp++ = e4[j]; - *fp++ = e5[j]; - } - break; - case 8: - e0 = e; - e1 = e0 + EQm; - e2 = e1 + EQm; - e3 = e2 + EQm; - e4 = e3 + EQm; - e5 = e4 + EQm; - e6 = e5 + EQm; - e7 = e6 + EQm; - for (int j = 0, j2 = 0; j < EQm; j++, j2 += 8) { - fp = &f[j2]; - fp[0] = e0[j]; - fp[1] = e1[j]; - fp[2] = e2[j]; - fp[3] = e3[j]; - fp[4] = e4[j]; - fp[5] = e5[j]; - fp[6] = e6[j]; - fp[7] = e7[j]; - } - break; + } break; default: - AssertFatal(1 == 0, "Should never be here!\n"); + AssertFatal(false, "Should be here!\n"); } -#endif } void nr_deinterleaving_ldpc(uint32_t E, uint8_t Qm, int16_t *e, int16_t *f) diff --git a/openair1/PHY/CODING/nrLDPC_defs.h b/openair1/PHY/CODING/nrLDPC_defs.h index 3db41a0e29..2298400078 100644 --- a/openair1/PHY/CODING/nrLDPC_defs.h +++ b/openair1/PHY/CODING/nrLDPC_defs.h @@ -28,7 +28,7 @@ /** \brief LDPC encoder parameter structure \var n_segments number of segments in the transport block - \var macro_num index of the subset of segments to encode + \var first_seg index of the first segment of the subset to encode within the transport block \var gen_code flag to generate parity check code 0 -> encoding @@ -52,7 +52,7 @@ */ typedef struct { unsigned int n_segments; // optim8seg - unsigned int macro_num; // optim8segmulti + unsigned int first_seg; // optim8segmulti unsigned char gen_code; //orig time_stats_t *tinput; time_stats_t *tprep; @@ -92,6 +92,6 @@ typedef int32_t(LDPC_decoderfunc_t)(t_nrLDPC_dec_params *p_decParams, int8_t *p_out, t_nrLDPC_time_stats *, decode_abort_t *ab); -typedef int32_t(LDPC_encoderfunc_t)(uint8_t **, uint8_t **, encoder_implemparams_t *); +typedef int32_t(LDPC_encoderfunc_t)(uint8_t **, uint8_t *, encoder_implemparams_t *); #endif diff --git a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder.c b/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder.c index f1f6e0af5a..055c0a7eb6 100644 --- a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder.c +++ b/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder.c @@ -41,11 +41,11 @@ #include "openair1/PHY/CODING/nrLDPC_extern.h" #include "ldpc_generate_coefficient.c" -int LDPCencoder(unsigned char **inputArray, unsigned char **outputArray, encoder_implemparams_t *impp) +int LDPCencoder(unsigned char **inputArray, unsigned char *outputArray, encoder_implemparams_t *impp) { const unsigned char *input = inputArray[0]; // channel input is the output of this function! - unsigned char *output = outputArray[0]; + unsigned char *output = outputArray; const int Zc = impp->Zc; const int Kb = impp->Kb; const short block_length = impp->K; diff --git a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8segmulti.c b/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8segmulti.c index 268d2f9dc6..b8f46969ec 100644 --- a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8segmulti.c +++ b/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8segmulti.c @@ -43,7 +43,7 @@ #include "ldpc_encode_parity_check.c" #include "ldpc_generate_coefficient.c" -int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp) +int LDPCencoder(uint8_t **input, uint8_t *output, encoder_implemparams_t *impp) { //set_log(PHY, 4); @@ -61,41 +61,10 @@ int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp) unsigned int macro_segment, macro_segment_end; - macro_segment = 8*impp->macro_num; - macro_segment_end = (impp->n_segments > 8*(impp->macro_num+1)) ? 8*(impp->macro_num+1) : impp->n_segments; + macro_segment = impp->first_seg; + macro_segment_end = (impp->n_segments > impp->first_seg + 8) ? impp->first_seg + 8 : impp->n_segments; ///printf("macro_segment: %d\n", macro_segment); ///printf("macro_segment_end: %d\n", macro_segment_end ); -#ifdef __aarch64__ - simde__m128i shufmask = simde_mm_set_epi64x(0x0101010101010101, 0x0000000000000000); - simde__m128i andmask = simde_mm_set1_epi64x(0x0102040810204080); // every 8 bits -> 8 bytes, pattern repeats. - simde__m128i zero128 = simde_mm_setzero_si128(); - simde__m128i masks[8]; - register simde__m128i c128; - masks[0] = simde_mm_set1_epi8(0x1); - masks[1] = simde_mm_set1_epi8(0x2); - masks[2] = simde_mm_set1_epi8(0x4); - masks[3] = simde_mm_set1_epi8(0x8); - masks[4] = simde_mm_set1_epi8(0x10); - masks[5] = simde_mm_set1_epi8(0x20); - masks[6] = simde_mm_set1_epi8(0x40); - masks[7] = simde_mm_set1_epi8(0x80); - -#else - simde__m256i shufmask = simde_mm256_set_epi64x(0x0303030303030303, 0x0202020202020202,0x0101010101010101, 0x0000000000000000); - simde__m256i andmask = simde_mm256_set1_epi64x(0x0102040810204080); // every 8 bits -> 8 bytes, pattern repeats. - simde__m256i zero256 = simde_mm256_setzero_si256(); - simde__m256i masks[8]; - register simde__m256i c256; - masks[0] = simde_mm256_set1_epi8(0x1); - masks[1] = simde_mm256_set1_epi8(0x2); - masks[2] = simde_mm256_set1_epi8(0x4); - masks[3] = simde_mm256_set1_epi8(0x8); - masks[4] = simde_mm256_set1_epi8(0x10); - masks[5] = simde_mm256_set1_epi8(0x20); - masks[6] = simde_mm256_set1_epi8(0x40); - masks[7] = simde_mm256_set1_epi8(0x80); -#endif - //determine number of bits in codeword if (BG==1) @@ -136,55 +105,90 @@ int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp) if(impp->tinput != NULL) start_meas(impp->tinput); //interleave up to 8 transport-block segements at a time -#if 0 - for (i=0; i>(i&7); - //printf("c(%d,%d)=%d\n",j,i,temp); - c[i] |= (temp << (j-macro_segment)); - } - } -#elif defined(__aarch64__) - for (int i=0; i>4; i++) { - c128 = simde_mm_and_si128(simde_mm_cmpeq_epi8(simde_mm_andnot_si128(simde_mm_shuffle_epi8(simde_mm_set1_epi16(((uint16_t*)input[macro_segment])[i]), shufmask),andmask),zero128),masks[0]); - //for (j=1; j>4)<<4;i>(i&7)))>>(7-(i&7)); - //printf("c(%d,%d)=%d\n",j,i,temp); - cc[i] |= (temp << (j-macro_segment)); + for (; i_byte < ((block_length >> 6) << 6); i_byte += 64) { + unsigned int i = i_byte >> 6; + c512 = _mm512_mask_blend_epi8(((uint64_t *)&input[macro_segment][0])[i], zero512, masks5[0]); + for (int j = macro_segment + 1; j < macro_segment_end; j++) { + c512 = _mm512_or_si512(c512, _mm512_mask_blend_epi8(((uint64_t *)&input[j][0])[i], zero512, masks5[j - macro_segment])); } + c512 = _mm512_permutexvar_epi8(*(__m512i*)perm, c512); + ((__m512i *)cc)[i] = c512; } -#else - for (int i=0; i>5; i++) { +#endif + +#ifndef __aarch64__ + simde__m256i shufmask = simde_mm256_set_epi64x(0x0303030303030303, 0x0202020202020202,0x0101010101010101, 0x0000000000000000); + simde__m256i andmask = simde_mm256_set1_epi64x(0x0102040810204080); // every 8 bits -> 8 bytes, pattern repeats. + simde__m256i zero256 = simde_mm256_setzero_si256(); + simde__m256i masks[8]; + register simde__m256i c256; + masks[0] = simde_mm256_set1_epi8(0x1); + masks[1] = simde_mm256_set1_epi8(0x2); + masks[2] = simde_mm256_set1_epi8(0x4); + masks[3] = simde_mm256_set1_epi8(0x8); + masks[4] = simde_mm256_set1_epi8(0x10); + masks[5] = simde_mm256_set1_epi8(0x20); + masks[6] = simde_mm256_set1_epi8(0x40); + masks[7] = simde_mm256_set1_epi8(0x80); + + for (; i_byte < ((block_length >> 5 ) << 5); i_byte += 32) { + unsigned int i = i_byte >> 5; c256 = simde_mm256_and_si256(simde_mm256_cmpeq_epi8(simde_mm256_andnot_si256(simde_mm256_shuffle_epi8(simde_mm256_set1_epi32(((uint32_t*)input[macro_segment])[i]), shufmask),andmask),zero256),masks[0]); - //for (j=1; j>5)<<5;i 8 bytes, pattern repeats. + simde__m128i zero128 = simde_mm_setzero_si128(); + simde__m128i masks[8]; + register simde__m128i c128; + masks[0] = simde_mm_set1_epi8(0x1); + masks[1] = simde_mm_set1_epi8(0x2); + masks[2] = simde_mm_set1_epi8(0x4); + masks[3] = simde_mm_set1_epi8(0x8); + masks[4] = simde_mm_set1_epi8(0x10); + masks[5] = simde_mm_set1_epi8(0x20); + masks[6] = simde_mm_set1_epi8(0x40); + masks[7] = simde_mm_set1_epi8(0x80); - temp = (input[j][i/8]&(128>>(i&7)))>>(7-(i&7)); - //printf("c(%d,%d)=%d\n",j,i,temp); + for (; i_byte < ((block_length >> 4 ) << 4); i_byte += 16) { + unsigned int i = i_byte >> 4; + c128 = simde_mm_and_si128(simde_mm_cmpeq_epi8(simde_mm_andnot_si128(simde_mm_shuffle_epi8(simde_mm_set1_epi16(((uint16_t*)input[macro_segment])[i]), shufmask),andmask),zero128),masks[0]); + for (int j=macro_segment+1; j < macro_segment_end; j++) { + c128 = simde_mm_or_si128(simde_mm_and_si128(simde_mm_cmpeq_epi8(simde_mm_andnot_si128(simde_mm_shuffle_epi8(simde_mm_set1_epi32(((uint16_t*)input[j])[i]), shufmask),andmask),zero128),masks[j-macro_segment]),c128); + } + ((simde__m128i *)cc)[i] = c128; + } +#endif + + for (; i_byte < block_length; i_byte++) { + unsigned int i = i_byte; + for (int j = macro_segment; j < macro_segment_end; j++) { + + temp = (input[j][i/8]&(128>>(i&7)))>>(7-(i&7)); cc[i] |= (temp << (j-macro_segment)); } } -#endif if(impp->tinput != NULL) stop_meas(impp->tinput); @@ -203,8 +207,8 @@ int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp) return(-1); } } - memcpy(output[0],&cc[2*Zc],(block_length-(2*Zc))); - memcpy(output[0]+block_length-(2*Zc),dd,((nrows-no_punctured_columns) * Zc-removed_bit)); + memcpy(output,&cc[2*Zc],(block_length-(2*Zc))); + memcpy(output+block_length-(2*Zc),dd,((nrows-no_punctured_columns) * Zc-removed_bit)); return 0; }