From d6ee29a193492b04944e13627bec8776cb13e8dd Mon Sep 17 00:00:00 2001 From: Romain Beurdouche Date: Fri, 2 May 2025 13:34:47 +0000 Subject: [PATCH 01/17] feat(nrLDPC_encoder): remove _optim and _optim8seg It is pointless to keep updating ldpc_optim and ldpc_optim8seg. So we remove them. --- doc/Doxyfile | 2 - openair1/PHY/CODING/CMakeLists.txt | 28 +-- openair1/PHY/CODING/DOC/LDPCImplementation.md | 8 +- .../nrLDPC_encoder/ldpc_encoder_optim.c | 128 ----------- .../nrLDPC_encoder/ldpc_encoder_optim8seg.c | 201 ------------------ 5 files changed, 11 insertions(+), 356 deletions(-) delete mode 100644 openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim.c delete mode 100644 openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8seg.c diff --git a/doc/Doxyfile b/doc/Doxyfile index a154051692..6b0b390f05 100644 --- a/doc/Doxyfile +++ b/doc/Doxyfile @@ -2133,10 +2133,8 @@ INPUT = \ @CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc_encode_parity_check.c \ @CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc_BG2_Zc176_byte.c \ @CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc_BG2_Zc352_byte.c \ -@CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim.c \ @CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc_BG2_Zc128_byte_128.c \ @CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc224_byte.c \ -@CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8seg.c \ @CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc_generate_coefficient.c \ @CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc_BG2_Zc104_byte.c \ @CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc320_byte_128.c \ diff --git a/openair1/PHY/CODING/CMakeLists.txt b/openair1/PHY/CODING/CMakeLists.txt index 4eee35a2cd..7d8ce40c14 100644 --- a/openair1/PHY/CODING/CMakeLists.txt +++ b/openair1/PHY/CODING/CMakeLists.txt @@ -17,20 +17,6 @@ add_library(ldpc_orig MODULE set_target_properties(ldpc_orig PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) target_link_libraries(ldpc_orig PRIVATE ldpc_segment ldpc_gen_HEADERS) -add_library(ldpc_optim MODULE - nrLDPC_decoder/nrLDPC_decoder.c - nrLDPC_encoder/ldpc_encoder_optim.c -) -set_target_properties(ldpc_optim PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) -target_link_libraries(ldpc_optim PRIVATE ldpc_segment ldpc_gen_HEADERS) - -add_library(ldpc_optim8seg MODULE - nrLDPC_decoder/nrLDPC_decoder.c - nrLDPC_encoder/ldpc_encoder_optim8seg.c -) -set_target_properties(ldpc_optim8seg PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) -target_link_libraries(ldpc_optim8seg PRIVATE ldpc_segment ldpc_gen_HEADERS) - add_library(ldpc MODULE nrLDPC_decoder/nrLDPC_decoder.c nrLDPC_encoder/ldpc_encoder_optim8segmulti.c @@ -38,13 +24,13 @@ add_library(ldpc MODULE set_target_properties(ldpc PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) target_link_libraries(ldpc PRIVATE ldpc_segment ldpc_gen_HEADERS) -add_dependencies(ldpctest ldpc ldpc_orig ldpc_optim ldpc_optim8seg) -add_dependencies(nr-softmodem ldpc ldpc_orig ldpc_optim ldpc_optim8seg) -add_dependencies(nr-uesoftmodem ldpc ldpc_orig ldpc_optim ldpc_optim8seg) -add_dependencies(nr_ulsim ldpc ldpc_orig ldpc_optim ldpc_optim8seg) -add_dependencies(nr_ulschsim ldpc ldpc_orig ldpc_optim ldpc_optim8seg) -add_dependencies(nr_dlsim ldpc ldpc_orig ldpc_optim ldpc_optim8seg) -add_dependencies(nr_dlschsim ldpc ldpc_orig ldpc_optim ldpc_optim8seg) +add_dependencies(ldpctest ldpc ldpc_orig) +add_dependencies(nr-softmodem ldpc ldpc_orig) +add_dependencies(nr-uesoftmodem ldpc ldpc_orig) +add_dependencies(nr_ulsim ldpc ldpc_orig) +add_dependencies(nr_ulschsim ldpc ldpc_orig) +add_dependencies(nr_dlsim ldpc ldpc_orig) +add_dependencies(nr_dlschsim ldpc ldpc_orig) add_custom_target(nrLDPC_decoder_kernels_CL COMMAND gcc nrLDPC_decoder/nrLDPC_decoder_CL.c -dD -DNRLDPC_KERNEL_SOURCE -E -o ${CMAKE_CURRENT_BINARY_DIR}/nrLDPC_decoder_kernels_CL.clc diff --git a/openair1/PHY/CODING/DOC/LDPCImplementation.md b/openair1/PHY/CODING/DOC/LDPCImplementation.md index 2880c45ecf..325b208a64 100644 --- a/openair1/PHY/CODING/DOC/LDPCImplementation.md +++ b/openair1/PHY/CODING/DOC/LDPCImplementation.md @@ -120,16 +120,16 @@ By default the function `int load_nrLDPClib(void)` looks for `libldpc.so`, this #### Examples of ldpc shared lib selection when running nr softmodem's: -loading `libldpc_optim8seg.so` instead of `libldpc.so`: +loading `libldpc_orig.so` instead of `libldpc.so`: ``` -./nr-softmodem -O libconfig:gnb.band78.tm1.106PRB.usrpx300.conf:dbgl5 --loader.ldpc.shlibversion _optim8seg +./nr-softmodem -O libconfig:gnb.band78.tm1.106PRB.usrpx300.conf:dbgl5 --loader.ldpc.shlibversion _orig ....................... [CONFIG] loader.ldpc.shlibversion set to default value "" [LIBCONFIG] loader.ldpc: 2/2 parameters successfully set, (1 to default value) -[CONFIG] shlibversion set to _optim8seg from command line +[CONFIG] shlibversion set to _orig from command line [CONFIG] loader.ldpc 1 options set from command line -[LOADER] library libldpc_optim8seg.so successfully loaded +[LOADER] library libldpc_orig.so successfully loaded ........................ ``` diff --git a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim.c b/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim.c deleted file mode 100644 index deaba29e0d..0000000000 --- a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim.c +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The OpenAirInterface Software Alliance licenses this file to You under - * the OAI Public License, Version 1.1 (the "License"); you may not use this file - * except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.openairinterface.org/?page_id=698 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *------------------------------------------------------------------------------- - * For more information about the OpenAirInterface (OAI) Software Alliance: - * contact@openairinterface.org - */ - -/*! \file ldpc_encoder_optim.c - * \brief Defines the optimized LDPC encoder - * \author Florian Kaltenberger, Raymond Knopp, Kien le Trung (Eurecom) - * \email openair_tech@eurecom.fr - * \date 27-03-2018 - * \version 1.0 - * \note - * \warning - */ - -#include -#include -#include -#include -#include "assertions.h" -#include "common/utils/LOG/log.h" -#include "time_meas.h" -#include "openair1/PHY/CODING/nrLDPC_defs.h" -#include "openair1/PHY/CODING/nrLDPC_extern.h" -#include "ldpc_encode_parity_check.c" -#include "ldpc_generate_coefficient.c" - -int LDPCencoder(uint8_t **test_input, uint8_t **channel_input, encoder_implemparams_t *impp) -{ - int Zc = impp->Zc; - int Kb = impp->Kb; - int block_length = impp->K; - int BG = impp->BG; - int nrows=0,ncols=0; - int rate=3; - int no_punctured_columns,removed_bit; - - int simd_size; - - //determine number of bits in codeword - //if (block_length>3840) - if (BG==1) - { - //BG=1; - nrows=46; //parity check bits - ncols=22; //info bits - rate=3; - } - //else if (block_length<=3840) - else if (BG==2) - { - //BG=2; - nrows=42; //parity check bits - ncols=10; // info bits - rate=5; - - } - - -#ifdef DEBUG_LDPC - LOG_D(PHY,"ldpc_encoder_optim_8seg: BG %d, Zc %d, Kb %d, block_length %d\n",BG,Zc,Kb,block_length); - LOG_D(PHY,"ldpc_encoder_optim_8seg: PDU %x %x %x %x\n",test_input[0][0],test_input[0][1],test_input[0][2],test_input[0][3]); -#endif - - if ((Zc&31) > 0) simd_size = 16; - else simd_size = 32; - - uint8_t c[22*Zc] __attribute__((aligned(32))); //padded input, unpacked, max size - uint8_t d[46*Zc] __attribute__((aligned(32))); //coded parity part output, unpacked, max size - - // calculate number of punctured bits - no_punctured_columns=(int)((nrows-2)*Zc+block_length-block_length*rate)/Zc; - removed_bit = (nrows - no_punctured_columns - 2) * Zc + block_length - block_length * rate; - // printf("%d\n",no_punctured_columns); - // printf("%d\n",removed_bit); - // unpack input - memset(c, 0, sizeof(c)); - memset(d, 0, sizeof(d)); - - if(impp->tinput != NULL) start_meas(impp->tinput); - for (int i=0; i>(i&7)))>>(7-(i&7)); - //printf("c(%d,%d)=%d\n",j,i,temp); - } - - if(impp->tinput != NULL) stop_meas(impp->tinput); - - if ((BG==1 && Zc>=176) || (BG==2 && Zc>=72)) { - // extend matrix - if(impp->tprep != NULL) start_meas(impp->tprep); - if(impp->tprep != NULL) stop_meas(impp->tprep); - //parity check part - if(impp->tparity != NULL) start_meas(impp->tparity); - encode_parity_check_part_optim(c, d, BG, Zc, Kb,simd_size, ncols); - if(impp->tparity != NULL) stop_meas(impp->tparity); - } - else { - if (encode_parity_check_part_orig(c, d, BG, Zc, Kb, block_length)!=0) { - printf("Problem with encoder\n"); - return(-1); - } - } - if(impp->toutput != NULL) start_meas(impp->toutput); - // information part and puncture columns - memcpy(&channel_input[0][0], &c[2*Zc], (block_length-2*Zc)*sizeof(unsigned char)); - memcpy(&channel_input[0][block_length-2*Zc], &d[0], ((nrows-no_punctured_columns) * Zc-removed_bit)*sizeof(unsigned char)); - - if(impp->toutput != NULL) stop_meas(impp->toutput); - return 0; -} - - diff --git a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8seg.c b/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8seg.c deleted file mode 100644 index a45aac2474..0000000000 --- a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8seg.c +++ /dev/null @@ -1,201 +0,0 @@ -/* - * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The OpenAirInterface Software Alliance licenses this file to You under - * the OAI Public License, Version 1.1 (the "License"); you may not use this file - * except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.openairinterface.org/?page_id=698 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *------------------------------------------------------------------------------- - * For more information about the OpenAirInterface (OAI) Software Alliance: - * contact@openairinterface.org - */ - -/*! \file ldpc_encoder_optim8seg.c - * \brief Defines the optimized LDPC encoder - * \author Florian Kaltenberger, Raymond Knopp, Kien le Trung (Eurecom) - * \email openair_tech@eurecom.fr - * \date 27-03-2018 - * \version 1.0 - * \note - * \warning - */ - -#include -#include -#include -#include -#include "assertions.h" -#include "common/utils/LOG/log.h" -#include "time_meas.h" -#include "openair1/PHY/CODING/nrLDPC_defs.h" -#include "openair1/PHY/CODING/nrLDPC_extern.h" -#include "ldpc_encode_parity_check.c" -#include "ldpc_generate_coefficient.c" -#include "PHY/sse_intrin.h" - -int LDPCencoder(uint8_t **test_input, uint8_t **channel_input, encoder_implemparams_t *impp) -{ - int Zc = impp->Zc; - int Kb = impp->Kb; - short block_length = impp->K; - short BG = impp->BG; - - short nrows=0,ncols=0; - int i,i1,j,rate=3; - int no_punctured_columns,removed_bit; - char temp; - int simd_size; - - simde__m256i shufmask = simde_mm256_set_epi64x(0x0303030303030303, 0x0202020202020202,0x0101010101010101, 0x0000000000000000); - simde__m256i andmask = simde_mm256_set1_epi64x(0x0102040810204080); // every 8 bits -> 8 bytes, pattern repeats. - simde__m256i zero256 = simde_mm256_setzero_si256(); - simde__m256i masks[8]; - register simde__m256i c256; - masks[0] = simde_mm256_set1_epi8(0x1); - masks[1] = simde_mm256_set1_epi8(0x2); - masks[2] = simde_mm256_set1_epi8(0x4); - masks[3] = simde_mm256_set1_epi8(0x8); - masks[4] = simde_mm256_set1_epi8(0x10); - masks[5] = simde_mm256_set1_epi8(0x20); - masks[6] = simde_mm256_set1_epi8(0x40); - masks[7] = simde_mm256_set1_epi8(0x80); - - AssertFatal((impp->n_segments>0&&impp->n_segments<=8),"0 < n_segments %d <= 8\n",impp->n_segments); - - //determine number of bits in codeword - //if (block_length>3840) - if (BG==1) - { - nrows=46; //parity check bits - ncols=22; //info bits - rate=3; - } - //else if (block_length<=3840) - else if (BG==2) - { - //BG=2; - nrows=42; //parity check bits - ncols=10; // info bits - rate=5; - - } - -#ifdef DEBUG_LDPC - LOG_D(PHY,"ldpc_encoder_optim_8seg: BG %d, Zc %d, Kb %d, block_length %d, segments %d\n",BG,Zc,Kb,block_length,n_segments); - LOG_D(PHY,"ldpc_encoder_optim_8seg: PDU (seg 0) %x %x %x %x\n",test_input[0][0],test_input[0][1],test_input[0][2],test_input[0][3]); -#endif - - AssertFatal(Zc>0,"no valid Zc found for block length %d\n",block_length); - - if ((Zc&31) > 0) simd_size = 16; - else simd_size = 32; - - unsigned char c[22*Zc] __attribute__((aligned(32))); //padded input, unpacked, max size - unsigned char d[46*Zc] __attribute__((aligned(32))); //coded parity part output, unpacked, max size - - - // calculate number of punctured bits - no_punctured_columns=(int)((nrows-2)*Zc+block_length-block_length*rate)/Zc; - removed_bit=(nrows-no_punctured_columns-2) * Zc+block_length-(int)(block_length*rate); - // printf("%d\n",no_punctured_columns); - // printf("%d\n",removed_bit); - // unpack input - memset(c,0,sizeof(unsigned char) * ncols * Zc); - memset(d,0,sizeof(unsigned char) * nrows * Zc); - - if(impp->tinput != NULL) start_meas(impp->tinput); -#if 0 - for (i=0; i>(i&7)))>>(7-(i&7)); - //printf("c(%d,%d)=%d\n",j,i,temp); - c[i] |= (temp << j); - } - } -#else - for (i=0; i>5; i++) { - c256 = simde_mm256_and_si256(simde_mm256_cmpeq_epi8(simde_mm256_andnot_si256(simde_mm256_shuffle_epi8(simde_mm256_set1_epi32(((uint32_t*)test_input[0])[i]), shufmask),andmask),zero256),masks[0]); - for (j=1; jn_segments; j++) { - c256 = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_cmpeq_epi8(simde_mm256_andnot_si256(simde_mm256_shuffle_epi8(simde_mm256_set1_epi32(((uint32_t*)test_input[j])[i]), shufmask),andmask),zero256),masks[j]),c256); - } - ((simde__m256i *)c)[i] = c256; - } - - for (i=(block_length>>5)<<5;in_segments; j++) { - - temp = (test_input[j][i/8]&(128>>(i&7)))>>(7-(i&7)); - //printf("c(%d,%d)=%d\n",j,i,temp); - c[i] |= (temp << j); - } - } -#endif - - if(impp->tinput != NULL) stop_meas(impp->tinput); - - if ((BG==1 && Zc>=176) || (BG==2 && Zc>=72)) { - // extend matrix - if(impp->tprep != NULL) start_meas(impp->tprep); - if(impp->tprep != NULL) stop_meas(impp->tprep); - //parity check part - if(impp->tparity != NULL) start_meas(impp->tparity); - encode_parity_check_part_optim(c, d, BG, Zc, Kb, simd_size, ncols); - if(impp->tparity != NULL) stop_meas(impp->tparity); - } - else { - if (encode_parity_check_part_orig(c, d, BG, Zc, Kb, block_length)!=0) { - printf("Problem with encoder\n"); - return(-1); - } - } - if(impp->toutput != NULL) start_meas(impp->toutput); - // information part and puncture columns - /* - memcpy(&channel_input[0], &c[2*Zc], (block_length-2*Zc)*sizeof(unsigned char)); - memcpy(&channel_input[block_length-2*Zc], &d[0], ((nrows-no_punctured_columns) * Zc-removed_bit)*sizeof(unsigned char)); - */ - if ((((2*Zc)&31) == 0) && (((block_length-(2*Zc))&31) == 0)) { - //AssertFatal(((2*Zc)&31) == 0,"2*Zc needs to be a multiple of 32 for now\n"); - //AssertFatal(((block_length-(2*Zc))&31) == 0,"block_length-(2*Zc) needs to be a multiple of 32 for now\n"); - uint32_t l1 = (block_length-(2*Zc))>>5; - uint32_t l2 = ((nrows-no_punctured_columns) * Zc-removed_bit)>>5; - simde__m256i *c256p = (simde__m256i *)&c[2*Zc]; - simde__m256i *d256p = (simde__m256i *)&d[0]; - // if (((block_length-(2*Zc))&31)>0) l1++; - - for (i=0;in_segments;j++) ((simde__m256i *)channel_input[j])[i] = simde_mm256_and_si256(simde_mm256_srai_epi16(c256p[i],j),masks[0]); - - // if ((((nrows-no_punctured_columns) * Zc-removed_bit)&31)>0) l2++; - - for (i1=0;i1n_segments;j++) ((simde__m256i *)channel_input[j])[i] = simde_mm256_and_si256(simde_mm256_srai_epi16(d256p[i1],j),masks[0]); - } - else { -#ifdef DEBUG_LDPC - LOG_W(PHY,"using non-optimized version\n"); -#endif - // do non-SIMD version - for (i=0;i<(block_length-2*Zc);i++) - for (j=0; jn_segments; j++) - channel_input[j][i] = (c[2*Zc+i]>>j)&1; - for (i=0;i<((nrows-no_punctured_columns) * Zc-removed_bit);i++) - for (j=0; jn_segments; j++) - channel_input[j][block_length-2*Zc+i] = (d[i]>>j)&1; - } - - if(impp->toutput != NULL) stop_meas(impp->toutput); - return 0; -} - - From c07bfe6db51edbb3b5a84453a25c9141fbd85518 Mon Sep 17 00:00:00 2001 From: Raymond Knopp Date: Fri, 9 May 2025 07:14:31 +0000 Subject: [PATCH 02/17] Put CPU affinity control in nr_ulsim. --- openair1/SIMULATION/NR_PHY/ulsim.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/openair1/SIMULATION/NR_PHY/ulsim.c b/openair1/SIMULATION/NR_PHY/ulsim.c index 83c1c31826..f6222c4c84 100644 --- a/openair1/SIMULATION/NR_PHY/ulsim.c +++ b/openair1/SIMULATION/NR_PHY/ulsim.c @@ -249,7 +249,9 @@ int main(int argc, char *argv[]) InitSinLUT(); int c; - while ((c = getopt(argc, argv, "--:O:a:b:c:d:ef:g:h:i:jk:m:n:p:q:r:s:t:u:v:w:y:z:A:C:F:G:H:I:M:N:PR:S:T:U:L:ZW:E:X:")) != -1) { + bool setAffinity=false; + char gNBthreads[128]="n"; + while ((c = getopt(argc, argv, "--:O:a:b:c:d:ef:g:h:i:jk:m:n:p:q:r:s:t:u:v:w:y:z:A:C:F:G:H:I:M:N:PR:S:T:U:L:ZW:E:X:Y:")) != -1) { /* ignore long options starting with '--', option '-O' and their arguments that are handled by configmodule */ /* with this opstring getopt returns 1 for non-option arguments, refer to 'man 3 getopt' */ if (c == 1 || c == '-' || c == 'O') @@ -494,6 +496,13 @@ int main(int argc, char *argv[]) AssertFatal(filename_csv != NULL, "strdup() error: errno %d\n", errno); break; + case 'Y': + threadCnt = sizeof(gNBthreads)-1; + strncpy(gNBthreads, optarg, threadCnt); + gNBthreads[threadCnt]=0; + setAffinity=true; + break; + case 'Z': transform_precoding = transformPrecoder_enabled; num_dmrs_cdm_grps_no_data = 2; @@ -592,7 +601,11 @@ int main(int argc, char *argv[]) gNB->RU_list[0] = calloc(1, sizeof(**gNB->RU_list)); gNB->RU_list[0]->rfdevice.openair0_cfg = openair0_cfg; - initFloatingCoresTpool(threadCnt, &gNB->threadPool, false, "gNB-tpool"); + if (setAffinity == false) + initFloatingCoresTpool(threadCnt, &gNB->threadPool, false, "gNB-tpool"); + else + initNamedTpool(gNBthreads, &gNB->threadPool, true, "gNB-tpool"); + initNotifiedFIFO(&gNB->respDecode); initNotifiedFIFO(&gNB->respPuschSymb); From 8964fdb6a2ae509012c92956616904bbfc6a9d9d Mon Sep 17 00:00:00 2001 From: Raymond Knopp Date: Mon, 16 Dec 2024 01:09:38 +0100 Subject: [PATCH 03/17] AVX512 modifications for LDPC encoding: interleaving, ZC384 BG1, output formatting for encoder, segmentation (memcpy) --- openair1/PHY/CODING/TESTBENCH/ldpctest.c | 2 +- .../nrLDPC_coding_segment_encoder.c | 2 +- .../nrLDPC_coding_segment/nr_rate_matching.c | 450 ++++++++++++------ .../nrLDPC_encoder/ldpc384_simd512_byte.c | 151 ++++++ .../nrLDPC_encoder/ldpc_encode_parity_check.c | 7 +- .../PHY/CODING/nrLDPC_encoder/ldpc_encoder.c | 8 +- .../ldpc_encoder_optim8segmulti.c | 51 +- openair1/PHY/CODING/nr_segmentation.c | 9 +- 8 files changed, 523 insertions(+), 157 deletions(-) create mode 100644 openair1/PHY/CODING/nrLDPC_encoder/ldpc384_simd512_byte.c diff --git a/openair1/PHY/CODING/TESTBENCH/ldpctest.c b/openair1/PHY/CODING/TESTBENCH/ldpctest.c index 5912c9d0b4..ee33c7f91a 100644 --- a/openair1/PHY/CODING/TESTBENCH/ldpctest.c +++ b/openair1/PHY/CODING/TESTBENCH/ldpctest.c @@ -282,7 +282,7 @@ one_measurement_t test_ldpc(short max_iterations, } encoder_implemparams_t impp = {.Zc = Zc, .Kb = Kb, .BG = BG, .K = K}; - impp.gen_code = 2; + impp.gen_code = 1; if (ntrials == 0) ldpc_orig.LDPCencoder(test_input, channel_input, &impp); diff --git a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c index b51712ed71..30ace31054 100644 --- a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c +++ b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c @@ -64,7 +64,7 @@ static void ldpc8blocks(void *p) // nrLDPC_encoder output is in "d" // let's make this interface happy! - uint8_t tmp[8][68 * 384] __attribute__((aligned(32))); + uint8_t tmp[8][68 * 384] __attribute__((aligned(64))); uint8_t *d[impp->n_segments]; for (int rr = impp->macro_num * 8, i = 0; rr < impp->n_segments && rr < (impp->macro_num + 1) * 8; rr++, i++) d[rr] = tmp[i]; diff --git a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.c b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.c index ebd1ddf2dd..8f111b5308 100644 --- a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.c +++ b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.c @@ -35,43 +35,139 @@ void nr_interleaving_ldpc(uint32_t E, uint8_t Qm, uint8_t *e, uint8_t *f) memset(f, 0, E * sizeof(uint8_t)); uint8_t *e0, *e1, *e2, *e3, *e4, *e5, *e6, *e7; uint8_t *fp; -#if 0 // def __WASAVX2__ - simde__m256i tmp0,tmp1,tmp2,tmp0b,tmp1b,tmp3,tmp4,tmp5; - simde__m256i *e0_256,*e1_256,*e2_256,*e3_256,*e4_256,*e5_256,*e6_256,*e7_256; + int j,k; +#if 1 //def __WASAVX2__ +#ifdef __AVX512VBMI__ + __m512i tmp0,tmp1,tmp2,tmp3,tmp4,tmp5; + __m512i *e0_512,*e1_512,*e2_512,*e3_512,*e4_512,*e5_512,*e6_512,*e7_512; - simde__m256i *f_256=(simde__m256i *)f; + __m512i *f_512=(__m512i *)f; +#else + simde__m128i tmp0,tmp1,tmp2,tmp3,tmp4,tmp5,tmp6,tmp7; + simde__m128i *e0_128,*e1_128,*e2_128,*e3_128,*e4_128,*e5_128,*e6_128,*e7_128; - uint8_t *fp2; + simde__m128i *f_128=(simde__m128i *)f; +#endif switch(Qm) { case 2: e0=e; e1=e0+EQm; - e0_256=(simde__m256i *)e0; - e1_256=(simde__m256i *)e1; - for (int k=0,j=0;j>5;j++,k+=2) { - f_256[k] = simde_mm256_unpacklo_epi8(e0_256[j],e1_256[j]); - f_256[k+1] = simde_mm256_unpackhi_epi8(e0_256[j],e1_256[j]); +#ifdef __AVX512VBMI__ + e0_512=(__m512i *)e0; + e1_512=(__m512i *)e1; + __m512i p8a = _mm512_set_epi8(95,31,94,30,93,29,92,28,91,27,90,26,89,25,88,24,87,23,86,22,85,21,84,20,83,19,82,18,81,17,80,16,79,15,78,14,77,13,76,12,75,11,74,10,73,9,72,8,71,7,70,6,69,5,68,4,67,3,66,2,65,1,64,0); + __m512i p8b = _mm512_set_epi8(127,63,126,62,125,61,124,60,123,59,122,58,121,57,120,56,119,55,118,54,117,53,116,52,115,51,114,50,113,49,112,48,111,47,110,46,109,45,108,44,107,43,106,42,105,41,104,40,103,39,102,38,101,37,100,36,99,35,98,34,97,33,96,32); + + __m512i e0j,e1j; + for (k=0,j=0;j>6;j++,k+=2) { + e0j = _mm512_loadu_si512(e0_512+j); + e1j = _mm512_loadu_si512(e1_512+j); + _mm512_storeu_si512(f_512+k,_mm512_permutex2var_epi8(e0j,p8a,e1j)); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15) + _mm512_storeu_si512(f_512+k+1,_mm512_permutex2var_epi8(e0j,p8b,e1j)); } + if ((j<<6) != EQm) { + int k2=k<<6; + fp = &f[k2]; + for (int j2=(j<<6) ; j2< EQm ; j2++) { + *fp++ = e0[j2]; + *fp++ = e1[j2]; + } + } +#else + e0_128=(simde__m128i *)e0; + e1_128=(simde__m128i *)e1; + simde__m128i e0j,e1j; + for (k=0,j=0;j>4;j++,k+=2) { + e0j=simde_mm_loadu_si128(e0_128+j); + e1j=simde_mm_loadu_si128(e1_128+j); + simde_mm_storeu_si128(f_128+k,simde_mm_unpacklo_epi8(e0j,e1j)); + simde_mm_storeu_si128(f_128+k+1,simde_mm_unpackhi_epi8(e0j,e1j)); + } + if ((j<<4) != EQm) { + int k2=k<<4; + fp = &f[k2]; + for (int j2=(j<<4) ; j2< EQm ; j2++) { + *fp++ = e0[j2]; + *fp++ = e1[j2]; + } + } +#endif break; case 4: e0=e; e1=e0+EQm; e2=e1+EQm; e3=e2+EQm; - e0_256=(simde__m256i *)e0; - e1_256=(simde__m256i *)e1; - e2_256=(simde__m256i *)e2; - e3_256=(simde__m256i *)e3; - for (int k=0,j=0;j>5;j++,k+=4) { - tmp0 = simde_mm256_unpacklo_epi8(e0_256[j],e1_256[j]); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15) - tmp1 = simde_mm256_unpacklo_epi8(e2_256[j],e3_256[j]); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15) - f_256[k] = simde_mm256_unpacklo_epi8(tmp0,tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) - f_256[k+1] = simde_mm256_unpackhi_epi8(tmp0,tmp1); // e0(i+8) e1(i+8) e2(i+8) e3(i+8) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) - tmp0 = simde_mm256_unpackhi_epi8(e0_256[j],e1_256[j]); // e0(i+16) e1(i+16) e0(i+17) e1(i+17) .... e0(i+31) e1(i+31) - tmp1 = simde_mm256_unpackhi_epi8(e2_256[j],e3_256[j]); // e2(i+16) e3(i+16) e2(i+17) e3(i+17) .... e2(i+31) e3(i+31) - f_256[k+2] = simde_mm256_unpacklo_epi8(tmp0,tmp1); - f_256[k+3] = simde_mm256_unpackhi_epi8(tmp0,tmp1); +#ifdef __AVX512VBMI__ + e0_512=(__m512i *)e0; + e1_512=(__m512i *)e1; + e2_512=(__m512i *)e2; + e3_512=(__m512i *)e3; + p8a = _mm512_set_epi8(95,31,94,30,93,29,92,28,91,27,90,26,89,25,88,24,87,23,86,22,85,21,84,20,83,19,82,18,81,17,80,16,79,15,78,14,77,13,76,12,75,11,74,10,73,9,72,8,71,7,70,6,69,5,68,4,67,3,66,2,65,1,64,0); + p8b = _mm512_set_epi8(127,63,126,62,125,61,124,60,123,59,122,58,121,57,120,56,119,55,118,54,117,53,116,52,115,51,114,50,113,49,112,48,111,47,110,46,109,45,108,44,107,43,106,42,105,41,104,40,103,39,102,38,101,37,100,36,99,35,98,34,97,33,96,32); + + __m512i p16a = _mm512_set_epi16(47,15,46,14,45,13,44,12,43,11,42,10,41,9,40,8,39,7,38,6,37,5,36,4,35,3,34,2,33,1,32,0); + __m512i p16b = _mm512_set_epi16(63,31,62,30,61,29,60,28,59,27,58,26,57,25,56,24,55,23,54,22,53,21,52,20,51,19,50,18,49,17,48,16); + + __m512i p32a = _mm512_set_epi32(23,7,22,6,21,5,20,4,19,3,18,2,17,1,16,0); + __m512i p32b = _mm512_set_epi32(31,15,30,14,29,13,28,12,27,11,26,10,25,9,24,8); + __m512i e2j,e3j; + for (k=0,j=0;j>6;j++,k+=4) { + e0j = _mm512_loadu_si512(e0_512+j); + e1j = _mm512_loadu_si512(e1_512+j); + e2j = _mm512_loadu_si512(e2_512+j); + e3j = _mm512_loadu_si512(e3_512+j); + tmp0 = _mm512_permutex2var_epi8(e0j,p8a,e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15) + tmp1 = _mm512_permutex2var_epi8(e2j,p8a,e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15) + _mm512_storeu_si512(f_512+k,_mm512_permutex2var_epi16(tmp0,p16a,tmp1)); // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) + _mm512_storeu_si512(f_512+k+1,_mm512_permutex2var_epi16(tmp0,p16b,tmp1)); + + tmp0 = _mm512_permutex2var_epi8(e0j,p8b,e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15) + tmp1 = _mm512_permutex2var_epi8(e2j,p8b,e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15) + _mm512_storeu_si512(f_512+k+2,_mm512_permutex2var_epi16(tmp0,p16a,tmp1)); // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) + _mm512_storeu_si512(f_512+k+3,_mm512_permutex2var_epi16(tmp0,p16b,tmp1)); } + if ((j<<6) != EQm) { + int k2=k<<6; + fp = &f[k2]; + for (int j2=(j<<6) ; j2< EQm ; j2++) { + *fp++ = e0[j2]; + *fp++ = e1[j2]; + *fp++ = e2[j2]; + *fp++ = e3[j2]; + } + } +#else + e0_128=(simde__m128i *)e0; + e1_128=(simde__m128i *)e1; + e2_128=(simde__m128i *)e2; + e3_128=(simde__m128i *)e3; + simde__m128i e2j,e3j; + for (k=0,j=0;j>4;j++,k+=4) { + e0j = simde_mm_loadu_si128(e0_128+j); + e1j = simde_mm_loadu_si128(e1_128+j); + e2j = simde_mm_loadu_si128(e2_128+j); + e3j = simde_mm_loadu_si128(e3_128+j); + tmp0 = simde_mm_unpacklo_epi8(e0j,e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+7) e1(i+7) + tmp1 = simde_mm_unpacklo_epi8(e2j,e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+7) e3(i+7) + simde_mm_storeu_si128(f_128+k,simde_mm_unpacklo_epi16(tmp0,tmp1)); // e0(i) e1(i) e2(i) e3(i) ... e0(i+3) e1(i+3) e2(i+3) e3(i+3) + simde_mm_storeu_si128(f_128+k+1,simde_mm_unpackhi_epi16(tmp0,tmp1)); // e0(i+4) e1(i+4) e2(i+4) e3(i+4) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) + tmp0 = simde_mm_unpackhi_epi8(e0j,e1j); // e0(i+8) e1(i+8) e0(i+9) e1(i+9) .... e0(i+15) e1(i+15) + tmp1 = simde_mm_unpackhi_epi8(e2j,e3j); // e2(i+8) e3(i+9) e2(i+10) e3(i+10) .... e2(i+31) e3(i+31) + simde_mm_storeu_si128(f_128+k+2,simde_mm_unpacklo_epi16(tmp0,tmp1)); + simde_mm_storeu_si128(f_128+k+3,simde_mm_unpackhi_epi16(tmp0,tmp1)); + } + if ((j<<4) != EQm) { + int k2=k<<4; + fp = &f[k2]; + for (int j2=(j<<4) ; j2< EQm ; j2++) { + *fp++ = e0[j2]; + *fp++ = e1[j2]; + *fp++ = e2[j2]; + *fp++ = e3[j2]; + } + } +#endif break; case 6: e0=e; @@ -80,95 +176,77 @@ void nr_interleaving_ldpc(uint32_t E, uint8_t Qm, uint8_t *e, uint8_t *f) e3=e2+EQm; e4=e3+EQm; e5=e4+EQm; - e0_256=(simde__m256i *)e0; - e1_256=(simde__m256i *)e1; - e2_256=(simde__m256i *)e2; - e3_256=(simde__m256i *)e3; - e4_256=(simde__m256i *)e4; - e5_256=(simde__m256i *)e5; +#ifdef __AVX512VBMI__ + e0_512=(__m512i *)e0; + e1_512=(__m512i *)e1; + e2_512=(__m512i *)e2; + e3_512=(__m512i *)e3; + e4_512=(__m512i *)e4; + e5_512=(__m512i *)e5; + __m512i e4j,e5j; + p8a = _mm512_set_epi8(95,31,94,30,93,29,92,28,91,27,90,26,89,25,88,24,87,23,86,22,85,21,84,20,83,19,82,18,81,17,80,16,79,15,78,14,77,13,76,12,75,11,74,10,73,9,72,8,71,7,70,6,69,5,68,4,67,3,66,2,65,1,64,0); + p8b = _mm512_set_epi8(127,63,126,62,125,61,124,60,123,59,122,58,121,57,120,56,119,55,118,54,117,53,116,52,115,51,114,50,113,49,112,48,111,47,110,46,109,45,108,44,107,43,106,42,105,41,104,40,103,39,102,38,101,37,100,36,99,35,98,34,97,33,96,32); - for (int j=0,k=0;j>5;j++,k+=192) { - fp = f+k; - fp2 = fp+96; + p16a = _mm512_set_epi16(47,15,46,14,45,13,44,12,43,11,42,10,41,9,40,8,39,7,38,6,37,5,36,4,35,3,34,2,33,1,32,0); + p16b = _mm512_set_epi16(63,31,62,30,61,29,60,28,59,27,58,26,57,25,56,24,55,23,54,22,53,21,52,20,51,19,50,18,49,17,48,16); - tmp0 = simde_mm256_unpacklo_epi8(e0_256[j],e1_256[j]); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15) - tmp1 = simde_mm256_unpacklo_epi8(e2_256[j],e3_256[j]); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15) - tmp0b = simde_mm256_unpacklo_epi16(tmp0,tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) - tmp1b = simde_mm256_unpackhi_epi16(tmp0,tmp1); // e0(i+8) e1(i+8) e2(i+8) e3(i+8) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) - tmp0 = simde_mm256_unpacklo_epi8(e4_256[j],e5_256[j]); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+15) e5(i+15) - *((uint32_t*)fp) = simde_mm256_extract_epi32(tmp0b,0); - *((uint16_t*)(fp+4)) = simde_mm256_extract_epi16(tmp0,0); - *((uint32_t*)(fp+6)) = simde_mm256_extract_epi32(tmp0b,1); - *((uint16_t*)(fp+10)) = simde_mm256_extract_epi16(tmp0,1); - *((uint32_t*)(fp+12)) = simde_mm256_extract_epi32(tmp0b,2); - *((uint16_t*)(fp+16)) = simde_mm256_extract_epi16(tmp0,2); - *((uint32_t*)(fp+18)) = simde_mm256_extract_epi32(tmp0b,3); - *((uint16_t*)(fp+22)) = simde_mm256_extract_epi16(tmp0,3); - *((uint32_t*)(fp+24)) = simde_mm256_extract_epi32(tmp0b,4); - *((uint16_t*)(fp+26)) = simde_mm256_extract_epi16(tmp0,4); - *((uint32_t*)(fp+30)) = simde_mm256_extract_epi32(tmp0b,5); - *((uint16_t*)(fp+34)) = simde_mm256_extract_epi16(tmp0,5); - *((uint32_t*)(fp+36)) = simde_mm256_extract_epi32(tmp0,6); - *((uint16_t*)(fp+40)) = simde_mm256_extract_epi16(tmp0,6); - *((uint32_t*)(fp+42)) = simde_mm256_extract_epi32(tmp0b,7); - *((uint16_t*)(fp+46)) = simde_mm256_extract_epi16(tmp0,7); + __m512i p16c = _mm512_set_epi16(21,20,41,19,18,40,17,16,39,15,14,38,13,12,37,11,10,36,9,8,35,7,6,34,5,4,33,3,2,32,1,0); + __m512i p16d = _mm512_set_epi16(10,52,9,8,51,7,6,50,5,4,49,3,2,48,1,0,47,31,30,46,29,28,45,27,26,44,25,24,43,23,22,42); + __m512i p16d2 = _mm512_set_epi16(32+10,30,32+9,32+8,27,32+7,32+6,24,32+5,32+4,21,32+3,32+2,18,32+1,32+0,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0); + __m512i p16e = _mm512_set_epi16(63,31,30,62,29,28,61,27,26,60,25,24,59,23,22,58,21,20,57,19,18,56,17,16,55,15,14,54,13,12,53,11); - *((uint32_t*)(fp+48)) = simde_mm256_extract_epi32(tmp1b,0); - *((uint16_t*)(fp+52)) = simde_mm256_extract_epi16(tmp0,8); - *((uint32_t*)(fp+56)) = simde_mm256_extract_epi32(tmp1b,1); - *((uint16_t*)(fp+60)) = simde_mm256_extract_epi16(tmp0,9); - *((uint32_t*)(fp+62)) = simde_mm256_extract_epi32(tmp1b,2); - *((uint16_t*)(fp+66)) = simde_mm256_extract_epi16(tmp0,10); - *((uint32_t*)(fp+68)) = simde_mm256_extract_epi32(tmp1b,3); - *((uint16_t*)(fp+72)) = simde_mm256_extract_epi16(tmp0,11); - *((uint32_t*)(fp+74)) = simde_mm256_extract_epi32(tmp1b,4); - *((uint16_t*)(fp+76)) = simde_mm256_extract_epi16(tmp0,12); - *((uint32_t*)(fp+80)) = simde_mm256_extract_epi32(tmp1b,5); - *((uint16_t*)(fp+82)) = simde_mm256_extract_epi16(tmp0,13); - *((uint32_t*)(fp+86)) = simde_mm256_extract_epi32(tmp1b,6); - *((uint16_t*)(fp+90)) = simde_mm256_extract_epi16(tmp0,14); - *((uint32_t*)(fp+92)) = simde_mm256_extract_epi32(tmp1b,7); - *((uint16_t*)(fp+94)) = simde_mm256_extract_epi16(tmp0,15); + for (j=0,k=0;j>6;j++,k+=6) { + e0j = _mm512_loadu_si512(e0_512+j); + e1j = _mm512_loadu_si512(e1_512+j); + e2j = _mm512_loadu_si512(e2_512+j); + e3j = _mm512_loadu_si512(e3_512+j); + e4j = _mm512_loadu_si512(e4_512+j); + e5j = _mm512_loadu_si512(e5_512+j); + tmp0 = _mm512_permutex2var_epi8(e0j,p8a,e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+31) e1(i+31) + tmp1 = _mm512_permutex2var_epi8(e2j,p8a,e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+31) e3(i+31) + tmp2 = _mm512_permutex2var_epi8(e4j,p8a,e5j); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+31) e5(i+31) + tmp3 = _mm512_permutex2var_epi16(tmp0,p16a,tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) + tmp4 = _mm512_permutex2var_epi16(tmp0,p16b,tmp1); // e0(i+16) e1(i+16) e2(i+16) e3(i+16) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31) + _mm512_storeu_si512(f_512+k,_mm512_permutex2var_epi16(tmp3,p16c,tmp2)); // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) ... e0(i+9) e1(i+9) e2(i+9) e3(i+9) e4(i+9) e5(i+9) e0(i+10) e1(i+10) + tmp5 = _mm512_permutex2var_epi16(tmp3,p16d,tmp2); // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) e4(i+15) e5(i+15) x x x x e4(i+16) e5(i+16) x x x x .... e4(i+20) e5(i+20) x x + _mm512_storeu_si512(f_512+k+1,_mm512_permutex2var_epi16(tmp5,p16d2,tmp4)); // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) e4(i+15) e5(i+15) e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16) e0(i+20) e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e0(i+21) e1(i+21) + _mm512_storeu_si512(f_512+k+2,_mm512_permutex2var_epi16(tmp4,p16e,tmp2)); // e2(i+21) e3(i+21) e4(i+21) e5(i+21) .... e0(i+31) e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31) - tmp0 = simde_mm256_unpackhi_epi8(e0_256[j],e1_256[j]); // e0(i+16) e1(i+16) e0(i+17) e1(i+17) .... e0(i+31) e1(i+31) - tmp1 = simde_mm256_unpackhi_epi8(e2_256[j],e3_256[j]); // e2(i+16) e3(i+16) e2(i+17) e3(i+17) .... e2(i+31) e3(i+31) - tmp0b = simde_mm256_unpacklo_epi16(tmp0,tmp1); // e0(i+16) e1(i+16) e2(i+16) e3(i+16) ... e0(i+23) e1(i+23) e2(i+23) e3(i+23) - tmp1b = simde_mm256_unpackhi_epi16(tmp0,tmp1); // e0(i+24) e1(i+24) e2(i+24) e3(i+24) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31) - tmp0 = simde_mm256_unpackhi_epi8(e4_256[j],e5_256[j]); // e4(i+16) e5(i+16) e4(i+17) e5(i+17) .... e4(i+31) e5(i+31) - *((uint32_t*)fp2) = simde_mm256_extract_epi32(tmp0b,0); - *((uint16_t*)(fp2+4)) = simde_mm256_extract_epi16(tmp0,0); - *((uint32_t*)(fp2+6)) = simde_mm256_extract_epi32(tmp0b,1); - *((uint16_t*)(fp2+10)) = simde_mm256_extract_epi16(tmp0,1); - *((uint32_t*)(fp2+12)) = simde_mm256_extract_epi32(tmp0b,2); - *((uint16_t*)(fp2+16)) = simde_mm256_extract_epi16(tmp0,2); - *((uint32_t*)(fp2+18)) = simde_mm256_extract_epi32(tmp0b,3); - *((uint16_t*)(fp2+22)) = simde_mm256_extract_epi16(tmp0,3); - *((uint32_t*)(fp2+24)) = simde_mm256_extract_epi32(tmp0b,4); - *((uint16_t*)(fp2+26)) = simde_mm256_extract_epi16(tmp0,4); - *((uint32_t*)(fp2+30)) = simde_mm256_extract_epi32(tmp0b,5); - *((uint16_t*)(fp2+34)) = simde_mm256_extract_epi16(tmp0,5); - *((uint32_t*)(fp2+36)) = simde_mm256_extract_epi32(tmp0,6); - *((uint16_t*)(fp2+40)) = simde_mm256_extract_epi16(tmp0,6); - *((uint32_t*)(fp2+42)) = simde_mm256_extract_epi32(tmp0b,7); - *((uint16_t*)(fp2+46)) = simde_mm256_extract_epi16(tmp0,7); + tmp0 = _mm512_permutex2var_epi8(e0j,p8b,e1j); // e0(i+32) e1(i+32) e0(i+32) e1(i+32) .... e0(i+63) e1(i+63) + tmp1 = _mm512_permutex2var_epi8(e2j,p8b,e3j); // e2(i+32) e3(i+32) e2(i+32) e3(i+32) .... e2(i+63) e3(i+63) + tmp2 = _mm512_permutex2var_epi8(e4j,p8b,e5j); // e4(i+32) e5(i+32) e4(i+32) e5(i+32) .... e4(i+63) e5(i+63) + tmp3 = _mm512_permutex2var_epi16(tmp0,p16a,tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) + tmp4 = _mm512_permutex2var_epi16(tmp0,p16b,tmp1); // e0(i+16) e1(i+16) e2(i+16) e3(i+16) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31) - *((uint32_t*)(fp2+48)) = simde_mm256_extract_epi32(tmp1b,0); - *((uint16_t*)(fp2+52)) = simde_mm256_extract_epi16(tmp0,8); - *((uint32_t*)(fp2+56)) = simde_mm256_extract_epi32(tmp1b,1); - *((uint16_t*)(fp2+60)) = simde_mm256_extract_epi16(tmp0,9); - *((uint32_t*)(fp2+62)) = simde_mm256_extract_epi32(tmp1b,2); - *((uint16_t*)(fp2+66)) = simde_mm256_extract_epi16(tmp0,10); - *((uint32_t*)(fp2+68)) = simde_mm256_extract_epi32(tmp1b,3); - *((uint16_t*)(fp2+72)) = simde_mm256_extract_epi16(tmp0,11); - *((uint32_t*)(fp2+74)) = simde_mm256_extract_epi32(tmp1b,4); - *((uint16_t*)(fp2+76)) = simde_mm256_extract_epi16(tmp0,12); - *((uint32_t*)(fp2+80)) = simde_mm256_extract_epi32(tmp1b,5); - *((uint16_t*)(fp2+82)) = simde_mm256_extract_epi16(tmp0,13); - *((uint32_t*)(fp2+86)) = simde_mm256_extract_epi32(tmp1b,6); - *((uint16_t*)(fp2+90)) = simde_mm256_extract_epi16(tmp0,14); - *((uint32_t*)(fp2+92)) = simde_mm256_extract_epi32(tmp1b,7); - *((uint16_t*)(fp2+94)) = simde_mm256_extract_epi16(tmp0,15); + _mm512_storeu_si512(f_512+k+3,_mm512_permutex2var_epi16(tmp3,p16c,tmp2)); // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) ... e0(i+9) e1(i+9) e2(i+9) e3(i+9) e4(i+9) e5(i+9) e0(i+10) e1(i+10) + tmp5 = _mm512_permutex2var_epi16(tmp3,p16d,tmp2); // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) e4(i+15) e5(i+15) x x x x e4(i+16) e5(i+16) x x x x .... e4(i+20) e5(i+20) x x + _mm512_storeu_si512(f_512+k+4,_mm512_permutex2var_epi16(tmp5,p16d2,tmp4)); // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) e4(i+15) e5(i+15) e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16) e0(i+20) e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e0(i+21) e1(i+21) + _mm512_storeu_si512(f_512+k+5,_mm512_permutex2var_epi16(tmp4,p16e,tmp2)); // e2(i+21) e3(i+21) e4(i+21) e5(i+21) .... e0(i+31) e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31) } + if ((j<<6) != EQm) { + int k2=k<<6; + fp = &f[k2]; + for (int j2=(j<<6) ; j2< EQm ; j2++) { + *fp++ = e0[j2]; + *fp++ = e1[j2]; + *fp++ = e2[j2]; + *fp++ = e3[j2]; + *fp++ = e4[j2]; + *fp++ = e5[j2]; + } + } +#else + fp = f; + for (int j = 0; j< EQm; j++){ + *fp++ = e0[j]; + *fp++ = e1[j]; + *fp++ = e2[j]; + *fp++ = e3[j]; + *fp++ = e4[j]; + *fp++ = e5[j]; + } + +#endif break; case 8: e0=e; @@ -180,43 +258,137 @@ void nr_interleaving_ldpc(uint32_t E, uint8_t Qm, uint8_t *e, uint8_t *f) e6=e5+EQm; e7=e6+EQm; - e0_256=(simde__m256i *)e0; - e1_256=(simde__m256i *)e1; - e2_256=(simde__m256i *)e2; - e3_256=(simde__m256i *)e3; - e4_256=(simde__m256i *)e4; - e5_256=(simde__m256i *)e5; - e6_256=(simde__m256i *)e6; - e7_256=(simde__m256i *)e7; - for (int k=0,j=0;j>5;j++,k+=8) { - tmp0 = simde_mm256_unpacklo_epi8(e0_256[j],e1_256[j]); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15) - tmp1 = simde_mm256_unpacklo_epi8(e2_256[j],e3_256[j]); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15) - tmp2 = simde_mm256_unpacklo_epi8(e4_256[j],e5_256[j]); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+15) e5(i+15) - tmp3 = simde_mm256_unpacklo_epi8(e6_256[j],e7_256[j]); // e6(i) e7(i) e6(i+1) e7(i+1) .... e6(i+15) e7(i+15) - tmp4 = simde_mm256_unpacklo_epi16(tmp0,tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) - tmp5 = simde_mm256_unpacklo_epi16(tmp2,tmp3); // e4(i) e5(i) e6(i) e7(i) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7) - f_256[k] = simde_mm256_unpacklo_epi16(tmp4,tmp5); // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i)... e0(i+3) e1(i+3) e2(i+3) e3(i+3) e4(i+3) e5(i+3) e6(i+3) e7(i+3)) - f_256[k+1] = simde_mm256_unpackhi_epi16(tmp4,tmp5); // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7(i+4)... e0(i+7) e1(i+7) e2(i+7) e3(i+7) e4(i+7) e5(i+7) e6(i+7) e7(i+7)) +#ifdef __AVX512VBMI__ + e0_512=(__m512i *)e0; + e1_512=(__m512i *)e1; + e2_512=(__m512i *)e2; + e3_512=(__m512i *)e3; + e4_512=(__m512i *)e4; + e5_512=(__m512i *)e5; + e6_512=(__m512i *)e6; + e7_512=(__m512i *)e7; + __m512i e6j,e7j; + p8a = _mm512_set_epi8(95,31,94,30,93,29,92,28,91,27,90,26,89,25,88,24,87,23,86,22,85,21,84,20,83,19,82,18,81,17,80,16,79,15,78,14,77,13,76,12,75,11,74,10,73,9,72,8,71,7,70,6,69,5,68,4,67,3,66,2,65,1,64,0); + p8b = _mm512_set_epi8(127,63,126,62,125,61,124,60,123,59,122,58,121,57,120,56,119,55,118,54,117,53,116,52,115,51,114,50,113,49,112,48,111,47,110,46,109,45,108,44,107,43,106,42,105,41,104,40,103,39,102,38,101,37,100,36,99,35,98,34,97,33,96,32); - tmp4 = simde_mm256_unpackhi_epi16(tmp0,tmp1); // e0(i+8) e1(i+8) e2(i+8) e3(i+8) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) - tmp5 = simde_mm256_unpackhi_epi16(tmp2,tmp3); // e4(i+8) e5(i+8) e6(i+8) e7(i+8) ... e4(i+15) e5(i+15) e6(i+15) e7(i+15) - f_256[k+2] = simde_mm256_unpacklo_epi16(tmp4,tmp5); // e0(i+8) e1(i+8) e2(i+8) e3(i+8) e4(i+8) e5(i+8) e6(i+8) e7(i+8)... e0(i+11) e1(i+11) e2(i+11) e3(i+11) e4(i+11) e5(i+11) e6(i+11) e7(i+11)) - f_256[k+3] = simde_mm256_unpackhi_epi16(tmp4,tmp5); // e0(i+12) e1(i+12) e2(i+12) e3(i+12) e4(i+12) e5(i+12) e6(i+12) e7(i+12)... e0(i+15) e1(i+15) e2(i+15) e3(i+15) e4(i+15) e5(i+15) e6(i+15) e7(i+15)) + p16a = _mm512_set_epi16(47,15,46,14,45,13,44,12,43,11,42,10,41,9,40,8,39,7,38,6,37,5,36,4,35,3,34,2,33,1,32,0); + p16b = _mm512_set_epi16(63,31,62,30,61,29,60,28,59,27,58,26,57,25,56,24,55,23,54,22,53,21,52,20,51,19,50,18,49,17,48,16); - tmp0 = simde_mm256_unpackhi_epi8(e0_256[j],e1_256[j]); // e0(i+16) e1(i+16) e0(i+17) e1(i+17) .... e0(i+31) e1(i+31) - tmp1 = simde_mm256_unpackhi_epi8(e2_256[j],e3_256[j]); // e2(i+16) e3(i+16) e2(i+17) e3(i+17) .... e2(i+31) e3(i+31) - tmp2 = simde_mm256_unpackhi_epi8(e4_256[j],e5_256[j]); // e4(i+16) e5(i+16) e4(i+17) e5(i+17) .... e4(i+31) e5(i+31) - tmp3 = simde_mm256_unpackhi_epi8(e6_256[j],e7_256[j]); // e6(i+16) e7(i+16) e6(i+17) e7(i+17) .... e6(i+31) e7(i+31) - tmp4 = simde_mm256_unpacklo_epi16(tmp0,tmp1); // e0(i+!6) e1(i+16) e2(i+16) e3(i+16) ... e0(i+23) e1(i+23) e2(i+23) e3(i+23) - tmp5 = simde_mm256_unpacklo_epi16(tmp2,tmp3); // e4(i+16) e5(i+16) e6(i+16) e7(i+16) ... e4(i+23) e5(i+23) e6(i+23) e7(i+23) - f_256[k+4] = simde_mm256_unpacklo_epi16(tmp4,tmp5); // e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16) e6(i+16) e7(i+16)... e0(i+19) e1(i+19) e2(i+19) e3(i+19) e4(i+19) e5(i+19) e6(i+19) e7(i+19)) - f_256[k+5] = simde_mm256_unpackhi_epi16(tmp4,tmp5); // e0(i+20) e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e6(i+20) e7(i+20)... e0(i+23) e1(i+23) e2(i+23) e3(i+23) e4(i+23) e5(i+23) e6(i+23) e7(i+23)) + p32a = _mm512_set_epi32(23,7,22,6,21,5,20,4,19,3,18,2,17,1,16,0); + p32b = _mm512_set_epi32(31,15,30,14,29,13,28,12,27,11,26,10,25,9,24,8); + for (k=0,j=0;j>6;j++,k+=8) { + e0j = _mm512_loadu_si512(e0_512+j); + e1j = _mm512_loadu_si512(e1_512+j); + e2j = _mm512_loadu_si512(e2_512+j); + e3j = _mm512_loadu_si512(e3_512+j); + e4j = _mm512_loadu_si512(e4_512+j); + e5j = _mm512_loadu_si512(e5_512+j); + e6j = _mm512_loadu_si512(e6_512+j); + e7j = _mm512_loadu_si512(e7_512+j); + tmp0 = _mm512_permutex2var_epi8(e0j,p8a,e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15) + tmp1 = _mm512_permutex2var_epi8(e2j,p8a,e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15) + tmp2 = _mm512_permutex2var_epi8(e4j,p8a,e5j); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+15) e5(i+15) + tmp3 = _mm512_permutex2var_epi8(e6j,p8a,e7j); // e6(i) e7(i) e6(i+1) e7(i+1) .... e6(i+15) e7(i+15) + tmp4 = _mm512_permutex2var_epi16(tmp0,p16a,tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) + tmp5 = _mm512_permutex2var_epi16(tmp2,p16a,tmp3); // e4(i) e5(i) e6(i) e7(i) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7) + _mm512_storeu_si512(f_512+k,_mm512_permutex2var_epi32(tmp4,p32a,tmp5)); // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i)... e0(i+3) e1(i+3) e2(i+3) e3(i+3) e4(i+3) e5(i+3) e6(i+3) e7(i+3)) + _mm512_storeu_si512(f_512+k+1,_mm512_permutex2var_epi32(tmp4,p32b,tmp5)); // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7(i+4)... e0(i+7) e1(i+7) e2(i+7) e3(i+7) e4(i+7) e5(i+7) e6(i+7) e7(i+7)) - tmp4 = simde_mm256_unpackhi_epi16(tmp0,tmp1); // e0(i+24) e1(i+24) e2(i+24) e3(i+24) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31) - tmp5 = simde_mm256_unpackhi_epi16(tmp2,tmp3); // e4(i+24) e5(i+24) e6(i+24) e7(i+24) ... e4(i+31) e5(i+31) e6(i+31) e7(i+31) - f_256[k+6] = simde_mm256_unpacklo_epi16(tmp4,tmp5); // e0(i+24) e1(i+24) e2(i+24) e3(i+24) e4(i+24) e5(i+24) e6(i+24) e7(i+24)... e0(i+27) e1(i+27) e2(i+27) e3(i+27) e4(i+27) e5(i+27) e6(i+27) e7(i+27)) - f_256[k+7] = simde_mm256_unpackhi_epi16(tmp4,tmp5); // e0(i+28) e1(i+28) e2(i+28) e3(i+28) e4(i+28) e5(i+28) e6(i+28) e7(i+28)... e0(i+31) e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31) e6(i+31) e7(i+31)) + tmp4 = _mm512_permutex2var_epi16(tmp0,p16b,tmp1); // e0(i+8) e1(i+8) e2(i+8) e3(i+8) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) + tmp5 = _mm512_permutex2var_epi16(tmp2,p16b,tmp3); // e4(i+8) e5(i+8) e6(i+8) e7(i+8) ... e4(i+15) e5(i+15) e6(i+15) e7(i+15) + _mm512_storeu_si512(f_512+k+2,_mm512_permutex2var_epi32(tmp4,p32a,tmp5)); // e0(i+8) e1(i+8) e2(i+8) e3(i+8) e4(i+8) e5(i+8) e6(i+8) e7(i+8)... e0(i+11) e1(i+11) e2(i+11) e3(i+11) e4(i+11) e5(i+11) e6(i+11) e7(i+11)) + _mm512_storeu_si512(f_512+k+3,_mm512_permutex2var_epi32(tmp4,p32b,tmp5)); // e0(i+12) e1(i+12) e2(i+12) e3(i+12) e4(i+12) e5(i+12) e6(i+12) e7(i+12)... e0(i+15) e1(i+15) e2(i+15) e3(i+15) e4(i+15) e5(i+15) e6(i+15) e7(i+15)) + + tmp0 = _mm512_permutex2var_epi8(e0j,p8b,e1j); // e0(i+16) e1(i+16) e0(i+17) e1(i+17) .... e0(i+31) e1(i+31) + tmp1 = _mm512_permutex2var_epi8(e2j,p8b,e3j); // e2(i+16) e3(i+16) e2(i+17) e3(i+17) .... e2(i+31) e3(i+31) + tmp2 = _mm512_permutex2var_epi8(e4j,p8b,e5j); // e4(i+16) e5(i+16) e4(i+17) e5(i+17) .... e4(i+31) e5(i+31) + tmp3 = _mm512_permutex2var_epi8(e6j,p8b,e7j); // e6(i+16) e7(i+16) e6(i+17) e7(i+17) .... e6(i+31) e7(i+31) + tmp4 = _mm512_permutex2var_epi16(tmp0,p16a,tmp1); // e0(i+!6) e1(i+16) e2(i+16) e3(i+16) ... e0(i+23) e1(i+23) e2(i+23) e3(i+23) + tmp5 = _mm512_permutex2var_epi16(tmp2,p16a,tmp3); // e4(i+16) e5(i+16) e6(i+16) e7(i+16) ... e4(i+23) e5(i+23) e6(i+23) e7(i+23) + _mm512_storeu_si512(f_512+k+4,_mm512_permutex2var_epi32(tmp4,p32a,tmp5)); // e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16) e6(i+16) e7(i+16)... e0(i+19) e1(i+19) e2(i+19) e3(i+19) e4(i+19) e5(i+19) e6(i+19) e7(i+19)) + _mm512_storeu_si512(f_512+k+5,_mm512_permutex2var_epi32(tmp4,p32b,tmp5)); // e0(i+20) e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e6(i+20) e7(i+20)... e0(i+23) e1(i+23) e2(i+23) e3(i+23) e4(i+23) e5(i+23) e6(i+23) e7(i+23)) + + tmp4 = _mm512_permutex2var_epi16(tmp0,p16b,tmp1); // e0(i+24) e1(i+24) e2(i+24) e3(i+24) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31) + tmp5 = _mm512_permutex2var_epi16(tmp2,p16b,tmp3); // e4(i+24) e5(i+24) e6(i+24) e7(i+24) ... e4(i+31) e5(i+31) e6(i+31) e7(i+31) + _mm512_storeu_si512(f_512+k+6,_mm512_permutex2var_epi32(tmp4,p32a,tmp5)); // e0(i+24) e1(i+24) e2(i+24) e3(i+24) e4(i+24) e5(i+24) e6(i+24) e7(i+24)... e0(i+27) e1(i+27) e2(i+27) e3(i+27) e4(i+27) e5(i+27) e6(i+27) e7(i+27)) + _mm512_storeu_si512(f_512+k+7,_mm512_permutex2var_epi32(tmp4,p32b,tmp5)); // e0(i+28) e1(i+28) e2(i+28) e3(i+28) e4(i+28) e5(i+28) e6(i+28) e7(i+28)... e0(i+31) e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31) e6(i+31) e7(i+31)) } + if ((j<<6) != EQm) { + int k2=k<<6; + fp=&f[k2]; + for (int j2=(j<<6) ; j2< EQm ; j2++) { + *fp++ = e0[j2]; + *fp++ = e1[j2]; + *fp++ = e2[j2]; + *fp++ = e3[j2]; + *fp++ = e4[j2]; + *fp++ = e5[j2]; + *fp++ = e6[j2]; + *fp++ = e7[j2]; + } + } +#else + e0_128=(simde__m128i *)e0; + e1_128=(simde__m128i *)e1; + e2_128=(simde__m128i *)e2; + e3_128=(simde__m128i *)e3; + e4_128=(simde__m128i *)e4; + e5_128=(simde__m128i *)e5; + e6_128=(simde__m128i *)e6; + e7_128=(simde__m128i *)e7; + simde__m128i e4j,e5j,e6j,e7j; + + for (k=0,j=0;j>4;j++,k+=8) { + e0j = simde_mm_loadu_si128(e0_128+j); + e1j = simde_mm_loadu_si128(e1_128+j); + e2j = simde_mm_loadu_si128(e2_128+j); + e3j = simde_mm_loadu_si128(e3_128+j); + e4j = simde_mm_loadu_si128(e4_128+j); + e5j = simde_mm_loadu_si128(e5_128+j); + e6j = simde_mm_loadu_si128(e6_128+j); + e7j = simde_mm_loadu_si128(e7_128+j); + tmp0 = simde_mm_unpacklo_epi8(e0j,e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+7) e1(i+7) + tmp1 = simde_mm_unpacklo_epi8(e2j,e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+7) e3(i+7) + tmp2 = simde_mm_unpacklo_epi16(tmp0,tmp1);// e0(i) e1(i) e2(i) e3(i) e0(i+1) e1(i+1) e2(i+1) e3(i+1) ... e0(i+3) e1(i+3) e2(i+3) e3(i+3) + tmp3 = simde_mm_unpackhi_epi16(tmp0,tmp1);// e0(i+4) e1(i+4) e2(i+4) e3(i+4) e0(i+5) e1(i+5) e2(i+5) e3(i+5) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) + tmp4 = simde_mm_unpacklo_epi8(e4j,e5j); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+7) e5(i+7) + tmp5 = simde_mm_unpacklo_epi8(e6j,e7j); // e6(i) e7(i) e6(i+1) e7(i+1) .... e6(i+7) e7(i+7) + tmp6 = simde_mm_unpacklo_epi16(tmp4,tmp5);// e4(i) e5(i) e6(i) e7(i) e4(i+1) e5(i+1) e6(i+1) e7(i+1) ... e4(i+3) e5(i+3) e6(i+) e7(i+3) + tmp7 = simde_mm_unpackhi_epi16(tmp4,tmp5);// e4(i+4) e5(i+4) e6(i+4) e7(i+4) e4(i+5) e5(i+5) e6(i+5) e7(i+5) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7) + simde_mm_storeu_si128(f_128+k,simde_mm_unpacklo_epi32(tmp2,tmp6)); // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i) e0(i+1) ... e7(i+1) + simde_mm_storeu_si128(f_128+k+1,simde_mm_unpackhi_epi32(tmp2,tmp6)); // e0(i+2) e1(i+2) e2(i+2) e3(i+2) e4(i+2) e5(i+2) e6(i+2) e7(i+2) e0(i+3) e1(i+3) ... e7(i+3) + simde_mm_storeu_si128(f_128+k+2,simde_mm_unpacklo_epi32(tmp3,tmp7)); // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7+4(i) e0(i+5) ... e7(i+5) + simde_mm_storeu_si128(f_128+k+3,simde_mm_unpackhi_epi32(tmp3,tmp7)); // e0(i+6) e1(i+6) e2(i+6) e3(i+6) e4(i+6) e5(i+6) e6(i+6) e7(i+6) e0(i+7) e0(i+7) ... e7(i+7) + + + tmp0 = simde_mm_unpackhi_epi8(e0j,e1j); // e0(i+8) e1(i+8) e0(i+9) e1(i+9) .... e0(i+15) e1(i+15) + tmp1 = simde_mm_unpackhi_epi8(e2j,e3j); // e2(i+8) e3(i+8) e2(i+9) e3(i+9) .... e2(i+15) e3(i+15) + tmp2 = simde_mm_unpacklo_epi16(tmp0,tmp1);// e0(i) e1(i) e2(i) e3(i) e0(i+1) e1(i+1) e2(i+1) e3(i+1) ... e0(i+3) e1(i+3) e2(i+3) e3(i+3) + tmp3 = simde_mm_unpackhi_epi16(tmp0,tmp1);// e0(i+4) e1(i+4) e2(i+4) e3(i+4) e0(i+5) e1(i+5) e2(i+5) e3(i+5) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) + tmp4 = simde_mm_unpackhi_epi8(e4j,e5j); // e4(i+8) e5(i+8) e4(i+9) e5(i+9) .... e4(i+15) e515i+7) + tmp5 = simde_mm_unpackhi_epi8(e6j,e7j); // e6(i+8) e7(i+8) e6(i+9) e7(i+9) .... e6(i+15) e7(i+15) + tmp6 = simde_mm_unpacklo_epi16(tmp4,tmp5);// e4(i) e5(i) e6(i) e7(i) e4(i+1) e5(i+1) e6(i+1) e7(i+1) ... e4(i+3) e5(i+3) e6(i+) e7(i+3) + tmp7 = simde_mm_unpackhi_epi16(tmp4,tmp5);// e4(i+4) e5(i+4) e6(i+4) e7(i+4) e4(i+5) e5(i+5) e6(i+5) e7(i+5) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7) + simde_mm_storeu_si128(f_128+k+4,simde_mm_unpacklo_epi32(tmp2,tmp6)); // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i) e0(i+1) ... e7(i+1) + simde_mm_storeu_si128(f_128+k+5,simde_mm_unpackhi_epi32(tmp2,tmp6)); // e0(i+2) e1(i+2) e2(i+2) e3(i+2) e4(i+2) e5(i+2) e6(i+2) e7(i+2) e0(i+3) e1(i+3) ... e7(i+3) + simde_mm_storeu_si128(f_128+k+6,simde_mm_unpacklo_epi32(tmp3,tmp7)); // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7+4(i) e0(i+5) ... e7(i+5) + simde_mm_storeu_si128(f_128+k+7,simde_mm_unpackhi_epi32(tmp3,tmp7)); // e0(i+6) e1(i+6) e2(i+6) e3(i+6) e4(i+6) e5(i+6) e6(i+6) e7(i+6) e0(i+7) e0(i+7) ... e7(i+7) + } + if ((j<<4) != EQm) { + int k2=k<<4; + fp=&f[k2]; + for (int j2=(j<<4) ; j2< EQm ; j2++) { + *fp++ = e0[j2]; + *fp++ = e1[j2]; + *fp++ = e2[j2]; + *fp++ = e3[j2]; + *fp++ = e4[j2]; + *fp++ = e5[j2]; + *fp++ = e6[j2]; + *fp++ = e7[j2]; + } + } +#endif break; default: AssertFatal(1==0,"Should be here!\n"); } diff --git a/openair1/PHY/CODING/nrLDPC_encoder/ldpc384_simd512_byte.c b/openair1/PHY/CODING/nrLDPC_encoder/ldpc384_simd512_byte.c new file mode 100644 index 0000000000..16ddbd3e48 --- /dev/null +++ b/openair1/PHY/CODING/nrLDPC_encoder/ldpc384_simd512_byte.c @@ -0,0 +1,151 @@ +#include "PHY/sse_intrin.h" +// generated code for Zc=384, byte encoding +static inline void ldpc384_byte(uint8_t *c,uint8_t *d) { + __m512i *csimd=(__m512i *)c,*dsimd=(__m512i *)d; + + __m512i *c2,*d2; + + int i2; + for (i2=0; i2<6; i2++) { + c2=&csimd[i2]; + d2=&dsimd[i2]; + +//row: 0 + d2[0]=_mm512_xor_si512(c2[13468],_mm512_xor_si512(c2[3169],_mm512_xor_si512(c2[3435],_mm512_xor_si512(c2[5284],_mm512_xor_si512(c2[5028],_mm512_xor_si512(c2[15327],_mm512_xor_si512(c2[6085],_mm512_xor_si512(c2[13224],_mm512_xor_si512(c2[3193],_mm512_xor_si512(c2[2141],_mm512_xor_si512(c2[12977],_mm512_xor_si512(c2[2413],_mm512_xor_si512(c2[36],_mm512_xor_si512(c2[8500],_mm512_xor_si512(c2[3221],_mm512_xor_si512(c2[5068],_mm512_xor_si512(c2[14054],_mm512_xor_si512(c2[4286],_mm512_xor_si512(c2[64],_mm512_xor_si512(c2[6411],_mm512_xor_si512(c2[8786],_mm512_xor_si512(c2[1923],_mm512_xor_si512(c2[2993],_mm512_xor_si512(c2[2992],_mm512_xor_si512(c2[6686],_mm512_xor_si512(c2[3005],_mm512_xor_si512(c2[8546],_mm512_xor_si512(c2[14880],_mm512_xor_si512(c2[16216],_mm512_xor_si512(c2[13310],_mm512_xor_si512(c2[16740],_mm512_xor_si512(c2[8572],_mm512_xor_si512(c2[386],_mm512_xor_si512(c2[1178],_mm512_xor_si512(c2[12013],_mm512_xor_si512(c2[10432],_mm512_xor_si512(c2[13072],_mm512_xor_si512(c2[4632],_mm512_xor_si512(c2[5957],_mm512_xor_si512(c2[10443],_mm512_xor_si512(c2[9929],_mm512_xor_si512(c2[2271],_mm512_xor_si512(c2[5705],_mm512_xor_si512(c2[6771],_mm512_xor_si512(c2[6505],_mm512_xor_si512(c2[5451],_mm512_xor_si512(c2[6255],_mm512_xor_si512(c2[9421],_mm512_xor_si512(c2[14172],_mm512_xor_si512(c2[11281],_mm512_xor_si512(c2[9173],_mm512_xor_si512(c2[12868],_mm512_xor_si512(c2[13405],_mm512_xor_si512(c2[998],_mm512_xor_si512(c2[11824],_mm512_xor_si512(c2[13419],_mm512_xor_si512(c2[12891],_mm512_xor_si512(c2[4180],_mm512_xor_si512(c2[13958],_mm512_xor_si512(c2[3137],_mm512_xor_si512(c2[3663],_mm512_xor_si512(c2[2885],_mm512_xor_si512(c2[3672],_mm512_xor_si512(c2[10536],_mm512_xor_si512(c2[7121],_mm512_xor_si512(c2[12925],c2[10025])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 1 + d2[6]=_mm512_xor_si512(c2[13468],_mm512_xor_si512(c2[13732],_mm512_xor_si512(c2[3433],_mm512_xor_si512(c2[3699],_mm512_xor_si512(c2[5548],_mm512_xor_si512(c2[5028],_mm512_xor_si512(c2[5292],_mm512_xor_si512(c2[15591],_mm512_xor_si512(c2[6349],_mm512_xor_si512(c2[13224],_mm512_xor_si512(c2[13488],_mm512_xor_si512(c2[3457],_mm512_xor_si512(c2[2405],_mm512_xor_si512(c2[12977],_mm512_xor_si512(c2[13241],_mm512_xor_si512(c2[2677],_mm512_xor_si512(c2[300],_mm512_xor_si512(c2[8764],_mm512_xor_si512(c2[3485],_mm512_xor_si512(c2[5332],_mm512_xor_si512(c2[14054],_mm512_xor_si512(c2[14318],_mm512_xor_si512(c2[4550],_mm512_xor_si512(c2[328],_mm512_xor_si512(c2[6411],_mm512_xor_si512(c2[6675],_mm512_xor_si512(c2[9050],_mm512_xor_si512(c2[2187],_mm512_xor_si512(c2[3257],_mm512_xor_si512(c2[3256],_mm512_xor_si512(c2[6950],_mm512_xor_si512(c2[3269],_mm512_xor_si512(c2[8810],_mm512_xor_si512(c2[15144],_mm512_xor_si512(c2[16216],_mm512_xor_si512(c2[16480],_mm512_xor_si512(c2[13574],_mm512_xor_si512(c2[109],_mm512_xor_si512(c2[8572],_mm512_xor_si512(c2[8836],_mm512_xor_si512(c2[650],_mm512_xor_si512(c2[1442],_mm512_xor_si512(c2[12013],_mm512_xor_si512(c2[12277],_mm512_xor_si512(c2[10696],_mm512_xor_si512(c2[13336],_mm512_xor_si512(c2[4632],_mm512_xor_si512(c2[4896],_mm512_xor_si512(c2[6221],_mm512_xor_si512(c2[10707],_mm512_xor_si512(c2[9929],_mm512_xor_si512(c2[10193],_mm512_xor_si512(c2[2535],_mm512_xor_si512(c2[5969],_mm512_xor_si512(c2[7035],_mm512_xor_si512(c2[6769],_mm512_xor_si512(c2[5715],_mm512_xor_si512(c2[6255],_mm512_xor_si512(c2[6519],_mm512_xor_si512(c2[9685],_mm512_xor_si512(c2[14436],_mm512_xor_si512(c2[11281],_mm512_xor_si512(c2[11545],_mm512_xor_si512(c2[9437],_mm512_xor_si512(c2[13132],_mm512_xor_si512(c2[13669],_mm512_xor_si512(c2[1262],_mm512_xor_si512(c2[12088],_mm512_xor_si512(c2[13419],_mm512_xor_si512(c2[13683],_mm512_xor_si512(c2[13155],_mm512_xor_si512(c2[4444],_mm512_xor_si512(c2[13958],_mm512_xor_si512(c2[14222],_mm512_xor_si512(c2[3401],_mm512_xor_si512(c2[3927],_mm512_xor_si512(c2[2885],_mm512_xor_si512(c2[3149],_mm512_xor_si512(c2[3936],_mm512_xor_si512(c2[10800],_mm512_xor_si512(c2[7121],_mm512_xor_si512(c2[7385],_mm512_xor_si512(c2[13189],c2[10289]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 2 + d2[12]=_mm512_xor_si512(c2[13732],_mm512_xor_si512(c2[3433],_mm512_xor_si512(c2[3435],_mm512_xor_si512(c2[3699],_mm512_xor_si512(c2[5284],_mm512_xor_si512(c2[5548],_mm512_xor_si512(c2[5292],_mm512_xor_si512(c2[15327],_mm512_xor_si512(c2[15591],_mm512_xor_si512(c2[6085],_mm512_xor_si512(c2[6349],_mm512_xor_si512(c2[13488],_mm512_xor_si512(c2[3457],_mm512_xor_si512(c2[2141],_mm512_xor_si512(c2[2405],_mm512_xor_si512(c2[13241],_mm512_xor_si512(c2[2677],_mm512_xor_si512(c2[36],_mm512_xor_si512(c2[300],_mm512_xor_si512(c2[8764],_mm512_xor_si512(c2[3221],_mm512_xor_si512(c2[3485],_mm512_xor_si512(c2[5068],_mm512_xor_si512(c2[5332],_mm512_xor_si512(c2[14318],_mm512_xor_si512(c2[4550],_mm512_xor_si512(c2[64],_mm512_xor_si512(c2[328],_mm512_xor_si512(c2[6675],_mm512_xor_si512(c2[8786],_mm512_xor_si512(c2[9050],_mm512_xor_si512(c2[1923],_mm512_xor_si512(c2[2187],_mm512_xor_si512(c2[3257],_mm512_xor_si512(c2[2992],_mm512_xor_si512(c2[3256],_mm512_xor_si512(c2[6686],_mm512_xor_si512(c2[6950],_mm512_xor_si512(c2[3269],_mm512_xor_si512(c2[8546],_mm512_xor_si512(c2[8810],_mm512_xor_si512(c2[14880],_mm512_xor_si512(c2[15144],_mm512_xor_si512(c2[16480],_mm512_xor_si512(c2[13574],_mm512_xor_si512(c2[16740],_mm512_xor_si512(c2[109],_mm512_xor_si512(c2[8836],_mm512_xor_si512(c2[386],_mm512_xor_si512(c2[650],_mm512_xor_si512(c2[1178],_mm512_xor_si512(c2[1442],_mm512_xor_si512(c2[12277],_mm512_xor_si512(c2[10696],_mm512_xor_si512(c2[13072],_mm512_xor_si512(c2[13336],_mm512_xor_si512(c2[4896],_mm512_xor_si512(c2[6221],_mm512_xor_si512(c2[10443],_mm512_xor_si512(c2[10707],_mm512_xor_si512(c2[10193],_mm512_xor_si512(c2[2271],_mm512_xor_si512(c2[2535],_mm512_xor_si512(c2[5705],_mm512_xor_si512(c2[5969],_mm512_xor_si512(c2[7035],_mm512_xor_si512(c2[6505],_mm512_xor_si512(c2[6769],_mm512_xor_si512(c2[5451],_mm512_xor_si512(c2[5715],_mm512_xor_si512(c2[6519],_mm512_xor_si512(c2[9685],_mm512_xor_si512(c2[14172],_mm512_xor_si512(c2[14436],_mm512_xor_si512(c2[11545],_mm512_xor_si512(c2[9437],_mm512_xor_si512(c2[12868],_mm512_xor_si512(c2[13132],_mm512_xor_si512(c2[13669],_mm512_xor_si512(c2[998],_mm512_xor_si512(c2[1262],_mm512_xor_si512(c2[11824],_mm512_xor_si512(c2[12088],_mm512_xor_si512(c2[13683],_mm512_xor_si512(c2[12891],_mm512_xor_si512(c2[13155],_mm512_xor_si512(c2[4180],_mm512_xor_si512(c2[4444],_mm512_xor_si512(c2[14222],_mm512_xor_si512(c2[3401],_mm512_xor_si512(c2[3663],_mm512_xor_si512(c2[3927],_mm512_xor_si512(c2[3149],_mm512_xor_si512(c2[3672],_mm512_xor_si512(c2[3936],_mm512_xor_si512(c2[10536],_mm512_xor_si512(c2[10800],_mm512_xor_si512(c2[7385],_mm512_xor_si512(c2[13189],_mm512_xor_si512(c2[10025],c2[10289])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 3 + d2[18]=_mm512_xor_si512(c2[13732],_mm512_xor_si512(c2[3433],_mm512_xor_si512(c2[3699],_mm512_xor_si512(c2[5284],_mm512_xor_si512(c2[5548],_mm512_xor_si512(c2[5292],_mm512_xor_si512(c2[15591],_mm512_xor_si512(c2[6085],_mm512_xor_si512(c2[6349],_mm512_xor_si512(c2[13488],_mm512_xor_si512(c2[3457],_mm512_xor_si512(c2[2405],_mm512_xor_si512(c2[13241],_mm512_xor_si512(c2[2677],_mm512_xor_si512(c2[36],_mm512_xor_si512(c2[300],_mm512_xor_si512(c2[8764],_mm512_xor_si512(c2[3485],_mm512_xor_si512(c2[5068],_mm512_xor_si512(c2[5332],_mm512_xor_si512(c2[14318],_mm512_xor_si512(c2[4550],_mm512_xor_si512(c2[328],_mm512_xor_si512(c2[6675],_mm512_xor_si512(c2[9050],_mm512_xor_si512(c2[1923],_mm512_xor_si512(c2[2187],_mm512_xor_si512(c2[3257],_mm512_xor_si512(c2[3256],_mm512_xor_si512(c2[6686],_mm512_xor_si512(c2[6950],_mm512_xor_si512(c2[3269],_mm512_xor_si512(c2[8810],_mm512_xor_si512(c2[14880],_mm512_xor_si512(c2[15144],_mm512_xor_si512(c2[16480],_mm512_xor_si512(c2[13574],_mm512_xor_si512(c2[109],_mm512_xor_si512(c2[8836],_mm512_xor_si512(c2[650],_mm512_xor_si512(c2[1178],_mm512_xor_si512(c2[1442],_mm512_xor_si512(c2[12277],_mm512_xor_si512(c2[10696],_mm512_xor_si512(c2[13072],_mm512_xor_si512(c2[13336],_mm512_xor_si512(c2[4896],_mm512_xor_si512(c2[6221],_mm512_xor_si512(c2[10443],_mm512_xor_si512(c2[10707],_mm512_xor_si512(c2[10193],_mm512_xor_si512(c2[2535],_mm512_xor_si512(c2[5705],_mm512_xor_si512(c2[5969],_mm512_xor_si512(c2[7035],_mm512_xor_si512(c2[6769],_mm512_xor_si512(c2[5451],_mm512_xor_si512(c2[5715],_mm512_xor_si512(c2[6519],_mm512_xor_si512(c2[9685],_mm512_xor_si512(c2[14436],_mm512_xor_si512(c2[11545],_mm512_xor_si512(c2[9437],_mm512_xor_si512(c2[12868],_mm512_xor_si512(c2[13132],_mm512_xor_si512(c2[13669],_mm512_xor_si512(c2[1262],_mm512_xor_si512(c2[11824],_mm512_xor_si512(c2[12088],_mm512_xor_si512(c2[13683],_mm512_xor_si512(c2[13155],_mm512_xor_si512(c2[4180],_mm512_xor_si512(c2[4444],_mm512_xor_si512(c2[14222],_mm512_xor_si512(c2[3401],_mm512_xor_si512(c2[3927],_mm512_xor_si512(c2[3149],_mm512_xor_si512(c2[3936],_mm512_xor_si512(c2[10536],_mm512_xor_si512(c2[10800],_mm512_xor_si512(c2[7385],_mm512_xor_si512(c2[13189],_mm512_xor_si512(c2[10025],c2[10289]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 4 + d2[24]=_mm512_xor_si512(c2[3173],c2[14006]); + +//row: 5 + d2[30]=_mm512_xor_si512(c2[4225],_mm512_xor_si512(c2[10827],_mm512_xor_si512(c2[11093],_mm512_xor_si512(c2[12936],_mm512_xor_si512(c2[795],_mm512_xor_si512(c2[12686],_mm512_xor_si512(c2[6084],_mm512_xor_si512(c2[13743],_mm512_xor_si512(c2[3708],_mm512_xor_si512(c2[3987],_mm512_xor_si512(c2[10851],_mm512_xor_si512(c2[9793],_mm512_xor_si512(c2[3734],_mm512_xor_si512(c2[10071],_mm512_xor_si512(c2[7694],_mm512_xor_si512(c2[13501],_mm512_xor_si512(c2[16152],_mm512_xor_si512(c2[10873],_mm512_xor_si512(c2[12720],_mm512_xor_si512(c2[4817],_mm512_xor_si512(c2[11944],_mm512_xor_si512(c2[7716],_mm512_xor_si512(c2[14069],_mm512_xor_si512(c2[16444],_mm512_xor_si512(c2[9581],_mm512_xor_si512(c2[10645],_mm512_xor_si512(c2[10644],_mm512_xor_si512(c2[14344],_mm512_xor_si512(c2[10657],_mm512_xor_si512(c2[16204],_mm512_xor_si512(c2[5643],_mm512_xor_si512(c2[6973],_mm512_xor_si512(c2[4073],_mm512_xor_si512(c2[7503],_mm512_xor_si512(c2[16224],_mm512_xor_si512(c2[8044],_mm512_xor_si512(c2[8836],_mm512_xor_si512(c2[2776],_mm512_xor_si512(c2[1189],_mm512_xor_si512(c2[3829],_mm512_xor_si512(c2[12290],_mm512_xor_si512(c2[13609],_mm512_xor_si512(c2[1200],_mm512_xor_si512(c2[10178],_mm512_xor_si512(c2[686],_mm512_xor_si512(c2[9929],_mm512_xor_si512(c2[13357],_mm512_xor_si512(c2[14429],_mm512_xor_si512(c2[14163],_mm512_xor_si512(c2[13109],_mm512_xor_si512(c2[13913],_mm512_xor_si512(c2[184],_mm512_xor_si512(c2[4935],_mm512_xor_si512(c2[2044],_mm512_xor_si512(c2[16825],_mm512_xor_si512(c2[3625],_mm512_xor_si512(c2[13131],_mm512_xor_si512(c2[4168],_mm512_xor_si512(c2[8656],_mm512_xor_si512(c2[2581],_mm512_xor_si512(c2[4176],_mm512_xor_si512(c2[3648],_mm512_xor_si512(c2[11832],_mm512_xor_si512(c2[4721],_mm512_xor_si512(c2[10789],_mm512_xor_si512(c2[11321],_mm512_xor_si512(c2[10537],_mm512_xor_si512(c2[11330],_mm512_xor_si512(c2[1299],_mm512_xor_si512(c2[14773],_mm512_xor_si512(c2[3688],_mm512_xor_si512(c2[782],c2[13716])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 6 + d2[36]=_mm512_xor_si512(c2[5812],_mm512_xor_si512(c2[340],_mm512_xor_si512(c2[384],_mm512_xor_si512(c2[8321],_mm512_xor_si512(c2[7549],_mm512_xor_si512(c2[16311],_mm512_xor_si512(c2[4968],c2[8955]))))))); + +//row: 7 + d2[42]=_mm512_xor_si512(c2[2376],_mm512_xor_si512(c2[16380],_mm512_xor_si512(c2[15892],_mm512_xor_si512(c2[3521],_mm512_xor_si512(c2[9076],c2[13369]))))); + +//row: 8 + d2[48]=_mm512_xor_si512(c2[14261],_mm512_xor_si512(c2[7921],_mm512_xor_si512(c2[3962],_mm512_xor_si512(c2[14523],_mm512_xor_si512(c2[4228],_mm512_xor_si512(c2[14525],_mm512_xor_si512(c2[14789],_mm512_xor_si512(c2[6077],_mm512_xor_si512(c2[16368],_mm512_xor_si512(c2[16632],_mm512_xor_si512(c2[13468],_mm512_xor_si512(c2[5821],_mm512_xor_si512(c2[16382],_mm512_xor_si512(c2[16120],_mm512_xor_si512(c2[9516],_mm512_xor_si512(c2[9780],_mm512_xor_si512(c2[6878],_mm512_xor_si512(c2[280],_mm512_xor_si512(c2[544],_mm512_xor_si512(c2[13478],_mm512_xor_si512(c2[14017],_mm512_xor_si512(c2[7683],_mm512_xor_si512(c2[3986],_mm512_xor_si512(c2[14547],_mm512_xor_si512(c2[2928],_mm512_xor_si512(c2[13225],_mm512_xor_si512(c2[13489],_mm512_xor_si512(c2[13764],_mm512_xor_si512(c2[7430],_mm512_xor_si512(c2[3206],_mm512_xor_si512(c2[13767],_mm512_xor_si512(c2[829],_mm512_xor_si512(c2[11126],_mm512_xor_si512(c2[11390],_mm512_xor_si512(c2[9806],_mm512_xor_si512(c2[9293],_mm512_xor_si512(c2[2953],_mm512_xor_si512(c2[4008],_mm512_xor_si512(c2[14305],_mm512_xor_si512(c2[14569],_mm512_xor_si512(c2[5861],_mm512_xor_si512(c2[16152],_mm512_xor_si512(c2[16416],_mm512_xor_si512(c2[14847],_mm512_xor_si512(c2[8513],_mm512_xor_si512(c2[5079],_mm512_xor_si512(c2[15640],_mm512_xor_si512(c2[857],_mm512_xor_si512(c2[11148],_mm512_xor_si512(c2[11412],_mm512_xor_si512(c2[7204],_mm512_xor_si512(c2[864],_mm512_xor_si512(c2[9579],_mm512_xor_si512(c2[2981],_mm512_xor_si512(c2[3245],_mm512_xor_si512(c2[2716],_mm512_xor_si512(c2[13013],_mm512_xor_si512(c2[13277],_mm512_xor_si512(c2[3780],_mm512_xor_si512(c2[14341],_mm512_xor_si512(c2[3785],_mm512_xor_si512(c2[14076],_mm512_xor_si512(c2[14340],_mm512_xor_si512(c2[7479],_mm512_xor_si512(c2[881],_mm512_xor_si512(c2[1145],_mm512_xor_si512(c2[3792],_mm512_xor_si512(c2[14353],_mm512_xor_si512(c2[9339],_mm512_xor_si512(c2[2741],_mm512_xor_si512(c2[3005],_mm512_xor_si512(c2[15673],_mm512_xor_si512(c2[9075],_mm512_xor_si512(c2[9339],_mm512_xor_si512(c2[108],_mm512_xor_si512(c2[10669],_mm512_xor_si512(c2[14103],_mm512_xor_si512(c2[7769],_mm512_xor_si512(c2[638],_mm512_xor_si512(c2[10935],_mm512_xor_si512(c2[11199],_mm512_xor_si512(c2[9365],_mm512_xor_si512(c2[3025],_mm512_xor_si512(c2[1179],_mm512_xor_si512(c2[11476],_mm512_xor_si512(c2[11740],_mm512_xor_si512(c2[1971],_mm512_xor_si512(c2[12268],_mm512_xor_si512(c2[12532],_mm512_xor_si512(c2[12806],_mm512_xor_si512(c2[6472],_mm512_xor_si512(c2[11225],_mm512_xor_si512(c2[4885],_mm512_xor_si512(c2[13865],_mm512_xor_si512(c2[7261],_mm512_xor_si512(c2[7525],_mm512_xor_si512(c2[5425],_mm512_xor_si512(c2[15986],_mm512_xor_si512(c2[6744],_mm512_xor_si512(c2[410],_mm512_xor_si512(c2[11236],_mm512_xor_si512(c2[4632],_mm512_xor_si512(c2[4896],_mm512_xor_si512(c2[4896],_mm512_xor_si512(c2[10716],_mm512_xor_si512(c2[4382],_mm512_xor_si512(c2[3064],_mm512_xor_si512(c2[13361],_mm512_xor_si512(c2[13625],_mm512_xor_si512(c2[6492],_mm512_xor_si512(c2[16789],_mm512_xor_si512(c2[158],_mm512_xor_si512(c2[7564],_mm512_xor_si512(c2[1224],_mm512_xor_si512(c2[7298],_mm512_xor_si512(c2[700],_mm512_xor_si512(c2[964],_mm512_xor_si512(c2[6244],_mm512_xor_si512(c2[16541],_mm512_xor_si512(c2[16805],_mm512_xor_si512(c2[7048],_mm512_xor_si512(c2[708],_mm512_xor_si512(c2[10214],_mm512_xor_si512(c2[3880],_mm512_xor_si512(c2[14965],_mm512_xor_si512(c2[8367],_mm512_xor_si512(c2[8631],_mm512_xor_si512(c2[12074],_mm512_xor_si512(c2[5740],_mm512_xor_si512(c2[9960],_mm512_xor_si512(c2[3626],_mm512_xor_si512(c2[13661],_mm512_xor_si512(c2[7057],_mm512_xor_si512(c2[7321],_mm512_xor_si512(c2[10488],_mm512_xor_si512(c2[14198],_mm512_xor_si512(c2[7864],_mm512_xor_si512(c2[1791],_mm512_xor_si512(c2[12088],_mm512_xor_si512(c2[12352],_mm512_xor_si512(c2[12617],_mm512_xor_si512(c2[6013],_mm512_xor_si512(c2[6277],_mm512_xor_si512(c2[14212],_mm512_xor_si512(c2[7872],_mm512_xor_si512(c2[13684],_mm512_xor_si512(c2[7080],_mm512_xor_si512(c2[7344],_mm512_xor_si512(c2[4973],_mm512_xor_si512(c2[15264],_mm512_xor_si512(c2[15528],_mm512_xor_si512(c2[14751],_mm512_xor_si512(c2[8417],_mm512_xor_si512(c2[3924],_mm512_xor_si512(c2[14485],_mm512_xor_si512(c2[4456],_mm512_xor_si512(c2[14753],_mm512_xor_si512(c2[15017],_mm512_xor_si512(c2[8679],_mm512_xor_si512(c2[3672],_mm512_xor_si512(c2[14233],_mm512_xor_si512(c2[4465],_mm512_xor_si512(c2[14762],_mm512_xor_si512(c2[15026],_mm512_xor_si512(c2[11329],_mm512_xor_si512(c2[4731],_mm512_xor_si512(c2[4995],_mm512_xor_si512(c2[7908],_mm512_xor_si512(c2[1574],_mm512_xor_si512(c2[13718],_mm512_xor_si512(c2[7384],_mm512_xor_si512(c2[10812],_mm512_xor_si512(c2[4214],_mm512_xor_si512(c2[4478],c2[12929])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 9 + d2[54]=_mm512_xor_si512(c2[12149],_mm512_xor_si512(c2[10575],_mm512_xor_si512(c2[389],_mm512_xor_si512(c2[1454],_mm512_xor_si512(c2[15204],_mm512_xor_si512(c2[12616],_mm512_xor_si512(c2[16848],c2[4993]))))))); + +//row: 10 + d2[60]=_mm512_xor_si512(c2[9781],_mm512_xor_si512(c2[5045],_mm512_xor_si512(c2[4804],_mm512_xor_si512(c2[12493],_mm512_xor_si512(c2[16733],c2[9149]))))); + +//row: 11 + d2[66]=_mm512_xor_si512(c2[268],_mm512_xor_si512(c2[10032],_mm512_xor_si512(c2[10296],_mm512_xor_si512(c2[6864],_mm512_xor_si512(c2[3],_mm512_xor_si512(c2[7130],_mm512_xor_si512(c2[269],_mm512_xor_si512(c2[8979],_mm512_xor_si512(c2[2112],_mm512_xor_si512(c2[12672],_mm512_xor_si512(c2[8729],_mm512_xor_si512(c2[1598],_mm512_xor_si512(c2[1862],_mm512_xor_si512(c2[2127],_mm512_xor_si512(c2[12161],_mm512_xor_si512(c2[9780],_mm512_xor_si512(c2[2919],_mm512_xor_si512(c2[10045],_mm512_xor_si512(c2[24],_mm512_xor_si512(c2[9794],_mm512_xor_si512(c2[10058],_mm512_xor_si512(c2[6888],_mm512_xor_si512(c2[27],_mm512_xor_si512(c2[5836],_mm512_xor_si512(c2[15864],_mm512_xor_si512(c2[16672],_mm512_xor_si512(c2[9541],_mm512_xor_si512(c2[9805],_mm512_xor_si512(c2[6108],_mm512_xor_si512(c2[16142],_mm512_xor_si512(c2[3737],_mm512_xor_si512(c2[13765],_mm512_xor_si512(c2[12195],_mm512_xor_si512(c2[5328],_mm512_xor_si512(c2[6916],_mm512_xor_si512(c2[49],_mm512_xor_si512(c2[8763],_mm512_xor_si512(c2[1896],_mm512_xor_si512(c2[854],_mm512_xor_si512(c2[10624],_mm512_xor_si512(c2[10888],_mm512_xor_si512(c2[7981],_mm512_xor_si512(c2[1120],_mm512_xor_si512(c2[3759],_mm512_xor_si512(c2[13793],_mm512_xor_si512(c2[10106],_mm512_xor_si512(c2[2981],_mm512_xor_si512(c2[3245],_mm512_xor_si512(c2[12481],_mm512_xor_si512(c2[5620],_mm512_xor_si512(c2[5618],_mm512_xor_si512(c2[15652],_mm512_xor_si512(c2[6688],_mm512_xor_si512(c2[16716],_mm512_xor_si512(c2[6687],_mm512_xor_si512(c2[16721],_mm512_xor_si512(c2[10381],_mm512_xor_si512(c2[3520],_mm512_xor_si512(c2[6700],_mm512_xor_si512(c2[16728],_mm512_xor_si512(c2[12241],_mm512_xor_si512(c2[5380],_mm512_xor_si512(c2[1680],_mm512_xor_si512(c2[11714],_mm512_xor_si512(c2[3016],_mm512_xor_si512(c2[12780],_mm512_xor_si512(c2[13044],_mm512_xor_si512(c2[110],_mm512_xor_si512(c2[10144],_mm512_xor_si512(c2[3540],_mm512_xor_si512(c2[13574],_mm512_xor_si512(c2[12267],_mm512_xor_si512(c2[5136],_mm512_xor_si512(c2[5400],_mm512_xor_si512(c2[4081],_mm512_xor_si512(c2[14115],_mm512_xor_si512(c2[4873],_mm512_xor_si512(c2[14907],_mm512_xor_si512(c2[15708],_mm512_xor_si512(c2[8583],_mm512_xor_si512(c2[8847],_mm512_xor_si512(c2[14127],_mm512_xor_si512(c2[7260],_mm512_xor_si512(c2[16767],_mm512_xor_si512(c2[9900],_mm512_xor_si512(c2[8333],_mm512_xor_si512(c2[1202],_mm512_xor_si512(c2[1466],_mm512_xor_si512(c2[9652],_mm512_xor_si512(c2[2785],_mm512_xor_si512(c2[14138],_mm512_xor_si512(c2[7277],_mm512_xor_si512(c2[2256],_mm512_xor_si512(c2[13624],_mm512_xor_si512(c2[6493],_mm512_xor_si512(c2[6757],_mm512_xor_si512(c2[5966],_mm512_xor_si512(c2[16000],_mm512_xor_si512(c2[9400],_mm512_xor_si512(c2[2533],_mm512_xor_si512(c2[10466],_mm512_xor_si512(c2[3605],_mm512_xor_si512(c2[10200],_mm512_xor_si512(c2[3339],_mm512_xor_si512(c2[9146],_mm512_xor_si512(c2[2285],_mm512_xor_si512(c2[9950],_mm512_xor_si512(c2[2825],_mm512_xor_si512(c2[3089],_mm512_xor_si512(c2[13116],_mm512_xor_si512(c2[6255],_mm512_xor_si512(c2[972],_mm512_xor_si512(c2[11006],_mm512_xor_si512(c2[14976],_mm512_xor_si512(c2[7851],_mm512_xor_si512(c2[8115],_mm512_xor_si512(c2[12868],_mm512_xor_si512(c2[6001],_mm512_xor_si512(c2[16563],_mm512_xor_si512(c2[9696],_mm512_xor_si512(c2[12600],_mm512_xor_si512(c2[205],_mm512_xor_si512(c2[10239],_mm512_xor_si512(c2[4693],_mm512_xor_si512(c2[14727],_mm512_xor_si512(c2[15519],_mm512_xor_si512(c2[8652],_mm512_xor_si512(c2[219],_mm512_xor_si512(c2[9989],_mm512_xor_si512(c2[10253],_mm512_xor_si512(c2[16586],_mm512_xor_si512(c2[9725],_mm512_xor_si512(c2[7875],_mm512_xor_si512(c2[1008],_mm512_xor_si512(c2[758],_mm512_xor_si512(c2[10528],_mm512_xor_si512(c2[10792],_mm512_xor_si512(c2[6832],_mm512_xor_si512(c2[16860],_mm512_xor_si512(c2[7358],_mm512_xor_si512(c2[497],_mm512_xor_si512(c2[6580],_mm512_xor_si512(c2[16344],_mm512_xor_si512(c2[16608],_mm512_xor_si512(c2[7373],_mm512_xor_si512(c2[506],_mm512_xor_si512(c2[14237],_mm512_xor_si512(c2[7370],_mm512_xor_si512(c2[10816],_mm512_xor_si512(c2[3685],_mm512_xor_si512(c2[3949],_mm512_xor_si512(c2[16620],_mm512_xor_si512(c2[9759],_mm512_xor_si512(c2[13720],_mm512_xor_si512(c2[6853],c2[16094]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 12 + d2[72]=_mm512_xor_si512(c2[3433],_mm512_xor_si512(c2[15326],_mm512_xor_si512(c2[12266],_mm512_xor_si512(c2[10695],_mm512_xor_si512(c2[13356],c2[2857]))))); + +//row: 13 + d2[78]=_mm512_xor_si512(c2[9772],_mm512_xor_si512(c2[10036],_mm512_xor_si512(c2[16632],_mm512_xor_si512(c2[3],_mm512_xor_si512(c2[1852],_mm512_xor_si512(c2[15052],_mm512_xor_si512(c2[1332],_mm512_xor_si512(c2[1596],_mm512_xor_si512(c2[11895],_mm512_xor_si512(c2[2653],_mm512_xor_si512(c2[9528],_mm512_xor_si512(c2[9792],_mm512_xor_si512(c2[16656],_mm512_xor_si512(c2[15604],_mm512_xor_si512(c2[9281],_mm512_xor_si512(c2[9545],_mm512_xor_si512(c2[15876],_mm512_xor_si512(c2[13505],_mm512_xor_si512(c2[12974],_mm512_xor_si512(c2[5068],_mm512_xor_si512(c2[16684],_mm512_xor_si512(c2[1636],_mm512_xor_si512(c2[10358],_mm512_xor_si512(c2[10622],_mm512_xor_si512(c2[854],_mm512_xor_si512(c2[13527],_mm512_xor_si512(c2[2715],_mm512_xor_si512(c2[2979],_mm512_xor_si512(c2[5354],_mm512_xor_si512(c2[15386],_mm512_xor_si512(c2[16456],_mm512_xor_si512(c2[16455],_mm512_xor_si512(c2[3254],_mm512_xor_si512(c2[2728],_mm512_xor_si512(c2[16468],_mm512_xor_si512(c2[5114],_mm512_xor_si512(c2[11448],_mm512_xor_si512(c2[12520],_mm512_xor_si512(c2[12784],_mm512_xor_si512(c2[9878],_mm512_xor_si512(c2[13308],_mm512_xor_si512(c2[4876],_mm512_xor_si512(c2[5140],_mm512_xor_si512(c2[13849],_mm512_xor_si512(c2[14641],_mm512_xor_si512(c2[8317],_mm512_xor_si512(c2[8581],_mm512_xor_si512(c2[7000],_mm512_xor_si512(c2[9640],_mm512_xor_si512(c2[936],_mm512_xor_si512(c2[1200],_mm512_xor_si512(c2[2525],_mm512_xor_si512(c2[7011],_mm512_xor_si512(c2[6233],_mm512_xor_si512(c2[6497],_mm512_xor_si512(c2[15734],_mm512_xor_si512(c2[2273],_mm512_xor_si512(c2[3339],_mm512_xor_si512(c2[3073],_mm512_xor_si512(c2[2019],_mm512_xor_si512(c2[2559],_mm512_xor_si512(c2[2823],_mm512_xor_si512(c2[5989],_mm512_xor_si512(c2[10740],_mm512_xor_si512(c2[7585],_mm512_xor_si512(c2[7849],_mm512_xor_si512(c2[5741],_mm512_xor_si512(c2[9436],_mm512_xor_si512(c2[9973],_mm512_xor_si512(c2[14461],_mm512_xor_si512(c2[8392],_mm512_xor_si512(c2[9723],_mm512_xor_si512(c2[9987],_mm512_xor_si512(c2[9459],_mm512_xor_si512(c2[748],_mm512_xor_si512(c2[10262],_mm512_xor_si512(c2[10526],_mm512_xor_si512(c2[16600],_mm512_xor_si512(c2[231],_mm512_xor_si512(c2[16084],_mm512_xor_si512(c2[16348],_mm512_xor_si512(c2[240],_mm512_xor_si512(c2[7104],_mm512_xor_si512(c2[13705],_mm512_xor_si512(c2[3425],_mm512_xor_si512(c2[3689],_mm512_xor_si512(c2[9493],c2[6593]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 14 + d2[84]=_mm512_xor_si512(c2[3698],_mm512_xor_si512(c2[14931],_mm512_xor_si512(c2[2558],_mm512_xor_si512(c2[6793],_mm512_xor_si512(c2[7337],c2[3420]))))); + +//row: 15 + d2[90]=_mm512_xor_si512(c2[267],_mm512_xor_si512(c2[6869],_mm512_xor_si512(c2[7129],_mm512_xor_si512(c2[8714],_mm512_xor_si512(c2[8978],_mm512_xor_si512(c2[12939],_mm512_xor_si512(c2[8728],_mm512_xor_si512(c2[2126],_mm512_xor_si512(c2[9521],_mm512_xor_si512(c2[9785],_mm512_xor_si512(c2[540],_mm512_xor_si512(c2[29],_mm512_xor_si512(c2[6893],_mm512_xor_si512(c2[5835],_mm512_xor_si512(c2[16671],_mm512_xor_si512(c2[6113],_mm512_xor_si512(c2[3472],_mm512_xor_si512(c2[3736],_mm512_xor_si512(c2[12194],_mm512_xor_si512(c2[6915],_mm512_xor_si512(c2[8498],_mm512_xor_si512(c2[8762],_mm512_xor_si512(c2[853],_mm512_xor_si512(c2[7980],_mm512_xor_si512(c2[3758],_mm512_xor_si512(c2[10105],_mm512_xor_si512(c2[12480],_mm512_xor_si512(c2[5353],_mm512_xor_si512(c2[5617],_mm512_xor_si512(c2[6687],_mm512_xor_si512(c2[6686],_mm512_xor_si512(c2[10116],_mm512_xor_si512(c2[10380],_mm512_xor_si512(c2[6699],_mm512_xor_si512(c2[12240],_mm512_xor_si512(c2[1421],_mm512_xor_si512(c2[1685],_mm512_xor_si512(c2[3015],_mm512_xor_si512(c2[109],_mm512_xor_si512(c2[3545],_mm512_xor_si512(c2[12266],_mm512_xor_si512(c2[4080],_mm512_xor_si512(c2[4608],_mm512_xor_si512(c2[4872],_mm512_xor_si512(c2[4875],_mm512_xor_si512(c2[15713],_mm512_xor_si512(c2[14126],_mm512_xor_si512(c2[16502],_mm512_xor_si512(c2[16766],_mm512_xor_si512(c2[8332],_mm512_xor_si512(c2[9651],_mm512_xor_si512(c2[13873],_mm512_xor_si512(c2[14137],_mm512_xor_si512(c2[13623],_mm512_xor_si512(c2[5965],_mm512_xor_si512(c2[9135],_mm512_xor_si512(c2[9399],_mm512_xor_si512(c2[16528],_mm512_xor_si512(c2[10465],_mm512_xor_si512(c2[10205],_mm512_xor_si512(c2[8881],_mm512_xor_si512(c2[9145],_mm512_xor_si512(c2[9949],_mm512_xor_si512(c2[13121],_mm512_xor_si512(c2[977],_mm512_xor_si512(c2[14981],_mm512_xor_si512(c2[12867],_mm512_xor_si512(c2[16298],_mm512_xor_si512(c2[16562],_mm512_xor_si512(c2[204],_mm512_xor_si512(c2[4692],_mm512_xor_si512(c2[15254],_mm512_xor_si512(c2[15518],_mm512_xor_si512(c2[218],_mm512_xor_si512(c2[16585],_mm512_xor_si512(c2[7610],_mm512_xor_si512(c2[7874],_mm512_xor_si512(c2[14736],_mm512_xor_si512(c2[757],_mm512_xor_si512(c2[6831],_mm512_xor_si512(c2[7357],_mm512_xor_si512(c2[6579],_mm512_xor_si512(c2[7372],_mm512_xor_si512(c2[13972],_mm512_xor_si512(c2[14236],_mm512_xor_si512(c2[10815],_mm512_xor_si512(c2[16625],_mm512_xor_si512(c2[13455],c2[13719])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 16 + d2[96]=_mm512_xor_si512(c2[11621],_mm512_xor_si512(c2[1322],_mm512_xor_si512(c2[1588],_mm512_xor_si512(c2[3437],_mm512_xor_si512(c2[3181],_mm512_xor_si512(c2[13480],_mm512_xor_si512(c2[4238],_mm512_xor_si512(c2[3444],_mm512_xor_si512(c2[11377],_mm512_xor_si512(c2[1346],_mm512_xor_si512(c2[288],_mm512_xor_si512(c2[11124],_mm512_xor_si512(c2[566],_mm512_xor_si512(c2[15084],_mm512_xor_si512(c2[4793],_mm512_xor_si512(c2[6653],_mm512_xor_si512(c2[1368],_mm512_xor_si512(c2[3221],_mm512_xor_si512(c2[12207],_mm512_xor_si512(c2[2439],_mm512_xor_si512(c2[15112],_mm512_xor_si512(c2[4564],_mm512_xor_si512(c2[6939],_mm512_xor_si512(c2[76],_mm512_xor_si512(c2[1140],_mm512_xor_si512(c2[1145],_mm512_xor_si512(c2[4839],_mm512_xor_si512(c2[1152],_mm512_xor_si512(c2[6699],_mm512_xor_si512(c2[13033],_mm512_xor_si512(c2[14369],_mm512_xor_si512(c2[11463],_mm512_xor_si512(c2[14893],_mm512_xor_si512(c2[6725],_mm512_xor_si512(c2[15434],_mm512_xor_si512(c2[16226],_mm512_xor_si512(c2[10166],_mm512_xor_si512(c2[8585],_mm512_xor_si512(c2[11225],_mm512_xor_si512(c2[15180],_mm512_xor_si512(c2[2785],_mm512_xor_si512(c2[4104],_mm512_xor_si512(c2[8596],_mm512_xor_si512(c2[8076],_mm512_xor_si512(c2[424],_mm512_xor_si512(c2[3852],_mm512_xor_si512(c2[4924],_mm512_xor_si512(c2[4658],_mm512_xor_si512(c2[3604],_mm512_xor_si512(c2[4408],_mm512_xor_si512(c2[7574],_mm512_xor_si512(c2[12325],_mm512_xor_si512(c2[9434],_mm512_xor_si512(c2[7320],_mm512_xor_si512(c2[11021],_mm512_xor_si512(c2[11558],_mm512_xor_si512(c2[16046],_mm512_xor_si512(c2[9977],_mm512_xor_si512(c2[11572],_mm512_xor_si512(c2[11044],_mm512_xor_si512(c2[2333],_mm512_xor_si512(c2[12111],_mm512_xor_si512(c2[1284],_mm512_xor_si512(c2[1816],_mm512_xor_si512(c2[1032],_mm512_xor_si512(c2[1825],_mm512_xor_si512(c2[8689],_mm512_xor_si512(c2[8956],_mm512_xor_si512(c2[5268],_mm512_xor_si512(c2[11078],c2[8172])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 17 + d2[102]=_mm512_xor_si512(c2[1060],_mm512_xor_si512(c2[12580],_mm512_xor_si512(c2[4681],_mm512_xor_si512(c2[10241],c2[14777])))); + +//row: 18 + d2[108]=_mm512_xor_si512(c2[542],_mm512_xor_si512(c2[9386],_mm512_xor_si512(c2[6496],_mm512_xor_si512(c2[1274],c2[1284])))); + +//row: 19 + d2[114]=_mm512_xor_si512(c2[4490],_mm512_xor_si512(c2[5559],_mm512_xor_si512(c2[6425],_mm512_xor_si512(c2[13299],c2[1443])))); + +//row: 20 + d2[120]=_mm512_xor_si512(c2[12149],_mm512_xor_si512(c2[1850],_mm512_xor_si512(c2[2116],_mm512_xor_si512(c2[3965],_mm512_xor_si512(c2[15578],_mm512_xor_si512(c2[3709],_mm512_xor_si512(c2[14008],_mm512_xor_si512(c2[4766],_mm512_xor_si512(c2[11905],_mm512_xor_si512(c2[1874],_mm512_xor_si512(c2[816],_mm512_xor_si512(c2[11652],_mm512_xor_si512(c2[1094],_mm512_xor_si512(c2[15612],_mm512_xor_si512(c2[3735],_mm512_xor_si512(c2[7181],_mm512_xor_si512(c2[1896],_mm512_xor_si512(c2[3749],_mm512_xor_si512(c2[12735],_mm512_xor_si512(c2[2967],_mm512_xor_si512(c2[15640],_mm512_xor_si512(c2[5092],_mm512_xor_si512(c2[7467],_mm512_xor_si512(c2[604],_mm512_xor_si512(c2[1668],_mm512_xor_si512(c2[1673],_mm512_xor_si512(c2[5367],_mm512_xor_si512(c2[1680],_mm512_xor_si512(c2[7227],_mm512_xor_si512(c2[13561],_mm512_xor_si512(c2[14897],_mm512_xor_si512(c2[11991],_mm512_xor_si512(c2[15421],_mm512_xor_si512(c2[2224],_mm512_xor_si512(c2[7253],_mm512_xor_si512(c2[15962],_mm512_xor_si512(c2[16754],_mm512_xor_si512(c2[10694],_mm512_xor_si512(c2[9113],_mm512_xor_si512(c2[11753],_mm512_xor_si512(c2[5681],_mm512_xor_si512(c2[3313],_mm512_xor_si512(c2[4632],_mm512_xor_si512(c2[9124],_mm512_xor_si512(c2[8604],_mm512_xor_si512(c2[952],_mm512_xor_si512(c2[4380],_mm512_xor_si512(c2[5452],_mm512_xor_si512(c2[5186],_mm512_xor_si512(c2[4132],_mm512_xor_si512(c2[4936],_mm512_xor_si512(c2[8102],_mm512_xor_si512(c2[12853],_mm512_xor_si512(c2[9962],_mm512_xor_si512(c2[7848],_mm512_xor_si512(c2[11549],_mm512_xor_si512(c2[12086],_mm512_xor_si512(c2[16574],_mm512_xor_si512(c2[10505],_mm512_xor_si512(c2[12100],_mm512_xor_si512(c2[11572],_mm512_xor_si512(c2[2861],_mm512_xor_si512(c2[12639],_mm512_xor_si512(c2[1812],_mm512_xor_si512(c2[2344],_mm512_xor_si512(c2[1560],_mm512_xor_si512(c2[2353],_mm512_xor_si512(c2[9217],_mm512_xor_si512(c2[5796],_mm512_xor_si512(c2[11606],c2[8700])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 21 + d2[126]=_mm512_xor_si512(c2[3447],_mm512_xor_si512(c2[10093],_mm512_xor_si512(c2[2309],_mm512_xor_si512(c2[5787],c2[8965])))); + +//row: 22 + d2[132]=_mm512_xor_si512(c2[7920],_mm512_xor_si512(c2[3048],_mm512_xor_si512(c2[10983],c2[6012]))); + +//row: 23 + d2[138]=_mm512_xor_si512(c2[6348],_mm512_xor_si512(c2[6625],_mm512_xor_si512(c2[16224],c2[7344]))); + +//row: 24 + d2[144]=_mm512_xor_si512(c2[7658],_mm512_xor_si512(c2[14260],_mm512_xor_si512(c2[14520],_mm512_xor_si512(c2[16369],_mm512_xor_si512(c2[11092],_mm512_xor_si512(c2[16119],_mm512_xor_si512(c2[9517],_mm512_xor_si512(c2[281],_mm512_xor_si512(c2[7420],_mm512_xor_si512(c2[14284],_mm512_xor_si512(c2[13226],_mm512_xor_si512(c2[7167],_mm512_xor_si512(c2[13504],_mm512_xor_si512(c2[11127],_mm512_xor_si512(c2[7958],_mm512_xor_si512(c2[2690],_mm512_xor_si512(c2[14306],_mm512_xor_si512(c2[16153],_mm512_xor_si512(c2[11403],_mm512_xor_si512(c2[8244],_mm512_xor_si512(c2[15377],_mm512_xor_si512(c2[11149],_mm512_xor_si512(c2[601],_mm512_xor_si512(c2[2976],_mm512_xor_si512(c2[13008],_mm512_xor_si512(c2[14078],_mm512_xor_si512(c2[14077],_mm512_xor_si512(c2[876],_mm512_xor_si512(c2[14090],_mm512_xor_si512(c2[2736],_mm512_xor_si512(c2[9076],_mm512_xor_si512(c2[10406],_mm512_xor_si512(c2[7500],_mm512_xor_si512(c2[10936],_mm512_xor_si512(c2[2762],_mm512_xor_si512(c2[11477],_mm512_xor_si512(c2[12269],_mm512_xor_si512(c2[6209],_mm512_xor_si512(c2[4622],_mm512_xor_si512(c2[7262],_mm512_xor_si512(c2[5153],_mm512_xor_si512(c2[15723],_mm512_xor_si512(c2[147],_mm512_xor_si512(c2[4633],_mm512_xor_si512(c2[4119],_mm512_xor_si512(c2[13356],_mm512_xor_si512(c2[16790],_mm512_xor_si512(c2[961],_mm512_xor_si512(c2[701],_mm512_xor_si512(c2[16536],_mm512_xor_si512(c2[445],_mm512_xor_si512(c2[3617],_mm512_xor_si512(c2[8368],_mm512_xor_si512(c2[5477],_mm512_xor_si512(c2[3363],_mm512_xor_si512(c2[7058],_mm512_xor_si512(c2[7601],_mm512_xor_si512(c2[12089],_mm512_xor_si512(c2[6014],_mm512_xor_si512(c2[7609],_mm512_xor_si512(c2[7081],_mm512_xor_si512(c2[15265],_mm512_xor_si512(c2[8148],_mm512_xor_si512(c2[14222],_mm512_xor_si512(c2[14748],_mm512_xor_si512(c2[13970],_mm512_xor_si512(c2[14763],_mm512_xor_si512(c2[4732],_mm512_xor_si512(c2[1311],_mm512_xor_si512(c2[7121],c2[4215])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 25 + d2[150]=_mm512_xor_si512(c2[2125],_mm512_xor_si512(c2[4560],_mm512_xor_si512(c2[16721],c2[14956]))); + +//row: 26 + d2[156]=_mm512_xor_si512(c2[1849],_mm512_xor_si512(c2[4513],_mm512_xor_si512(c2[3217],c2[2294]))); + +//row: 27 + d2[162]=_mm512_xor_si512(c2[543],_mm512_xor_si512(c2[603],c2[9865])); + +//row: 28 + d2[168]=_mm512_xor_si512(c2[7923],_mm512_xor_si512(c2[5064],_mm512_xor_si512(c2[13959],c2[5008]))); + +//row: 29 + d2[174]=_mm512_xor_si512(c2[528],_mm512_xor_si512(c2[7130],_mm512_xor_si512(c2[7396],_mm512_xor_si512(c2[8981],_mm512_xor_si512(c2[9245],_mm512_xor_si512(c2[8989],_mm512_xor_si512(c2[2393],_mm512_xor_si512(c2[9782],_mm512_xor_si512(c2[10046],_mm512_xor_si512(c2[15855],_mm512_xor_si512(c2[290],_mm512_xor_si512(c2[7154],_mm512_xor_si512(c2[6096],_mm512_xor_si512(c2[37],_mm512_xor_si512(c2[6374],_mm512_xor_si512(c2[3733],_mm512_xor_si512(c2[3997],_mm512_xor_si512(c2[12461],_mm512_xor_si512(c2[7176],_mm512_xor_si512(c2[8765],_mm512_xor_si512(c2[9029],_mm512_xor_si512(c2[1120],_mm512_xor_si512(c2[8247],_mm512_xor_si512(c2[4025],_mm512_xor_si512(c2[10372],_mm512_xor_si512(c2[12747],_mm512_xor_si512(c2[5620],_mm512_xor_si512(c2[5884],_mm512_xor_si512(c2[6948],_mm512_xor_si512(c2[6953],_mm512_xor_si512(c2[10383],_mm512_xor_si512(c2[10647],_mm512_xor_si512(c2[6960],_mm512_xor_si512(c2[12507],_mm512_xor_si512(c2[1682],_mm512_xor_si512(c2[1946],_mm512_xor_si512(c2[3276],_mm512_xor_si512(c2[376],_mm512_xor_si512(c2[3806],_mm512_xor_si512(c2[12533],_mm512_xor_si512(c2[4347],_mm512_xor_si512(c2[4875],_mm512_xor_si512(c2[5139],_mm512_xor_si512(c2[15974],_mm512_xor_si512(c2[14393],_mm512_xor_si512(c2[16769],_mm512_xor_si512(c2[132],_mm512_xor_si512(c2[8593],_mm512_xor_si512(c2[9912],_mm512_xor_si512(c2[14140],_mm512_xor_si512(c2[14404],_mm512_xor_si512(c2[13884],_mm512_xor_si512(c2[6232],_mm512_xor_si512(c2[9396],_mm512_xor_si512(c2[9660],_mm512_xor_si512(c2[10732],_mm512_xor_si512(c2[10466],_mm512_xor_si512(c2[9148],_mm512_xor_si512(c2[9412],_mm512_xor_si512(c2[1488],_mm512_xor_si512(c2[10216],_mm512_xor_si512(c2[13382],_mm512_xor_si512(c2[1238],_mm512_xor_si512(c2[15242],_mm512_xor_si512(c2[13128],_mm512_xor_si512(c2[16565],_mm512_xor_si512(c2[16829],_mm512_xor_si512(c2[471],_mm512_xor_si512(c2[4959],_mm512_xor_si512(c2[15521],_mm512_xor_si512(c2[15785],_mm512_xor_si512(c2[485],_mm512_xor_si512(c2[16852],_mm512_xor_si512(c2[7877],_mm512_xor_si512(c2[8141],_mm512_xor_si512(c2[5234],_mm512_xor_si512(c2[1024],_mm512_xor_si512(c2[7092],_mm512_xor_si512(c2[7624],_mm512_xor_si512(c2[6840],_mm512_xor_si512(c2[7633],_mm512_xor_si512(c2[14233],_mm512_xor_si512(c2[14497],_mm512_xor_si512(c2[11076],_mm512_xor_si512(c2[16886],_mm512_xor_si512(c2[13716],c2[13980])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 30 + d2[180]=_mm512_xor_si512(c2[3696],_mm512_xor_si512(c2[10298],_mm512_xor_si512(c2[10300],_mm512_xor_si512(c2[10564],_mm512_xor_si512(c2[12149],_mm512_xor_si512(c2[12413],_mm512_xor_si512(c2[8186],_mm512_xor_si512(c2[12157],_mm512_xor_si512(c2[5297],_mm512_xor_si512(c2[5561],_mm512_xor_si512(c2[12950],_mm512_xor_si512(c2[13214],_mm512_xor_si512(c2[3458],_mm512_xor_si512(c2[10322],_mm512_xor_si512(c2[9000],_mm512_xor_si512(c2[9264],_mm512_xor_si512(c2[3205],_mm512_xor_si512(c2[9542],_mm512_xor_si512(c2[6901],_mm512_xor_si512(c2[7165],_mm512_xor_si512(c2[15629],_mm512_xor_si512(c2[10080],_mm512_xor_si512(c2[10344],_mm512_xor_si512(c2[11933],_mm512_xor_si512(c2[12197],_mm512_xor_si512(c2[4288],_mm512_xor_si512(c2[11415],_mm512_xor_si512(c2[6929],_mm512_xor_si512(c2[7193],_mm512_xor_si512(c2[13540],_mm512_xor_si512(c2[15651],_mm512_xor_si512(c2[15915],_mm512_xor_si512(c2[8788],_mm512_xor_si512(c2[9052],_mm512_xor_si512(c2[10116],_mm512_xor_si512(c2[9857],_mm512_xor_si512(c2[10121],_mm512_xor_si512(c2[13551],_mm512_xor_si512(c2[13815],_mm512_xor_si512(c2[10128],_mm512_xor_si512(c2[15411],_mm512_xor_si512(c2[15675],_mm512_xor_si512(c2[4850],_mm512_xor_si512(c2[5114],_mm512_xor_si512(c2[6444],_mm512_xor_si512(c2[3544],_mm512_xor_si512(c2[6710],_mm512_xor_si512(c2[6974],_mm512_xor_si512(c2[15701],_mm512_xor_si512(c2[7251],_mm512_xor_si512(c2[7515],_mm512_xor_si512(c2[8043],_mm512_xor_si512(c2[8307],_mm512_xor_si512(c2[9891],_mm512_xor_si512(c2[2247],_mm512_xor_si512(c2[660],_mm512_xor_si512(c2[3036],_mm512_xor_si512(c2[3300],_mm512_xor_si512(c2[11761],_mm512_xor_si512(c2[13080],_mm512_xor_si512(c2[413],_mm512_xor_si512(c2[677],_mm512_xor_si512(c2[157],_mm512_xor_si512(c2[9136],_mm512_xor_si512(c2[9400],_mm512_xor_si512(c2[12564],_mm512_xor_si512(c2[12828],_mm512_xor_si512(c2[1216],_mm512_xor_si512(c2[13900],_mm512_xor_si512(c2[13370],_mm512_xor_si512(c2[13634],_mm512_xor_si512(c2[12316],_mm512_xor_si512(c2[12580],_mm512_xor_si512(c2[13384],_mm512_xor_si512(c2[16550],_mm512_xor_si512(c2[4142],_mm512_xor_si512(c2[4406],_mm512_xor_si512(c2[1515],_mm512_xor_si512(c2[16296],_mm512_xor_si512(c2[2832],_mm512_xor_si512(c2[3096],_mm512_xor_si512(c2[3639],_mm512_xor_si512(c2[7863],_mm512_xor_si512(c2[8127],_mm512_xor_si512(c2[1788],_mm512_xor_si512(c2[2052],_mm512_xor_si512(c2[3653],_mm512_xor_si512(c2[2861],_mm512_xor_si512(c2[3125],_mm512_xor_si512(c2[11045],_mm512_xor_si512(c2[11309],_mm512_xor_si512(c2[4192],_mm512_xor_si512(c2[10260],_mm512_xor_si512(c2[10528],_mm512_xor_si512(c2[10792],_mm512_xor_si512(c2[10008],_mm512_xor_si512(c2[10537],_mm512_xor_si512(c2[10801],_mm512_xor_si512(c2[506],_mm512_xor_si512(c2[770],_mm512_xor_si512(c2[14244],_mm512_xor_si512(c2[3159],_mm512_xor_si512(c2[16884],c2[253]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 31 + d2[186]=_mm512_xor_si512(c2[13994],_mm512_xor_si512(c2[13730],_mm512_xor_si512(c2[3701],_mm512_xor_si512(c2[3437],_mm512_xor_si512(c2[3961],_mm512_xor_si512(c2[3697],_mm512_xor_si512(c2[5810],_mm512_xor_si512(c2[5282],_mm512_xor_si512(c2[5546],_mm512_xor_si512(c2[5560],_mm512_xor_si512(c2[5296],_mm512_xor_si512(c2[15853],_mm512_xor_si512(c2[15589],_mm512_xor_si512(c2[6617],_mm512_xor_si512(c2[6089],_mm512_xor_si512(c2[6353],_mm512_xor_si512(c2[9517],_mm512_xor_si512(c2[13756],_mm512_xor_si512(c2[13492],_mm512_xor_si512(c2[3725],_mm512_xor_si512(c2[3461],_mm512_xor_si512(c2[2667],_mm512_xor_si512(c2[2403],_mm512_xor_si512(c2[13503],_mm512_xor_si512(c2[13239],_mm512_xor_si512(c2[2945],_mm512_xor_si512(c2[2681],_mm512_xor_si512(c2[568],_mm512_xor_si512(c2[40],_mm512_xor_si512(c2[304],_mm512_xor_si512(c2[9026],_mm512_xor_si512(c2[8762],_mm512_xor_si512(c2[3747],_mm512_xor_si512(c2[3483],_mm512_xor_si512(c2[5594],_mm512_xor_si512(c2[5066],_mm512_xor_si512(c2[5330],_mm512_xor_si512(c2[14580],_mm512_xor_si512(c2[14316],_mm512_xor_si512(c2[4812],_mm512_xor_si512(c2[4548],_mm512_xor_si512(c2[590],_mm512_xor_si512(c2[326],_mm512_xor_si512(c2[6937],_mm512_xor_si512(c2[6673],_mm512_xor_si512(c2[9312],_mm512_xor_si512(c2[9048],_mm512_xor_si512(c2[2449],_mm512_xor_si512(c2[1921],_mm512_xor_si512(c2[2185],_mm512_xor_si512(c2[3519],_mm512_xor_si512(c2[3255],_mm512_xor_si512(c2[3518],_mm512_xor_si512(c2[3254],_mm512_xor_si512(c2[7212],_mm512_xor_si512(c2[6684],_mm512_xor_si512(c2[6948],_mm512_xor_si512(c2[6159],_mm512_xor_si512(c2[3531],_mm512_xor_si512(c2[3267],_mm512_xor_si512(c2[9072],_mm512_xor_si512(c2[8808],_mm512_xor_si512(c2[15412],_mm512_xor_si512(c2[14884],_mm512_xor_si512(c2[15148],_mm512_xor_si512(c2[16742],_mm512_xor_si512(c2[16478],_mm512_xor_si512(c2[13836],_mm512_xor_si512(c2[13572],_mm512_xor_si512(c2[377],_mm512_xor_si512(c2[113],_mm512_xor_si512(c2[9098],_mm512_xor_si512(c2[8834],_mm512_xor_si512(c2[912],_mm512_xor_si512(c2[648],_mm512_xor_si512(c2[1704],_mm512_xor_si512(c2[1176],_mm512_xor_si512(c2[1440],_mm512_xor_si512(c2[12545],_mm512_xor_si512(c2[12281],_mm512_xor_si512(c2[10958],_mm512_xor_si512(c2[10694],_mm512_xor_si512(c2[13598],_mm512_xor_si512(c2[13070],_mm512_xor_si512(c2[13334],_mm512_xor_si512(c2[5164],_mm512_xor_si512(c2[4900],_mm512_xor_si512(c2[6483],_mm512_xor_si512(c2[6219],_mm512_xor_si512(c2[10969],_mm512_xor_si512(c2[10441],_mm512_xor_si512(c2[10705],_mm512_xor_si512(c2[10455],_mm512_xor_si512(c2[10191],_mm512_xor_si512(c2[2797],_mm512_xor_si512(c2[2533],_mm512_xor_si512(c2[6231],_mm512_xor_si512(c2[5703],_mm512_xor_si512(c2[5967],_mm512_xor_si512(c2[7297],_mm512_xor_si512(c2[7033],_mm512_xor_si512(c2[7037],_mm512_xor_si512(c2[6773],_mm512_xor_si512(c2[5977],_mm512_xor_si512(c2[5449],_mm512_xor_si512(c2[5713],_mm512_xor_si512(c2[6781],_mm512_xor_si512(c2[6517],_mm512_xor_si512(c2[9953],_mm512_xor_si512(c2[9689],_mm512_xor_si512(c2[14704],_mm512_xor_si512(c2[14440],_mm512_xor_si512(c2[11813],_mm512_xor_si512(c2[11549],_mm512_xor_si512(c2[9699],_mm512_xor_si512(c2[9435],_mm512_xor_si512(c2[13394],_mm512_xor_si512(c2[12866],_mm512_xor_si512(c2[13130],_mm512_xor_si512(c2[13937],_mm512_xor_si512(c2[13673],_mm512_xor_si512(c2[1524],_mm512_xor_si512(c2[1260],_mm512_xor_si512(c2[12350],_mm512_xor_si512(c2[11822],_mm512_xor_si512(c2[12086],_mm512_xor_si512(c2[13945],_mm512_xor_si512(c2[13681],_mm512_xor_si512(c2[13417],_mm512_xor_si512(c2[13153],_mm512_xor_si512(c2[4706],_mm512_xor_si512(c2[4178],_mm512_xor_si512(c2[4442],_mm512_xor_si512(c2[14484],_mm512_xor_si512(c2[14220],_mm512_xor_si512(c2[3663],_mm512_xor_si512(c2[3399],_mm512_xor_si512(c2[4189],_mm512_xor_si512(c2[3925],_mm512_xor_si512(c2[3411],_mm512_xor_si512(c2[3147],_mm512_xor_si512(c2[4204],_mm512_xor_si512(c2[3940],_mm512_xor_si512(c2[11068],_mm512_xor_si512(c2[10540],_mm512_xor_si512(c2[10804],_mm512_xor_si512(c2[7647],_mm512_xor_si512(c2[7383],_mm512_xor_si512(c2[13457],_mm512_xor_si512(c2[13193],_mm512_xor_si512(c2[10551],_mm512_xor_si512(c2[10023],c2[10287])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 32 + d2[192]=_mm512_xor_si512(c2[5019],_mm512_xor_si512(c2[11621],_mm512_xor_si512(c2[11617],_mm512_xor_si512(c2[11881],_mm512_xor_si512(c2[13466],_mm512_xor_si512(c2[13730],_mm512_xor_si512(c2[10033],_mm512_xor_si512(c2[13480],_mm512_xor_si512(c2[6614],_mm512_xor_si512(c2[6878],_mm512_xor_si512(c2[14273],_mm512_xor_si512(c2[14537],_mm512_xor_si512(c2[4781],_mm512_xor_si512(c2[11645],_mm512_xor_si512(c2[10323],_mm512_xor_si512(c2[10587],_mm512_xor_si512(c2[4528],_mm512_xor_si512(c2[10865],_mm512_xor_si512(c2[8224],_mm512_xor_si512(c2[8488],_mm512_xor_si512(c2[51],_mm512_xor_si512(c2[11403],_mm512_xor_si512(c2[11667],_mm512_xor_si512(c2[13250],_mm512_xor_si512(c2[13514],_mm512_xor_si512(c2[5605],_mm512_xor_si512(c2[12732],_mm512_xor_si512(c2[8246],_mm512_xor_si512(c2[8510],_mm512_xor_si512(c2[14857],_mm512_xor_si512(c2[73],_mm512_xor_si512(c2[337],_mm512_xor_si512(c2[10105],_mm512_xor_si512(c2[10369],_mm512_xor_si512(c2[11439],_mm512_xor_si512(c2[11174],_mm512_xor_si512(c2[11438],_mm512_xor_si512(c2[14868],_mm512_xor_si512(c2[15132],_mm512_xor_si512(c2[11451],_mm512_xor_si512(c2[16728],_mm512_xor_si512(c2[97],_mm512_xor_si512(c2[6173],_mm512_xor_si512(c2[6437],_mm512_xor_si512(c2[7767],_mm512_xor_si512(c2[4861],_mm512_xor_si512(c2[8033],_mm512_xor_si512(c2[8297],_mm512_xor_si512(c2[123],_mm512_xor_si512(c2[8568],_mm512_xor_si512(c2[8832],_mm512_xor_si512(c2[9360],_mm512_xor_si512(c2[9624],_mm512_xor_si512(c2[3564],_mm512_xor_si512(c2[1983],_mm512_xor_si512(c2[4359],_mm512_xor_si512(c2[4623],_mm512_xor_si512(c2[13084],_mm512_xor_si512(c2[14403],_mm512_xor_si512(c2[1730],_mm512_xor_si512(c2[1994],_mm512_xor_si512(c2[2523],_mm512_xor_si512(c2[1480],_mm512_xor_si512(c2[10453],_mm512_xor_si512(c2[10717],_mm512_xor_si512(c2[13887],_mm512_xor_si512(c2[14151],_mm512_xor_si512(c2[15217],_mm512_xor_si512(c2[14693],_mm512_xor_si512(c2[14957],_mm512_xor_si512(c2[13633],_mm512_xor_si512(c2[13897],_mm512_xor_si512(c2[12578],_mm512_xor_si512(c2[14701],_mm512_xor_si512(c2[972],_mm512_xor_si512(c2[5465],_mm512_xor_si512(c2[5729],_mm512_xor_si512(c2[2832],_mm512_xor_si512(c2[724],_mm512_xor_si512(c2[4155],_mm512_xor_si512(c2[4419],_mm512_xor_si512(c2[4956],_mm512_xor_si512(c2[9180],_mm512_xor_si512(c2[9444],_mm512_xor_si512(c2[3111],_mm512_xor_si512(c2[3375],_mm512_xor_si512(c2[4970],_mm512_xor_si512(c2[4178],_mm512_xor_si512(c2[4442],_mm512_xor_si512(c2[12362],_mm512_xor_si512(c2[12626],_mm512_xor_si512(c2[5509],_mm512_xor_si512(c2[11583],_mm512_xor_si512(c2[11845],_mm512_xor_si512(c2[12109],_mm512_xor_si512(c2[11331],_mm512_xor_si512(c2[11860],_mm512_xor_si512(c2[12124],_mm512_xor_si512(c2[1829],_mm512_xor_si512(c2[2093],_mm512_xor_si512(c2[15567],_mm512_xor_si512(c2[4476],_mm512_xor_si512(c2[1312],c2[1576]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 33 + d2[198]=_mm512_xor_si512(c2[809],_mm512_xor_si512(c2[2136],_mm512_xor_si512(c2[10961],c2[11077]))); + +//row: 34 + d2[204]=_mm512_xor_si512(c2[10035],_mm512_xor_si512(c2[5366],_mm512_xor_si512(c2[2823],c2[14992]))); + +//row: 35 + d2[210]=_mm512_xor_si512(c2[16106],_mm512_xor_si512(c2[5813],_mm512_xor_si512(c2[6073],_mm512_xor_si512(c2[7922],_mm512_xor_si512(c2[7672],_mm512_xor_si512(c2[1070],_mm512_xor_si512(c2[8729],_mm512_xor_si512(c2[17],_mm512_xor_si512(c2[15868],_mm512_xor_si512(c2[5837],_mm512_xor_si512(c2[4779],_mm512_xor_si512(c2[15615],_mm512_xor_si512(c2[5057],_mm512_xor_si512(c2[2680],_mm512_xor_si512(c2[11138],_mm512_xor_si512(c2[5859],_mm512_xor_si512(c2[7706],_mm512_xor_si512(c2[16692],_mm512_xor_si512(c2[6924],_mm512_xor_si512(c2[2702],_mm512_xor_si512(c2[9049],_mm512_xor_si512(c2[11424],_mm512_xor_si512(c2[4561],_mm512_xor_si512(c2[4037],_mm512_xor_si512(c2[5631],_mm512_xor_si512(c2[5630],_mm512_xor_si512(c2[9324],_mm512_xor_si512(c2[5643],_mm512_xor_si512(c2[11184],_mm512_xor_si512(c2[629],_mm512_xor_si512(c2[1959],_mm512_xor_si512(c2[15948],_mm512_xor_si512(c2[2489],_mm512_xor_si512(c2[11210],_mm512_xor_si512(c2[3024],_mm512_xor_si512(c2[3816],_mm512_xor_si512(c2[14657],_mm512_xor_si512(c2[13070],_mm512_xor_si512(c2[15710],_mm512_xor_si512(c2[7276],_mm512_xor_si512(c2[8595],_mm512_xor_si512(c2[13081],_mm512_xor_si512(c2[672],_mm512_xor_si512(c2[12567],_mm512_xor_si512(c2[4909],_mm512_xor_si512(c2[8343],_mm512_xor_si512(c2[9409],_mm512_xor_si512(c2[9149],_mm512_xor_si512(c2[8089],_mm512_xor_si512(c2[8893],_mm512_xor_si512(c2[12065],_mm512_xor_si512(c2[16816],_mm512_xor_si512(c2[13925],_mm512_xor_si512(c2[11811],_mm512_xor_si512(c2[15506],_mm512_xor_si512(c2[16049],_mm512_xor_si512(c2[3636],_mm512_xor_si512(c2[14462],_mm512_xor_si512(c2[16057],_mm512_xor_si512(c2[15529],_mm512_xor_si512(c2[6818],_mm512_xor_si512(c2[16596],_mm512_xor_si512(c2[5775],_mm512_xor_si512(c2[6301],_mm512_xor_si512(c2[5523],_mm512_xor_si512(c2[6316],_mm512_xor_si512(c2[13180],_mm512_xor_si512(c2[9759],_mm512_xor_si512(c2[15569],c2[12663]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 36 + d2[216]=_mm512_xor_si512(c2[4755],_mm512_xor_si512(c2[15220],_mm512_xor_si512(c2[11008],c2[5760]))); + +//row: 37 + d2[222]=_mm512_xor_si512(c2[10032],_mm512_xor_si512(c2[10296],_mm512_xor_si512(c2[3],_mm512_xor_si512(c2[269],_mm512_xor_si512(c2[2112],_mm512_xor_si512(c2[1598],_mm512_xor_si512(c2[1862],_mm512_xor_si512(c2[12161],_mm512_xor_si512(c2[2919],_mm512_xor_si512(c2[3448],_mm512_xor_si512(c2[9794],_mm512_xor_si512(c2[10058],_mm512_xor_si512(c2[27],_mm512_xor_si512(c2[15864],_mm512_xor_si512(c2[9541],_mm512_xor_si512(c2[9805],_mm512_xor_si512(c2[16142],_mm512_xor_si512(c2[13765],_mm512_xor_si512(c2[5328],_mm512_xor_si512(c2[49],_mm512_xor_si512(c2[1896],_mm512_xor_si512(c2[10624],_mm512_xor_si512(c2[10888],_mm512_xor_si512(c2[1120],_mm512_xor_si512(c2[13793],_mm512_xor_si512(c2[2981],_mm512_xor_si512(c2[3245],_mm512_xor_si512(c2[5620],_mm512_xor_si512(c2[15652],_mm512_xor_si512(c2[16716],_mm512_xor_si512(c2[16721],_mm512_xor_si512(c2[3520],_mm512_xor_si512(c2[16728],_mm512_xor_si512(c2[5380],_mm512_xor_si512(c2[11714],_mm512_xor_si512(c2[12780],_mm512_xor_si512(c2[13044],_mm512_xor_si512(c2[10144],_mm512_xor_si512(c2[13574],_mm512_xor_si512(c2[5136],_mm512_xor_si512(c2[5400],_mm512_xor_si512(c2[14115],_mm512_xor_si512(c2[14907],_mm512_xor_si512(c2[8583],_mm512_xor_si512(c2[8847],_mm512_xor_si512(c2[7260],_mm512_xor_si512(c2[9900],_mm512_xor_si512(c2[1202],_mm512_xor_si512(c2[1466],_mm512_xor_si512(c2[2785],_mm512_xor_si512(c2[7277],_mm512_xor_si512(c2[6493],_mm512_xor_si512(c2[6757],_mm512_xor_si512(c2[16000],_mm512_xor_si512(c2[2533],_mm512_xor_si512(c2[4909],_mm512_xor_si512(c2[3605],_mm512_xor_si512(c2[3339],_mm512_xor_si512(c2[2285],_mm512_xor_si512(c2[2825],_mm512_xor_si512(c2[3089],_mm512_xor_si512(c2[6255],_mm512_xor_si512(c2[11006],_mm512_xor_si512(c2[7851],_mm512_xor_si512(c2[8115],_mm512_xor_si512(c2[6001],_mm512_xor_si512(c2[9696],_mm512_xor_si512(c2[10239],_mm512_xor_si512(c2[14727],_mm512_xor_si512(c2[8652],_mm512_xor_si512(c2[9989],_mm512_xor_si512(c2[10253],_mm512_xor_si512(c2[9725],_mm512_xor_si512(c2[1008],_mm512_xor_si512(c2[10528],_mm512_xor_si512(c2[10792],_mm512_xor_si512(c2[16860],_mm512_xor_si512(c2[497],_mm512_xor_si512(c2[16344],_mm512_xor_si512(c2[16608],_mm512_xor_si512(c2[506],_mm512_xor_si512(c2[7370],_mm512_xor_si512(c2[3685],_mm512_xor_si512(c2[3949],_mm512_xor_si512(c2[9759],c2[6853]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 38 + d2[228]=_mm512_xor_si512(c2[15050],_mm512_xor_si512(c2[13046],_mm512_xor_si512(c2[8836],c2[5955]))); + +//row: 39 + d2[234]=_mm512_xor_si512(c2[544],_mm512_xor_si512(c2[7693],_mm512_xor_si512(c2[6953],c2[11056]))); + +//row: 40 + d2[240]=_mm512_xor_si512(c2[12410],_mm512_xor_si512(c2[9864],c2[14992])); + +//row: 41 + d2[246]=_mm512_xor_si512(c2[13740],_mm512_xor_si512(c2[15352],_mm512_xor_si512(c2[3014],c2[8668]))); + +//row: 42 + d2[252]=_mm512_xor_si512(c2[3698],_mm512_xor_si512(c2[10300],_mm512_xor_si512(c2[10296],_mm512_xor_si512(c2[10560],_mm512_xor_si512(c2[12145],_mm512_xor_si512(c2[12409],_mm512_xor_si512(c2[12937],_mm512_xor_si512(c2[12159],_mm512_xor_si512(c2[5293],_mm512_xor_si512(c2[5557],_mm512_xor_si512(c2[12952],_mm512_xor_si512(c2[13216],_mm512_xor_si512(c2[3460],_mm512_xor_si512(c2[10324],_mm512_xor_si512(c2[9002],_mm512_xor_si512(c2[9266],_mm512_xor_si512(c2[3207],_mm512_xor_si512(c2[9544],_mm512_xor_si512(c2[6903],_mm512_xor_si512(c2[7167],_mm512_xor_si512(c2[15625],_mm512_xor_si512(c2[10082],_mm512_xor_si512(c2[10346],_mm512_xor_si512(c2[11929],_mm512_xor_si512(c2[12193],_mm512_xor_si512(c2[3744],_mm512_xor_si512(c2[4284],_mm512_xor_si512(c2[11417],_mm512_xor_si512(c2[6925],_mm512_xor_si512(c2[7189],_mm512_xor_si512(c2[13536],_mm512_xor_si512(c2[15653],_mm512_xor_si512(c2[15917],_mm512_xor_si512(c2[8784],_mm512_xor_si512(c2[9048],_mm512_xor_si512(c2[10118],_mm512_xor_si512(c2[9853],_mm512_xor_si512(c2[10117],_mm512_xor_si512(c2[13553],_mm512_xor_si512(c2[13817],_mm512_xor_si512(c2[10130],_mm512_xor_si512(c2[15413],_mm512_xor_si512(c2[15677],_mm512_xor_si512(c2[4852],_mm512_xor_si512(c2[5116],_mm512_xor_si512(c2[6446],_mm512_xor_si512(c2[3540],_mm512_xor_si512(c2[6712],_mm512_xor_si512(c2[6976],_mm512_xor_si512(c2[15697],_mm512_xor_si512(c2[7253],_mm512_xor_si512(c2[7517],_mm512_xor_si512(c2[8045],_mm512_xor_si512(c2[8309],_mm512_xor_si512(c2[2249],_mm512_xor_si512(c2[662],_mm512_xor_si512(c2[3038],_mm512_xor_si512(c2[3302],_mm512_xor_si512(c2[11763],_mm512_xor_si512(c2[13082],_mm512_xor_si512(c2[409],_mm512_xor_si512(c2[673],_mm512_xor_si512(c2[159],_mm512_xor_si512(c2[9132],_mm512_xor_si512(c2[9396],_mm512_xor_si512(c2[12566],_mm512_xor_si512(c2[12830],_mm512_xor_si512(c2[13896],_mm512_xor_si512(c2[13372],_mm512_xor_si512(c2[13636],_mm512_xor_si512(c2[12312],_mm512_xor_si512(c2[12576],_mm512_xor_si512(c2[13380],_mm512_xor_si512(c2[16552],_mm512_xor_si512(c2[4144],_mm512_xor_si512(c2[4408],_mm512_xor_si512(c2[1517],_mm512_xor_si512(c2[16298],_mm512_xor_si512(c2[2834],_mm512_xor_si512(c2[3098],_mm512_xor_si512(c2[3641],_mm512_xor_si512(c2[7865],_mm512_xor_si512(c2[8129],_mm512_xor_si512(c2[1790],_mm512_xor_si512(c2[2054],_mm512_xor_si512(c2[3649],_mm512_xor_si512(c2[2857],_mm512_xor_si512(c2[3121],_mm512_xor_si512(c2[11041],_mm512_xor_si512(c2[11305],_mm512_xor_si512(c2[4188],_mm512_xor_si512(c2[10262],_mm512_xor_si512(c2[10524],_mm512_xor_si512(c2[10788],_mm512_xor_si512(c2[10010],_mm512_xor_si512(c2[10539],_mm512_xor_si512(c2[10803],_mm512_xor_si512(c2[508],_mm512_xor_si512(c2[772],_mm512_xor_si512(c2[14246],_mm512_xor_si512(c2[3161],_mm512_xor_si512(c2[16886],c2[255])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 43 + d2[258]=_mm512_xor_si512(c2[7393],_mm512_xor_si512(c2[13995],_mm512_xor_si512(c2[14261],_mm512_xor_si512(c2[15840],_mm512_xor_si512(c2[16104],_mm512_xor_si512(c2[15854],_mm512_xor_si512(c2[9252],_mm512_xor_si512(c2[16647],_mm512_xor_si512(c2[16],_mm512_xor_si512(c2[12949],_mm512_xor_si512(c2[7155],_mm512_xor_si512(c2[14019],_mm512_xor_si512(c2[12961],_mm512_xor_si512(c2[6902],_mm512_xor_si512(c2[13239],_mm512_xor_si512(c2[10598],_mm512_xor_si512(c2[10862],_mm512_xor_si512(c2[2425],_mm512_xor_si512(c2[14041],_mm512_xor_si512(c2[15624],_mm512_xor_si512(c2[15888],_mm512_xor_si512(c2[7985],_mm512_xor_si512(c2[15112],_mm512_xor_si512(c2[10884],_mm512_xor_si512(c2[336],_mm512_xor_si512(c2[2717],_mm512_xor_si512(c2[12485],_mm512_xor_si512(c2[12749],_mm512_xor_si512(c2[13813],_mm512_xor_si512(c2[13812],_mm512_xor_si512(c2[353],_mm512_xor_si512(c2[617],_mm512_xor_si512(c2[13825],_mm512_xor_si512(c2[2477],_mm512_xor_si512(c2[8547],_mm512_xor_si512(c2[8811],_mm512_xor_si512(c2[10141],_mm512_xor_si512(c2[7241],_mm512_xor_si512(c2[10671],_mm512_xor_si512(c2[2497],_mm512_xor_si512(c2[11212],_mm512_xor_si512(c2[11740],_mm512_xor_si512(c2[12004],_mm512_xor_si512(c2[5944],_mm512_xor_si512(c2[4357],_mm512_xor_si512(c2[6733],_mm512_xor_si512(c2[6997],_mm512_xor_si512(c2[15458],_mm512_xor_si512(c2[16777],_mm512_xor_si512(c2[4104],_mm512_xor_si512(c2[4368],_mm512_xor_si512(c2[3854],_mm512_xor_si512(c2[13097],_mm512_xor_si512(c2[16261],_mm512_xor_si512(c2[16525],_mm512_xor_si512(c2[696],_mm512_xor_si512(c2[436],_mm512_xor_si512(c2[16013],_mm512_xor_si512(c2[16277],_mm512_xor_si512(c2[180],_mm512_xor_si512(c2[3352],_mm512_xor_si512(c2[8103],_mm512_xor_si512(c2[5212],_mm512_xor_si512(c2[3098],_mm512_xor_si512(c2[6529],_mm512_xor_si512(c2[6793],_mm512_xor_si512(c2[1250],_mm512_xor_si512(c2[7336],_mm512_xor_si512(c2[11824],_mm512_xor_si512(c2[5485],_mm512_xor_si512(c2[5749],_mm512_xor_si512(c2[7344],_mm512_xor_si512(c2[6816],_mm512_xor_si512(c2[14736],_mm512_xor_si512(c2[15000],_mm512_xor_si512(c2[13417],_mm512_xor_si512(c2[7889],_mm512_xor_si512(c2[13957],_mm512_xor_si512(c2[14489],_mm512_xor_si512(c2[13705],_mm512_xor_si512(c2[14498],_mm512_xor_si512(c2[4203],_mm512_xor_si512(c2[4467],_mm512_xor_si512(c2[1046],_mm512_xor_si512(c2[6856],_mm512_xor_si512(c2[3686],c2[3950])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 44 + d2[264]=_mm512_xor_si512(c2[1323],_mm512_xor_si512(c2[7925],_mm512_xor_si512(c2[8185],_mm512_xor_si512(c2[10034],_mm512_xor_si512(c2[4225],_mm512_xor_si512(c2[9784],_mm512_xor_si512(c2[3182],_mm512_xor_si512(c2[10841],_mm512_xor_si512(c2[1085],_mm512_xor_si512(c2[7949],_mm512_xor_si512(c2[6891],_mm512_xor_si512(c2[832],_mm512_xor_si512(c2[7169],_mm512_xor_si512(c2[4792],_mm512_xor_si512(c2[13250],_mm512_xor_si512(c2[7971],_mm512_xor_si512(c2[9818],_mm512_xor_si512(c2[1909],_mm512_xor_si512(c2[9036],_mm512_xor_si512(c2[4814],_mm512_xor_si512(c2[11161],_mm512_xor_si512(c2[13536],_mm512_xor_si512(c2[6673],_mm512_xor_si512(c2[7743],_mm512_xor_si512(c2[7742],_mm512_xor_si512(c2[11436],_mm512_xor_si512(c2[3781],_mm512_xor_si512(c2[7755],_mm512_xor_si512(c2[13296],_mm512_xor_si512(c2[2741],_mm512_xor_si512(c2[4071],_mm512_xor_si512(c2[1165],_mm512_xor_si512(c2[4601],_mm512_xor_si512(c2[9350],_mm512_xor_si512(c2[13322],_mm512_xor_si512(c2[5136],_mm512_xor_si512(c2[5928],_mm512_xor_si512(c2[16769],_mm512_xor_si512(c2[15182],_mm512_xor_si512(c2[927],_mm512_xor_si512(c2[9388],_mm512_xor_si512(c2[10707],_mm512_xor_si512(c2[15193],_mm512_xor_si512(c2[14679],_mm512_xor_si512(c2[7021],_mm512_xor_si512(c2[10455],_mm512_xor_si512(c2[11521],_mm512_xor_si512(c2[11261],_mm512_xor_si512(c2[10201],_mm512_xor_si512(c2[11005],_mm512_xor_si512(c2[14177],_mm512_xor_si512(c2[2033],_mm512_xor_si512(c2[16037],_mm512_xor_si512(c2[13923],_mm512_xor_si512(c2[723],_mm512_xor_si512(c2[1260],_mm512_xor_si512(c2[5748],_mm512_xor_si512(c2[16574],_mm512_xor_si512(c2[1274],_mm512_xor_si512(c2[746],_mm512_xor_si512(c2[8930],_mm512_xor_si512(c2[1813],_mm512_xor_si512(c2[7887],_mm512_xor_si512(c2[8413],_mm512_xor_si512(c2[7635],_mm512_xor_si512(c2[8428],_mm512_xor_si512(c2[15292],_mm512_xor_si512(c2[11871],_mm512_xor_si512(c2[780],c2[14775]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))); + +//row: 45 + d2[270]=_mm512_xor_si512(c2[1862],_mm512_xor_si512(c2[5618],c2[4080])); + } +} diff --git a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encode_parity_check.c b/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encode_parity_check.c index 55e02eea6a..94eb0a4d92 100644 --- a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encode_parity_check.c +++ b/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encode_parity_check.c @@ -37,7 +37,11 @@ #include "common/utils/LOG/log.h" +#ifdef __AVX512F__ +#include "ldpc384_simd512_byte.c" +#else #include "ldpc384_byte.c" +#endif #include "ldpc384_byte_128.c" #include "ldpc352_byte.c" #include "ldpc352_byte_128.c" @@ -87,7 +91,7 @@ static void encode_parity_check_part_optim(uint8_t *cc, uint8_t *d, short BG,short Zc,short Kb, int simd_size, int ncols) { - unsigned char c[2 * 22 * Zc * simd_size] __attribute__((aligned(32))); //double size matrix of c + unsigned char c[2 * 22 * Zc * simd_size] __attribute__((aligned(64))); //double size matrix of c for (int i1 = 0; i1 < ncols; i1++) { memcpy(&c[2 * i1 * Zc], &cc[i1 * Zc], Zc * sizeof(unsigned char)); memcpy(&c[(2 * i1 + 1) * Zc], &cc[i1 * Zc], Zc * sizeof(unsigned char)); @@ -95,7 +99,6 @@ static void encode_parity_check_part_optim(uint8_t *cc, uint8_t *d, short BG,sho for (int i1 = 1; i1 < simd_size; i1++) { memcpy(&c[(2 * ncols * Zc * i1)], &c[i1], (2 * ncols * Zc * sizeof(unsigned char)) - i1); } - if (BG == 1) { switch (Zc) { case 176: diff --git a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder.c b/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder.c index ebcb6bf4f5..f1f6e0af5a 100644 --- a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder.c +++ b/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder.c @@ -114,7 +114,13 @@ int LDPCencoder(unsigned char **inputArray, unsigned char **outputArray, encoder fprintf(fd,"#include \"PHY/sse_intrin.h\"\n"); fprintf(fd2,"#include \"PHY/sse_intrin.h\"\n"); - if (gen_code == 1 && (Zc&31)==0) { + if (gen_code == 1 && (Zc&63)==0) { + shift=6; + mask=63; + strcpy(data_type,"__m512i"); + strcpy(xor_command,"_mm512_xor_si512"); + } + else if (gen_code == 1 && (Zc&31)==0) { shift=5; // AVX2 - 256-bit SIMD mask=31; strcpy(data_type,"simde__m256i"); diff --git a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8segmulti.c b/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8segmulti.c index b91d1f5825..5e327df957 100644 --- a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8segmulti.c +++ b/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8segmulti.c @@ -103,10 +103,12 @@ int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp) AssertFatal(Zc > 0, "no valid Zc found for block length %d\n", block_length); if ((Zc&31) > 0) simd_size = 16; - else - simd_size = 32; - unsigned char cc[22*Zc] __attribute__((aligned(32))); //padded input, unpacked, max size - unsigned char dd[46*Zc] __attribute__((aligned(32))); //coded parity part output, unpacked, max size +#ifdef __AVX512F__ + else if ((BG==1) && Zc==384) simd_size =64; +#endif + else simd_size=32; + unsigned char cc[22*Zc] __attribute__((aligned(64))); //padded input, unpacked, max size + unsigned char dd[46*Zc] __attribute__((aligned(64))); //coded parity part output, unpacked, max size // calculate number of punctured bits no_punctured_columns=(int)((nrows-2)*Zc+block_length-block_length*rate)/Zc; @@ -114,8 +116,8 @@ int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp) //printf("%d\n",no_punctured_columns); //printf("%d\n",removed_bit); // unpack input - memset(cc,0,sizeof(unsigned char) * ncols * Zc); - memset(dd,0,sizeof(unsigned char) * nrows * Zc); + memset(cc,0,sizeof(cc)); + memset(dd,0,sizeof(dd)); if(impp->tinput != NULL) start_meas(impp->tinput); #if 0 @@ -172,6 +174,42 @@ int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp) memcpy(&output[0], &c[2*Zc], (block_length-2*Zc)*sizeof(unsigned char)); memcpy(&output[block_length-2*Zc], &d[0], ((nrows-no_punctured_columns) * Zc-removed_bit)*sizeof(unsigned char)); */ +#ifdef __AVX512F__ + + uint32_t l1 = (block_length-(2*Zc)); + uint32_t l2 = ((nrows-no_punctured_columns) * Zc-removed_bit); + if ((((2*Zc)&63) == 0) && (((block_length-(2*Zc))&63) == 0)) { + __m512i *c512p = (__m512i *)&cc[2*Zc]; + __m512i *d512p = (__m512i *)&dd[0]; + __m512i mask0 = _mm512_set1_epi8(0x1); + + for (int i=0;i>6;i++) + for (int j=macro_segment; j < macro_segment_end; j++) + ((__m512i *)output[j])[i] = _mm512_and_si512(_mm512_srai_epi16(c512p[i],j-macro_segment),mask0); + for (int i1=0, i=l1>>6;i1>6;i1++,i++) + for (int j=macro_segment; j < macro_segment_end; j++) + ((__m512i *)output[j])[i] = _mm512_and_si512(_mm512_srai_epi16(d512p[i1],j-macro_segment),mask0); + } + else if ((((2*Zc)&31) == 0) && (((block_length-(2*Zc))&31) == 0)) { + uint32_t l1 = (block_length-(2*Zc))>>5; + uint32_t l2 = ((nrows-no_punctured_columns) * Zc-removed_bit)>>5; + simde__m256i *c256p = (simde__m256i *)&cc[2*Zc]; + simde__m256i *d256p = (simde__m256i *)&dd[0]; + // if (((block_length-(2*Zc))&31)>0) l1++; + + for (int i=0;i0) l2++; + + for (int i1=0, i=l1;i1>3)) { - output_buffers[r][k] = input_buffer[s]; - //printf("encoding segment %d : byte %d (%d) => %d\n",r,k,(Kprime-L)>>3,input_buffer[s]); - k++; - s++; - } + memcpy(output_buffers[r],input_buffer+s,(Kprime-L)>>3); + s+=(Kprime-L)>>3; if (*C > 1) { // add CRC crc = crc24b(output_buffers[r],Kprime-L)>>8; From a4c950774b3e11fd1b3d129c209a0d9e053e1a11 Mon Sep 17 00:00:00 2001 From: Raymond Knopp Date: Sun, 29 Dec 2024 09:49:00 +0000 Subject: [PATCH 04/17] feat(ldpc_encoder_optim8segmulti): Improvements for Arm Neoverse Use 128 bit SIMD in packing and unpacking segment bits in bytes Adapt compiler options for Neoverse. --- CMakeLists.txt | 16 +++++- .../ldpc_encoder_optim8segmulti.c | 56 ++++++++++++++++++- 2 files changed, 68 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index feb7e41a4f..491dcd7e0b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -108,6 +108,7 @@ add_list_string_option(CMAKE_BUILD_TYPE "RelWithDebInfo" "Choose the type of bui # -so to avoid AVX512 instructions generation by gcc if(EXISTS "/proc/cpuinfo" AND NOT CROSS_COMPILE) file(STRINGS "/proc/cpuinfo" CPUFLAGS REGEX flags LIMIT_COUNT 1) + file(STRINGS "/proc/cpuinfo" CPUPART REGEX "CPU part" LIMIT_COUNT 1) else() message(WARNING "did not find /proc/cpuinfo -- not setting any x86-specific compilation variables") endif() @@ -146,9 +147,20 @@ elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -DSIMDE_X86_SSSE3_NATIVE") endif() elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "armv7l") - set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -gdwarf-2 -mfloat-abi=hard -mfpu=neon -lgcc -lrt") + set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -gdwarf-2 -mfloat-abi=hard -mfpu=neon -lgcc -lrt") elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64") - set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -gdwarf-2 -lgcc -lrt") +# Neoverse CPU part codes available at https://github.com/util-linux/util-linux/blob/master/sys-utils/lscpu-arm.c + if (CPUPART MATCHES "0xd40") # Neoverse-V1 + set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -gdwarf-2 -mcpu=neoverse-v1 -lgcc -lrt") + elseif (CPUPART MATCHES "0xd4f") # Neoverse-V2 + set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -gdwarf-2 -mcpu=neoverse-v2 -ftree-vectorize -lgcc -lrt") + elseif (CPUPART MATCHES "0xd0c") # Neoverse-N1 + set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -gdwarf-2 -mcpu=neoverse-n1 -lgcc -lrt") + elseif (CPUPART MATCHES "0xd49") # Neoverse-N2 + set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -gdwarf-2 -mcpu=neoverse-n2 -ftree-vectorize -lgcc -lrt") + else () + set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -gdwarf-2 -mcpu=native -lgcc -lrt") + endif() else() message(FATAL_ERROR "compile for CPU architecture ${CPUARCH}, CMAKE_SYSTEM_PROCESSOR ${CMAKE_SYSTEM_PROCESSOR}") endif() diff --git a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8segmulti.c b/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8segmulti.c index 5e327df957..bd25ecf646 100644 --- a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8segmulti.c +++ b/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8segmulti.c @@ -65,7 +65,22 @@ int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp) macro_segment_end = (impp->n_segments > 8*(impp->macro_num+1)) ? 8*(impp->macro_num+1) : impp->n_segments; ///printf("macro_segment: %d\n", macro_segment); ///printf("macro_segment_end: %d\n", macro_segment_end ); +#ifdef __aarch64__ + simde__m128i shufmask = simde_mm_set_epi64x(0x0101010101010101, 0x0000000000000000); + simde__m128i andmask = simde_mm_set1_epi64x(0x0102040810204080); // every 8 bits -> 8 bytes, pattern repeats. + simde__m128i zero128 = simde_mm_setzero_si128(); + simde__m128i masks[8]; + register simde__m128i c128; + masks[0] = simde_mm_set1_epi8(0x1); + masks[1] = simde_mm_set1_epi8(0x2); + masks[2] = simde_mm_set1_epi8(0x4); + masks[3] = simde_mm_set1_epi8(0x8); + masks[4] = simde_mm_set1_epi8(0x10); + masks[5] = simde_mm_set1_epi8(0x20); + masks[6] = simde_mm_set1_epi8(0x40); + masks[7] = simde_mm_set1_epi8(0x80); +#else simde__m256i shufmask = simde_mm256_set_epi64x(0x0303030303030303, 0x0202020202020202,0x0101010101010101, 0x0000000000000000); simde__m256i andmask = simde_mm256_set1_epi64x(0x0102040810204080); // every 8 bits -> 8 bytes, pattern repeats. simde__m256i zero256 = simde_mm256_setzero_si256(); @@ -79,7 +94,7 @@ int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp) masks[5] = simde_mm256_set1_epi8(0x20); masks[6] = simde_mm256_set1_epi8(0x40); masks[7] = simde_mm256_set1_epi8(0x80); - +#endif //determine number of bits in codeword @@ -120,6 +135,7 @@ int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp) memset(dd,0,sizeof(dd)); if(impp->tinput != NULL) start_meas(impp->tinput); + //interleave up to 8 transport-block segements at a time #if 0 for (i=0; i>4; i++) { + c128 = simde_mm_and_si128(simde_mm_cmpeq_epi8(simde_mm_andnot_si128(simde_mm_shuffle_epi8(simde_mm_set1_epi16(((uint16_t*)input[macro_segment])[i]), shufmask),andmask),zero128),masks[0]); + //for (j=1; j>4)<<4;i>(i&7)))>>(7-(i&7)); + //printf("c(%d,%d)=%d\n",j,i,temp); + cc[i] |= (temp << (j-macro_segment)); + } + } #else for (int i=0; i>5; i++) { c256 = simde_mm256_and_si256(simde_mm256_cmpeq_epi8(simde_mm256_andnot_si256(simde_mm256_shuffle_epi8(simde_mm256_set1_epi32(((uint32_t*)input[macro_segment])[i]), shufmask),andmask),zero256),masks[0]); @@ -208,7 +243,24 @@ int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp) //for (j=0;j>4; + uint32_t l2 = ((nrows-no_punctured_columns) * Zc-removed_bit)>>4; + simde__m128i *c128p = (simde__m128i *)&cc[2*Zc]; + simde__m128i *d128p = (simde__m128i *)&dd[0]; + // if (((block_length-(2*Zc))&31)>0) l1++; + + for (int i=0;i Date: Fri, 9 May 2025 07:15:23 +0000 Subject: [PATCH 05/17] feat(cnProc_gen_BG1_128): Put back small change in 128-bit SIMD code generator for LDPC decoder (cnProc). Avoid the use of simde_mm_sign_epi8 which is inefficient on aarch64. Decoding times are significantly improved on aarch64. Co-authored-by: Romain Beurdouche --- .../generator_cnProc/cnProc_gen_BG1_128.c | 92 +++++++++++++++++-- 1 file changed, 83 insertions(+), 9 deletions(-) diff --git a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/generator_cnProc/cnProc_gen_BG1_128.c b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/generator_cnProc/cnProc_gen_BG1_128.c index e4d971472f..9e3d212697 100644 --- a/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/generator_cnProc/cnProc_gen_BG1_128.c +++ b/openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/generator_cnProc/cnProc_gen_BG1_128.c @@ -24,7 +24,9 @@ #include #include "../../nrLDPCdecoder_defs.h" -#define AVOID_MM256_SIGN 1 +#ifdef __aarch64__ +#define AVOID_SIGN 1 +#endif #define DROP_MAXLLR 1 void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) { @@ -126,7 +128,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) // ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i]; fprintf(fd," ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[0]>>4)+lut_idxCnProcG3[j][0]*2); // sgn = simde_mm_sign_epi8(ones, ymm0); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(ones, ymm0);\n"); +#else fprintf(fd," sgn = simde_mm_sign_epi8(ones, ymm0);\n"); +#endif // min = simde_mm_abs_epi8(ymm0); fprintf(fd," min = simde_mm_abs_epi8(ymm0);\n"); @@ -138,7 +144,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) fprintf(fd," min = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n"); // sgn = simde_mm_sign_epi8(sgn, ymm0); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(sgn, ymm0);\n"); +#else fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#endif // Store result // min = simde_mm_min_epu8(min, maxLLR); // 128 in epi8 is -127 #ifndef DROP_MAXLLR @@ -185,7 +195,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) // ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i]; fprintf(fd," ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[1]>>4)+lut_idxCnProcG4[j][0]*2); // sgn = simde_mm_sign_epi8(ones, ymm0); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(ones, ymm0);\n"); +#else fprintf(fd," sgn = simde_mm_sign_epi8(ones, ymm0);\n"); +#endif // min = simde_mm_abs_epi8(ymm0); fprintf(fd," min = simde_mm_abs_epi8(ymm0);\n"); @@ -199,7 +213,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) fprintf(fd," min = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n"); // sgn = simde_mm_sign_epi8(sgn, ymm0); - fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(sgn, ymm0);\n"); +#else + fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#endif } // Store result @@ -252,7 +270,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) // ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i]; fprintf(fd," ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[2]>>4)+lut_idxCnProcG5[j][0]*2); // sgn = simde_mm_sign_epi8(ones, ymm0); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(ones, ymm0);\n"); +#else fprintf(fd," sgn = simde_mm_sign_epi8(ones, ymm0);\n"); +#endif // min = simde_mm_abs_epi8(ymm0); fprintf(fd," min = simde_mm_abs_epi8(ymm0);\n"); @@ -266,7 +288,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) fprintf(fd," min = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n"); // sgn = simde_mm_sign_epi8(sgn, ymm0); - fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(sgn, ymm0);\n"); +#else + fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#endif } // Store result @@ -319,7 +345,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) // ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i]; fprintf(fd," ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[3]>>4)+lut_idxCnProcG6[j][0]*2); // sgn = simde_mm_sign_epi8(ones, ymm0); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(ones, ymm0);\n"); +#else fprintf(fd," sgn = simde_mm_sign_epi8(ones, ymm0);\n"); +#endif // min = simde_mm_abs_epi8(ymm0); fprintf(fd," min = simde_mm_abs_epi8(ymm0);\n"); @@ -333,7 +363,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) fprintf(fd," min = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n"); // sgn = simde_mm_sign_epi8(sgn, ymm0); - fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(sgn, ymm0);\n"); +#else + fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#endif } // Store result @@ -389,7 +423,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) // ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i]; fprintf(fd," ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[4]>>4)+lut_idxCnProcG7[j][0]*2); // sgn = simde_mm_sign_epi8(ones, ymm0); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(ones, ymm0);\n"); +#else fprintf(fd," sgn = simde_mm_sign_epi8(ones, ymm0);\n"); +#endif // min = simde_mm_abs_epi8(ymm0); fprintf(fd," min = simde_mm_abs_epi8(ymm0);\n"); @@ -403,7 +441,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) fprintf(fd," min = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n"); // sgn = simde_mm_sign_epi8(sgn, ymm0); - fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(sgn, ymm0);\n"); +#else + fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#endif } // Store result @@ -460,7 +502,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) // ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i]; fprintf(fd," ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[5]>>4)+lut_idxCnProcG8[j][0]*2); // sgn = simde_mm_sign_epi8(ones, ymm0); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(ones, ymm0);\n"); +#else fprintf(fd," sgn = simde_mm_sign_epi8(ones, ymm0);\n"); +#endif // min = simde_mm_abs_epi8(ymm0); fprintf(fd," min = simde_mm_abs_epi8(ymm0);\n"); @@ -474,7 +520,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) fprintf(fd," min = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n"); // sgn = simde_mm_sign_epi8(sgn, ymm0); - fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(sgn, ymm0);\n"); +#else + fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#endif } // Store result @@ -531,7 +581,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) // ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i]; fprintf(fd," ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[6]>>4)+lut_idxCnProcG9[j][0]*2); // sgn = simde_mm_sign_epi8(ones, ymm0); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(ones, ymm0);\n"); +#else fprintf(fd," sgn = simde_mm_sign_epi8(ones, ymm0);\n"); +#endif // min = simde_mm_abs_epi8(ymm0); fprintf(fd," min = simde_mm_abs_epi8(ymm0);\n"); @@ -545,7 +599,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) fprintf(fd," min = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n"); // sgn = simde_mm_sign_epi8(sgn, ymm0); - fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(sgn, ymm0);\n"); +#else + fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#endif } // Store result @@ -603,7 +661,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) // ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i]; fprintf(fd," ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[7]>>4)+lut_idxCnProcG10[j][0]*2); // sgn = simde_mm_sign_epi8(ones, ymm0); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(ones, ymm0);\n"); +#else fprintf(fd," sgn = simde_mm_sign_epi8(ones, ymm0);\n"); +#endif // min = simde_mm_abs_epi8(ymm0); fprintf(fd," min = simde_mm_abs_epi8(ymm0);\n"); @@ -617,7 +679,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) fprintf(fd," min = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n"); // sgn = simde_mm_sign_epi8(sgn, ymm0); - fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(sgn, ymm0);\n"); +#else + fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#endif } // Store result @@ -678,7 +744,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) // ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i]; fprintf(fd," ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[8]>>4)+lut_idxCnProcG19[j][0]*2); // sgn = simde_mm_sign_epi8(ones, ymm0); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(ones, ymm0);\n"); +#else fprintf(fd," sgn = simde_mm_sign_epi8(ones, ymm0);\n"); +#endif // min = simde_mm_abs_epi8(ymm0); fprintf(fd," min = simde_mm_abs_epi8(ymm0);\n"); @@ -692,7 +762,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R) fprintf(fd," min = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n"); // sgn = simde_mm_sign_epi8(sgn, ymm0); - fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#ifdef AVOID_SIGN + fprintf(fd," sgn = simde_mm_xor_si128(sgn, ymm0);\n"); +#else + fprintf(fd," sgn = simde_mm_sign_epi8(sgn, ymm0);\n"); +#endif } // Store result From bbbbda70f5352d99224d0b144b0ab0c4ab75d753 Mon Sep 17 00:00:00 2001 From: Raymond Knopp Date: Tue, 13 May 2025 13:20:34 +0000 Subject: [PATCH 06/17] feat(nrLDPC_coding_segment_encoder): Dense Rate Matching and Dense and Concatenated Output 1. 3GPP Rate-matching/interleaving on 8-segment blocks, deinterleaving at output of multi-segment processing. 2. Bit-packing done in nrLDPC_coding_segment_encoder.c now instead of nr_scrambling. Interestingly, it improves performance significantly too. Tested for AVX512 only for now. --- .../nrLDPC_coding_segment_encoder.c | 485 +++++++++++++++--- .../ldpc_encoder_optim8segmulti.c | 98 +--- openair1/PHY/NR_TRANSPORT/nr_scrambling.c | 25 +- openair1/SIMULATION/NR_PHY/dlschsim.c | 6 +- 4 files changed, 443 insertions(+), 171 deletions(-) diff --git a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c index 30ace31054..c962074560 100644 --- a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c +++ b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c @@ -41,8 +41,8 @@ #include -// #define DEBUG_LDPC_ENCODING -// #define DEBUG_LDPC_ENCODING_FREE 1 +#define DEBUG_LDPC_ENCODING +//#define DEBUG_LDPC_ENCODING_FREE 1 typedef struct ldpc8blocks_args_s { nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters; @@ -64,98 +64,442 @@ static void ldpc8blocks(void *p) // nrLDPC_encoder output is in "d" // let's make this interface happy! - uint8_t tmp[8][68 * 384] __attribute__((aligned(64))); - uint8_t *d[impp->n_segments]; - for (int rr = impp->macro_num * 8, i = 0; rr < impp->n_segments && rr < (impp->macro_num + 1) * 8; rr++, i++) - d[rr] = tmp[i]; - + uint8_t d[68 * 384] __attribute__((aligned(64))); + uint8_t *dp[2]; + dp[0] = &d[0]; uint8_t *c[nrLDPC_TB_encoding_parameters->C]; + unsigned int macro_segment, macro_segment_end; + + + macro_segment = 8*impp->macro_num; + macro_segment_end = (impp->n_segments > 8*(impp->macro_num+1)) ? 8*(impp->macro_num+1) : impp->n_segments; for (int r = 0; r < nrLDPC_TB_encoding_parameters->C; r++) c[r] = nrLDPC_TB_encoding_parameters->segments[r].c; start_meas(&nrLDPC_TB_encoding_parameters->segments[impp->macro_num * 8].ts_ldpc_encode); - LDPCencoder(c, d, impp); + LDPCencoder(c, dp, impp); stop_meas(&nrLDPC_TB_encoding_parameters->segments[impp->macro_num * 8].ts_ldpc_encode); // Compute where to place in output buffer that is concatenation of all segments - uint32_t r_offset = 0; - for (int i = 0; i < impp->macro_num * 8; i++) - r_offset += nrLDPC_TB_encoding_parameters->segments[i].E; - for (int rr = impp->macro_num * 8; rr < impp->n_segments && rr < (impp->macro_num + 1) * 8; rr++) { - if (impp->F > 0) { - // writing into positions d[r][k-2Zc] as in clause 5.3.2 step 2) in 38.212 - memset(&d[rr][impp->K - impp->F - 2 * impp->Zc], NR_NULL, impp->F); - } + + if (impp->F>0) { + // writing into positions d[k-2Zc] as in clause 5.3.2 step 2) in 38.212 + memset(&d[impp->K - impp->F - 2 * impp->Zc], NR_NULL, impp->F); + } #ifdef DEBUG_LDPC_ENCODING - LOG_D(PHY, "rvidx in encoding = %d\n", rel15->rvIndex[0]); + LOG_D(PHY, "rvidx in encoding = %d\n", nrLDPC_TB_encoding_parameters->rv_index); #endif - uint32_t E = nrLDPC_TB_encoding_parameters->segments[rr].E; - LOG_D(NR_PHY, - "Rate Matching, Code segment %d/%d (coded bits (G) %u, E %d, Filler bits %d, Filler offset %d mod_order %d, nb_rb " + const uint32_t E = nrLDPC_TB_encoding_parameters->segments[macro_segment].E; + uint32_t E2=E,E2_first_segment=macro_segment_end-macro_segment; + bool Eshift=false; + for (int s=macro_segment;ssegments[s].E != E) { + E2=nrLDPC_TB_encoding_parameters->segments[s].E; + Eshift=true; + E2_first_segment = s-macro_segment; + break; + } + + + LOG_D(NR_PHY, + "Rate Matching, Code segment %d...%d/%d (coded bits (G) %u, E %d, E2 %d Filler bits %d, Filler offset %d mod_order %d, nb_rb " "%d,nrOfLayer %d)...\n", - rr, - impp->n_segments, - G, - E, - impp->F, - impp->K - impp->F - 2 * impp->Zc, - mod_order, - nb_rb, - nrLDPC_TB_encoding_parameters->nb_layers); + macro_segment, + macro_segment_end, + impp->n_segments, + G, + E,E2, + impp->F, + impp->K - impp->F - 2 * impp->Zc, + mod_order, + nb_rb, + nrLDPC_TB_encoding_parameters->nb_layers); +/* + printf("Rate Matching, Code segment %d...%d/%d (coded bits (G) %u, E %d, E2 %d Filler bits %d, Filler offset %d mod_order %d, nb_rb " + "%d,nrOfLayer %d)...\n", + macro_segment, + macro_segment_end, + impp->n_segments, + G, + E,E2, + impp->F, + impp->K - impp->F - 2 * impp->Zc, + mod_order, + nb_rb, + nrLDPC_TB_encoding_parameters->nb_layers); +*/ + uint32_t Tbslbrm = nrLDPC_TB_encoding_parameters->tbslbrm; - uint32_t Tbslbrm = nrLDPC_TB_encoding_parameters->tbslbrm; - - uint8_t e[E]; - bzero(e, E); - start_meas(&nrLDPC_TB_encoding_parameters->segments[rr].ts_rate_match); + uint8_t e[E] __attribute__((aligned(64))); + uint8_t f[E+64] __attribute__((aligned(64))); + uint8_t e2[E2] __attribute__((aligned(64))); + uint8_t f2[E2+64] __attribute__((aligned(64))); + bzero(e, E); + if (Eshift) bzero(e2,E2); + start_meas(&nrLDPC_TB_encoding_parameters->segments[macro_segment].ts_rate_match); + nr_rate_matching_ldpc(Tbslbrm, + impp->BG, + impp->Zc, + d, + e, + impp->n_segments, + impp->F, + impp->K - impp->F - 2 * impp->Zc, + nrLDPC_TB_encoding_parameters->rv_index, + E); + if (Eshift) nr_rate_matching_ldpc(Tbslbrm, impp->BG, impp->Zc, - d[rr], - e, + d, + e2, impp->n_segments, impp->F, impp->K - impp->F - 2 * impp->Zc, nrLDPC_TB_encoding_parameters->rv_index, - E); - stop_meas(&nrLDPC_TB_encoding_parameters->segments[rr].ts_rate_match); - if (impp->K - impp->F - 2 * impp->Zc > E) { - LOG_E(PHY, "dlsch coding A %d K %d G %d (nb_rb %d, mod_order %d)\n", A, impp->K, G, nb_rb, (int)mod_order); + E2); - LOG_E(NR_PHY, - "Rate Matching, Code segment %d/%d (coded bits (G) %u, E %d, K %d, Filler bits %d, Filler offset %d mod_order %d, " - "nb_rb %d)...\n", - rr, - impp->n_segments, - G, - E, - impp->K, - impp->F, - impp->K - impp->F - 2 * impp->Zc, - mod_order, - nb_rb); - } -#ifdef DEBUG_LDPC_ENCODING - - for (int i = 0; i < 16; i++) - printf("output ratematching e[%d]= %d r_offset %u\n", i, e[i], r_offset); + stop_meas(&nrLDPC_TB_encoding_parameters->segments[macro_segment].ts_rate_match); + if (impp->K - impp->F - 2 * impp->Zc > E) { + LOG_E(PHY, + "dlsch coding A %d Kr %d G %d (nb_rb %d, mod_order %d)\n", + A, + impp->K, + G, + nb_rb, + (int)mod_order); + LOG_E(NR_PHY, + "Rate Matching, Code segments %d...%d/%d (coded bits (G) %u, E %d, Kr %d, Filler bits %d, Filler offset %d mod_order %d, " + "nb_rb %d)...\n", + macro_segment, + macro_segment_end, + impp->n_segments, + G, + E, + impp->K, + impp->F, + impp->K - impp->F - 2 * impp->Zc, + mod_order, + nb_rb); + } + start_meas(&nrLDPC_TB_encoding_parameters->segments[macro_segment].ts_interleave); + nr_interleaving_ldpc(E, + mod_order, + e, + f); + if (Eshift) + nr_interleaving_ldpc(E2, + mod_order, + e2, + f2); + stop_meas(&nrLDPC_TB_encoding_parameters->segments[macro_segment].ts_interleave); + if(impp->toutput != NULL) start_meas(impp->toutput); +/* + for (int i=0;i<16;i++) + for (int s=0;s>s)&1); +*/ + // information part and puncture columns + + uint32_t Eoffset=0; + for (int s=0; ssegments[s].E); +#ifdef __AVX512VBMI__ + uint64_t *output_p = (uint64_t*)impp->output; + int i=0; + __m512i bitperm[8]; + uint32_t Eoffset2[8]; + uint32_t Eoffset2_bit[8]; + bitperm[0] = _mm512_set1_epi64(0x3830282018100800); + __m512i inc = _mm512_set1_epi8(0x1); + Eoffset2[0] = Eoffset>>6; + Eoffset2_bit[0] = Eoffset&63; + //printf("E2_first_segment %d,Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E2_first_segment,Eoffset,Eoffset2[0],Eoffset2_bit[0]); + for (int n=1;n>6; + Eoffset2_bit[n] = Eoffset&63; + //printf("E %d : n %d, Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E,n,Eoffset,Eoffset2[n],Eoffset2_bit[n]); + } + if (Eshift) { + bitperm[E2_first_segment] = _mm512_add_epi8(bitperm[E2_first_segment-1],inc); + Eoffset += E; + Eoffset2[E2_first_segment] = Eoffset>>6; + Eoffset2_bit[E2_first_segment] = Eoffset &63; + //printf("E2 %d : Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E2,Eoffset,Eoffset2[E2_first_segment],Eoffset2_bit[E2_first_segment]); + } + for (int n=E2_first_segment+1;n>6; + Eoffset2_bit[n] = Eoffset&63; + //printf("E2 %d (macro_segment_end %d, macro_segment %d) : n %d, Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E2,macro_segment_end,macro_segment,n,Eoffset,Eoffset2[n],Eoffset2_bit[n]); + } + int i2=0; + __m64 tmp; + for (i=0;i>6), + (Eoffset2_bit[j] + i)&63, + (i<=(E-64)) ? 63 : (Eoffset2_bit[j] + E-1)&63); + if (Eoffset2_bit[j] > 0) + printf("segment %d : qword %d, first bit %d last bit %d\n",j, + 1+Eoffset2[j]+(i>>6), + 0, + ((Eoffset2_bit[j] + i)&63) - 1); +#endif + // Note: Here and below for AVX2, we are using the 64-bit SIMD instruction + // instead of C >>/<< because when the Eoffset2_bit is 64 or 0, the << + // and >> operations are undefined and in fact don't give "0" which is + // what we want here. The SIMD version do give 0 when the shift is 64 + tmp = (__m64)_mm512_bitshuffle_epi64_mask(((__m512i *)f)[i2],bitperm[j]); + *(__m64*)(output_p + Eoffset2[j]) = _mm_or_si64(*(__m64*)(output_p + Eoffset2[j]),_mm_slli_si64(tmp,Eoffset2_bit[j])); + *(__m64*)(output_p + Eoffset2[j]+1) = _mm_or_si64(*(__m64*)(output_p + Eoffset2[j]+1),_mm_srli_si64(tmp,(64-Eoffset2_bit[j]))); +#ifdef DEBUG_BIT_INTERLEAVE + if (j<=1) { + printf("pos : %d, i2 %d, j %d\n",(int)(output_p + Eoffset2[j]-(uint64_t*)impp->output),i2,j); + printf("i : %x.%x.%x.%x.%x.%x.%x.%x\n", + ((uint8_t*)(output_p + Eoffset2[j]))[0], + ((uint8_t*)(output_p + Eoffset2[j]))[1], + ((uint8_t*)(output_p + Eoffset2[j]))[2], + ((uint8_t*)(output_p + Eoffset2[j]))[3], + ((uint8_t*)(output_p + Eoffset2[j]))[4], + ((uint8_t*)(output_p + Eoffset2[j]))[5], + ((uint8_t*)(output_p + Eoffset2[j]))[6], + ((uint8_t*)(output_p + Eoffset2[j]))[7]); + printf("i+1 : %x.%x.%x.%x.%x.%x.%x.%x\n", + ((uint8_t*)(output_p + Eoffset2[j]+1))[0], + ((uint8_t*)(output_p + Eoffset2[j]+1))[1], + ((uint8_t*)(output_p + Eoffset2[j]+1))[2], + ((uint8_t*)(output_p + Eoffset2[j]+1))[3], + ((uint8_t*)(output_p + Eoffset2[j]+1))[4], + ((uint8_t*)(output_p + Eoffset2[j]+1))[5], + ((uint8_t*)(output_p + Eoffset2[j]+1))[6], + ((uint8_t*)(output_p + Eoffset2[j]+1))[7]); + printf("%x.%x.%x.%x.%x.%x.%x.%x\n", + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),0)<<(0-j))&1)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),1)<<(1-j))&2)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),2)<<(2-j))&4)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),3)<<(3-j))&8)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),4)<<(4-j))&16)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),5)<<(5-j))&32)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),6)<<(6-j))&64)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),7)<<(7-j))&128), + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),8)<<(0-j))&1)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),9)<<(1-j))&2)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),10)<<(2-j))&4)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),11)<<(3-j))&8)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),12)<<(4-j))&16)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),13)<<(5-j))&32)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),14)<<(6-j))&64)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),15)<<(7-j))&128), + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),16)<<(0-j))&1)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),17)<<(1-j))&2)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),18)<<(2-j))&4)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),19)<<(3-j))&8)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),20)<<(4-j))&16)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),21)<<(5-j))&32)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),22)<<(6-j))&64)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),23)<<(7-j))&128), + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),24)<<(0-j))&1)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),25)<<(1-j))&2)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),26)<<(2-j))&4)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),27)<<(3-j))&8)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),28)<<(4-j))&16)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),29)<<(5-j))&32)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),30)<<(6-j))&64)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),31)<<(7-j))&128), + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),0)<<(0-j))&1)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),1)<<(1-j))&2)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),2)<<(2-j))&4)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),3)<<(3-j))&8)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),4)<<(4-j))&16)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),5)<<(5-j))&32)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),6)<<(6-j))&64)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),7)<<(7-j))&128), + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),8)<<(0-j))&1)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),9)<<(1-j))&2)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),10)<<(2-j))&4)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),11)<<(3-j))&8)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),12)<<(4-j))&16)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),13)<<(5-j))&32)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),14)<<(6-j))&64)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),15)<<(7-j))&128), + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),16)<<(0-j))&1)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),17)<<(1-j))&2)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),18)<<(2-j))&4)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),19)<<(3-j))&8)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),20)<<(4-j))&16)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),21)<<(5-j))&32)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),22)<<(6-j))&64)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),23)<<(7-j))&128), + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),24)<<(0-j))&1)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),25)<<(1-j))&2)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),26)<<(2-j))&4)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),27)<<(3-j))&8)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),28)<<(4-j))&16)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),29)<<(5-j))&32)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),30)<<(6-j))&64)+ + ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),31)<<(7-j))&128)); + } #endif - start_meas(&nrLDPC_TB_encoding_parameters->segments[rr].ts_interleave); - nr_interleaving_ldpc(E, mod_order, e, impp->output + r_offset); - stop_meas(&nrLDPC_TB_encoding_parameters->segments[rr].ts_interleave); -#ifdef DEBUG_LDPC_ENCODING - - for (int i = 0; i < 16; i++) - printf("output interleaving f[%d]= %d r_offset %u\n", i, impp->output[i + r_offset], r_offset); - - if (r == impp->n_segments - 1) - write_output("enc_output.m", "enc", impp->output, G, 1, 4); - + } //for (int j=0; + } // if (i < E) + for (int j=E2_first_segment; j < macro_segment_end-macro_segment; j++) { +#ifdef DEBUG_BIT_INTERLEAVE + printf("segment %d : qword %d, first bit %d last bit %d\n",j, + Eoffset2[j]+(i>>6), + (Eoffset2_bit[j] + i)&63, + (i<=(E-64)) ? 63 : (Eoffset2_bit[j] + E-1)&63); + if (Eoffset2_bit[j] > 0) + printf("segment %d : qword %d, first bit %d last bit %d\n",j, + 1+Eoffset2[j]+(i>>6), + 0, + ((Eoffset2_bit[j] + i)&63) - 1); #endif - r_offset += E; + tmp = (__m64)_mm512_bitshuffle_epi64_mask(((__m512i *)f2)[i2],bitperm[j]); + *(__m64*)(output_p + Eoffset2[j]) = _mm_or_si64(*(__m64*)(output_p + Eoffset2[j]), + _mm_slli_si64(tmp,Eoffset2_bit[j])); + *(__m64*)(output_p + Eoffset2[j]+1) = _mm_or_si64(*(__m64*)(output_p + Eoffset2[j]+1),_mm_srli_si64(tmp,(64-Eoffset2_bit[j]))); + } + output_p++; } +#elif defined(__aarch64__) + uint16_t *output_p = (uint16_t*)impp->output; + int i=0; + uint32_t Eoffset2[8]; + uint32_t Eoffset2_bit[8]; + Eoffset2[0] = Eoffset>>4; + Eoffset2_bit[0] = Eoffset&15; + //printf("E2_first_segment %d,Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E2_first_segment,Eoffset,Eoffset2[0],Eoffset2_bit[0]); + for (int n=1;n>4; + Eoffset2_bit[n] = Eoffset&15; + //printf("E %d : n %d, Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E,n,Eoffset,Eoffset2[n],Eoffset2_bit[n]); + } + if (Eshift) { + Eoffset += E; + Eoffset2[E2_first_segment] = Eoffset>>4; + Eoffset2_bit[E2_first_segment] = Eoffset &15; + //printf("E2 %d : Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E2,Eoffset,Eoffset2[E2_first_segment],Eoffset2_bit[E2_first_segment]); + } + for (int n=E2_first_segment+1;n>4; + Eoffset2_bit[n] = Eoffset&15; + //printf("E2 %d (macro_segment_end %d, macro_segment %d) : n %d, Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E2,macro_segment_end,macro_segment,n,Eoffset,Eoffset2[n],Eoffset2_bit[n]); + } + int i2=0; + const int8_t __attribute__ ((aligned (16))) ucShift[8][16] = { + {0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7}, // segment 0 + {-1,0,1,2,3,4,5,6,-1,0,1,2,3,4,5,6}, // segment 1 + {-2,-1,0,1,2,3,4,5,-2,-1,0,1,2,3,4,5}, // segment 2 + {-3,-2,-1,0,1,2,3,4,-3,-2,-1,0,1,2,3,4}, // segment 3 + {-4,-3,-2,-1,0,1,2,3,-4,-3,-2,-1,0,1,2,3}, // segment 4 + {-5,-4,-3,-2,-1,0,1,2,-5,-4,-3,-2,-1,0,1,2}, // segment 5 + {-6,-5,-4,-3,-2,-1,0,1,-6,-5,-4,-3,-2,-1,0,1}, // segment 6 + {-7,-6,-5,-4,-3,-2,-1,0,-7,-6,-5,-4,-3,-2,-1,0}}; // segment 7 + const uint8_t __attribute__ ((aligned (16))) masks[16] = + {0x1,0x2,0x4,0x8,0x10,0x20,0x40,0x80,0x1,0x2,0x4,0x8,0x10,0x20,0x40,0x80}; + int8x16_t vshift[8]; + for (int n=0;n<8;n++) vshift[n] = vld1q_s8(ucShift[n]); + uint8x16_t vmask = vld1q_u8(masks); + + int32_t tmp; + uint8x16_t cshift; + for (i=0;i>(16-Eoffset2_bit[j])); + } + } + for (int j=E2_first_segment; j < macro_segment_end-macro_segment; j++) { + cshift = vandq_u8(vshlq_u8(((uint8x16_t*)f2)[i2],vshift[j]),vmask); + tmp = (int)vaddv_u8(vget_low_u8(cshift)); + tmp += (int)(vaddv_u8(vget_high_u8(cshift))<<8); + *(output_p + Eoffset2[j]) |= (uint16_t)(tmp<>(16-Eoffset2_bit[j])); + } + output_p++; + } + +#else + uint32_t *output_p = (uint32_t*)impp->output; + int i=0; + uint32_t Eoffset2[8]; + uint32_t Eoffset2_bit[8]; + Eoffset2[0] = Eoffset>>5; + Eoffset2_bit[0] = Eoffset&31; + //printf("E2_first_segment %d,Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E2_first_segment,Eoffset,Eoffset2[0],Eoffset2_bit[0]); + for (int n=1;n>5; + Eoffset2_bit[n] = Eoffset&31; + //printf("E %d : n %d, Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E,n,Eoffset,Eoffset2[n],Eoffset2_bit[n]); + } + if (Eshift) { + Eoffset += E; + Eoffset2[E2_first_segment] = Eoffset>>5; + Eoffset2_bit[E2_first_segment] = Eoffset &31; + //printf("E2 %d : Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E2,Eoffset,Eoffset2[E2_first_segment],Eoffset2_bit[E2_first_segment]); + } + for (int n=E2_first_segment+1;n>5; + Eoffset2_bit[n] = Eoffset&31; + //printf("E2 %d (macro_segment_end %d, macro_segment %d) : n %d, Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E2,macro_segment_end,macro_segment,n,Eoffset,Eoffset2[n],Eoffset2_bit[n]); + } + int i2=0; + int tmp; + __m64 tmp64,tmp64b,tmp64c; + + for (i=0;i>/<< because when the Eoffset2_bit is 64 or 0, the << + // and >> operations are undefined and in fact don't give "0" which is + // what we want here. The SIMD version do give 0 when the shift is 64 + tmp = _mm256_movemask_epi8(_mm256_slli_epi16(((__m256i *)f)[i2],7-j)); + tmp64=_mm_set1_pi32(tmp); + tmp64b = _mm_or_si64(*(__m64*)(output_p + Eoffset2[j]),_mm_slli_pi32(tmp64,Eoffset2_bit[j])); + tmp64c = _mm_or_si64(*(__m64*)(output_p + Eoffset2[j]),_mm_srli_pi32(tmp64,(32-Eoffset2_bit[j]))); + *(output_p + Eoffset2[j]) = _m_to_int(tmp64b); + *(output_p + Eoffset2[j]+1) = _m_to_int(_mm_srli_si64(tmp64c,32)); + } + } + for (int j=E2_first_segment; j < macro_segment_end-macro_segment; j++) { + tmp = _mm256_movemask_epi8(_mm256_slli_epi16(((__m256i *)f2)[i2],7-j)); + tmp64=_mm_set1_pi32(tmp); + tmp64b = _mm_or_si64(*(__m64*)(output_p + Eoffset2[j]),_mm_slli_pi32(tmp64,Eoffset2_bit[j])); + tmp64c = _mm_or_si64(*(__m64*)(output_p + Eoffset2[j]),_mm_srli_pi32(tmp64,(32-Eoffset2_bit[j]))); + *(output_p + Eoffset2[j]) = _m_to_int(tmp64b); + *(output_p + Eoffset2[j]+1) = _m_to_int(_mm_srli_si64(tmp64c,32)); + } + output_p++; + } + + +#endif + + if(impp->toutput != NULL) stop_meas(impp->toutput); +/* + printf("E %d (mod16 %d) F %d\n",E,E&15,impp->F); + for (int i=0;i<16;i++) + for (int s=0;sans); } @@ -176,7 +520,7 @@ static int nrLDPC_prepare_TB_encoding(nrLDPC_slot_encoding_parameters_t *nrLDPC_ impp.Kb = nrLDPC_TB_encoding_parameters->Kb; impp.Zc = nrLDPC_TB_encoding_parameters->Z; impp.BG = nrLDPC_TB_encoding_parameters->BG; - impp.output = nrLDPC_TB_encoding_parameters->segments->output; + impp.output = nrLDPC_TB_encoding_parameters->segments->output; impp.K = nrLDPC_TB_encoding_parameters->K; impp.F = nrLDPC_TB_encoding_parameters->F; @@ -201,6 +545,7 @@ static int nrLDPC_prepare_TB_encoding(nrLDPC_slot_encoding_parameters_t *nrLDPC_ int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters) { int nbTasks = 0; + for (int dlsch_id = 0; dlsch_id < nrLDPC_slot_encoding_parameters->nb_TBs; dlsch_id++) { nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id]; size_t n_seg = (nrLDPC_TB_encoding_parameters->C / 8 + ((nrLDPC_TB_encoding_parameters->C & 7) == 0 ? 0 : 1)); diff --git a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8segmulti.c b/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8segmulti.c index bd25ecf646..268d2f9dc6 100644 --- a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8segmulti.c +++ b/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8segmulti.c @@ -203,102 +203,8 @@ int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp) return(-1); } } - if(impp->toutput != NULL) start_meas(impp->toutput); - // information part and puncture columns - /* - memcpy(&output[0], &c[2*Zc], (block_length-2*Zc)*sizeof(unsigned char)); - memcpy(&output[block_length-2*Zc], &d[0], ((nrows-no_punctured_columns) * Zc-removed_bit)*sizeof(unsigned char)); - */ -#ifdef __AVX512F__ - - uint32_t l1 = (block_length-(2*Zc)); - uint32_t l2 = ((nrows-no_punctured_columns) * Zc-removed_bit); - if ((((2*Zc)&63) == 0) && (((block_length-(2*Zc))&63) == 0)) { - __m512i *c512p = (__m512i *)&cc[2*Zc]; - __m512i *d512p = (__m512i *)&dd[0]; - __m512i mask0 = _mm512_set1_epi8(0x1); - - for (int i=0;i>6;i++) - for (int j=macro_segment; j < macro_segment_end; j++) - ((__m512i *)output[j])[i] = _mm512_and_si512(_mm512_srai_epi16(c512p[i],j-macro_segment),mask0); - for (int i1=0, i=l1>>6;i1>6;i1++,i++) - for (int j=macro_segment; j < macro_segment_end; j++) - ((__m512i *)output[j])[i] = _mm512_and_si512(_mm512_srai_epi16(d512p[i1],j-macro_segment),mask0); - } - else if ((((2*Zc)&31) == 0) && (((block_length-(2*Zc))&31) == 0)) { - uint32_t l1 = (block_length-(2*Zc))>>5; - uint32_t l2 = ((nrows-no_punctured_columns) * Zc-removed_bit)>>5; - simde__m256i *c256p = (simde__m256i *)&cc[2*Zc]; - simde__m256i *d256p = (simde__m256i *)&dd[0]; - // if (((block_length-(2*Zc))&31)>0) l1++; - - for (int i=0;i0) l2++; - - for (int i1=0, i=l1;i1>4; - uint32_t l2 = ((nrows-no_punctured_columns) * Zc-removed_bit)>>4; - simde__m128i *c128p = (simde__m128i *)&cc[2*Zc]; - simde__m128i *d128p = (simde__m128i *)&dd[0]; - // if (((block_length-(2*Zc))&31)>0) l1++; - - for (int i=0;i>5; - uint32_t l2 = ((nrows-no_punctured_columns) * Zc-removed_bit)>>5; - simde__m256i *c256p = (simde__m256i *)&cc[2*Zc]; - simde__m256i *d256p = (simde__m256i *)&dd[0]; - // if (((block_length-(2*Zc))&31)>0) l1++; - - for (int i=0;i0) l2++; - - for (int i1=0, i=l1;i1>(j-macro_segment))&1; - for (int i=0;i<((nrows-no_punctured_columns) * Zc-removed_bit);i++) - //for (j=0; j>(j-macro_segment))&1; - } - - if(impp->toutput != NULL) stop_meas(impp->toutput); + memcpy(output[0],&cc[2*Zc],(block_length-(2*Zc))); + memcpy(output[0]+block_length-(2*Zc),dd,((nrows-no_punctured_columns) * Zc-removed_bit)); return 0; } diff --git a/openair1/PHY/NR_TRANSPORT/nr_scrambling.c b/openair1/PHY/NR_TRANSPORT/nr_scrambling.c index b71ee0cce0..821d92a44d 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_scrambling.c +++ b/openair1/PHY/NR_TRANSPORT/nr_scrambling.c @@ -33,12 +33,33 @@ void nr_codeword_scrambling(uint8_t *in, { const int roundedSz = (size + 31) / 32; uint32_t *seq = gold_cache((n_RNTI << 15) + (q << 14) + Nid, roundedSz); + for (int i = 0; i < roundedSz; i++) + out[i] = ((uint32_t*)in)[i] ^ seq[i]; +#if 0 +//#elif __AVX2__ for (int i = 0; i < roundedSz; i++) { - simde__m256i c = ((simde__m256i*)in)[i]; - uint32_t in32 = simde_mm256_movemask_epi8(simde_mm256_slli_epi16(c, 7)); + __m256i c = ((__m256i*)in)[i]; + uint32_t in32 = _mm256_movemask_epi8(_mm256_slli_epi16(c, 7)); out[i] = in32 ^ seq[i]; DEBUG_SCRAMBLING(LOG_D(PHY, "in[%d] %x => %x\n", i, in32, out[i])); } +//#elif defined (__aarch64__) + const int8_t __attribute__ ((aligned (16))) ucShift[] = {0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7}; + int8x16_t vshift = vld1q_s8(ucShift); + uint8x16_t *c = (uint8x16_t*)in; + uint8x16_t cshift; + uint32_t in32; + for (int i = 0; i < roundedSz; i++) { + cshift = vshlq_u8(*c++, vshift); + in32 = vaddv_u8(vget_low_u8(cshift)); + in32 += (vaddv_u8(vget_high_u8(cshift)) << 8); + cshift = vshlq_u8(*c++, vshift); + in32 += (vaddv_u8(vget_low_u8(cshift)) << 16); + in32 += (vaddv_u8(vget_high_u8(cshift)) << 24); + out[i] = in32 ^ seq[i]; + DEBUG_SCRAMBLING(LOG_D(PHY, "in[%d] %x => %x\n", i, in32, out[i])); + } +#endif } void nr_codeword_unscrambling(int16_t* llr, uint32_t size, uint8_t q, uint32_t Nid, uint32_t n_RNTI) diff --git a/openair1/SIMULATION/NR_PHY/dlschsim.c b/openair1/SIMULATION/NR_PHY/dlschsim.c index 36fe9daa50..f7b10e5fb1 100644 --- a/openair1/SIMULATION/NR_PHY/dlschsim.c +++ b/openair1/SIMULATION/NR_PHY/dlschsim.c @@ -512,7 +512,7 @@ int main(int argc, char **argv) //printf("crc32: [0]->0x%08x\n",crc24c(test_input, 32)); // generate signal - unsigned char output[rel15->rbSize * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * 8 * NR_MAX_NB_LAYERS] __attribute__((aligned(32))); + unsigned char output[rel15->rbSize * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * 8 * NR_MAX_NB_LAYERS] __attribute__((aligned(64))); bzero(output,rel15->rbSize * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * 8 * NR_MAX_NB_LAYERS); if (input_fd == NULL) { msgDataTx.num_pdsch_slot = 1; @@ -533,12 +533,12 @@ int main(int argc, char **argv) //if (i<16) // printf("encoder output f[%d] = %d\n",i,dlsch->harq_processes[0]->f[i]); - if (output[i] == 0) + if ((output[i]&1) == 0) modulated_input[i] = 1.0; ///sqrt(2); //QPSK else modulated_input[i] = -1.0; ///sqrt(2); - //if (i<16) printf("modulated_input[%d] = %d\n",i,modulated_input[i]); + //if (i<16) printf("modulated_input[%d] = %f (%d)\n",i,modulated_input[i],output[i]); //SNR =10; SNR_lin = pow(10, SNR / 10.0); sigma = 1.0 / sqrt(2 * SNR_lin); From f3daf0ed833f950c8b766290da79f271029e1495 Mon Sep 17 00:00:00 2001 From: Romain Beurdouche Date: Wed, 5 Mar 2025 09:32:32 +0000 Subject: [PATCH 07/17] fix(nrLDPC_coding): Miscellaneous fixes 1. nrLDPC_coding_segment_encoder: pointer not aligned on 64 bits An undefined behavior was happening because of a shift on a __m64 which was actually a uint32_t which was not aligned on 64 bits. This commit suggests a fix by properly loading into a __m64 with _mm_set_pi32 before performing the shift. 2. nr_rate_matching: Filler bits detection Previously, if the bit selection was reaching Ncb and restarting selection from the beginning of encoded data, the filler bits were detected by testing the bytes again NR_NULL which was equal to 2. But on this branch the bits of up to 8 segments are stored in one byte while doing rate matching so that one byte could be tested to be a filler while it is not. So this commit changes the way to test filler bits. Instead of relying on NR_NULL, bit selection now relies on the offset and size of filler bits. 3. nr_ulsch_coding: missing filling f with zeros The array `f` of the UE UL HARQ process was not filled with zeros between transmissions. This is a problem if the encoder does not properly overwrite its output. In order to avoid such situation, this commit adds a memset to fill `f` with zeros before passing it to the encoder. 4. nrLDPC_coding_segment_encoder: missing init of f to zeros In the encoder, `f` is not filled with zeros before being used. `f` is instead somehow filled with arbitrary values. When writing to `output` by 64 bit vectors, some arbitrary bits may be added after the segment. Then the arbitrary added ones are not overwritten when writing the next segment because of writing with a logical or. This commit properly fills `f` and `f2` with zeros. 5. nr_ulschsim & nr_dlschsim: Adapt modulation to new encoder output The QPSK symbols that are feed to the channel model of nr_dlschsim were determined based on the bytes of the encoder output. With the new output of the encoder, it had to be reworked to determine the QPSK symbols based on the bits of the encoder output. --- .../nrLDPC_coding_segment_encoder.c | 50 +++++++++---------- .../nrLDPC_coding_segment/nr_rate_matching.c | 7 ++- .../nrLDPC_coding_segment/nr_rate_matching.h | 2 - .../PHY/NR_UE_TRANSPORT/nr_ulsch_coding.c | 4 ++ openair1/SIMULATION/NR_PHY/dlschsim.c | 6 +-- openair1/SIMULATION/NR_PHY/ulschsim.c | 2 +- 6 files changed, 38 insertions(+), 33 deletions(-) diff --git a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c index c962074560..da4502d657 100644 --- a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c +++ b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c @@ -80,12 +80,6 @@ static void ldpc8blocks(void *p) stop_meas(&nrLDPC_TB_encoding_parameters->segments[impp->macro_num * 8].ts_ldpc_encode); // Compute where to place in output buffer that is concatenation of all segments - - if (impp->F>0) { - // writing into positions d[k-2Zc] as in clause 5.3.2 step 2) in 38.212 - memset(&d[impp->K - impp->F - 2 * impp->Zc], NR_NULL, impp->F); - } - #ifdef DEBUG_LDPC_ENCODING LOG_D(PHY, "rvidx in encoding = %d\n", nrLDPC_TB_encoding_parameters->rv_index); #endif @@ -135,7 +129,11 @@ static void ldpc8blocks(void *p) uint8_t e2[E2] __attribute__((aligned(64))); uint8_t f2[E2+64] __attribute__((aligned(64))); bzero(e, E); - if (Eshift) bzero(e2,E2); + bzero(f, E + 64); + if (Eshift) { + bzero(e2, E2); + bzero(f2, E2 + 64); + } start_meas(&nrLDPC_TB_encoding_parameters->segments[macro_segment].ts_rate_match); nr_rate_matching_ldpc(Tbslbrm, impp->BG, @@ -460,32 +458,34 @@ static void ldpc8blocks(void *p) Eoffset2_bit[n] = Eoffset&31; //printf("E2 %d (macro_segment_end %d, macro_segment %d) : n %d, Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E2,macro_segment_end,macro_segment,n,Eoffset,Eoffset2[n],Eoffset2_bit[n]); } - int i2=0; + int i2 = 0; int tmp; - __m64 tmp64,tmp64b,tmp64c; + __m64 tmp64, tmp64b, tmp64c, out64; - for (i=0;i>/<< because when the Eoffset2_bit is 64 or 0, the << // and >> operations are undefined and in fact don't give "0" which is // what we want here. The SIMD version do give 0 when the shift is 64 - tmp = _mm256_movemask_epi8(_mm256_slli_epi16(((__m256i *)f)[i2],7-j)); - tmp64=_mm_set1_pi32(tmp); - tmp64b = _mm_or_si64(*(__m64*)(output_p + Eoffset2[j]),_mm_slli_pi32(tmp64,Eoffset2_bit[j])); - tmp64c = _mm_or_si64(*(__m64*)(output_p + Eoffset2[j]),_mm_srli_pi32(tmp64,(32-Eoffset2_bit[j]))); - *(output_p + Eoffset2[j]) = _m_to_int(tmp64b); - *(output_p + Eoffset2[j]+1) = _m_to_int(_mm_srli_si64(tmp64c,32)); + tmp = _mm256_movemask_epi8(_mm256_slli_epi16(((__m256i *)f)[i2], 7 - j)); + tmp64 = _mm_set1_pi32(tmp); + out64 = _mm_set_pi32(*(output_p + Eoffset2[j] + 1), *(output_p + Eoffset2[j])); + tmp64b = _mm_or_si64(out64, _mm_slli_pi32(tmp64, Eoffset2_bit[j])); + tmp64c = _mm_or_si64(out64, _mm_srli_pi32(tmp64, (32 - Eoffset2_bit[j]))); + *(output_p + Eoffset2[j]) = _m_to_int(tmp64b); + *(output_p + Eoffset2[j] + 1) = _m_to_int(_mm_srli_si64(tmp64c, 32)); } } - for (int j=E2_first_segment; j < macro_segment_end-macro_segment; j++) { - tmp = _mm256_movemask_epi8(_mm256_slli_epi16(((__m256i *)f2)[i2],7-j)); - tmp64=_mm_set1_pi32(tmp); - tmp64b = _mm_or_si64(*(__m64*)(output_p + Eoffset2[j]),_mm_slli_pi32(tmp64,Eoffset2_bit[j])); - tmp64c = _mm_or_si64(*(__m64*)(output_p + Eoffset2[j]),_mm_srli_pi32(tmp64,(32-Eoffset2_bit[j]))); - *(output_p + Eoffset2[j]) = _m_to_int(tmp64b); - *(output_p + Eoffset2[j]+1) = _m_to_int(_mm_srli_si64(tmp64c,32)); + for (int j = E2_first_segment; j < macro_segment_end - macro_segment; j++) { + tmp = _mm256_movemask_epi8(_mm256_slli_epi16(((__m256i *)f2)[i2], 7 - j)); + tmp64 = _mm_set1_pi32(tmp); + out64 = _mm_set_pi32(*(output_p + Eoffset2[j] + 1), *(output_p + Eoffset2[j])); + tmp64b = _mm_or_si64(out64, _mm_slli_pi32(tmp64, Eoffset2_bit[j])); + tmp64c = _mm_or_si64(out64, _mm_srli_pi32(tmp64, (32 - Eoffset2_bit[j]))); + *(output_p + Eoffset2[j]) = _m_to_int(tmp64b); + *(output_p + Eoffset2[j] + 1) = _m_to_int(_mm_srli_si64(tmp64c, 32)); } output_p++; } diff --git a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.c b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.c index 8f111b5308..4e5a31f31f 100644 --- a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.c +++ b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.c @@ -626,8 +626,11 @@ int nr_rate_matching_ldpc(uint32_t Tbslbrm, printf("RM_TX k%u Ind: %u (%d)\n", k, ind, d[ind]); #endif - if (d[ind] != NR_NULL) - e[k++] = d[ind]; + if (ind == Foffset) + ind = F + Foffset; // skip filler bits + + e[k++] = d[ind]; + } } diff --git a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.h b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.h index ac6a496801..597a53070a 100644 --- a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.h +++ b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.h @@ -24,8 +24,6 @@ #include -#define NR_NULL 2 - /** * \brief interleave a code segment after encoding and rate matching * \param E size of the code segment in bits diff --git a/openair1/PHY/NR_UE_TRANSPORT/nr_ulsch_coding.c b/openair1/PHY/NR_UE_TRANSPORT/nr_ulsch_coding.c index bed86bf852..dcf18ceff5 100644 --- a/openair1/PHY/NR_UE_TRANSPORT/nr_ulsch_coding.c +++ b/openair1/PHY/NR_UE_TRANSPORT/nr_ulsch_coding.c @@ -156,6 +156,10 @@ int nr_ulsch_encoding(PHY_VARS_NR_UE *ue, NR_UL_UE_HARQ_t *harq_process = &ue->ul_harq_processes[harq_pid]; TB_parameters->segments = segments[pusch_id]; + const nfapi_nr_ue_pusch_pdu_t *pusch_pdu = &ulsch->pusch_pdu; + uint16_t nb_rb = pusch_pdu->rb_size; + memset(harq_process->f, 0, 14 * nb_rb * 12 * 16); + int r_offset = 0; for (int r = 0; r < TB_parameters->C; r++) { nrLDPC_segment_encoding_parameters_t *segment_parameters = &TB_parameters->segments[r]; diff --git a/openair1/SIMULATION/NR_PHY/dlschsim.c b/openair1/SIMULATION/NR_PHY/dlschsim.c index f7b10e5fb1..c475bfc039 100644 --- a/openair1/SIMULATION/NR_PHY/dlschsim.c +++ b/openair1/SIMULATION/NR_PHY/dlschsim.c @@ -512,8 +512,8 @@ int main(int argc, char **argv) //printf("crc32: [0]->0x%08x\n",crc24c(test_input, 32)); // generate signal - unsigned char output[rel15->rbSize * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * 8 * NR_MAX_NB_LAYERS] __attribute__((aligned(64))); - bzero(output,rel15->rbSize * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * 8 * NR_MAX_NB_LAYERS); + unsigned char output[rel15->rbSize * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * NR_MAX_NB_LAYERS] __attribute__((aligned(64))); + bzero(output, sizeof(output)); if (input_fd == NULL) { msgDataTx.num_pdsch_slot = 1; nr_dlsch_encoding(gNB, &msgDataTx, frame, slot, frame_parms, output, NULL, NULL, NULL, NULL, NULL, NULL, NULL); @@ -533,7 +533,7 @@ int main(int argc, char **argv) //if (i<16) // printf("encoder output f[%d] = %d\n",i,dlsch->harq_processes[0]->f[i]); - if ((output[i]&1) == 0) + if ((output[i >> 3] & (1 << (i & 7))) == 0) modulated_input[i] = 1.0; ///sqrt(2); //QPSK else modulated_input[i] = -1.0; ///sqrt(2); diff --git a/openair1/SIMULATION/NR_PHY/ulschsim.c b/openair1/SIMULATION/NR_PHY/ulschsim.c index f68f6885a6..ef6fd34ee4 100644 --- a/openair1/SIMULATION/NR_PHY/ulschsim.c +++ b/openair1/SIMULATION/NR_PHY/ulschsim.c @@ -542,7 +542,7 @@ int main(int argc, char **argv) } */ - if (harq_process_ul_ue->f[i] == 0) + if ((harq_process_ul_ue->f[i >> 3] & (1 << (i & 7))) == 0) modulated_input[i] = 1.0; ///sqrt(2); //QPSK else modulated_input[i] = -1.0; ///sqrt(2); From 1b31ac417bd116340bb7d90ccfb44fa0160d7e74 Mon Sep 17 00:00:00 2001 From: Romain Beurdouche Date: Fri, 14 Mar 2025 07:36:45 +0000 Subject: [PATCH 08/17] fix(nrLDPC_coding_interface): Move the encoding output to the TB Since this branch aims to bring all the encoded segments packed and contiguous, it doesn't make anymore sense to have one output array per segment. Therefore this commit moves the output array to the TB instead of the segment and adapts its size. TODO: Adapt the size of f where it is relevant. --- .../nrLDPC_coding/nrLDPC_coding_interface.h | 4 ++-- .../nrLDPC_coding_segment_encoder.c | 2 +- .../nrLDPC_coding_t2/nrLDPC_coding_t2.c | 2 +- openair1/PHY/NR_TRANSPORT/nr_dlsch.c | 20 +++++++++---------- openair1/PHY/NR_TRANSPORT/nr_dlsch_coding.c | 10 ++++------ .../PHY/NR_UE_TRANSPORT/nr_ulsch_coding.c | 4 +--- 6 files changed, 19 insertions(+), 23 deletions(-) diff --git a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_interface.h b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_interface.h index 7544f8cf55..e99e1badbc 100644 --- a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_interface.h +++ b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_interface.h @@ -140,7 +140,6 @@ typedef struct nrLDPC_slot_decoding_parameters_s{ * \struct nrLDPC_segment_encoding_parameters_s * \brief encoding parameter of segments * \var E input llr segment size - * \var output input llr segment array * \var c Pointers to code blocks before LDPC encoding (38.212 V15.4.0 section 5.2.2) * flag indicating that the decoding of the segment was successful * IT MUST BE FILLED BY THE IMPLEMENTATION @@ -150,7 +149,6 @@ typedef struct nrLDPC_slot_decoding_parameters_s{ */ typedef struct nrLDPC_segment_encoding_parameters_s{ int E; - unsigned char *output; uint8_t *c; time_stats_t ts_interleave; time_stats_t ts_rate_match; @@ -179,6 +177,7 @@ typedef struct nrLDPC_segment_encoding_parameters_s{ * \var F filler bits size * \var C number of segments * \var segments array of segments parameters + * \var output input llr TB array */ typedef struct nrLDPC_TB_encoding_parameters_s{ @@ -202,6 +201,7 @@ typedef struct nrLDPC_TB_encoding_parameters_s{ uint32_t C; nrLDPC_segment_encoding_parameters_t *segments; + unsigned char *output; } nrLDPC_TB_encoding_parameters_t; /** diff --git a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c index da4502d657..5e44eaf2ae 100644 --- a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c +++ b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c @@ -520,7 +520,7 @@ static int nrLDPC_prepare_TB_encoding(nrLDPC_slot_encoding_parameters_t *nrLDPC_ impp.Kb = nrLDPC_TB_encoding_parameters->Kb; impp.Zc = nrLDPC_TB_encoding_parameters->Z; impp.BG = nrLDPC_TB_encoding_parameters->BG; - impp.output = nrLDPC_TB_encoding_parameters->segments->output; + impp.output = nrLDPC_TB_encoding_parameters->output; impp.K = nrLDPC_TB_encoding_parameters->K; impp.F = nrLDPC_TB_encoding_parameters->F; diff --git a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_t2/nrLDPC_coding_t2.c b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_t2/nrLDPC_coding_t2.c index c16b21bfdc..97792ed1fc 100644 --- a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_t2/nrLDPC_coding_t2.c +++ b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_t2/nrLDPC_coding_t2.c @@ -656,7 +656,7 @@ retrieve_ldpc_enc_op(struct rte_bbdev_enc_op **ops, struct rte_bbdev_op_data *output = &ops[j]->ldpc_enc.output; struct rte_mbuf *m = output->data; uint16_t data_len = rte_pktmbuf_data_len(m) - output->offset; - uint8_t *out = nrLDPC_slot_encoding_parameters->TBs[h].segments[i].output; + uint8_t *out = nrLDPC_slot_encoding_parameters->TBs[h].output; const char *data = m->buf_addr + m->data_off; const char *end = data + data_len; while (data < end) { diff --git a/openair1/PHY/NR_TRANSPORT/nr_dlsch.c b/openair1/PHY/NR_TRANSPORT/nr_dlsch.c index 517d0ab974..6e68a459c9 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_dlsch.c +++ b/openair1/PHY/NR_TRANSPORT/nr_dlsch.c @@ -600,14 +600,14 @@ static int do_one_dlsch(unsigned char *input_ptr, PHY_VARS_gNB *gNB, NR_gNB_DLSC printf("\nEncoded payload:\n"); for (int i = 0; i < encoded_length; i += 8) { for (int j = 0; j < 8; j++) - printf("%d", input_ptr[i + j]); + printf("%d", (input_ptr[i >> 3] >> j) & 1); printf("\t"); } printf("\n"); #endif if (IS_SOFTMODEM_DLSIM) - memcpy(harq->f, input_ptr, encoded_length); + memcpy(harq->f, input_ptr, (encoded_length + 7) >> 3); c16_t mod_symbs[rel15->NrOfCodewords][encoded_length] __attribute__((aligned(64))); for (int codeWord = 0; codeWord < rel15->NrOfCodewords; codeWord++) { @@ -763,11 +763,11 @@ static int do_one_dlsch(unsigned char *input_ptr, PHY_VARS_gNB *gNB, NR_gNB_DLSC } stop_meas(&gNB->dlsch_precoding_stats); - /* output and its parts for each dlsch should be aligned on 64 bytes - * should remain a multiple of 64 with enough offset to fit each dlsch + /* output and its parts for each dlsch should be aligned on 64 bytes (or 8 * 64 bits) + * should remain a multiple of 8 * 64 with enough offset to fit each dlsch */ uint32_t size_output_tb = rel15->rbSize * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * Qm * rel15->nrOfLayers; - return ceil_mod(size_output_tb,64); + return ((size_output_tb + 511) >> 9) << 6; } void nr_generate_pdsch(processingData_L1tx_t *msgTx, int frame, int slot) @@ -817,15 +817,15 @@ void nr_generate_pdsch(processingData_L1tx_t *msgTx, int frame, int slot) /// CRC, coding, interleaving and rate matching AssertFatal(harq->pdu != NULL, "%4d.%2d no HARQ PDU for PDSCH generation\n", msgTx->frame, msgTx->slot); - /* output and its parts for each dlsch should be aligned on 64 bytes - * => size_output is a sum of parts sizes rounded up to a multiple of 64 + /* output and its parts for each dlsch should be aligned on 64 bytes (or 8 * 64 bits) + * => size_output is a sum of parts sizes rounded up to a multiple of 8 * 64 */ size_t size_output_tb = rel15->rbSize * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * Qm * rel15->nrOfLayers; - size_output += ceil_mod(size_output_tb, 64); + size_output += ceil_mod(size_output_tb, 8 * 64); } - unsigned char output[size_output] __attribute__((aligned(64))); - bzero(output, size_output); + unsigned char output[size_output >> 3] __attribute__((aligned(64))); + bzero(output, sizeof(output)); start_meas(dlsch_encoding_stats); if (nr_dlsch_encoding(gNB, diff --git a/openair1/PHY/NR_TRANSPORT/nr_dlsch_coding.c b/openair1/PHY/NR_TRANSPORT/nr_dlsch_coding.c index 29ea38232c..56a849a4af 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_dlsch_coding.c +++ b/openair1/PHY/NR_TRANSPORT/nr_dlsch_coding.c @@ -265,15 +265,13 @@ int nr_dlsch_encoding(PHY_VARS_gNB *gNB, TB_parameters->tbslbrm = rel15->maintenance_parms_v3.tbSizeLbrmBytes; + TB_parameters->output = &output[dlsch_offset >> 3]; TB_parameters->segments = &segments[segments_offset]; - size_t r_offset = 0; for (int r = 0; r < TB_parameters->C; r++) { nrLDPC_segment_encoding_parameters_t *segment_parameters = &TB_parameters->segments[r]; segment_parameters->c = harq->c[r]; segment_parameters->E = nr_get_E(TB_parameters->G, TB_parameters->C, TB_parameters->Qm, rel15->nrOfLayers, r); - segment_parameters->output = &output[dlsch_offset + r_offset]; - r_offset += segment_parameters->E; reset_meas(&segment_parameters->ts_interleave); reset_meas(&segment_parameters->ts_rate_match); @@ -282,11 +280,11 @@ int nr_dlsch_encoding(PHY_VARS_gNB *gNB, segments_offset += TB_parameters->C; - /* output and its parts for each dlsch should be aligned on 64 bytes - * => dlsch_offset should remain a multiple of 64 with enough offset to fit each dlsch + /* output and its parts for each dlsch should be aligned on 64 bytes (or 8 * 64 bits) + * => dlsch_offset should remain a multiple of 8 * 64 with enough offset to fit each dlsch */ const size_t dlsch_size = rel15->rbSize * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * rel15->qamModOrder[0] * rel15->nrOfLayers; - dlsch_offset += ceil_mod(dlsch_size, 64); + dlsch_offset += ceil_mod(dlsch_size, 8 * 64); } gNB->nrLDPC_coding_interface.nrLDPC_coding_encoder(&slot_parameters); diff --git a/openair1/PHY/NR_UE_TRANSPORT/nr_ulsch_coding.c b/openair1/PHY/NR_UE_TRANSPORT/nr_ulsch_coding.c index dcf18ceff5..2eeb27d457 100644 --- a/openair1/PHY/NR_UE_TRANSPORT/nr_ulsch_coding.c +++ b/openair1/PHY/NR_UE_TRANSPORT/nr_ulsch_coding.c @@ -159,8 +159,8 @@ int nr_ulsch_encoding(PHY_VARS_NR_UE *ue, const nfapi_nr_ue_pusch_pdu_t *pusch_pdu = &ulsch->pusch_pdu; uint16_t nb_rb = pusch_pdu->rb_size; memset(harq_process->f, 0, 14 * nb_rb * 12 * 16); + TB_parameters->output = harq_process->f; - int r_offset = 0; for (int r = 0; r < TB_parameters->C; r++) { nrLDPC_segment_encoding_parameters_t *segment_parameters = &TB_parameters->segments[r]; segment_parameters->c = harq_process->c[r]; @@ -169,8 +169,6 @@ int nr_ulsch_encoding(PHY_VARS_NR_UE *ue, TB_parameters->Qm, TB_parameters->nb_layers, r); - segment_parameters->output = harq_process->f + r_offset; - r_offset += segment_parameters->E; reset_meas(&segment_parameters->ts_interleave); reset_meas(&segment_parameters->ts_rate_match); From 06c81030dc560ee47c9f4660203d6261655b1f48 Mon Sep 17 00:00:00 2001 From: Romain Beurdouche Date: Tue, 8 Apr 2025 12:42:21 +0000 Subject: [PATCH 09/17] fix(nrLDPC_coding_segment_encoder): address review 1. nrLDPC_coding_segment_encoding: Clarifying encoder parameters The way the encoder implementation params was used could be made clearer: * `macro_num` was always used in order to calculate segment indexes. It is clearer to pass directly the first segment index. * The initialisation of a common implementation parameters structure can be made clearer. 2. nrLDPC_coding_segment_encoder: Remove multi segment dimension in encoder output The output of `LDPCencoder` was a 2D unsigned char array. But all the segments were actually packed in the first segment, 8 bits of different segments per byte of the first segment. So the first dimension was useless. This commit removes this dimension. 3. nrLDPC_coding_segment_encoder: Rename `nrLDPC_prepare_TB_encoding` into `nrLDPC_launch_TB_encoding` 4. nrLDPC_coding_segment_encoder: Remove the useless second rate matching for E2, one rate matching with max(E, E2) does the job. 5. nr_interleaving_ldpc: cleanup Rewrite code to use different SIMD sizes in a linear fashion. This means cascading preprocessor directives are used to start from a high SIMD size an decrease it progressively. 6. ldpc_encoder_optim8segmulti: linear usage of SIMD Use cascading preprocessor directives to include different SIMD sizes and types depending on which are available on the system. The different loops use a common index that is progressively incremented. Also add AVX512 version of bits distribution from 8 segments into one byte. Co-authored-by: Laurent Thomas --- openair1/PHY/CODING/TESTBENCH/ldpctest.c | 30 +- .../nrLDPC_coding_segment_encoder.c | 78 +- .../nrLDPC_coding_segment/nr_rate_matching.c | 807 +++++++++--------- openair1/PHY/CODING/nrLDPC_defs.h | 6 +- .../PHY/CODING/nrLDPC_encoder/ldpc_encoder.c | 4 +- .../ldpc_encoder_optim8segmulti.c | 142 +-- 6 files changed, 515 insertions(+), 552 deletions(-) diff --git a/openair1/PHY/CODING/TESTBENCH/ldpctest.c b/openair1/PHY/CODING/TESTBENCH/ldpctest.c index ee33c7f91a..46534c37e3 100644 --- a/openair1/PHY/CODING/TESTBENCH/ldpctest.c +++ b/openair1/PHY/CODING/TESTBENCH/ldpctest.c @@ -123,7 +123,7 @@ one_measurement_t test_ldpc(short max_iterations, uint8_t estimated_output[MAX_NUM_DLSCH_SEGMENTS][Kprime]; memset(estimated_output, 0, sizeof(estimated_output)); uint8_t *channel_input[MAX_NUM_DLSCH_SEGMENTS]; - uint8_t *channel_input_optim[MAX_NUM_DLSCH_SEGMENTS]; + uint8_t *channel_input_optim; // double channel_output[68 * 384]; double modulated_input[MAX_NUM_DLSCH_SEGMENTS][68 * 384] = {0}; int8_t channel_output_fixed[MAX_NUM_DLSCH_SEGMENTS][68 * 384] = {0}; @@ -265,9 +265,9 @@ one_measurement_t test_ldpc(short max_iterations, memset(test_input[j], 0, ((K + 7) & ~7) / 8); channel_input[j] = malloc16(68 * 384); memset(channel_input[j], 0, 68 * 384); - channel_input_optim[j] = malloc16(68 * 384); - memset(channel_input_optim[j], 0, 68 * 384); } + channel_input_optim = malloc16(68 * 384); + memset(channel_input_optim, 0, 68 * 384); // Fill input segments with random values for (int j = 0; j < MAX_NUM_DLSCH_SEGMENTS; j++) { @@ -285,7 +285,7 @@ one_measurement_t test_ldpc(short max_iterations, impp.gen_code = 1; if (ntrials == 0) - ldpc_orig.LDPCencoder(test_input, channel_input, &impp); + ldpc_orig.LDPCencoder(test_input, channel_input[0], &impp); impp.gen_code = 0; decode_abort_t dec_abort; init_abort(&dec_abort); @@ -294,23 +294,21 @@ one_measurement_t test_ldpc(short max_iterations, //// encoder start_meas(&time); for (int j = 0; j < n_segments; j++) { - ldpc_orig.LDPCencoder(&test_input[j], &channel_input[j], &impp); + ldpc_orig.LDPCencoder(&test_input[j], channel_input[j], &impp); } stop_meas(&time); impp.n_segments = n_segments; - for (int j = 0; j < (n_segments / 8 + 1); j++) { - start_meas(&ret.time_optim); - impp.macro_num = j; - ldpc_toCompare.LDPCencoder(test_input, channel_input_optim, &impp); - stop_meas(&ret.time_optim); - } + start_meas(&ret.time_optim); + impp.first_seg = 0; + ldpc_toCompare.LDPCencoder(test_input, channel_input_optim, &impp); + stop_meas(&ret.time_optim); if (ntrials == 1) for (int j = 0; j < n_segments; j++) for (int i = 0; i < K + (nrows - no_punctured_columns) * Zc - removed_bit; i++) { - if (channel_input[j][i] != channel_input_optim[j][i]) { - printf("differ in seg %d pos %d (%u,%u)\n", j, i, channel_input[j][i], channel_input_optim[j][i]); + if (channel_input[j][i] != ((channel_input_optim[i] >> j) & 0x1)) { + printf("differ in seg %d pos %d (%u,%u)\n", j, i, channel_input[j][i], (channel_input_optim[i] >> j) & 0x1); return ret; } } @@ -322,7 +320,7 @@ one_measurement_t test_ldpc(short max_iterations, printf("\ne %d..%d: ", i, i + 15); #endif - if (channel_input_optim[j][i - 2 * Zc] == 0) + if (((channel_input_optim[i - 2 * Zc] >> j) & 0x1) == 0) modulated_input[j][i] = 1.0; /// sqrt(2); //QPSK else modulated_input[j][i] = -1.0; /// sqrt(2); @@ -332,7 +330,7 @@ one_measurement_t test_ldpc(short max_iterations, // Uncoded BER uint8_t channel_output_uncoded = channel_output_fixed[j][i] < 0 ? 1 /* QPSK demod */ : 0; - if (channel_output_uncoded != channel_input_optim[j][i - 2 * Zc]) + if (channel_output_uncoded != ((channel_input_optim[i - 2 * Zc] >> j) & 0x1)) ret.errors_bit_uncoded++; } @@ -396,8 +394,8 @@ one_measurement_t test_ldpc(short max_iterations, for (int j = 0; j < MAX_NUM_DLSCH_SEGMENTS; j++) { free(test_input[j]); free(channel_input[j]); - free(channel_input_optim[j]); } + free(channel_input_optim); print_meas(&time, "ldpc_encoder", NULL, NULL); print_meas(&ret.time_optim, "ldpc_encoder_optim", NULL, NULL); diff --git a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c index 5e44eaf2ae..554b399f4b 100644 --- a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c +++ b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c @@ -65,19 +65,17 @@ static void ldpc8blocks(void *p) // nrLDPC_encoder output is in "d" // let's make this interface happy! uint8_t d[68 * 384] __attribute__((aligned(64))); - uint8_t *dp[2]; - dp[0] = &d[0]; uint8_t *c[nrLDPC_TB_encoding_parameters->C]; unsigned int macro_segment, macro_segment_end; - macro_segment = 8*impp->macro_num; - macro_segment_end = (impp->n_segments > 8*(impp->macro_num+1)) ? 8*(impp->macro_num+1) : impp->n_segments; + macro_segment = impp->first_seg; + macro_segment_end = (impp->n_segments > impp->first_seg + 8) ? impp->first_seg + 8 : impp->n_segments; for (int r = 0; r < nrLDPC_TB_encoding_parameters->C; r++) c[r] = nrLDPC_TB_encoding_parameters->segments[r].c; - start_meas(&nrLDPC_TB_encoding_parameters->segments[impp->macro_num * 8].ts_ldpc_encode); - LDPCencoder(c, dp, impp); - stop_meas(&nrLDPC_TB_encoding_parameters->segments[impp->macro_num * 8].ts_ldpc_encode); + start_meas(&nrLDPC_TB_encoding_parameters->segments[impp->first_seg].ts_ldpc_encode); + LDPCencoder(c, d, impp); + stop_meas(&nrLDPC_TB_encoding_parameters->segments[impp->first_seg].ts_ldpc_encode); // Compute where to place in output buffer that is concatenation of all segments #ifdef DEBUG_LDPC_ENCODING @@ -86,11 +84,14 @@ static void ldpc8blocks(void *p) const uint32_t E = nrLDPC_TB_encoding_parameters->segments[macro_segment].E; uint32_t E2=E,E2_first_segment=macro_segment_end-macro_segment; bool Eshift=false; + uint32_t Emax = E; for (int s=macro_segment;ssegments[s].E != E) { E2=nrLDPC_TB_encoding_parameters->segments[s].E; Eshift=true; E2_first_segment = s-macro_segment; + if(E2 > Emax) + Emax = E2; break; } @@ -124,14 +125,12 @@ static void ldpc8blocks(void *p) */ uint32_t Tbslbrm = nrLDPC_TB_encoding_parameters->tbslbrm; - uint8_t e[E] __attribute__((aligned(64))); + uint8_t e[Emax] __attribute__((aligned(64))); uint8_t f[E+64] __attribute__((aligned(64))); - uint8_t e2[E2] __attribute__((aligned(64))); uint8_t f2[E2+64] __attribute__((aligned(64))); - bzero(e, E); + bzero(e, Emax); bzero(f, E + 64); if (Eshift) { - bzero(e2, E2); bzero(f2, E2 + 64); } start_meas(&nrLDPC_TB_encoding_parameters->segments[macro_segment].ts_rate_match); @@ -144,18 +143,7 @@ static void ldpc8blocks(void *p) impp->F, impp->K - impp->F - 2 * impp->Zc, nrLDPC_TB_encoding_parameters->rv_index, - E); - if (Eshift) - nr_rate_matching_ldpc(Tbslbrm, - impp->BG, - impp->Zc, - d, - e2, - impp->n_segments, - impp->F, - impp->K - impp->F - 2 * impp->Zc, - nrLDPC_TB_encoding_parameters->rv_index, - E2); + Emax); stop_meas(&nrLDPC_TB_encoding_parameters->segments[macro_segment].ts_rate_match); if (impp->K - impp->F - 2 * impp->Zc > E) { @@ -189,7 +177,7 @@ static void ldpc8blocks(void *p) if (Eshift) nr_interleaving_ldpc(E2, mod_order, - e2, + e, f2); stop_meas(&nrLDPC_TB_encoding_parameters->segments[macro_segment].ts_interleave); if(impp->toutput != NULL) start_meas(impp->toutput); @@ -504,42 +492,42 @@ static void ldpc8blocks(void *p) completed_task_ans(impp->ans); } -static int nrLDPC_prepare_TB_encoding(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters, +static int nrLDPC_launch_TB_encoding(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters, int dlsch_id, thread_info_tm_t *t_info) { nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id]; - encoder_implemparams_t impp = {0}; + encoder_implemparams_t common_segment_params = { + .n_segments = nrLDPC_TB_encoding_parameters->C, + .tinput = nrLDPC_slot_encoding_parameters->tinput, + .tprep = nrLDPC_slot_encoding_parameters->tprep, + .tparity = nrLDPC_slot_encoding_parameters->tparity, + .toutput = nrLDPC_slot_encoding_parameters->toutput, + .Kb = nrLDPC_TB_encoding_parameters->Kb, + .Zc = nrLDPC_TB_encoding_parameters->Z, + .BG = nrLDPC_TB_encoding_parameters->BG, + .output = nrLDPC_TB_encoding_parameters->output, + .K = nrLDPC_TB_encoding_parameters->K, + .F = nrLDPC_TB_encoding_parameters->F, + }; - impp.n_segments = nrLDPC_TB_encoding_parameters->C; - impp.tinput = nrLDPC_slot_encoding_parameters->tinput; - impp.tprep = nrLDPC_slot_encoding_parameters->tprep; - impp.tparity = nrLDPC_slot_encoding_parameters->tparity; - impp.toutput = nrLDPC_slot_encoding_parameters->toutput; - impp.Kb = nrLDPC_TB_encoding_parameters->Kb; - impp.Zc = nrLDPC_TB_encoding_parameters->Z; - impp.BG = nrLDPC_TB_encoding_parameters->BG; - impp.output = nrLDPC_TB_encoding_parameters->output; - impp.K = nrLDPC_TB_encoding_parameters->K; - impp.F = nrLDPC_TB_encoding_parameters->F; + const size_t n_tasks = (common_segment_params.n_segments + 7) >> 3; - size_t const n_seg = (impp.n_segments / 8 + ((impp.n_segments & 7) == 0 ? 0 : 1)); - - for (int j = 0; j < n_seg; j++) { + for (int j = 0; j < n_tasks; j++) { ldpc8blocks_args_t *perJobImpp = &((ldpc8blocks_args_t *)t_info->buf)[t_info->len]; DevAssert(t_info->len < t_info->cap); - impp.ans = t_info->ans; t_info->len += 1; - impp.macro_num = j; - perJobImpp->impp = impp; + perJobImpp->impp = common_segment_params; + perJobImpp->impp.first_seg = j * 8; + perJobImpp->impp.ans = t_info->ans; perJobImpp->nrLDPC_TB_encoding_parameters = nrLDPC_TB_encoding_parameters; task_t t = {.func = ldpc8blocks, .args = perJobImpp}; pushTpool(nrLDPC_slot_encoding_parameters->threadPool, t); } - return n_seg; + return n_tasks; } int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters) @@ -558,7 +546,7 @@ int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encodin int nbEncode = 0; for (int dlsch_id = 0; dlsch_id < nrLDPC_slot_encoding_parameters->nb_TBs; dlsch_id++) { - nbEncode += nrLDPC_prepare_TB_encoding(nrLDPC_slot_encoding_parameters, dlsch_id, &t_info); + nbEncode += nrLDPC_launch_TB_encoding(nrLDPC_slot_encoding_parameters, dlsch_id, &t_info); } if (nbEncode < nbTasks) { completed_many_task_ans(&ans, nbTasks - nbEncode); diff --git a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.c b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.c index 4e5a31f31f..4a507fa381 100644 --- a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.c +++ b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.c @@ -29,446 +29,419 @@ static const uint8_t index_k0[2][4] = {{0, 17, 33, 56}, {0, 13, 25, 43}}; void nr_interleaving_ldpc(uint32_t E, uint8_t Qm, uint8_t *e, uint8_t *f) { - uint32_t EQm; - - EQm = E / Qm; + const uint32_t EQm = E / Qm; memset(f, 0, E * sizeof(uint8_t)); - uint8_t *e0, *e1, *e2, *e3, *e4, *e5, *e6, *e7; - uint8_t *fp; - int j,k; -#if 1 //def __WASAVX2__ -#ifdef __AVX512VBMI__ - __m512i tmp0,tmp1,tmp2,tmp3,tmp4,tmp5; - __m512i *e0_512,*e1_512,*e2_512,*e3_512,*e4_512,*e5_512,*e6_512,*e7_512; - - __m512i *f_512=(__m512i *)f; -#else - simde__m128i tmp0,tmp1,tmp2,tmp3,tmp4,tmp5,tmp6,tmp7; - simde__m128i *e0_128,*e1_128,*e2_128,*e3_128,*e4_128,*e5_128,*e6_128,*e7_128; - - simde__m128i *f_128=(simde__m128i *)f; -#endif switch(Qm) { - case 2: - e0=e; - e1=e0+EQm; + case 2: { + uint8_t *e0 = e; + uint8_t *e1 = e0 + EQm; + int i = 0; #ifdef __AVX512VBMI__ - e0_512=(__m512i *)e0; - e1_512=(__m512i *)e1; - __m512i p8a = _mm512_set_epi8(95,31,94,30,93,29,92,28,91,27,90,26,89,25,88,24,87,23,86,22,85,21,84,20,83,19,82,18,81,17,80,16,79,15,78,14,77,13,76,12,75,11,74,10,73,9,72,8,71,7,70,6,69,5,68,4,67,3,66,2,65,1,64,0); - __m512i p8b = _mm512_set_epi8(127,63,126,62,125,61,124,60,123,59,122,58,121,57,120,56,119,55,118,54,117,53,116,52,115,51,114,50,113,49,112,48,111,47,110,46,109,45,108,44,107,43,106,42,105,41,104,40,103,39,102,38,101,37,100,36,99,35,98,34,97,33,96,32); - - __m512i e0j,e1j; - for (k=0,j=0;j>6;j++,k+=2) { - e0j = _mm512_loadu_si512(e0_512+j); - e1j = _mm512_loadu_si512(e1_512+j); - _mm512_storeu_si512(f_512+k,_mm512_permutex2var_epi8(e0j,p8a,e1j)); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15) - _mm512_storeu_si512(f_512+k+1,_mm512_permutex2var_epi8(e0j,p8b,e1j)); - } - if ((j<<6) != EQm) { - int k2=k<<6; - fp = &f[k2]; - for (int j2=(j<<6) ; j2< EQm ; j2++) { - *fp++ = e0[j2]; - *fp++ = e1[j2]; + // clang-format off + const __m512i p8a = _mm512_set_epi8(95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, + 87, 23, 86, 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, + 79, 15, 78, 14, 77, 13, 76, 12, 75, 11, 74, 10, 73, 9, 72, 8, + 71, 7, 70, 6, 69, 5, 68, 4, 67, 3, 66, 2, 65, 1, 64, 0); + const __m512i p8b = _mm512_set_epi8(127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, + 56, 119, 55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, + 48, 111, 47, 110, 46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, + 40, 103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32); + // clang-format on + __m512i *f_512 = (__m512i *)f; + __m512i *e0_512 = (__m512i *)e0; + __m512i *e1_512 = (__m512i *)e1; + for (; i < (EQm & ~63); i += 64) { + __m512i e0j = _mm512_loadu_si512(e0_512++); + __m512i e1j = _mm512_loadu_si512(e1_512++); + // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi8(e0j, p8a, e1j)); + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi8(e0j, p8b, e1j)); } - } -#else - e0_128=(simde__m128i *)e0; - e1_128=(simde__m128i *)e1; - simde__m128i e0j,e1j; - for (k=0,j=0;j>4;j++,k+=2) { - e0j=simde_mm_loadu_si128(e0_128+j); - e1j=simde_mm_loadu_si128(e1_128+j); - simde_mm_storeu_si128(f_128+k,simde_mm_unpacklo_epi8(e0j,e1j)); - simde_mm_storeu_si128(f_128+k+1,simde_mm_unpackhi_epi8(e0j,e1j)); - } - if ((j<<4) != EQm) { - int k2=k<<4; - fp = &f[k2]; - for (int j2=(j<<4) ; j2< EQm ; j2++) { - *fp++ = e0[j2]; - *fp++ = e1[j2]; - } - } + e0 = (uint8_t *)e0_512; + e1 = (uint8_t *)e1_512; + f = (uint8_t *)f_512; #endif - break; - case 4: - e0=e; - e1=e0+EQm; - e2=e1+EQm; - e3=e2+EQm; +#ifdef USE128BIT + simde__m128i *f_128 = (simde__m128i *)f; + simde__m128i *e0_128 = (simde__m128i *)e0; + simde__m128i *e1_128 = (simde__m128i *)e1; + for (; i < (EQm & ~15); i += 64) { + simde__m128i e0j = simde_mm_loadu_si128(e0_128++); + simde__m128i e1j = simde_mm_loadu_si128(e1_128++); + simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi8(e0j, e1j)); + simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi8(e0j, e1j)); + } + e0 = (uint8_t *)e0_128; + e1 = (uint8_t *)e1_128; + f = (uint8_t *)f_128; +#endif + for (; i < EQm; i++) { + *f++ = *e0++; + *f++ = *e1++; + } + } break; + case 4: { + uint8_t *e0 = e; + uint8_t *e1 = e0 + EQm; + uint8_t *e2 = e1 + EQm; + uint8_t *e3 = e2 + EQm; + int i = 0; #ifdef __AVX512VBMI__ - e0_512=(__m512i *)e0; - e1_512=(__m512i *)e1; - e2_512=(__m512i *)e2; - e3_512=(__m512i *)e3; - p8a = _mm512_set_epi8(95,31,94,30,93,29,92,28,91,27,90,26,89,25,88,24,87,23,86,22,85,21,84,20,83,19,82,18,81,17,80,16,79,15,78,14,77,13,76,12,75,11,74,10,73,9,72,8,71,7,70,6,69,5,68,4,67,3,66,2,65,1,64,0); - p8b = _mm512_set_epi8(127,63,126,62,125,61,124,60,123,59,122,58,121,57,120,56,119,55,118,54,117,53,116,52,115,51,114,50,113,49,112,48,111,47,110,46,109,45,108,44,107,43,106,42,105,41,104,40,103,39,102,38,101,37,100,36,99,35,98,34,97,33,96,32); - - __m512i p16a = _mm512_set_epi16(47,15,46,14,45,13,44,12,43,11,42,10,41,9,40,8,39,7,38,6,37,5,36,4,35,3,34,2,33,1,32,0); - __m512i p16b = _mm512_set_epi16(63,31,62,30,61,29,60,28,59,27,58,26,57,25,56,24,55,23,54,22,53,21,52,20,51,19,50,18,49,17,48,16); - - __m512i p32a = _mm512_set_epi32(23,7,22,6,21,5,20,4,19,3,18,2,17,1,16,0); - __m512i p32b = _mm512_set_epi32(31,15,30,14,29,13,28,12,27,11,26,10,25,9,24,8); - __m512i e2j,e3j; - for (k=0,j=0;j>6;j++,k+=4) { - e0j = _mm512_loadu_si512(e0_512+j); - e1j = _mm512_loadu_si512(e1_512+j); - e2j = _mm512_loadu_si512(e2_512+j); - e3j = _mm512_loadu_si512(e3_512+j); - tmp0 = _mm512_permutex2var_epi8(e0j,p8a,e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15) - tmp1 = _mm512_permutex2var_epi8(e2j,p8a,e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15) - _mm512_storeu_si512(f_512+k,_mm512_permutex2var_epi16(tmp0,p16a,tmp1)); // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) - _mm512_storeu_si512(f_512+k+1,_mm512_permutex2var_epi16(tmp0,p16b,tmp1)); - - tmp0 = _mm512_permutex2var_epi8(e0j,p8b,e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15) - tmp1 = _mm512_permutex2var_epi8(e2j,p8b,e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15) - _mm512_storeu_si512(f_512+k+2,_mm512_permutex2var_epi16(tmp0,p16a,tmp1)); // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) - _mm512_storeu_si512(f_512+k+3,_mm512_permutex2var_epi16(tmp0,p16b,tmp1)); - } - if ((j<<6) != EQm) { - int k2=k<<6; - fp = &f[k2]; - for (int j2=(j<<6) ; j2< EQm ; j2++) { - *fp++ = e0[j2]; - *fp++ = e1[j2]; - *fp++ = e2[j2]; - *fp++ = e3[j2]; + // clang-format off + const __m512i p8a = _mm512_set_epi8(95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, 87, 23, 86, + 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, 79, 15, 78, 14, 77, 13, + 76, 12, 75, 11, 74, 10, 73, 9, 72, 8, 71, 7, 70, 6, 69, 5, 68, 4, 67, 3, 66, 2, 65, 1, 64, 0); + const __m512i p8b = _mm512_set_epi8(127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56, 119, + 55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48, 111, 47, + 110, 46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40, 103, 39, 102, + 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32); + const __m512i p16a = _mm512_set_epi16(47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7, 38, 6, + 37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0); + const __m512i p16b = _mm512_set_epi16(63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23, 54, + 22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16); + // clang-format on + __m512i *e0_512 = (__m512i *)e0; + __m512i *e1_512 = (__m512i *)e1; + __m512i *e2_512 = (__m512i *)e2; + __m512i *e3_512 = (__m512i *)e3; + __m512i *f_512 = (__m512i *)f; + for (; i < (EQm & ~63); i += 64) { + __m512i e0j = _mm512_loadu_si512(e0_512++); + __m512i e1j = _mm512_loadu_si512(e1_512++); + __m512i e2j = _mm512_loadu_si512(e2_512++); + __m512i e3j = _mm512_loadu_si512(e3_512++); + __m512i tmp0 = _mm512_permutex2var_epi8(e0j, p8a, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15) + __m512i tmp1 = _mm512_permutex2var_epi8(e2j, p8a, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15) + // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp0, p16a, tmp1)); + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp0, p16b, tmp1)); + tmp0 = _mm512_permutex2var_epi8(e0j, p8b, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15) + tmp1 = _mm512_permutex2var_epi8(e2j, p8b, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15) + // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp0, p16a, tmp1)); + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp0, p16b, tmp1)); } - } -#else - e0_128=(simde__m128i *)e0; - e1_128=(simde__m128i *)e1; - e2_128=(simde__m128i *)e2; - e3_128=(simde__m128i *)e3; - simde__m128i e2j,e3j; - for (k=0,j=0;j>4;j++,k+=4) { - e0j = simde_mm_loadu_si128(e0_128+j); - e1j = simde_mm_loadu_si128(e1_128+j); - e2j = simde_mm_loadu_si128(e2_128+j); - e3j = simde_mm_loadu_si128(e3_128+j); - tmp0 = simde_mm_unpacklo_epi8(e0j,e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+7) e1(i+7) - tmp1 = simde_mm_unpacklo_epi8(e2j,e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+7) e3(i+7) - simde_mm_storeu_si128(f_128+k,simde_mm_unpacklo_epi16(tmp0,tmp1)); // e0(i) e1(i) e2(i) e3(i) ... e0(i+3) e1(i+3) e2(i+3) e3(i+3) - simde_mm_storeu_si128(f_128+k+1,simde_mm_unpackhi_epi16(tmp0,tmp1)); // e0(i+4) e1(i+4) e2(i+4) e3(i+4) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) - tmp0 = simde_mm_unpackhi_epi8(e0j,e1j); // e0(i+8) e1(i+8) e0(i+9) e1(i+9) .... e0(i+15) e1(i+15) - tmp1 = simde_mm_unpackhi_epi8(e2j,e3j); // e2(i+8) e3(i+9) e2(i+10) e3(i+10) .... e2(i+31) e3(i+31) - simde_mm_storeu_si128(f_128+k+2,simde_mm_unpacklo_epi16(tmp0,tmp1)); - simde_mm_storeu_si128(f_128+k+3,simde_mm_unpackhi_epi16(tmp0,tmp1)); - } - if ((j<<4) != EQm) { - int k2=k<<4; - fp = &f[k2]; - for (int j2=(j<<4) ; j2< EQm ; j2++) { - *fp++ = e0[j2]; - *fp++ = e1[j2]; - *fp++ = e2[j2]; - *fp++ = e3[j2]; - } - } + e0 = (uint8_t *)e0_512; + e1 = (uint8_t *)e1_512; + e2 = (uint8_t *)e2_512; + e3 = (uint8_t *)e3_512; + f = (uint8_t *)f_512; #endif - break; - case 6: - e0=e; - e1=e0+EQm; - e2=e1+EQm; - e3=e2+EQm; - e4=e3+EQm; - e5=e4+EQm; +#ifdef USE128BIT + simde__m128i *e0_128 = (simde__m128i *)e0; + simde__m128i *e1_128 = (simde__m128i *)e1; + simde__m128i *e2_128 = (simde__m128i *)e2; + simde__m128i *e3_128 = (simde__m128i *)e3; + simde__m128i *f_128 = (simde__m128i *)f; + for (; i < (EQm & ~15); i += 16) { + simde__m128i e0j = simde_mm_loadu_si128(e0_128++); + simde__m128i e1j = simde_mm_loadu_si128(e1_128++); + simde__m128i e2j = simde_mm_loadu_si128(e2_128++); + simde__m128i e3j = simde_mm_loadu_si128(e3_128++); + simde__m128i tmp0 = simde_mm_unpacklo_epi8(e0j, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+7) e1(i+7) + simde__m128i tmp1 = simde_mm_unpacklo_epi8(e2j, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+7) e3(i+7) + simde_mm_storeu_si128(f_128++, + simde_mm_unpacklo_epi16(tmp0, tmp1)); // e0(i) e1(i) e2(i) e3(i) ... e0(i+3) e1(i+3) e2(i+3) e3(i+3) + simde_mm_storeu_si128( + f_128++, + simde_mm_unpackhi_epi16(tmp0, tmp1)); // e0(i+4) e1(i+4) e2(i+4) e3(i+4) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) + tmp0 = simde_mm_unpackhi_epi8(e0j, e1j); // e0(i+8) e1(i+8) e0(i+9) e1(i+9) .... e0(i+15) e1(i+15) + tmp1 = simde_mm_unpackhi_epi8(e2j, e3j); // e2(i+8) e3(i+9) e2(i+10) e3(i+10) .... e2(i+31) e3(i+31) + simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi16(tmp0, tmp1)); + simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi16(tmp0, tmp1)); + } + e0 = (uint8_t *)e0_128; + e1 = (uint8_t *)e1_128; + e2 = (uint8_t *)e2_128; + e3 = (uint8_t *)e3_128; + f = (uint8_t *)f_128; +#endif + for (; i < EQm; i++) { + *f++ = *e0++; + *f++ = *e1++; + *f++ = *e2++; + *f++ = *e3++; + } + } break; + case 6: { + uint8_t *e0 = e; + uint8_t *e1 = e0 + EQm; + uint8_t *e2 = e1 + EQm; + uint8_t *e3 = e2 + EQm; + uint8_t *e4 = e3 + EQm; + uint8_t *e5 = e4 + EQm; + int i = 0; #ifdef __AVX512VBMI__ - e0_512=(__m512i *)e0; - e1_512=(__m512i *)e1; - e2_512=(__m512i *)e2; - e3_512=(__m512i *)e3; - e4_512=(__m512i *)e4; - e5_512=(__m512i *)e5; - __m512i e4j,e5j; - p8a = _mm512_set_epi8(95,31,94,30,93,29,92,28,91,27,90,26,89,25,88,24,87,23,86,22,85,21,84,20,83,19,82,18,81,17,80,16,79,15,78,14,77,13,76,12,75,11,74,10,73,9,72,8,71,7,70,6,69,5,68,4,67,3,66,2,65,1,64,0); - p8b = _mm512_set_epi8(127,63,126,62,125,61,124,60,123,59,122,58,121,57,120,56,119,55,118,54,117,53,116,52,115,51,114,50,113,49,112,48,111,47,110,46,109,45,108,44,107,43,106,42,105,41,104,40,103,39,102,38,101,37,100,36,99,35,98,34,97,33,96,32); + // clang-format off + const __m512i p8a = _mm512_set_epi8(95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, 87, 23, + 86, 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, 79, 15, 78, 14, + 77, 13, 76, 12, 75, 11, 74, 10, 73, 9, 72, 8, 71, 7, 70, 6, 69, 5, 68, 4, 67, 3, 66, 2, 65, 1, 64, 0); + const __m512i p8b = _mm512_set_epi8(127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56, 119, + 55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48, 111, 47, 110, + 46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40, 103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32); + const __m512i p16a = _mm512_set_epi16(47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7, 38, 6, 37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0); + const __m512i p16b = _mm512_set_epi16(63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23, 54, 22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16); + const __m512i p16c = _mm512_set_epi16(21, 20, 41, 19, 18, 40, 17, 16, 39, 15, 14, 38, 13, 12, 37, 11, 10, 36, 9, 8, 35, 7, 6, 34, 5, 4, 33, 3, 2, 32, 1, 0); + const __m512i p16d = _mm512_set_epi16(10, 52, 9, 8, 51, 7, 6, 50, 5, 4, 49, 3, 2, 48, 1, 0, 47, 31, 30, 46, 29, 28, 45, 27, 26, 44, 25, 24, 43, 23, 22, 42); + const __m512i p16d2 = _mm512_set_epi16(32 + 10, 30, 32 + 9, 32 + 8, 27, 32 + 7, 32 + 6, 24, 32 + 5, 32 + 4, 21, 32 + 3, 32 + 2, 18, 32 + 1, 32 + 0, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + const __m512i p16e = _mm512_set_epi16(63, 31, 30, 62, 29, 28, 61, 27, 26, 60, 25, 24, 59, 23, 22, 58, 21, 20, 57, 19, 18, 56, 17, 16, 55, 15, 14, 54, 13, 12, 53, 11); + // clang-format on + __m512i *e0_512 = (__m512i *)e0; + __m512i *e1_512 = (__m512i *)e1; + __m512i *e2_512 = (__m512i *)e2; + __m512i *e3_512 = (__m512i *)e3; + __m512i *e4_512 = (__m512i *)e4; + __m512i *e5_512 = (__m512i *)e5; + __m512i *f_512 = (__m512i *)f; + for (; i < (EQm & ~63); i += 64) { + __m512i e0j = _mm512_loadu_si512(e0_512++); + __m512i e1j = _mm512_loadu_si512(e1_512++); + __m512i e2j = _mm512_loadu_si512(e2_512++); + __m512i e3j = _mm512_loadu_si512(e3_512++); + __m512i e4j = _mm512_loadu_si512(e4_512++); + __m512i e5j = _mm512_loadu_si512(e5_512++); + __m512i tmp0 = _mm512_permutex2var_epi8(e0j, p8a, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+31) e1(i+31) + __m512i tmp1 = _mm512_permutex2var_epi8(e2j, p8a, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+31) e3(i+31) + __m512i tmp2 = _mm512_permutex2var_epi8(e4j, p8a, e5j); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+31) e5(i+31) + // e0(i) e1(i) e2(i) e3(i) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) + __m512i tmp3 = _mm512_permutex2var_epi16(tmp0, p16a, tmp1); + // e0(i+16) e1(i+16) e2(i+16) e3(i+16) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31) + __m512i tmp4 = _mm512_permutex2var_epi16(tmp0, p16b, tmp1); + // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) ... e0(i+9) e1(i+9) e2(i+9) e3(i+9) e4(i+9) e5(i+9) e0(i+10) e1(i+10) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp3, p16c, tmp2)); + // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) e4(i+15) + // e5(i+15) x x x x e4(i+16) e5(i+16) x x x x .... e4(i+20) e5(i+20) x x + __m512i tmp5 = _mm512_permutex2var_epi16(tmp3, p16d, tmp2); + // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) + // e4(i+15) e5(i+15) e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16) e0(i+20) + // e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e0(i+21) e1(i+21) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp5, p16d2, tmp4)); + // e2(i+21) e3(i+21) e4(i+21) e5(i+21) .... e0(i+31) e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp4, p16e, tmp2)); - p16a = _mm512_set_epi16(47,15,46,14,45,13,44,12,43,11,42,10,41,9,40,8,39,7,38,6,37,5,36,4,35,3,34,2,33,1,32,0); - p16b = _mm512_set_epi16(63,31,62,30,61,29,60,28,59,27,58,26,57,25,56,24,55,23,54,22,53,21,52,20,51,19,50,18,49,17,48,16); - - __m512i p16c = _mm512_set_epi16(21,20,41,19,18,40,17,16,39,15,14,38,13,12,37,11,10,36,9,8,35,7,6,34,5,4,33,3,2,32,1,0); - __m512i p16d = _mm512_set_epi16(10,52,9,8,51,7,6,50,5,4,49,3,2,48,1,0,47,31,30,46,29,28,45,27,26,44,25,24,43,23,22,42); - __m512i p16d2 = _mm512_set_epi16(32+10,30,32+9,32+8,27,32+7,32+6,24,32+5,32+4,21,32+3,32+2,18,32+1,32+0,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0); - __m512i p16e = _mm512_set_epi16(63,31,30,62,29,28,61,27,26,60,25,24,59,23,22,58,21,20,57,19,18,56,17,16,55,15,14,54,13,12,53,11); - - for (j=0,k=0;j>6;j++,k+=6) { - e0j = _mm512_loadu_si512(e0_512+j); - e1j = _mm512_loadu_si512(e1_512+j); - e2j = _mm512_loadu_si512(e2_512+j); - e3j = _mm512_loadu_si512(e3_512+j); - e4j = _mm512_loadu_si512(e4_512+j); - e5j = _mm512_loadu_si512(e5_512+j); - tmp0 = _mm512_permutex2var_epi8(e0j,p8a,e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+31) e1(i+31) - tmp1 = _mm512_permutex2var_epi8(e2j,p8a,e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+31) e3(i+31) - tmp2 = _mm512_permutex2var_epi8(e4j,p8a,e5j); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+31) e5(i+31) - tmp3 = _mm512_permutex2var_epi16(tmp0,p16a,tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) - tmp4 = _mm512_permutex2var_epi16(tmp0,p16b,tmp1); // e0(i+16) e1(i+16) e2(i+16) e3(i+16) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31) - _mm512_storeu_si512(f_512+k,_mm512_permutex2var_epi16(tmp3,p16c,tmp2)); // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) ... e0(i+9) e1(i+9) e2(i+9) e3(i+9) e4(i+9) e5(i+9) e0(i+10) e1(i+10) - tmp5 = _mm512_permutex2var_epi16(tmp3,p16d,tmp2); // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) e4(i+15) e5(i+15) x x x x e4(i+16) e5(i+16) x x x x .... e4(i+20) e5(i+20) x x - _mm512_storeu_si512(f_512+k+1,_mm512_permutex2var_epi16(tmp5,p16d2,tmp4)); // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) e4(i+15) e5(i+15) e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16) e0(i+20) e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e0(i+21) e1(i+21) - _mm512_storeu_si512(f_512+k+2,_mm512_permutex2var_epi16(tmp4,p16e,tmp2)); // e2(i+21) e3(i+21) e4(i+21) e5(i+21) .... e0(i+31) e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31) - - tmp0 = _mm512_permutex2var_epi8(e0j,p8b,e1j); // e0(i+32) e1(i+32) e0(i+32) e1(i+32) .... e0(i+63) e1(i+63) - tmp1 = _mm512_permutex2var_epi8(e2j,p8b,e3j); // e2(i+32) e3(i+32) e2(i+32) e3(i+32) .... e2(i+63) e3(i+63) - tmp2 = _mm512_permutex2var_epi8(e4j,p8b,e5j); // e4(i+32) e5(i+32) e4(i+32) e5(i+32) .... e4(i+63) e5(i+63) - tmp3 = _mm512_permutex2var_epi16(tmp0,p16a,tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) - tmp4 = _mm512_permutex2var_epi16(tmp0,p16b,tmp1); // e0(i+16) e1(i+16) e2(i+16) e3(i+16) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31) - - _mm512_storeu_si512(f_512+k+3,_mm512_permutex2var_epi16(tmp3,p16c,tmp2)); // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) ... e0(i+9) e1(i+9) e2(i+9) e3(i+9) e4(i+9) e5(i+9) e0(i+10) e1(i+10) - tmp5 = _mm512_permutex2var_epi16(tmp3,p16d,tmp2); // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) e4(i+15) e5(i+15) x x x x e4(i+16) e5(i+16) x x x x .... e4(i+20) e5(i+20) x x - _mm512_storeu_si512(f_512+k+4,_mm512_permutex2var_epi16(tmp5,p16d2,tmp4)); // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) e4(i+15) e5(i+15) e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16) e0(i+20) e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e0(i+21) e1(i+21) - _mm512_storeu_si512(f_512+k+5,_mm512_permutex2var_epi16(tmp4,p16e,tmp2)); // e2(i+21) e3(i+21) e4(i+21) e5(i+21) .... e0(i+31) e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31) - } - if ((j<<6) != EQm) { - int k2=k<<6; - fp = &f[k2]; - for (int j2=(j<<6) ; j2< EQm ; j2++) { - *fp++ = e0[j2]; - *fp++ = e1[j2]; - *fp++ = e2[j2]; - *fp++ = e3[j2]; - *fp++ = e4[j2]; - *fp++ = e5[j2]; + tmp0 = _mm512_permutex2var_epi8(e0j, p8b, e1j); // e0(i+32) e1(i+32) e0(i+32) e1(i+32) .... e0(i+63) e1(i+63) + tmp1 = _mm512_permutex2var_epi8(e2j, p8b, e3j); // e2(i+32) e3(i+32) e2(i+32) e3(i+32) .... e2(i+63) e3(i+63) + tmp2 = _mm512_permutex2var_epi8(e4j, p8b, e5j); // e4(i+32) e5(i+32) e4(i+32) e5(i+32) .... e4(i+63) e5(i+63) + tmp3 = _mm512_permutex2var_epi16(tmp0, p16a, tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) + // e0(i+16) e1(i+16) e2(i+16) e3(i+16) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31) + tmp4 = _mm512_permutex2var_epi16(tmp0, p16b, tmp1); + // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) ... e0(i+9) e1(i+9) e2(i+9) e3(i+9) e4(i+9) e5(i+9) e0(i+10) e1(i+10) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp3, p16c, tmp2)); + // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) + // e4(i+15) e5(i+15) x x x x e4(i+16) e5(i+16) x x x x .... e4(i+20) e5(i+20) x x + tmp5 = _mm512_permutex2var_epi16(tmp3, p16d, tmp2); + // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) + // e4(i+15) e5(i+15) e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16) e0(i+20) + // e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e0(i+21) e1(i+21) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp5, p16d2, tmp4)); + // e2(i+21) e3(i+21) e4(i+21) e5(i+21) .... e0(i+31) e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp4, p16e, tmp2)); } - } -#else - fp = f; - for (int j = 0; j< EQm; j++){ - *fp++ = e0[j]; - *fp++ = e1[j]; - *fp++ = e2[j]; - *fp++ = e3[j]; - *fp++ = e4[j]; - *fp++ = e5[j]; - } - + e0 = (uint8_t *)e0_512; + e1 = (uint8_t *)e1_512; + e2 = (uint8_t *)e2_512; + e3 = (uint8_t *)e3_512; + e4 = (uint8_t *)e4_512; + e5 = (uint8_t *)e5_512; + f = (uint8_t *)f_512; #endif - break; - case 8: - e0=e; - e1=e0+EQm; - e2=e1+EQm; - e3=e2+EQm; - e4=e3+EQm; - e5=e4+EQm; - e6=e5+EQm; - e7=e6+EQm; - + for (; i < EQm; i++) { + *f++ = *e0++; + *f++ = *e1++; + *f++ = *e2++; + *f++ = *e3++; + *f++ = *e4++; + *f++ = *e5++; + } + } break; + case 8: { + uint8_t *e0 = e; + uint8_t *e1 = e0 + EQm; + uint8_t *e2 = e1 + EQm; + uint8_t *e3 = e2 + EQm; + uint8_t *e4 = e3 + EQm; + uint8_t *e5 = e4 + EQm; + uint8_t *e6 = e5 + EQm; + uint8_t *e7 = e6 + EQm; + + int i = 0; #ifdef __AVX512VBMI__ - e0_512=(__m512i *)e0; - e1_512=(__m512i *)e1; - e2_512=(__m512i *)e2; - e3_512=(__m512i *)e3; - e4_512=(__m512i *)e4; - e5_512=(__m512i *)e5; - e6_512=(__m512i *)e6; - e7_512=(__m512i *)e7; - __m512i e6j,e7j; - p8a = _mm512_set_epi8(95,31,94,30,93,29,92,28,91,27,90,26,89,25,88,24,87,23,86,22,85,21,84,20,83,19,82,18,81,17,80,16,79,15,78,14,77,13,76,12,75,11,74,10,73,9,72,8,71,7,70,6,69,5,68,4,67,3,66,2,65,1,64,0); - p8b = _mm512_set_epi8(127,63,126,62,125,61,124,60,123,59,122,58,121,57,120,56,119,55,118,54,117,53,116,52,115,51,114,50,113,49,112,48,111,47,110,46,109,45,108,44,107,43,106,42,105,41,104,40,103,39,102,38,101,37,100,36,99,35,98,34,97,33,96,32); + // clang-format off + const __m512i p8a = _mm512_set_epi8(95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, 87, 23, + 86, 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, 79, 15, 78, 14, 77, 13, + 76, 12, 75, 11, 74, 10, 73, 9, 72, 8, 71, 7, 70, 6, 69, 5, 68, 4, 67, 3, 66, 2, 65, 1, 64, 0); + const __m512i p8b = _mm512_set_epi8(127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56, 119, 55, + 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48, 111, 47, 110, 46, + 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40, 103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32); - p16a = _mm512_set_epi16(47,15,46,14,45,13,44,12,43,11,42,10,41,9,40,8,39,7,38,6,37,5,36,4,35,3,34,2,33,1,32,0); - p16b = _mm512_set_epi16(63,31,62,30,61,29,60,28,59,27,58,26,57,25,56,24,55,23,54,22,53,21,52,20,51,19,50,18,49,17,48,16); + const __m512i p16a = _mm512_set_epi16(47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7, 38, 6, 37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0); + const __m512i p16b = _mm512_set_epi16(63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23, 54, 22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16); - p32a = _mm512_set_epi32(23,7,22,6,21,5,20,4,19,3,18,2,17,1,16,0); - p32b = _mm512_set_epi32(31,15,30,14,29,13,28,12,27,11,26,10,25,9,24,8); - for (k=0,j=0;j>6;j++,k+=8) { - e0j = _mm512_loadu_si512(e0_512+j); - e1j = _mm512_loadu_si512(e1_512+j); - e2j = _mm512_loadu_si512(e2_512+j); - e3j = _mm512_loadu_si512(e3_512+j); - e4j = _mm512_loadu_si512(e4_512+j); - e5j = _mm512_loadu_si512(e5_512+j); - e6j = _mm512_loadu_si512(e6_512+j); - e7j = _mm512_loadu_si512(e7_512+j); - tmp0 = _mm512_permutex2var_epi8(e0j,p8a,e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15) - tmp1 = _mm512_permutex2var_epi8(e2j,p8a,e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15) - tmp2 = _mm512_permutex2var_epi8(e4j,p8a,e5j); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+15) e5(i+15) - tmp3 = _mm512_permutex2var_epi8(e6j,p8a,e7j); // e6(i) e7(i) e6(i+1) e7(i+1) .... e6(i+15) e7(i+15) - tmp4 = _mm512_permutex2var_epi16(tmp0,p16a,tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) - tmp5 = _mm512_permutex2var_epi16(tmp2,p16a,tmp3); // e4(i) e5(i) e6(i) e7(i) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7) - _mm512_storeu_si512(f_512+k,_mm512_permutex2var_epi32(tmp4,p32a,tmp5)); // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i)... e0(i+3) e1(i+3) e2(i+3) e3(i+3) e4(i+3) e5(i+3) e6(i+3) e7(i+3)) - _mm512_storeu_si512(f_512+k+1,_mm512_permutex2var_epi32(tmp4,p32b,tmp5)); // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7(i+4)... e0(i+7) e1(i+7) e2(i+7) e3(i+7) e4(i+7) e5(i+7) e6(i+7) e7(i+7)) + const __m512i p32a = _mm512_set_epi32(23, 7, 22, 6, 21, 5, 20, 4, 19, 3, 18, 2, 17, 1, 16, 0); + const __m512i p32b = _mm512_set_epi32(31, 15, 30, 14, 29, 13, 28, 12, 27, 11, 26, 10, 25, 9, 24, 8); + // clang-format on + __m512i *e0_512 = (__m512i *)e0; + __m512i *e1_512 = (__m512i *)e1; + __m512i *e2_512 = (__m512i *)e2; + __m512i *e3_512 = (__m512i *)e3; + __m512i *e4_512 = (__m512i *)e4; + __m512i *e5_512 = (__m512i *)e5; + __m512i *e6_512 = (__m512i *)e6; + __m512i *e7_512 = (__m512i *)e7; + __m512i *f_512 = (__m512i *)f; + for (; i < (EQm & ~63); i += 64) { + __m512i e0j = _mm512_loadu_si512(e0_512++); + __m512i e1j = _mm512_loadu_si512(e1_512++); + __m512i e2j = _mm512_loadu_si512(e2_512++); + __m512i e3j = _mm512_loadu_si512(e3_512++); + __m512i e4j = _mm512_loadu_si512(e4_512++); + __m512i e5j = _mm512_loadu_si512(e5_512++); + __m512i e6j = _mm512_loadu_si512(e6_512++); + __m512i e7j = _mm512_loadu_si512(e7_512++); + __m512i tmp0 = _mm512_permutex2var_epi8(e0j, p8a, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15) + __m512i tmp1 = _mm512_permutex2var_epi8(e2j, p8a, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15) + __m512i tmp2 = _mm512_permutex2var_epi8(e4j, p8a, e5j); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+15) e5(i+15) + __m512i tmp3 = _mm512_permutex2var_epi8(e6j, p8a, e7j); // e6(i) e7(i) e6(i+1) e7(i+1) .... e6(i+15) e7(i+15) + __m512i tmp4 = _mm512_permutex2var_epi16(tmp0, p16a, tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) + // e4(i) e5(i) e6(i) e7(i) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7) + // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i)... e0(i+3) e1(i+3) + __m512i tmp5 = _mm512_permutex2var_epi16(tmp2, p16a, tmp3); + // e2(i+3) e3(i+3) e4(i+3) e5(i+3) e6(i+3) e7(i+3)) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32a, tmp5)); + // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7(i+4)... + // e0(i+7) e1(i+7) e2(i+7) e3(i+7) e4(i+7) e5(i+7) e6(i+7) e7(i+7)) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32b, tmp5)); + // e0(i+8) e1(i+8) e2(i+8) e3(i+8) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) + tmp4 = _mm512_permutex2var_epi16(tmp0, p16b, tmp1); + // e4(i+8) e5(i+8) e6(i+8) e7(i+8) ... e4(i+15) e5(i+15) e6(i+15) e7(i+15) + tmp5 = _mm512_permutex2var_epi16(tmp2, p16b, tmp3); + // e0(i+8) e1(i+8) e2(i+8) e3(i+8) e4(i+8) e5(i+8) e6(i+8) e7(i+8)... e0(i+11) + // e1(i+11) e2(i+11) e3(i+11) e4(i+11) e5(i+11) e6(i+11) e7(i+11)) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32a, tmp5)); + // e0(i+12) e1(i+12) e2(i+12) e3(i+12) e4(i+12) e5(i+12) e6(i+12) e7(i+12)... e0(i+15) + // e1(i+15) e2(i+15) e3(i+15) e4(i+15) e5(i+15) e6(i+15) e7(i+15)) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32b, tmp5)); - tmp4 = _mm512_permutex2var_epi16(tmp0,p16b,tmp1); // e0(i+8) e1(i+8) e2(i+8) e3(i+8) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) - tmp5 = _mm512_permutex2var_epi16(tmp2,p16b,tmp3); // e4(i+8) e5(i+8) e6(i+8) e7(i+8) ... e4(i+15) e5(i+15) e6(i+15) e7(i+15) - _mm512_storeu_si512(f_512+k+2,_mm512_permutex2var_epi32(tmp4,p32a,tmp5)); // e0(i+8) e1(i+8) e2(i+8) e3(i+8) e4(i+8) e5(i+8) e6(i+8) e7(i+8)... e0(i+11) e1(i+11) e2(i+11) e3(i+11) e4(i+11) e5(i+11) e6(i+11) e7(i+11)) - _mm512_storeu_si512(f_512+k+3,_mm512_permutex2var_epi32(tmp4,p32b,tmp5)); // e0(i+12) e1(i+12) e2(i+12) e3(i+12) e4(i+12) e5(i+12) e6(i+12) e7(i+12)... e0(i+15) e1(i+15) e2(i+15) e3(i+15) e4(i+15) e5(i+15) e6(i+15) e7(i+15)) - - tmp0 = _mm512_permutex2var_epi8(e0j,p8b,e1j); // e0(i+16) e1(i+16) e0(i+17) e1(i+17) .... e0(i+31) e1(i+31) - tmp1 = _mm512_permutex2var_epi8(e2j,p8b,e3j); // e2(i+16) e3(i+16) e2(i+17) e3(i+17) .... e2(i+31) e3(i+31) - tmp2 = _mm512_permutex2var_epi8(e4j,p8b,e5j); // e4(i+16) e5(i+16) e4(i+17) e5(i+17) .... e4(i+31) e5(i+31) - tmp3 = _mm512_permutex2var_epi8(e6j,p8b,e7j); // e6(i+16) e7(i+16) e6(i+17) e7(i+17) .... e6(i+31) e7(i+31) - tmp4 = _mm512_permutex2var_epi16(tmp0,p16a,tmp1); // e0(i+!6) e1(i+16) e2(i+16) e3(i+16) ... e0(i+23) e1(i+23) e2(i+23) e3(i+23) - tmp5 = _mm512_permutex2var_epi16(tmp2,p16a,tmp3); // e4(i+16) e5(i+16) e6(i+16) e7(i+16) ... e4(i+23) e5(i+23) e6(i+23) e7(i+23) - _mm512_storeu_si512(f_512+k+4,_mm512_permutex2var_epi32(tmp4,p32a,tmp5)); // e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16) e6(i+16) e7(i+16)... e0(i+19) e1(i+19) e2(i+19) e3(i+19) e4(i+19) e5(i+19) e6(i+19) e7(i+19)) - _mm512_storeu_si512(f_512+k+5,_mm512_permutex2var_epi32(tmp4,p32b,tmp5)); // e0(i+20) e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e6(i+20) e7(i+20)... e0(i+23) e1(i+23) e2(i+23) e3(i+23) e4(i+23) e5(i+23) e6(i+23) e7(i+23)) - - tmp4 = _mm512_permutex2var_epi16(tmp0,p16b,tmp1); // e0(i+24) e1(i+24) e2(i+24) e3(i+24) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31) - tmp5 = _mm512_permutex2var_epi16(tmp2,p16b,tmp3); // e4(i+24) e5(i+24) e6(i+24) e7(i+24) ... e4(i+31) e5(i+31) e6(i+31) e7(i+31) - _mm512_storeu_si512(f_512+k+6,_mm512_permutex2var_epi32(tmp4,p32a,tmp5)); // e0(i+24) e1(i+24) e2(i+24) e3(i+24) e4(i+24) e5(i+24) e6(i+24) e7(i+24)... e0(i+27) e1(i+27) e2(i+27) e3(i+27) e4(i+27) e5(i+27) e6(i+27) e7(i+27)) - _mm512_storeu_si512(f_512+k+7,_mm512_permutex2var_epi32(tmp4,p32b,tmp5)); // e0(i+28) e1(i+28) e2(i+28) e3(i+28) e4(i+28) e5(i+28) e6(i+28) e7(i+28)... e0(i+31) e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31) e6(i+31) e7(i+31)) - } - if ((j<<6) != EQm) { - int k2=k<<6; - fp=&f[k2]; - for (int j2=(j<<6) ; j2< EQm ; j2++) { - *fp++ = e0[j2]; - *fp++ = e1[j2]; - *fp++ = e2[j2]; - *fp++ = e3[j2]; - *fp++ = e4[j2]; - *fp++ = e5[j2]; - *fp++ = e6[j2]; - *fp++ = e7[j2]; + tmp0 = _mm512_permutex2var_epi8(e0j, p8b, e1j); // e0(i+16) e1(i+16) e0(i+17) e1(i+17) .... e0(i+31) e1(i+31) + tmp1 = _mm512_permutex2var_epi8(e2j, p8b, e3j); // e2(i+16) e3(i+16) e2(i+17) e3(i+17) .... e2(i+31) e3(i+31) + tmp2 = _mm512_permutex2var_epi8(e4j, p8b, e5j); // e4(i+16) e5(i+16) e4(i+17) e5(i+17) .... e4(i+31) e5(i+31) + tmp3 = _mm512_permutex2var_epi8(e6j, p8b, e7j); // e6(i+16) e7(i+16) e6(i+17) e7(i+17) .... e6(i+31) e7(i+31) + // e0(i+!6) e1(i+16) e2(i+16) e3(i+16) ... e0(i+23) e1(i+23) e2(i+23) e3(i+23) + tmp4 = _mm512_permutex2var_epi16(tmp0, p16a, tmp1); + // e4(i+16) e5(i+16) e6(i+16) e7(i+16) ... e4(i+23) e5(i+23) e6(i+23) e7(i+23) + tmp5 = _mm512_permutex2var_epi16(tmp2, p16a, tmp3); + // e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16) e6(i+16) e7(i+16)... e0(i+19) + // e1(i+19) e2(i+19) e3(i+19) e4(i+19) e5(i+19) e6(i+19) e7(i+19)) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32a, tmp5)); + // e0(i+20) e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e6(i+20) e7(i+20)... e0(i+23) + // e1(i+23) e2(i+23) e3(i+23) e4(i+23) e5(i+23) e6(i+23) e7(i+23)) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32b, tmp5)); + // e0(i+24) e1(i+24) e2(i+24) e3(i+24) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31) + tmp4 = _mm512_permutex2var_epi16(tmp0, p16b, tmp1); + // e4(i+24) e5(i+24) e6(i+24) e7(i+24) ... e4(i+31) e5(i+31) e6(i+31) e7(i+31) + tmp5 = _mm512_permutex2var_epi16(tmp2, p16b, tmp3); + // e0(i+24) e1(i+24) e2(i+24) e3(i+24) e4(i+24) e5(i+24) e6(i+24) e7(i+24)... e0(i+27) + // e1(i+27) e2(i+27) e3(i+27) e4(i+27) e5(i+27) e6(i+27) e7(i+27)) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32a, tmp5)); + // e0(i+28) e1(i+28) e2(i+28) e3(i+28) e4(i+28) e5(i+28) e6(i+28) e7(i+28)... e0(i+31) + // e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31) e6(i+31) e7(i+31)) + _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32b, tmp5)); } - } -#else - e0_128=(simde__m128i *)e0; - e1_128=(simde__m128i *)e1; - e2_128=(simde__m128i *)e2; - e3_128=(simde__m128i *)e3; - e4_128=(simde__m128i *)e4; - e5_128=(simde__m128i *)e5; - e6_128=(simde__m128i *)e6; - e7_128=(simde__m128i *)e7; - simde__m128i e4j,e5j,e6j,e7j; - - for (k=0,j=0;j>4;j++,k+=8) { - e0j = simde_mm_loadu_si128(e0_128+j); - e1j = simde_mm_loadu_si128(e1_128+j); - e2j = simde_mm_loadu_si128(e2_128+j); - e3j = simde_mm_loadu_si128(e3_128+j); - e4j = simde_mm_loadu_si128(e4_128+j); - e5j = simde_mm_loadu_si128(e5_128+j); - e6j = simde_mm_loadu_si128(e6_128+j); - e7j = simde_mm_loadu_si128(e7_128+j); - tmp0 = simde_mm_unpacklo_epi8(e0j,e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+7) e1(i+7) - tmp1 = simde_mm_unpacklo_epi8(e2j,e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+7) e3(i+7) - tmp2 = simde_mm_unpacklo_epi16(tmp0,tmp1);// e0(i) e1(i) e2(i) e3(i) e0(i+1) e1(i+1) e2(i+1) e3(i+1) ... e0(i+3) e1(i+3) e2(i+3) e3(i+3) - tmp3 = simde_mm_unpackhi_epi16(tmp0,tmp1);// e0(i+4) e1(i+4) e2(i+4) e3(i+4) e0(i+5) e1(i+5) e2(i+5) e3(i+5) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) - tmp4 = simde_mm_unpacklo_epi8(e4j,e5j); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+7) e5(i+7) - tmp5 = simde_mm_unpacklo_epi8(e6j,e7j); // e6(i) e7(i) e6(i+1) e7(i+1) .... e6(i+7) e7(i+7) - tmp6 = simde_mm_unpacklo_epi16(tmp4,tmp5);// e4(i) e5(i) e6(i) e7(i) e4(i+1) e5(i+1) e6(i+1) e7(i+1) ... e4(i+3) e5(i+3) e6(i+) e7(i+3) - tmp7 = simde_mm_unpackhi_epi16(tmp4,tmp5);// e4(i+4) e5(i+4) e6(i+4) e7(i+4) e4(i+5) e5(i+5) e6(i+5) e7(i+5) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7) - simde_mm_storeu_si128(f_128+k,simde_mm_unpacklo_epi32(tmp2,tmp6)); // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i) e0(i+1) ... e7(i+1) - simde_mm_storeu_si128(f_128+k+1,simde_mm_unpackhi_epi32(tmp2,tmp6)); // e0(i+2) e1(i+2) e2(i+2) e3(i+2) e4(i+2) e5(i+2) e6(i+2) e7(i+2) e0(i+3) e1(i+3) ... e7(i+3) - simde_mm_storeu_si128(f_128+k+2,simde_mm_unpacklo_epi32(tmp3,tmp7)); // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7+4(i) e0(i+5) ... e7(i+5) - simde_mm_storeu_si128(f_128+k+3,simde_mm_unpackhi_epi32(tmp3,tmp7)); // e0(i+6) e1(i+6) e2(i+6) e3(i+6) e4(i+6) e5(i+6) e6(i+6) e7(i+6) e0(i+7) e0(i+7) ... e7(i+7) - - - tmp0 = simde_mm_unpackhi_epi8(e0j,e1j); // e0(i+8) e1(i+8) e0(i+9) e1(i+9) .... e0(i+15) e1(i+15) - tmp1 = simde_mm_unpackhi_epi8(e2j,e3j); // e2(i+8) e3(i+8) e2(i+9) e3(i+9) .... e2(i+15) e3(i+15) - tmp2 = simde_mm_unpacklo_epi16(tmp0,tmp1);// e0(i) e1(i) e2(i) e3(i) e0(i+1) e1(i+1) e2(i+1) e3(i+1) ... e0(i+3) e1(i+3) e2(i+3) e3(i+3) - tmp3 = simde_mm_unpackhi_epi16(tmp0,tmp1);// e0(i+4) e1(i+4) e2(i+4) e3(i+4) e0(i+5) e1(i+5) e2(i+5) e3(i+5) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) - tmp4 = simde_mm_unpackhi_epi8(e4j,e5j); // e4(i+8) e5(i+8) e4(i+9) e5(i+9) .... e4(i+15) e515i+7) - tmp5 = simde_mm_unpackhi_epi8(e6j,e7j); // e6(i+8) e7(i+8) e6(i+9) e7(i+9) .... e6(i+15) e7(i+15) - tmp6 = simde_mm_unpacklo_epi16(tmp4,tmp5);// e4(i) e5(i) e6(i) e7(i) e4(i+1) e5(i+1) e6(i+1) e7(i+1) ... e4(i+3) e5(i+3) e6(i+) e7(i+3) - tmp7 = simde_mm_unpackhi_epi16(tmp4,tmp5);// e4(i+4) e5(i+4) e6(i+4) e7(i+4) e4(i+5) e5(i+5) e6(i+5) e7(i+5) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7) - simde_mm_storeu_si128(f_128+k+4,simde_mm_unpacklo_epi32(tmp2,tmp6)); // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i) e0(i+1) ... e7(i+1) - simde_mm_storeu_si128(f_128+k+5,simde_mm_unpackhi_epi32(tmp2,tmp6)); // e0(i+2) e1(i+2) e2(i+2) e3(i+2) e4(i+2) e5(i+2) e6(i+2) e7(i+2) e0(i+3) e1(i+3) ... e7(i+3) - simde_mm_storeu_si128(f_128+k+6,simde_mm_unpacklo_epi32(tmp3,tmp7)); // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7+4(i) e0(i+5) ... e7(i+5) - simde_mm_storeu_si128(f_128+k+7,simde_mm_unpackhi_epi32(tmp3,tmp7)); // e0(i+6) e1(i+6) e2(i+6) e3(i+6) e4(i+6) e5(i+6) e6(i+6) e7(i+6) e0(i+7) e0(i+7) ... e7(i+7) - } - if ((j<<4) != EQm) { - int k2=k<<4; - fp=&f[k2]; - for (int j2=(j<<4) ; j2< EQm ; j2++) { - *fp++ = e0[j2]; - *fp++ = e1[j2]; - *fp++ = e2[j2]; - *fp++ = e3[j2]; - *fp++ = e4[j2]; - *fp++ = e5[j2]; - *fp++ = e6[j2]; - *fp++ = e7[j2]; - } - } + e0 = (uint8_t *)e0_512; + e1 = (uint8_t *)e1_512; + e2 = (uint8_t *)e2_512; + e3 = (uint8_t *)e3_512; + e4 = (uint8_t *)e4_512; + e5 = (uint8_t *)e5_512; + e6 = (uint8_t *)e6_512; + e7 = (uint8_t *)e7_512; + f = (uint8_t *)f_512; #endif - break; - default: AssertFatal(1==0,"Should be here!\n"); - } +#ifdef USE128BIT + e0_128 = (simde__m128i *)e0; + e1_128 = (simde__m128i *)e1; + e2_128 = (simde__m128i *)e2; + e3_128 = (simde__m128i *)e3; + e4_128 = (simde__m128i *)e4; + e5_128 = (simde__m128i *)e5; + e6_128 = (simde__m128i *)e6; + e7_128 = (simde__m128i *)e7; + for (; i < (EQm & ~15); i += 16) { + simde__m128i e0j = simde_mm_loadu_si128(e0_128++); + simde__m128i e1j = simde_mm_loadu_si128(e1_128++); + simde__m128i e2j = simde_mm_loadu_si128(e2_128++); + simde__m128i e3j = simde_mm_loadu_si128(e3_128++); + simde__m128i e4j = simde_mm_loadu_si128(e4_128++); + simde__m128i e5j = simde_mm_loadu_si128(e5_128++); + simde__m128i e6j = simde_mm_loadu_si128(e6_128++); + simde__m128i e7j = simde_mm_loadu_si128(e7_128++); + simde__m128i tmp0 = simde_mm_unpacklo_epi8(e0j, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+7) e1(i+7) + simde__m128i tmp1 = simde_mm_unpacklo_epi8(e2j, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+7) e3(i+7) + // e0(i) e1(i) e2(i) e3(i) e0(i+1) e1(i+1) e2(i+1) e3(i+1) ... e0(i+3) e1(i+3) e2(i+3) e3(i+3) + simde__m128i tmp2 = simde_mm_unpacklo_epi16(tmp0, tmp1); + // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e0(i+5) e1(i+5) e2(i+5) e3(i+5) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) + simde__m128i tmp3 = simde_mm_unpackhi_epi16(tmp0, tmp1); + simde__m128i tmp4 = simde_mm_unpacklo_epi8(e4j, e5j); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+7) e5(i+7) + simde__m128i tmp5 = simde_mm_unpacklo_epi8(e6j, e7j); // e6(i) e7(i) e6(i+1) e7(i+1) .... e6(i+7) e7(i+7) + // e4(i) e5(i) e6(i) e7(i) e4(i+1) e5(i+1) e6(i+1) e7(i+1) ... e4(i+3) e5(i+3) e6(i+) e7(i+3) + simde__m128i tmp6 = simde_mm_unpacklo_epi16(tmp4, tmp5); + // e4(i+4) e5(i+4) e6(i+4) e7(i+4) e4(i+5) e5(i+5) e6(i+5) e7(i+5) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7) + simde__m128i tmp7 = simde_mm_unpackhi_epi16(tmp4, tmp5); + // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i) e0(i+1) ... e7(i+1) + simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi32(tmp2, tmp6)); + // e0(i+2) e1(i+2) e2(i+2) e3(i+2) e4(i+2) e5(i+2) e6(i+2) e7(i+2) e0(i+3) e1(i+3) ... e7(i+3) + simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi32(tmp2, tmp6)); + // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7+4(i) e0(i+5) ... e7(i+5) + simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi32(tmp3, tmp7)); + // e0(i+6) e1(i+6) e2(i+6) e3(i+6) e4(i+6) e5(i+6) e6(i+6) e7(i+6) e0(i+7) e0(i+7) ... e7(i+7) + simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi32(tmp3, tmp7)); -#else - // original unoptimized loops - /* - for (int j = 0; j< EQm; j++,j2+=2){ - for (int i = 0; i< Qm; i++){ - f[(i+j*Qm)] = e[(i*EQm + j)]; - } - } - */ - - fp = f; - switch (Qm) { - case 2: - e0 = e; - e1 = e0 + EQm; - for (int j = 0, j2 = 0; j < EQm; j++, j2 += 2) { - fp = &f[j2]; - fp[0] = e0[j]; - fp[1] = e1[j]; + tmp0 = simde_mm_unpackhi_epi8(e0j, e1j); // e0(i+8) e1(i+8) e0(i+9) e1(i+9) .... e0(i+15) e1(i+15) + tmp1 = simde_mm_unpackhi_epi8(e2j, e3j); // e2(i+8) e3(i+8) e2(i+9) e3(i+9) .... e2(i+15) e3(i+15) + // e0(i) e1(i) e2(i) e3(i) e0(i+1) e1(i+1) e2(i+1) e3(i+1) ... e0(i+3) e1(i+3) e2(i+3) e3(i+3) + tmp2 = simde_mm_unpacklo_epi16(tmp0, tmp1); + // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e0(i+5) e1(i+5) e2(i+5) e3(i+5) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7) + tmp3 = simde_mm_unpackhi_epi16(tmp0, tmp1); + tmp4 = simde_mm_unpackhi_epi8(e4j, e5j); // e4(i+8) e5(i+8) e4(i+9) e5(i+9) .... e4(i+15) e515i+7) + tmp5 = simde_mm_unpackhi_epi8(e6j, e7j); // e6(i+8) e7(i+8) e6(i+9) e7(i+9) .... e6(i+15) e7(i+15) + // e4(i) e5(i) e6(i) e7(i) e4(i+1) e5(i+1) e6(i+1) e7(i+1) ... e4(i+3) e5(i+3) e6(i+) e7(i+3) + tmp6 = simde_mm_unpacklo_epi16(tmp4, tmp5); + // e4(i+4) e5(i+4) e6(i+4) e7(i+4) e4(i+5) e5(i+5) e6(i+5) e7(i+5) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7) + tmp7 = simde_mm_unpackhi_epi16(tmp4, tmp5); + // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i) e0(i+1) ... e7(i+1) + simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi32(tmp2, tmp6)); + // e0(i+2) e1(i+2) e2(i+2) e3(i+2) e4(i+2) e5(i+2) e6(i+2) e7(i+2) e0(i+3) e1(i+3) ... e7(i+3) + simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi32(tmp2, tmp6)); + // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7+4(i) e0(i+5) ... e7(i+5) + simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi32(tmp3, tmp7)); + // e0(i+6) e1(i+6) e2(i+6) e3(i+6) e4(i+6) e5(i+6) e6(i+6) e7(i+6) e0(i+7) e0(i+7) ... e7(i+7) + simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi32(tmp3, tmp7)); } - break; - case 4: - e0 = e; - e1 = e0 + EQm; - e2 = e1 + EQm; - e3 = e2 + EQm; - for (int j = 0, j2 = 0; j < EQm; j++, j2 += 4) { - fp = &f[j2]; - fp[0] = e0[j]; - fp[1] = e1[j]; - fp[2] = e2[j]; - fp[3] = e3[j]; + e0 = (uint8_t *)e0_128; + e1 = (uint8_t *)e1_128; + e2 = (uint8_t *)e2_128; + e3 = (uint8_t *)e3_128; + e4 = (uint8_t *)e4_128; + e5 = (uint8_t *)e5_128; + e6 = (uint8_t *)e6_128; + e7 = (uint8_t *)e7_128; + f = (uint8_t *)f_128; +#endif + for (; i < EQm; i++) { + *f++ = *e0++; + *f++ = *e1++; + *f++ = *e2++; + *f++ = *e3++; + *f++ = *e4++; + *f++ = *e5++; + *f++ = *e6++; + *f++ = *e7++; } - break; - case 6: - e0 = e; - e1 = e0 + EQm; - e2 = e1 + EQm; - e3 = e2 + EQm; - e4 = e3 + EQm; - e5 = e4 + EQm; - fp = f; - for (int j = 0; j < EQm; j++) { - *fp++ = e0[j]; - *fp++ = e1[j]; - *fp++ = e2[j]; - *fp++ = e3[j]; - *fp++ = e4[j]; - *fp++ = e5[j]; - } - break; - case 8: - e0 = e; - e1 = e0 + EQm; - e2 = e1 + EQm; - e3 = e2 + EQm; - e4 = e3 + EQm; - e5 = e4 + EQm; - e6 = e5 + EQm; - e7 = e6 + EQm; - for (int j = 0, j2 = 0; j < EQm; j++, j2 += 8) { - fp = &f[j2]; - fp[0] = e0[j]; - fp[1] = e1[j]; - fp[2] = e2[j]; - fp[3] = e3[j]; - fp[4] = e4[j]; - fp[5] = e5[j]; - fp[6] = e6[j]; - fp[7] = e7[j]; - } - break; + } break; default: - AssertFatal(1 == 0, "Should never be here!\n"); + AssertFatal(false, "Should be here!\n"); } -#endif } void nr_deinterleaving_ldpc(uint32_t E, uint8_t Qm, int16_t *e, int16_t *f) diff --git a/openair1/PHY/CODING/nrLDPC_defs.h b/openair1/PHY/CODING/nrLDPC_defs.h index 3db41a0e29..2298400078 100644 --- a/openair1/PHY/CODING/nrLDPC_defs.h +++ b/openair1/PHY/CODING/nrLDPC_defs.h @@ -28,7 +28,7 @@ /** \brief LDPC encoder parameter structure \var n_segments number of segments in the transport block - \var macro_num index of the subset of segments to encode + \var first_seg index of the first segment of the subset to encode within the transport block \var gen_code flag to generate parity check code 0 -> encoding @@ -52,7 +52,7 @@ */ typedef struct { unsigned int n_segments; // optim8seg - unsigned int macro_num; // optim8segmulti + unsigned int first_seg; // optim8segmulti unsigned char gen_code; //orig time_stats_t *tinput; time_stats_t *tprep; @@ -92,6 +92,6 @@ typedef int32_t(LDPC_decoderfunc_t)(t_nrLDPC_dec_params *p_decParams, int8_t *p_out, t_nrLDPC_time_stats *, decode_abort_t *ab); -typedef int32_t(LDPC_encoderfunc_t)(uint8_t **, uint8_t **, encoder_implemparams_t *); +typedef int32_t(LDPC_encoderfunc_t)(uint8_t **, uint8_t *, encoder_implemparams_t *); #endif diff --git a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder.c b/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder.c index f1f6e0af5a..055c0a7eb6 100644 --- a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder.c +++ b/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder.c @@ -41,11 +41,11 @@ #include "openair1/PHY/CODING/nrLDPC_extern.h" #include "ldpc_generate_coefficient.c" -int LDPCencoder(unsigned char **inputArray, unsigned char **outputArray, encoder_implemparams_t *impp) +int LDPCencoder(unsigned char **inputArray, unsigned char *outputArray, encoder_implemparams_t *impp) { const unsigned char *input = inputArray[0]; // channel input is the output of this function! - unsigned char *output = outputArray[0]; + unsigned char *output = outputArray; const int Zc = impp->Zc; const int Kb = impp->Kb; const short block_length = impp->K; diff --git a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8segmulti.c b/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8segmulti.c index 268d2f9dc6..b8f46969ec 100644 --- a/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8segmulti.c +++ b/openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8segmulti.c @@ -43,7 +43,7 @@ #include "ldpc_encode_parity_check.c" #include "ldpc_generate_coefficient.c" -int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp) +int LDPCencoder(uint8_t **input, uint8_t *output, encoder_implemparams_t *impp) { //set_log(PHY, 4); @@ -61,41 +61,10 @@ int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp) unsigned int macro_segment, macro_segment_end; - macro_segment = 8*impp->macro_num; - macro_segment_end = (impp->n_segments > 8*(impp->macro_num+1)) ? 8*(impp->macro_num+1) : impp->n_segments; + macro_segment = impp->first_seg; + macro_segment_end = (impp->n_segments > impp->first_seg + 8) ? impp->first_seg + 8 : impp->n_segments; ///printf("macro_segment: %d\n", macro_segment); ///printf("macro_segment_end: %d\n", macro_segment_end ); -#ifdef __aarch64__ - simde__m128i shufmask = simde_mm_set_epi64x(0x0101010101010101, 0x0000000000000000); - simde__m128i andmask = simde_mm_set1_epi64x(0x0102040810204080); // every 8 bits -> 8 bytes, pattern repeats. - simde__m128i zero128 = simde_mm_setzero_si128(); - simde__m128i masks[8]; - register simde__m128i c128; - masks[0] = simde_mm_set1_epi8(0x1); - masks[1] = simde_mm_set1_epi8(0x2); - masks[2] = simde_mm_set1_epi8(0x4); - masks[3] = simde_mm_set1_epi8(0x8); - masks[4] = simde_mm_set1_epi8(0x10); - masks[5] = simde_mm_set1_epi8(0x20); - masks[6] = simde_mm_set1_epi8(0x40); - masks[7] = simde_mm_set1_epi8(0x80); - -#else - simde__m256i shufmask = simde_mm256_set_epi64x(0x0303030303030303, 0x0202020202020202,0x0101010101010101, 0x0000000000000000); - simde__m256i andmask = simde_mm256_set1_epi64x(0x0102040810204080); // every 8 bits -> 8 bytes, pattern repeats. - simde__m256i zero256 = simde_mm256_setzero_si256(); - simde__m256i masks[8]; - register simde__m256i c256; - masks[0] = simde_mm256_set1_epi8(0x1); - masks[1] = simde_mm256_set1_epi8(0x2); - masks[2] = simde_mm256_set1_epi8(0x4); - masks[3] = simde_mm256_set1_epi8(0x8); - masks[4] = simde_mm256_set1_epi8(0x10); - masks[5] = simde_mm256_set1_epi8(0x20); - masks[6] = simde_mm256_set1_epi8(0x40); - masks[7] = simde_mm256_set1_epi8(0x80); -#endif - //determine number of bits in codeword if (BG==1) @@ -136,55 +105,90 @@ int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp) if(impp->tinput != NULL) start_meas(impp->tinput); //interleave up to 8 transport-block segements at a time -#if 0 - for (i=0; i>(i&7); - //printf("c(%d,%d)=%d\n",j,i,temp); - c[i] |= (temp << (j-macro_segment)); - } - } -#elif defined(__aarch64__) - for (int i=0; i>4; i++) { - c128 = simde_mm_and_si128(simde_mm_cmpeq_epi8(simde_mm_andnot_si128(simde_mm_shuffle_epi8(simde_mm_set1_epi16(((uint16_t*)input[macro_segment])[i]), shufmask),andmask),zero128),masks[0]); - //for (j=1; j>4)<<4;i>(i&7)))>>(7-(i&7)); - //printf("c(%d,%d)=%d\n",j,i,temp); - cc[i] |= (temp << (j-macro_segment)); + for (; i_byte < ((block_length >> 6) << 6); i_byte += 64) { + unsigned int i = i_byte >> 6; + c512 = _mm512_mask_blend_epi8(((uint64_t *)&input[macro_segment][0])[i], zero512, masks5[0]); + for (int j = macro_segment + 1; j < macro_segment_end; j++) { + c512 = _mm512_or_si512(c512, _mm512_mask_blend_epi8(((uint64_t *)&input[j][0])[i], zero512, masks5[j - macro_segment])); } + c512 = _mm512_permutexvar_epi8(*(__m512i*)perm, c512); + ((__m512i *)cc)[i] = c512; } -#else - for (int i=0; i>5; i++) { +#endif + +#ifndef __aarch64__ + simde__m256i shufmask = simde_mm256_set_epi64x(0x0303030303030303, 0x0202020202020202,0x0101010101010101, 0x0000000000000000); + simde__m256i andmask = simde_mm256_set1_epi64x(0x0102040810204080); // every 8 bits -> 8 bytes, pattern repeats. + simde__m256i zero256 = simde_mm256_setzero_si256(); + simde__m256i masks[8]; + register simde__m256i c256; + masks[0] = simde_mm256_set1_epi8(0x1); + masks[1] = simde_mm256_set1_epi8(0x2); + masks[2] = simde_mm256_set1_epi8(0x4); + masks[3] = simde_mm256_set1_epi8(0x8); + masks[4] = simde_mm256_set1_epi8(0x10); + masks[5] = simde_mm256_set1_epi8(0x20); + masks[6] = simde_mm256_set1_epi8(0x40); + masks[7] = simde_mm256_set1_epi8(0x80); + + for (; i_byte < ((block_length >> 5 ) << 5); i_byte += 32) { + unsigned int i = i_byte >> 5; c256 = simde_mm256_and_si256(simde_mm256_cmpeq_epi8(simde_mm256_andnot_si256(simde_mm256_shuffle_epi8(simde_mm256_set1_epi32(((uint32_t*)input[macro_segment])[i]), shufmask),andmask),zero256),masks[0]); - //for (j=1; j>5)<<5;i 8 bytes, pattern repeats. + simde__m128i zero128 = simde_mm_setzero_si128(); + simde__m128i masks[8]; + register simde__m128i c128; + masks[0] = simde_mm_set1_epi8(0x1); + masks[1] = simde_mm_set1_epi8(0x2); + masks[2] = simde_mm_set1_epi8(0x4); + masks[3] = simde_mm_set1_epi8(0x8); + masks[4] = simde_mm_set1_epi8(0x10); + masks[5] = simde_mm_set1_epi8(0x20); + masks[6] = simde_mm_set1_epi8(0x40); + masks[7] = simde_mm_set1_epi8(0x80); - temp = (input[j][i/8]&(128>>(i&7)))>>(7-(i&7)); - //printf("c(%d,%d)=%d\n",j,i,temp); + for (; i_byte < ((block_length >> 4 ) << 4); i_byte += 16) { + unsigned int i = i_byte >> 4; + c128 = simde_mm_and_si128(simde_mm_cmpeq_epi8(simde_mm_andnot_si128(simde_mm_shuffle_epi8(simde_mm_set1_epi16(((uint16_t*)input[macro_segment])[i]), shufmask),andmask),zero128),masks[0]); + for (int j=macro_segment+1; j < macro_segment_end; j++) { + c128 = simde_mm_or_si128(simde_mm_and_si128(simde_mm_cmpeq_epi8(simde_mm_andnot_si128(simde_mm_shuffle_epi8(simde_mm_set1_epi32(((uint16_t*)input[j])[i]), shufmask),andmask),zero128),masks[j-macro_segment]),c128); + } + ((simde__m128i *)cc)[i] = c128; + } +#endif + + for (; i_byte < block_length; i_byte++) { + unsigned int i = i_byte; + for (int j = macro_segment; j < macro_segment_end; j++) { + + temp = (input[j][i/8]&(128>>(i&7)))>>(7-(i&7)); cc[i] |= (temp << (j-macro_segment)); } } -#endif if(impp->tinput != NULL) stop_meas(impp->tinput); @@ -203,8 +207,8 @@ int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp) return(-1); } } - memcpy(output[0],&cc[2*Zc],(block_length-(2*Zc))); - memcpy(output[0]+block_length-(2*Zc),dd,((nrows-no_punctured_columns) * Zc-removed_bit)); + memcpy(output,&cc[2*Zc],(block_length-(2*Zc))); + memcpy(output+block_length-(2*Zc),dd,((nrows-no_punctured_columns) * Zc-removed_bit)); return 0; } From b1e1c161ec0a3003642efa95340c9be291a6dffa Mon Sep 17 00:00:00 2001 From: Romain Beurdouche Date: Fri, 11 Apr 2025 14:15:00 +0000 Subject: [PATCH 10/17] feat(nrLDPC_coding_segment_encoder): Mitigate race condition There was a risk of a race condition between encoding tasks on the output array. It could be that two task compete in writing a word at the border of their respective segment groups. This commit offers to solve this issue by retaining the tail word in a struct that is returned after task completion so that segment group border can be fixed after completion of all the tasks. Comes with additional cleanups of nrLDPC_coding_segment_encoder: 1. Move the section of code dedicated to writing to the encoding library output to a separate function. 2. Remove bit interleave debug in AVX512VBMI case of output writing for improving code readability. --- .../nrLDPC_coding_segment_encoder.c | 576 +++++++++--------- 1 file changed, 285 insertions(+), 291 deletions(-) diff --git a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c index 554b399f4b..bbdda8c8bc 100644 --- a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c +++ b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c @@ -44,9 +44,268 @@ #define DEBUG_LDPC_ENCODING //#define DEBUG_LDPC_ENCODING_FREE 1 +/** + * \typedef group_tail_t + * \struct group_tail_s + * \brief index and content of the last word written by a task to the encoding library output + * It is returned after task completion to fix the borders of segment groups + * \var index index of the last word written by the task + * \var content content of the last word written by the task + */ +typedef struct group_tail_s +{ + uint32_t index; +#if defined(__AVX512VBMI__) + uint64_t content; +#elif defined(__aarch64__) + uint16_t content; +#else + uint32_t content; +#endif +} group_tail_t; + +/** + * \brief fix segment groups first and last words + * The last word of a segment group is not written by encoding tasks in order to avoid race conditions + * This function fixes the segment group borders after encoding tasks completion + * \param group_tail store segment groups tail to fix segment borders + * \param nbGroups number of groups + * \params output encoding library output to write to + */ +static void fix_group_borders(group_tail_t *group_tail, int nbGroups, uint8_t *output) +{ + for(int i = 0; i < nbGroups; i++) { +#if defined(__AVX512VBMI__) + ((uint64_t *)output)[group_tail[i].index] = group_tail[i].content | ((uint64_t *)output)[group_tail[i].index]; +#elif defined(__aarch64__) + ((uint16_t *)output)[group_tail[i].index] = group_tail[i].content | ((uint16_t *)output)[group_tail[i].index]; +#else + ((uint32_t *)output)[group_tail[i].index] = group_tail[i].content | ((uint32_t *)output)[group_tail[i].index]; +#endif + } +} + +/** + * \brief determine the value of E for the current segment + * \param E number of bits per segment in f + * \param E2 number of bits per segment in f2 + * \param Eshift flag indicating if there was a E shift within the segment group + * \param segment index of the segment + * \param E2_first_segment index of the first segment of size E2 if there is a E shift in the segment group + */ +static inline uint32_t select_E(uint32_t E, uint32_t E2, bool Eshift, uint32_t segment, uint32_t E2_first_segment) +{ + return (Eshift && (segment >= E2_first_segment)) ? E2 : E; +} + +/** + * \brief write nrLDPC_coding_segment_encoder output with interleaver output + * the interleaver output has 8 bits from 8 different segments per byte + * nrLDPC_coding_segment_encoder is made of concatenated packed segments + * \param f input interleaved segment, 8 bits from 8 segments per byte + * \param E number of bits per segment in f + * \param f2 input interleaved segment, 8 bits from 8 segments per byte + * interleaved with E2 if there was a E shift in the segment group + * \param E2 number of bits per segment in f2 + * \param Eshift flag indicating if there was a E shift within the segment group + * \param E2_first_segment index of the first segment of size E2 if there is a E shift in the segment group + * \param nb_segments number of segments in the group + * \param output nrLDPC_coding_segment_encoder with concatenated segments and packed bits + * \param Eoffset offset in number of bits of the first segment of the segment group within output + * \param group_tail store segment group last words to fix segment group borders + */ +static void write_task_output(uint8_t *f, + uint32_t E, + uint8_t *f2, + uint32_t E2, + bool Eshift, + uint32_t E2_first_segment, + uint32_t nb_segments, + uint8_t *output, + uint32_t Eoffset, + group_tail_t *group_tail) +{ + +#if defined(__AVX512VBMI__) + uint64_t *output_p = (uint64_t*)output; + int i=0; + __m512i bitperm[8]; + uint32_t Eoffset2[8]; + uint32_t Eoffset2_bit[8]; + bitperm[0] = _mm512_set1_epi64(0x3830282018100800); + __m512i inc = _mm512_set1_epi8(0x1); + Eoffset2[0] = Eoffset >> 6; + Eoffset2_bit[0] = Eoffset & 63; + Eoffset += select_E(E, E2, Eshift, 0, E2_first_segment); + for (int n = 1; n < nb_segments; n++) { + bitperm[n] = _mm512_add_epi8(bitperm[n-1] ,inc); + Eoffset2[n] = Eoffset >> 6; + Eoffset2_bit[n] = Eoffset & 63; + Eoffset += select_E(E, E2, Eshift, n, E2_first_segment); + } + group_tail->index = Eoffset2[nb_segments - 1]; + int i2=0; + __m64 tmp; + for (i=0;i>/<< because when the Eoffset2_bit is 64 or 0, the << + // and >> operations are undefined and in fact don't give "0" which is + // what we want here. The SIMD version do give 0 when the shift is 64 + tmp = (__m64)_mm512_bitshuffle_epi64_mask(((__m512i *)f)[i2],bitperm[j]); + + *(__m64*)(output_p + Eoffset2[j]) = _mm_or_si64(*(__m64*)(output_p + Eoffset2[j]),_mm_slli_si64(tmp,Eoffset2_bit[j])); + + *(__m64*)(output_p + Eoffset2[j] + 1) = _mm_or_si64(*(__m64*)(output_p + Eoffset2[j]+1),_mm_srli_si64(tmp,(64-Eoffset2_bit[j]))); + + } //for (int j=0; + } // if (i < E) + for (int j=E2_first_segment; j < nb_segments && (j < (nb_segments - 1) || i < (E2 - 64)); j++) { + tmp = (__m64)_mm512_bitshuffle_epi64_mask(((__m512i *)f2)[i2],bitperm[j]); + + *(__m64*)(output_p + Eoffset2[j]) = _mm_or_si64(*(__m64*)(output_p + Eoffset2[j]),_mm_slli_si64(tmp,Eoffset2_bit[j])); + + *(__m64*)(output_p + Eoffset2[j]+1) = _mm_or_si64(*(__m64*)(output_p + Eoffset2[j]+1),_mm_srli_si64(tmp,(64-Eoffset2_bit[j]))); + } + output_p++; + group_tail->index++; + } + + // Do not write the last word of the segment group but keep it for fixing segment borders + if (Eshift) { + tmp = (__m64)_mm512_bitshuffle_epi64_mask(((__m512i *)f2)[(E2 - 1) >> 6], bitperm[nb_segments - 1]); + } else { + tmp = (__m64)_mm512_bitshuffle_epi64_mask(((__m512i *)f)[(E2 - 1) >> 6], bitperm[nb_segments - 1]); + } + *(__m64*)(output_p - 1 + Eoffset2[nb_segments - 1]) = _mm_or_si64(*(__m64*)(output_p - 1 + Eoffset2[nb_segments - 1]), _mm_slli_si64(tmp, Eoffset2_bit[nb_segments - 1])); + group_tail->content = (uint64_t)_mm_srli_si64(tmp, (64 - Eoffset2_bit[nb_segments - 1])); + +#elif defined(__aarch64__) + uint16_t *output_p = (uint16_t*)output; + int i=0; + uint32_t Eoffset2[8]; + uint32_t Eoffset2_bit[8]; + for (int n = 0; n < nb_segments; n++) { + Eoffset2[n] = Eoffset >> 4; + Eoffset2_bit[n] = Eoffset & 15; + Eoffset += select_E(E, E2, Eshift, n, E2_first_segment); + } + group_tail->index = Eoffset2[nb_segments - 1]; + int i2=0; + const int8_t __attribute__ ((aligned (16))) ucShift[8][16] = { + {0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7}, // segment 0 + {-1,0,1,2,3,4,5,6,-1,0,1,2,3,4,5,6}, // segment 1 + {-2,-1,0,1,2,3,4,5,-2,-1,0,1,2,3,4,5}, // segment 2 + {-3,-2,-1,0,1,2,3,4,-3,-2,-1,0,1,2,3,4}, // segment 3 + {-4,-3,-2,-1,0,1,2,3,-4,-3,-2,-1,0,1,2,3}, // segment 4 + {-5,-4,-3,-2,-1,0,1,2,-5,-4,-3,-2,-1,0,1,2}, // segment 5 + {-6,-5,-4,-3,-2,-1,0,1,-6,-5,-4,-3,-2,-1,0,1}, // segment 6 + {-7,-6,-5,-4,-3,-2,-1,0,-7,-6,-5,-4,-3,-2,-1,0}}; // segment 7 + const uint8_t __attribute__ ((aligned (16))) masks[16] = + {0x1,0x2,0x4,0x8,0x10,0x20,0x40,0x80,0x1,0x2,0x4,0x8,0x10,0x20,0x40,0x80}; + int8x16_t vshift[8]; + for (int n=0;n<8;n++) vshift[n] = vld1q_s8(ucShift[n]); + uint8x16_t vmask = vld1q_u8(masks); + + int32_t tmp; + uint8x16_t cshift; + for (i=0;i>(16-Eoffset2_bit[j])); + } + } + for (int j=E2_first_segment; j < nb_segments && (j < (nb_segments - 1) || i < (E2 - 16)); j++) { + cshift = vandq_u8(vshlq_u8(((uint8x16_t*)f2)[i2],vshift[j]),vmask); + tmp = (int)vaddv_u8(vget_low_u8(cshift)); + tmp += (int)(vaddv_u8(vget_high_u8(cshift))<<8); + *(output_p + Eoffset2[j]) |= (uint16_t)(tmp<>(16-Eoffset2_bit[j])); + } + output_p++; + group_tail->index++; + } + + // Do not write the last word of the segment group but keep it for fixing segment borders + if (Eshift) { + cshift = vandq_u8(vshlq_u8(((uint8x16_t*)f2)[(E2 - 1) >> 4], vshift[nb_segments - 1]), vmask); + } else { + cshift = vandq_u8(vshlq_u8(((uint8x16_t*)f)[(E2 - 1) >> 4], vshift[nb_segments - 1]), vmask); + } + tmp = (int)vaddv_u8(vget_low_u8(cshift)); + tmp += (int)(vaddv_u8(vget_high_u8(cshift)) << 8); + *(output_p - 1 + Eoffset2[nb_segments - 1]) |= (uint16_t)(tmp << Eoffset2_bit[nb_segments - 1]); + group_tail->content = (uint16_t)(tmp >> (16 - Eoffset2_bit[nb_segments - 1])); + +#else + uint32_t *output_p = (uint32_t*)output; + int i=0; + uint32_t Eoffset2[8]; + uint32_t Eoffset2_bit[8]; + for (int n = 0; n < nb_segments; n++) { + Eoffset2[n] = Eoffset >> 5; + Eoffset2_bit[n] = Eoffset & 31; + Eoffset += select_E(E, E2, Eshift, n, E2_first_segment); + } + group_tail->index = Eoffset2[nb_segments - 1]; + int i2 = 0; + int tmp; + __m64 tmp64, tmp64b, tmp64c, out64; + + for (i=0; i < E2; i += 32, i2++) { + if (i < E) { + for (int j = 0; j < E2_first_segment && (j < (nb_segments - 1) || i < (E2 - 32)); j++) { + // Note: Here and below, we are using the 64-bit SIMD instruction + // instead of C >>/<< because when the Eoffset2_bit is 64 or 0, the << + // and >> operations are undefined and in fact don't give "0" which is + // what we want here. The SIMD version do give 0 when the shift is 64 + tmp = _mm256_movemask_epi8(_mm256_slli_epi16(((__m256i *)f)[i2], 7 - j)); + tmp64 = _mm_set1_pi32(tmp); + out64 = _mm_set_pi32(*(output_p + Eoffset2[j] + 1), *(output_p + Eoffset2[j])); + tmp64b = _mm_or_si64(out64, _mm_slli_pi32(tmp64, Eoffset2_bit[j])); + tmp64c = _mm_or_si64(out64, _mm_srli_pi32(tmp64, (32 - Eoffset2_bit[j]))); + *(output_p + Eoffset2[j]) = _m_to_int(tmp64b); + *(output_p + Eoffset2[j] + 1) = _m_to_int(_mm_srli_si64(tmp64c, 32)); + } + } + for (int j = E2_first_segment; j < nb_segments && (j < (nb_segments - 1) || i < (E2 - 32)); j++) { + tmp = _mm256_movemask_epi8(_mm256_slli_epi16(((__m256i *)f2)[i2], 7 - j)); + tmp64 = _mm_set1_pi32(tmp); + out64 = _mm_set_pi32(*(output_p + Eoffset2[j] + 1), *(output_p + Eoffset2[j])); + tmp64b = _mm_or_si64(out64, _mm_slli_pi32(tmp64, Eoffset2_bit[j])); + tmp64c = _mm_or_si64(out64, _mm_srli_pi32(tmp64, (32 - Eoffset2_bit[j]))); + *(output_p + Eoffset2[j]) = _m_to_int(tmp64b); + *(output_p + Eoffset2[j] + 1) = _m_to_int(_mm_srli_si64(tmp64c, 32)); + } + output_p++; + group_tail->index++; + } + + // Do not write the last word of the segment group but keep it for fixing segment borders + if (Eshift) { + tmp = _mm256_movemask_epi8(_mm256_slli_epi16(((__m256i *)f2)[(E2 - 1) >> 5], 7 - nb_segments + 1)); + } else { + tmp = _mm256_movemask_epi8(_mm256_slli_epi16(((__m256i *)f)[(E2 - 1) >> 5], 7 - nb_segments + 1)); + } + tmp64 = _mm_set1_pi32(tmp); + out64 = _mm_set_pi32(*(output_p - 1 + Eoffset2[nb_segments - 1] + 1), *(output_p - 1 + Eoffset2[nb_segments - 1])); + tmp64b = _mm_or_si64(out64, _mm_slli_pi32(tmp64, Eoffset2_bit[nb_segments - 1])); + tmp64c = _mm_srli_pi32(tmp64, (32 - Eoffset2_bit[nb_segments - 1])); + *(output_p - 1 + Eoffset2[nb_segments - 1]) = _m_to_int(tmp64b); + group_tail->content = _m_to_int(tmp64c); + +#endif +} + typedef struct ldpc8blocks_args_s { nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters; encoder_implemparams_t impp; + group_tail_t *group_tail; } ldpc8blocks_args_t; static void ldpc8blocks(void *p) @@ -191,295 +450,17 @@ static void ldpc8blocks(void *p) uint32_t Eoffset=0; for (int s=0; ssegments[s].E); -#ifdef __AVX512VBMI__ - uint64_t *output_p = (uint64_t*)impp->output; - int i=0; - __m512i bitperm[8]; - uint32_t Eoffset2[8]; - uint32_t Eoffset2_bit[8]; - bitperm[0] = _mm512_set1_epi64(0x3830282018100800); - __m512i inc = _mm512_set1_epi8(0x1); - Eoffset2[0] = Eoffset>>6; - Eoffset2_bit[0] = Eoffset&63; - //printf("E2_first_segment %d,Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E2_first_segment,Eoffset,Eoffset2[0],Eoffset2_bit[0]); - for (int n=1;n>6; - Eoffset2_bit[n] = Eoffset&63; - //printf("E %d : n %d, Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E,n,Eoffset,Eoffset2[n],Eoffset2_bit[n]); - } - if (Eshift) { - bitperm[E2_first_segment] = _mm512_add_epi8(bitperm[E2_first_segment-1],inc); - Eoffset += E; - Eoffset2[E2_first_segment] = Eoffset>>6; - Eoffset2_bit[E2_first_segment] = Eoffset &63; - //printf("E2 %d : Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E2,Eoffset,Eoffset2[E2_first_segment],Eoffset2_bit[E2_first_segment]); - } - for (int n=E2_first_segment+1;n>6; - Eoffset2_bit[n] = Eoffset&63; - //printf("E2 %d (macro_segment_end %d, macro_segment %d) : n %d, Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E2,macro_segment_end,macro_segment,n,Eoffset,Eoffset2[n],Eoffset2_bit[n]); - } - int i2=0; - __m64 tmp; - for (i=0;i>6), - (Eoffset2_bit[j] + i)&63, - (i<=(E-64)) ? 63 : (Eoffset2_bit[j] + E-1)&63); - if (Eoffset2_bit[j] > 0) - printf("segment %d : qword %d, first bit %d last bit %d\n",j, - 1+Eoffset2[j]+(i>>6), - 0, - ((Eoffset2_bit[j] + i)&63) - 1); -#endif - // Note: Here and below for AVX2, we are using the 64-bit SIMD instruction - // instead of C >>/<< because when the Eoffset2_bit is 64 or 0, the << - // and >> operations are undefined and in fact don't give "0" which is - // what we want here. The SIMD version do give 0 when the shift is 64 - tmp = (__m64)_mm512_bitshuffle_epi64_mask(((__m512i *)f)[i2],bitperm[j]); - *(__m64*)(output_p + Eoffset2[j]) = _mm_or_si64(*(__m64*)(output_p + Eoffset2[j]),_mm_slli_si64(tmp,Eoffset2_bit[j])); - *(__m64*)(output_p + Eoffset2[j]+1) = _mm_or_si64(*(__m64*)(output_p + Eoffset2[j]+1),_mm_srli_si64(tmp,(64-Eoffset2_bit[j]))); -#ifdef DEBUG_BIT_INTERLEAVE - if (j<=1) { - printf("pos : %d, i2 %d, j %d\n",(int)(output_p + Eoffset2[j]-(uint64_t*)impp->output),i2,j); - printf("i : %x.%x.%x.%x.%x.%x.%x.%x\n", - ((uint8_t*)(output_p + Eoffset2[j]))[0], - ((uint8_t*)(output_p + Eoffset2[j]))[1], - ((uint8_t*)(output_p + Eoffset2[j]))[2], - ((uint8_t*)(output_p + Eoffset2[j]))[3], - ((uint8_t*)(output_p + Eoffset2[j]))[4], - ((uint8_t*)(output_p + Eoffset2[j]))[5], - ((uint8_t*)(output_p + Eoffset2[j]))[6], - ((uint8_t*)(output_p + Eoffset2[j]))[7]); - printf("i+1 : %x.%x.%x.%x.%x.%x.%x.%x\n", - ((uint8_t*)(output_p + Eoffset2[j]+1))[0], - ((uint8_t*)(output_p + Eoffset2[j]+1))[1], - ((uint8_t*)(output_p + Eoffset2[j]+1))[2], - ((uint8_t*)(output_p + Eoffset2[j]+1))[3], - ((uint8_t*)(output_p + Eoffset2[j]+1))[4], - ((uint8_t*)(output_p + Eoffset2[j]+1))[5], - ((uint8_t*)(output_p + Eoffset2[j]+1))[6], - ((uint8_t*)(output_p + Eoffset2[j]+1))[7]); - printf("%x.%x.%x.%x.%x.%x.%x.%x\n", - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),0)<<(0-j))&1)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),1)<<(1-j))&2)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),2)<<(2-j))&4)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),3)<<(3-j))&8)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),4)<<(4-j))&16)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),5)<<(5-j))&32)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),6)<<(6-j))&64)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),7)<<(7-j))&128), - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),8)<<(0-j))&1)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),9)<<(1-j))&2)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),10)<<(2-j))&4)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),11)<<(3-j))&8)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),12)<<(4-j))&16)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),13)<<(5-j))&32)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),14)<<(6-j))&64)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),15)<<(7-j))&128), - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),16)<<(0-j))&1)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),17)<<(1-j))&2)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),18)<<(2-j))&4)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),19)<<(3-j))&8)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),20)<<(4-j))&16)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),21)<<(5-j))&32)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),22)<<(6-j))&64)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),23)<<(7-j))&128), - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),24)<<(0-j))&1)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),25)<<(1-j))&2)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),26)<<(2-j))&4)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),27)<<(3-j))&8)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),28)<<(4-j))&16)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),29)<<(5-j))&32)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),30)<<(6-j))&64)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],0),31)<<(7-j))&128), - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),0)<<(0-j))&1)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),1)<<(1-j))&2)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),2)<<(2-j))&4)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),3)<<(3-j))&8)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),4)<<(4-j))&16)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),5)<<(5-j))&32)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),6)<<(6-j))&64)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),7)<<(7-j))&128), - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),8)<<(0-j))&1)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),9)<<(1-j))&2)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),10)<<(2-j))&4)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),11)<<(3-j))&8)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),12)<<(4-j))&16)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),13)<<(5-j))&32)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),14)<<(6-j))&64)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),15)<<(7-j))&128), - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),16)<<(0-j))&1)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),17)<<(1-j))&2)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),18)<<(2-j))&4)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),19)<<(3-j))&8)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),20)<<(4-j))&16)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),21)<<(5-j))&32)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),22)<<(6-j))&64)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),23)<<(7-j))&128), - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),24)<<(0-j))&1)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),25)<<(1-j))&2)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),26)<<(2-j))&4)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),27)<<(3-j))&8)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),28)<<(4-j))&16)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),29)<<(5-j))&32)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),30)<<(6-j))&64)+ - ((_mm256_extract_epi8(_mm512_extracti32x8_epi32(((__m512i *)f)[i2],1),31)<<(7-j))&128)); - } -#endif - } //for (int j=0; - } // if (i < E) - for (int j=E2_first_segment; j < macro_segment_end-macro_segment; j++) { -#ifdef DEBUG_BIT_INTERLEAVE - printf("segment %d : qword %d, first bit %d last bit %d\n",j, - Eoffset2[j]+(i>>6), - (Eoffset2_bit[j] + i)&63, - (i<=(E-64)) ? 63 : (Eoffset2_bit[j] + E-1)&63); - if (Eoffset2_bit[j] > 0) - printf("segment %d : qword %d, first bit %d last bit %d\n",j, - 1+Eoffset2[j]+(i>>6), - 0, - ((Eoffset2_bit[j] + i)&63) - 1); -#endif - tmp = (__m64)_mm512_bitshuffle_epi64_mask(((__m512i *)f2)[i2],bitperm[j]); - *(__m64*)(output_p + Eoffset2[j]) = _mm_or_si64(*(__m64*)(output_p + Eoffset2[j]), - _mm_slli_si64(tmp,Eoffset2_bit[j])); - *(__m64*)(output_p + Eoffset2[j]+1) = _mm_or_si64(*(__m64*)(output_p + Eoffset2[j]+1),_mm_srli_si64(tmp,(64-Eoffset2_bit[j]))); - } - output_p++; - } -#elif defined(__aarch64__) - uint16_t *output_p = (uint16_t*)impp->output; - int i=0; - uint32_t Eoffset2[8]; - uint32_t Eoffset2_bit[8]; - Eoffset2[0] = Eoffset>>4; - Eoffset2_bit[0] = Eoffset&15; - //printf("E2_first_segment %d,Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E2_first_segment,Eoffset,Eoffset2[0],Eoffset2_bit[0]); - for (int n=1;n>4; - Eoffset2_bit[n] = Eoffset&15; - //printf("E %d : n %d, Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E,n,Eoffset,Eoffset2[n],Eoffset2_bit[n]); - } - if (Eshift) { - Eoffset += E; - Eoffset2[E2_first_segment] = Eoffset>>4; - Eoffset2_bit[E2_first_segment] = Eoffset &15; - //printf("E2 %d : Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E2,Eoffset,Eoffset2[E2_first_segment],Eoffset2_bit[E2_first_segment]); - } - for (int n=E2_first_segment+1;n>4; - Eoffset2_bit[n] = Eoffset&15; - //printf("E2 %d (macro_segment_end %d, macro_segment %d) : n %d, Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E2,macro_segment_end,macro_segment,n,Eoffset,Eoffset2[n],Eoffset2_bit[n]); - } - int i2=0; - const int8_t __attribute__ ((aligned (16))) ucShift[8][16] = { - {0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7}, // segment 0 - {-1,0,1,2,3,4,5,6,-1,0,1,2,3,4,5,6}, // segment 1 - {-2,-1,0,1,2,3,4,5,-2,-1,0,1,2,3,4,5}, // segment 2 - {-3,-2,-1,0,1,2,3,4,-3,-2,-1,0,1,2,3,4}, // segment 3 - {-4,-3,-2,-1,0,1,2,3,-4,-3,-2,-1,0,1,2,3}, // segment 4 - {-5,-4,-3,-2,-1,0,1,2,-5,-4,-3,-2,-1,0,1,2}, // segment 5 - {-6,-5,-4,-3,-2,-1,0,1,-6,-5,-4,-3,-2,-1,0,1}, // segment 6 - {-7,-6,-5,-4,-3,-2,-1,0,-7,-6,-5,-4,-3,-2,-1,0}}; // segment 7 - const uint8_t __attribute__ ((aligned (16))) masks[16] = - {0x1,0x2,0x4,0x8,0x10,0x20,0x40,0x80,0x1,0x2,0x4,0x8,0x10,0x20,0x40,0x80}; - int8x16_t vshift[8]; - for (int n=0;n<8;n++) vshift[n] = vld1q_s8(ucShift[n]); - uint8x16_t vmask = vld1q_u8(masks); - - int32_t tmp; - uint8x16_t cshift; - for (i=0;i>(16-Eoffset2_bit[j])); - } - } - for (int j=E2_first_segment; j < macro_segment_end-macro_segment; j++) { - cshift = vandq_u8(vshlq_u8(((uint8x16_t*)f2)[i2],vshift[j]),vmask); - tmp = (int)vaddv_u8(vget_low_u8(cshift)); - tmp += (int)(vaddv_u8(vget_high_u8(cshift))<<8); - *(output_p + Eoffset2[j]) |= (uint16_t)(tmp<>(16-Eoffset2_bit[j])); - } - output_p++; - } - -#else - uint32_t *output_p = (uint32_t*)impp->output; - int i=0; - uint32_t Eoffset2[8]; - uint32_t Eoffset2_bit[8]; - Eoffset2[0] = Eoffset>>5; - Eoffset2_bit[0] = Eoffset&31; - //printf("E2_first_segment %d,Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E2_first_segment,Eoffset,Eoffset2[0],Eoffset2_bit[0]); - for (int n=1;n>5; - Eoffset2_bit[n] = Eoffset&31; - //printf("E %d : n %d, Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E,n,Eoffset,Eoffset2[n],Eoffset2_bit[n]); - } - if (Eshift) { - Eoffset += E; - Eoffset2[E2_first_segment] = Eoffset>>5; - Eoffset2_bit[E2_first_segment] = Eoffset &31; - //printf("E2 %d : Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E2,Eoffset,Eoffset2[E2_first_segment],Eoffset2_bit[E2_first_segment]); - } - for (int n=E2_first_segment+1;n>5; - Eoffset2_bit[n] = Eoffset&31; - //printf("E2 %d (macro_segment_end %d, macro_segment %d) : n %d, Eoffset %d, Eoffset2 %d, Eoffset2_bit %d\n",E2,macro_segment_end,macro_segment,n,Eoffset,Eoffset2[n],Eoffset2_bit[n]); - } - int i2 = 0; - int tmp; - __m64 tmp64, tmp64b, tmp64c, out64; - - for (i=0; i < E2; i += 32, i2++) { - if (i < E) { - for (int j = 0; j < E2_first_segment; j++) { - // Note: Here and below, we are using the 64-bit SIMD instruction - // instead of C >>/<< because when the Eoffset2_bit is 64 or 0, the << - // and >> operations are undefined and in fact don't give "0" which is - // what we want here. The SIMD version do give 0 when the shift is 64 - tmp = _mm256_movemask_epi8(_mm256_slli_epi16(((__m256i *)f)[i2], 7 - j)); - tmp64 = _mm_set1_pi32(tmp); - out64 = _mm_set_pi32(*(output_p + Eoffset2[j] + 1), *(output_p + Eoffset2[j])); - tmp64b = _mm_or_si64(out64, _mm_slli_pi32(tmp64, Eoffset2_bit[j])); - tmp64c = _mm_or_si64(out64, _mm_srli_pi32(tmp64, (32 - Eoffset2_bit[j]))); - *(output_p + Eoffset2[j]) = _m_to_int(tmp64b); - *(output_p + Eoffset2[j] + 1) = _m_to_int(_mm_srli_si64(tmp64c, 32)); - } - } - for (int j = E2_first_segment; j < macro_segment_end - macro_segment; j++) { - tmp = _mm256_movemask_epi8(_mm256_slli_epi16(((__m256i *)f2)[i2], 7 - j)); - tmp64 = _mm_set1_pi32(tmp); - out64 = _mm_set_pi32(*(output_p + Eoffset2[j] + 1), *(output_p + Eoffset2[j])); - tmp64b = _mm_or_si64(out64, _mm_slli_pi32(tmp64, Eoffset2_bit[j])); - tmp64c = _mm_or_si64(out64, _mm_srli_pi32(tmp64, (32 - Eoffset2_bit[j]))); - *(output_p + Eoffset2[j]) = _m_to_int(tmp64b); - *(output_p + Eoffset2[j] + 1) = _m_to_int(_mm_srli_si64(tmp64c, 32)); - } - output_p++; - } - - -#endif + write_task_output(f, + E, + f2, + E2, + Eshift, + E2_first_segment, + macro_segment_end - macro_segment, + impp->output, + Eoffset, + args->group_tail); if(impp->toutput != NULL) stop_meas(impp->toutput); /* @@ -493,8 +474,9 @@ static void ldpc8blocks(void *p) } static int nrLDPC_launch_TB_encoding(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters, - int dlsch_id, - thread_info_tm_t *t_info) + int dlsch_id, + thread_info_tm_t *t_info, + group_tail_t *group_tail) { nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id]; @@ -523,6 +505,7 @@ static int nrLDPC_launch_TB_encoding(nrLDPC_slot_encoding_parameters_t *nrLDPC_s perJobImpp->impp.first_seg = j * 8; perJobImpp->impp.ans = t_info->ans; perJobImpp->nrLDPC_TB_encoding_parameters = nrLDPC_TB_encoding_parameters; + perJobImpp->group_tail = &group_tail[j]; task_t t = {.func = ldpc8blocks, .args = perJobImpp}; pushTpool(nrLDPC_slot_encoding_parameters->threadPool, t); @@ -544,9 +527,12 @@ int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encodin init_task_ans(&ans, nbTasks); thread_info_tm_t t_info = {.buf = (uint8_t *)arr, .len = 0, .cap = nbTasks, .ans = &ans}; + group_tail_t group_tail[nbTasks]; + memset(group_tail, 0, nbTasks * sizeof(group_tail_t)); + int nbEncode = 0; for (int dlsch_id = 0; dlsch_id < nrLDPC_slot_encoding_parameters->nb_TBs; dlsch_id++) { - nbEncode += nrLDPC_launch_TB_encoding(nrLDPC_slot_encoding_parameters, dlsch_id, &t_info); + nbEncode += nrLDPC_launch_TB_encoding(nrLDPC_slot_encoding_parameters, dlsch_id, &t_info, &group_tail[dlsch_id]); } if (nbEncode < nbTasks) { completed_many_task_ans(&ans, nbTasks - nbEncode); @@ -554,5 +540,13 @@ int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encodin // Execute thread pool tasks join_task_ans(&ans); + int n_seg_sum = 0; + for (int dlsch_id = 0; dlsch_id < nrLDPC_slot_encoding_parameters->nb_TBs; dlsch_id++) { + nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id]; + int n_seg = (nrLDPC_TB_encoding_parameters->C / 8 + ((nrLDPC_TB_encoding_parameters->C & 7) == 0 ? 0 : 1)); + fix_group_borders(&group_tail[n_seg_sum], n_seg, nrLDPC_TB_encoding_parameters->output); + n_seg_sum += n_seg; + } + return 0; } From 0aa3dba5b7950c8f3fb883fd2674611253d5a4fe Mon Sep 17 00:00:00 2001 From: Romain Beurdouche Date: Tue, 6 May 2025 08:53:57 +0000 Subject: [PATCH 11/17] feat(nrLDPC_coding_segment): Encoding with sequential output write 1. The encoder writes the output in sequence after the encoding tasks completed instead of writing within the tasks. 2. Simplify the output writing to make it more readable. Co-authored-by: Laurent Thomas --- .../nrLDPC_coding_segment_encoder.c | 405 ++++++++---------- 1 file changed, 178 insertions(+), 227 deletions(-) diff --git a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c index bbdda8c8bc..5d0e90e46b 100644 --- a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c +++ b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c @@ -44,60 +44,6 @@ #define DEBUG_LDPC_ENCODING //#define DEBUG_LDPC_ENCODING_FREE 1 -/** - * \typedef group_tail_t - * \struct group_tail_s - * \brief index and content of the last word written by a task to the encoding library output - * It is returned after task completion to fix the borders of segment groups - * \var index index of the last word written by the task - * \var content content of the last word written by the task - */ -typedef struct group_tail_s -{ - uint32_t index; -#if defined(__AVX512VBMI__) - uint64_t content; -#elif defined(__aarch64__) - uint16_t content; -#else - uint32_t content; -#endif -} group_tail_t; - -/** - * \brief fix segment groups first and last words - * The last word of a segment group is not written by encoding tasks in order to avoid race conditions - * This function fixes the segment group borders after encoding tasks completion - * \param group_tail store segment groups tail to fix segment borders - * \param nbGroups number of groups - * \params output encoding library output to write to - */ -static void fix_group_borders(group_tail_t *group_tail, int nbGroups, uint8_t *output) -{ - for(int i = 0; i < nbGroups; i++) { -#if defined(__AVX512VBMI__) - ((uint64_t *)output)[group_tail[i].index] = group_tail[i].content | ((uint64_t *)output)[group_tail[i].index]; -#elif defined(__aarch64__) - ((uint16_t *)output)[group_tail[i].index] = group_tail[i].content | ((uint16_t *)output)[group_tail[i].index]; -#else - ((uint32_t *)output)[group_tail[i].index] = group_tail[i].content | ((uint32_t *)output)[group_tail[i].index]; -#endif - } -} - -/** - * \brief determine the value of E for the current segment - * \param E number of bits per segment in f - * \param E2 number of bits per segment in f2 - * \param Eshift flag indicating if there was a E shift within the segment group - * \param segment index of the segment - * \param E2_first_segment index of the first segment of size E2 if there is a E shift in the segment group - */ -static inline uint32_t select_E(uint32_t E, uint32_t E2, bool Eshift, uint32_t segment, uint32_t E2_first_segment) -{ - return (Eshift && (segment >= E2_first_segment)) ? E2 : E; -} - /** * \brief write nrLDPC_coding_segment_encoder output with interleaver output * the interleaver output has 8 bits from 8 different segments per byte @@ -112,7 +58,6 @@ static inline uint32_t select_E(uint32_t E, uint32_t E2, bool Eshift, uint32_t s * \param nb_segments number of segments in the group * \param output nrLDPC_coding_segment_encoder with concatenated segments and packed bits * \param Eoffset offset in number of bits of the first segment of the segment group within output - * \param group_tail store segment group last words to fix segment group borders */ static void write_task_output(uint8_t *f, uint32_t E, @@ -122,77 +67,50 @@ static void write_task_output(uint8_t *f, uint32_t E2_first_segment, uint32_t nb_segments, uint8_t *output, - uint32_t Eoffset, - group_tail_t *group_tail) + uint32_t Eoffset) { #if defined(__AVX512VBMI__) uint64_t *output_p = (uint64_t*)output; - int i=0; - __m512i bitperm[8]; - uint32_t Eoffset2[8]; - uint32_t Eoffset2_bit[8]; - bitperm[0] = _mm512_set1_epi64(0x3830282018100800); __m512i inc = _mm512_set1_epi8(0x1); - Eoffset2[0] = Eoffset >> 6; - Eoffset2_bit[0] = Eoffset & 63; - Eoffset += select_E(E, E2, Eshift, 0, E2_first_segment); - for (int n = 1; n < nb_segments; n++) { - bitperm[n] = _mm512_add_epi8(bitperm[n-1] ,inc); - Eoffset2[n] = Eoffset >> 6; - Eoffset2_bit[n] = Eoffset & 63; - Eoffset += select_E(E, E2, Eshift, n, E2_first_segment); - } - group_tail->index = Eoffset2[nb_segments - 1]; - int i2=0; - __m64 tmp; - for (i=0;i>/<< because when the Eoffset2_bit is 64 or 0, the << // and >> operations are undefined and in fact don't give "0" which is // what we want here. The SIMD version do give 0 when the shift is 64 - tmp = (__m64)_mm512_bitshuffle_epi64_mask(((__m512i *)f)[i2],bitperm[j]); - - *(__m64*)(output_p + Eoffset2[j]) = _mm_or_si64(*(__m64*)(output_p + Eoffset2[j]),_mm_slli_si64(tmp,Eoffset2_bit[j])); - - *(__m64*)(output_p + Eoffset2[j] + 1) = _mm_or_si64(*(__m64*)(output_p + Eoffset2[j]+1),_mm_srli_si64(tmp,(64-Eoffset2_bit[j]))); - - } //for (int j=0; - } // if (i < E) - for (int j=E2_first_segment; j < nb_segments && (j < (nb_segments - 1) || i < (E2 - 64)); j++) { - tmp = (__m64)_mm512_bitshuffle_epi64_mask(((__m512i *)f2)[i2],bitperm[j]); - - *(__m64*)(output_p + Eoffset2[j]) = _mm_or_si64(*(__m64*)(output_p + Eoffset2[j]),_mm_slli_si64(tmp,Eoffset2_bit[j])); - - *(__m64*)(output_p + Eoffset2[j]+1) = _mm_or_si64(*(__m64*)(output_p + Eoffset2[j]+1),_mm_srli_si64(tmp,(64-Eoffset2_bit[j]))); + uint32_t Eoffset2_byte = Eoffset2 >> 6; + uint32_t Eoffset2_bit = Eoffset2 & 63; + __m64 tmp = (__m64)_mm512_bitshuffle_epi64_mask(((__m512i *)f)[i >> 6],bitperm); + *(__m64*)(output_p + Eoffset2_byte) = _mm_or_si64(*(__m64*)(output_p + Eoffset2_byte),_mm_slli_si64(tmp,Eoffset2_bit)); + *(__m64*)(output_p + Eoffset2_byte+1) = _mm_or_si64(*(__m64*)(output_p + Eoffset2_byte+1),_mm_srli_si64(tmp,(64-Eoffset2_bit))); + Eoffset2 += E; + bitperm = _mm512_add_epi8(bitperm ,inc); + } + } else { + for (int j=0; j < E2_first_segment; j++) { + Eoffset2 += E; + bitperm = _mm512_add_epi8(bitperm ,inc); + } + } + for (int j=E2_first_segment; j < nb_segments; j++) { + uint32_t Eoffset2_byte = Eoffset2 >> 6; + uint32_t Eoffset2_bit = Eoffset2 & 63; + __m64 tmp = (__m64)_mm512_bitshuffle_epi64_mask(((__m512i *)f2)[i >> 6],bitperm); + *(__m64*)(output_p + Eoffset2_byte) = _mm_or_si64(*(__m64*)(output_p + Eoffset2_byte),_mm_slli_si64(tmp,Eoffset2_bit)); + *(__m64*)(output_p + Eoffset2_byte+1) = _mm_or_si64(*(__m64*)(output_p + Eoffset2_byte+1),_mm_srli_si64(tmp,(64-Eoffset2_bit))); + Eoffset2 += E2; + bitperm = _mm512_add_epi8(bitperm ,inc); } output_p++; - group_tail->index++; } - // Do not write the last word of the segment group but keep it for fixing segment borders - if (Eshift) { - tmp = (__m64)_mm512_bitshuffle_epi64_mask(((__m512i *)f2)[(E2 - 1) >> 6], bitperm[nb_segments - 1]); - } else { - tmp = (__m64)_mm512_bitshuffle_epi64_mask(((__m512i *)f)[(E2 - 1) >> 6], bitperm[nb_segments - 1]); - } - *(__m64*)(output_p - 1 + Eoffset2[nb_segments - 1]) = _mm_or_si64(*(__m64*)(output_p - 1 + Eoffset2[nb_segments - 1]), _mm_slli_si64(tmp, Eoffset2_bit[nb_segments - 1])); - group_tail->content = (uint64_t)_mm_srli_si64(tmp, (64 - Eoffset2_bit[nb_segments - 1])); - #elif defined(__aarch64__) uint16_t *output_p = (uint16_t*)output; - int i=0; - uint32_t Eoffset2[8]; - uint32_t Eoffset2_bit[8]; - for (int n = 0; n < nb_segments; n++) { - Eoffset2[n] = Eoffset >> 4; - Eoffset2_bit[n] = Eoffset & 15; - Eoffset += select_E(E, E2, Eshift, n, E2_first_segment); - } - group_tail->index = Eoffset2[nb_segments - 1]; - int i2=0; const int8_t __attribute__ ((aligned (16))) ucShift[8][16] = { {0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7}, // segment 0 {-1,0,1,2,3,4,5,6,-1,0,1,2,3,4,5,6}, // segment 1 @@ -208,104 +126,99 @@ static void write_task_output(uint8_t *f, for (int n=0;n<8;n++) vshift[n] = vld1q_s8(ucShift[n]); uint8x16_t vmask = vld1q_u8(masks); - int32_t tmp; - uint8x16_t cshift; - for (i=0;i> 4; + uint32_t Eoffset2_bit = Eoffset2 & 15; + uint8x16_t cshift = vandq_u8(vshlq_u8(((uint8x16_t*)f)[i >> 4],vshift[j]),vmask); + int32_t tmp = (int)vaddv_u8(vget_low_u8(cshift)); tmp += (int)(vaddv_u8(vget_high_u8(cshift))<<8); - *(output_p + Eoffset2[j]) |= (uint16_t)(tmp<>(16-Eoffset2_bit[j])); + *(output_p + Eoffset2_byte) |= (uint16_t)(tmp<>(16-Eoffset2_bit)); + Eoffset2 += E; + } + } else { + for (int j=0; j < E2_first_segment; j++) { + Eoffset2 += E; } } - for (int j=E2_first_segment; j < nb_segments && (j < (nb_segments - 1) || i < (E2 - 16)); j++) { - cshift = vandq_u8(vshlq_u8(((uint8x16_t*)f2)[i2],vshift[j]),vmask); - tmp = (int)vaddv_u8(vget_low_u8(cshift)); + for (int j=E2_first_segment; j < nb_segments; j++) { + uint32_t Eoffset2_byte = Eoffset2 >> 4; + uint32_t Eoffset2_bit = Eoffset2 & 15; + uint8x16_t cshift = vandq_u8(vshlq_u8(((uint8x16_t*)f2)[i >> 4],vshift[j]),vmask); + int32_t tmp = (int)vaddv_u8(vget_low_u8(cshift)); tmp += (int)(vaddv_u8(vget_high_u8(cshift))<<8); - *(output_p + Eoffset2[j]) |= (uint16_t)(tmp<>(16-Eoffset2_bit[j])); + *(output_p + Eoffset2_byte) |= (uint16_t)(tmp<>(16-Eoffset2_bit)); + Eoffset2 += E2; } output_p++; - group_tail->index++; } - - // Do not write the last word of the segment group but keep it for fixing segment borders - if (Eshift) { - cshift = vandq_u8(vshlq_u8(((uint8x16_t*)f2)[(E2 - 1) >> 4], vshift[nb_segments - 1]), vmask); - } else { - cshift = vandq_u8(vshlq_u8(((uint8x16_t*)f)[(E2 - 1) >> 4], vshift[nb_segments - 1]), vmask); - } - tmp = (int)vaddv_u8(vget_low_u8(cshift)); - tmp += (int)(vaddv_u8(vget_high_u8(cshift)) << 8); - *(output_p - 1 + Eoffset2[nb_segments - 1]) |= (uint16_t)(tmp << Eoffset2_bit[nb_segments - 1]); - group_tail->content = (uint16_t)(tmp >> (16 - Eoffset2_bit[nb_segments - 1])); #else uint32_t *output_p = (uint32_t*)output; - int i=0; - uint32_t Eoffset2[8]; - uint32_t Eoffset2_bit[8]; - for (int n = 0; n < nb_segments; n++) { - Eoffset2[n] = Eoffset >> 5; - Eoffset2_bit[n] = Eoffset & 31; - Eoffset += select_E(E, E2, Eshift, n, E2_first_segment); - } - group_tail->index = Eoffset2[nb_segments - 1]; - int i2 = 0; - int tmp; - __m64 tmp64, tmp64b, tmp64c, out64; - - for (i=0; i < E2; i += 32, i2++) { + + for (int i=0; i < E2; i += 32) { + uint32_t Eoffset2 = Eoffset; if (i < E) { - for (int j = 0; j < E2_first_segment && (j < (nb_segments - 1) || i < (E2 - 32)); j++) { + for (int j = 0; j < E2_first_segment; j++) { // Note: Here and below, we are using the 64-bit SIMD instruction // instead of C >>/<< because when the Eoffset2_bit is 64 or 0, the << // and >> operations are undefined and in fact don't give "0" which is // what we want here. The SIMD version do give 0 when the shift is 64 - tmp = _mm256_movemask_epi8(_mm256_slli_epi16(((__m256i *)f)[i2], 7 - j)); - tmp64 = _mm_set1_pi32(tmp); - out64 = _mm_set_pi32(*(output_p + Eoffset2[j] + 1), *(output_p + Eoffset2[j])); - tmp64b = _mm_or_si64(out64, _mm_slli_pi32(tmp64, Eoffset2_bit[j])); - tmp64c = _mm_or_si64(out64, _mm_srli_pi32(tmp64, (32 - Eoffset2_bit[j]))); - *(output_p + Eoffset2[j]) = _m_to_int(tmp64b); - *(output_p + Eoffset2[j] + 1) = _m_to_int(_mm_srli_si64(tmp64c, 32)); + uint32_t Eoffset2_byte = Eoffset2 >> 5; + uint32_t Eoffset2_bit = Eoffset2 & 31; + int tmp = _mm256_movemask_epi8(_mm256_slli_epi16(((__m256i *)f)[i >> 5], 7 - j)); + __m64 tmp64 = _mm_set1_pi32(tmp); + __m64 out64 = _mm_set_pi32(*(output_p + Eoffset2_byte + 1), *(output_p + Eoffset2_byte)); + __m64 tmp64b = _mm_or_si64(out64, _mm_slli_pi32(tmp64, Eoffset2_bit)); + __m64 tmp64c = _mm_or_si64(out64, _mm_srli_pi32(tmp64, (32 - Eoffset2_bit))); + *(output_p + Eoffset2_byte) = _m_to_int(tmp64b); + *(output_p + Eoffset2_byte + 1) = _m_to_int(_mm_srli_si64(tmp64c, 32)); + Eoffset2 += E; + } + } else { + for (int j = 0; j < E2_first_segment; j++) { + Eoffset2 += E; } } - for (int j = E2_first_segment; j < nb_segments && (j < (nb_segments - 1) || i < (E2 - 32)); j++) { - tmp = _mm256_movemask_epi8(_mm256_slli_epi16(((__m256i *)f2)[i2], 7 - j)); - tmp64 = _mm_set1_pi32(tmp); - out64 = _mm_set_pi32(*(output_p + Eoffset2[j] + 1), *(output_p + Eoffset2[j])); - tmp64b = _mm_or_si64(out64, _mm_slli_pi32(tmp64, Eoffset2_bit[j])); - tmp64c = _mm_or_si64(out64, _mm_srli_pi32(tmp64, (32 - Eoffset2_bit[j]))); - *(output_p + Eoffset2[j]) = _m_to_int(tmp64b); - *(output_p + Eoffset2[j] + 1) = _m_to_int(_mm_srli_si64(tmp64c, 32)); + for (int j = E2_first_segment; j < nb_segments; j++) { + uint32_t Eoffset2_byte = Eoffset2 >> 5; + uint32_t Eoffset2_bit = Eoffset2 & 31; + int tmp = _mm256_movemask_epi8(_mm256_slli_epi16(((__m256i *)f2)[i >> 5], 7 - j)); + __m64 tmp64 = _mm_set1_pi32(tmp); + __m64 out64 = _mm_set_pi32(*(output_p + Eoffset2_byte + 1), *(output_p + Eoffset2_byte)); + __m64 tmp64b = _mm_or_si64(out64, _mm_slli_pi32(tmp64, Eoffset2_bit)); + __m64 tmp64c = _mm_or_si64(out64, _mm_srli_pi32(tmp64, (32 - Eoffset2_bit))); + *(output_p + Eoffset2_byte) = _m_to_int(tmp64b); + *(output_p + Eoffset2_byte + 1) = _m_to_int(_mm_srli_si64(tmp64c, 32)); + Eoffset2 += E2; } output_p++; - group_tail->index++; } - // Do not write the last word of the segment group but keep it for fixing segment borders - if (Eshift) { - tmp = _mm256_movemask_epi8(_mm256_slli_epi16(((__m256i *)f2)[(E2 - 1) >> 5], 7 - nb_segments + 1)); - } else { - tmp = _mm256_movemask_epi8(_mm256_slli_epi16(((__m256i *)f)[(E2 - 1) >> 5], 7 - nb_segments + 1)); - } - tmp64 = _mm_set1_pi32(tmp); - out64 = _mm_set_pi32(*(output_p - 1 + Eoffset2[nb_segments - 1] + 1), *(output_p - 1 + Eoffset2[nb_segments - 1])); - tmp64b = _mm_or_si64(out64, _mm_slli_pi32(tmp64, Eoffset2_bit[nb_segments - 1])); - tmp64c = _mm_srli_pi32(tmp64, (32 - Eoffset2_bit[nb_segments - 1])); - *(output_p - 1 + Eoffset2[nb_segments - 1]) = _m_to_int(tmp64b); - group_tail->content = _m_to_int(tmp64c); #endif } +/** + * \typedef ldpc8blocks_args_t + * \struct ldpc8blocks_args_s + * \brief Arguments of an encoding task + * encode up to 8 code blocks + * \var nrLDPC_TB_encoding_parameters TB encoding parameters as defined in the coding library interface + * \var impp encoder implementation specific parameters for the task + * \var f first interleaver output to be filled by the task + * \var f2 second interleaver output to be filled by the task + * in case of a shift of E in the code blocks group processed by the task + */ typedef struct ldpc8blocks_args_s { nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters; encoder_implemparams_t impp; - group_tail_t *group_tail; + uint8_t *f; + uint8_t *f2; } ldpc8blocks_args_t; static void ldpc8blocks(void *p) @@ -341,14 +254,13 @@ static void ldpc8blocks(void *p) LOG_D(PHY, "rvidx in encoding = %d\n", nrLDPC_TB_encoding_parameters->rv_index); #endif const uint32_t E = nrLDPC_TB_encoding_parameters->segments[macro_segment].E; - uint32_t E2=E,E2_first_segment=macro_segment_end-macro_segment; + uint32_t E2=E; bool Eshift=false; uint32_t Emax = E; for (int s=macro_segment;ssegments[s].E != E) { E2=nrLDPC_TB_encoding_parameters->segments[s].E; Eshift=true; - E2_first_segment = s-macro_segment; if(E2 > Emax) Emax = E2; break; @@ -385,12 +297,13 @@ static void ldpc8blocks(void *p) uint32_t Tbslbrm = nrLDPC_TB_encoding_parameters->tbslbrm; uint8_t e[Emax] __attribute__((aligned(64))); - uint8_t f[E+64] __attribute__((aligned(64))); - uint8_t f2[E2+64] __attribute__((aligned(64))); + // Interleaver outputs are stored in the output arrays + uint8_t *f = args->f; + uint8_t *f2 = args->f2; bzero(e, Emax); - bzero(f, E + 64); + bzero(f, ceil_mod(E, 64)); if (Eshift) { - bzero(f2, E2 + 64); + bzero(f2, ceil_mod(E2, 64)); } start_meas(&nrLDPC_TB_encoding_parameters->segments[macro_segment].ts_rate_match); nr_rate_matching_ldpc(Tbslbrm, @@ -439,44 +352,15 @@ static void ldpc8blocks(void *p) e, f2); stop_meas(&nrLDPC_TB_encoding_parameters->segments[macro_segment].ts_interleave); - if(impp->toutput != NULL) start_meas(impp->toutput); -/* - for (int i=0;i<16;i++) - for (int s=0;s>s)&1); -*/ - // information part and puncture columns - - uint32_t Eoffset=0; - for (int s=0; ssegments[s].E); - write_task_output(f, - E, - f2, - E2, - Eshift, - E2_first_segment, - macro_segment_end - macro_segment, - impp->output, - Eoffset, - args->group_tail); - - if(impp->toutput != NULL) stop_meas(impp->toutput); -/* - printf("E %d (mod16 %d) F %d\n",E,E&15,impp->F); - for (int i=0;i<16;i++) - for (int s=0;sans); } static int nrLDPC_launch_TB_encoding(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters, - int dlsch_id, - thread_info_tm_t *t_info, - group_tail_t *group_tail) + int dlsch_id, + thread_info_tm_t *t_info, + uint8_t **f, + uint8_t **f2) { nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id]; @@ -505,7 +389,8 @@ static int nrLDPC_launch_TB_encoding(nrLDPC_slot_encoding_parameters_t *nrLDPC_s perJobImpp->impp.first_seg = j * 8; perJobImpp->impp.ans = t_info->ans; perJobImpp->nrLDPC_TB_encoding_parameters = nrLDPC_TB_encoding_parameters; - perJobImpp->group_tail = &group_tail[j]; + perJobImpp->f = f[j]; + perJobImpp->f2 = f2[j]; task_t t = {.func = ldpc8blocks, .args = perJobImpp}; pushTpool(nrLDPC_slot_encoding_parameters->threadPool, t); @@ -517,22 +402,48 @@ int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encodin { int nbTasks = 0; + uint32_t Emax = 0; for (int dlsch_id = 0; dlsch_id < nrLDPC_slot_encoding_parameters->nb_TBs; dlsch_id++) { + // Compute number of tasks to encode TB nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id]; size_t n_seg = (nrLDPC_TB_encoding_parameters->C / 8 + ((nrLDPC_TB_encoding_parameters->C & 7) == 0 ? 0 : 1)); nbTasks += n_seg; + + // Search for maximum E for sizing encoder output f and f2 + for (int seg_id = 0; seg_id < nrLDPC_TB_encoding_parameters->C; seg_id++) { + uint32_t E = nrLDPC_TB_encoding_parameters->segments[seg_id].E; + Emax = E > Emax ? E : Emax; + } } + + // Create f and f2 to old encoding tasks outputs + uint32_t Emax_ceil_mod = ceil_mod(Emax, 64); + uint8_t f[nbTasks][Emax_ceil_mod] __attribute__((aligned(64))); + uint8_t f2[nbTasks][Emax_ceil_mod] __attribute__((aligned(64))); + ldpc8blocks_args_t arr[nbTasks]; task_ans_t ans; init_task_ans(&ans, nbTasks); thread_info_tm_t t_info = {.buf = (uint8_t *)arr, .len = 0, .cap = nbTasks, .ans = &ans}; - group_tail_t group_tail[nbTasks]; - memset(group_tail, 0, nbTasks * sizeof(group_tail_t)); - int nbEncode = 0; + nbTasks = 0; for (int dlsch_id = 0; dlsch_id < nrLDPC_slot_encoding_parameters->nb_TBs; dlsch_id++) { - nbEncode += nrLDPC_launch_TB_encoding(nrLDPC_slot_encoding_parameters, dlsch_id, &t_info, &group_tail[dlsch_id]); + // For easier indexing we store the pointers to sub arrays of f and f2 in pointer arrays + // Then a function to which we pass the pointer arrays can directly use f_2d[j] ans f2_2d[j] + nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id]; + size_t n_seg = (nrLDPC_TB_encoding_parameters->C / 8 + ((nrLDPC_TB_encoding_parameters->C & 7) == 0 ? 0 : 1)); + uint8_t *f_2d[n_seg]; + uint8_t *f2_2d[n_seg]; + for (int j = 0; j < n_seg; j++) { + f_2d[j] = &f[nbTasks + j][0]; + f2_2d[j] = &f2[nbTasks + j][0]; + } + + // Launch tasks encoding TB + nbEncode += nrLDPC_launch_TB_encoding(nrLDPC_slot_encoding_parameters, dlsch_id, &t_info, f_2d, f2_2d); + + nbTasks += n_seg; } if (nbEncode < nbTasks) { completed_many_task_ans(&ans, nbTasks - nbEncode); @@ -540,12 +451,52 @@ int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encodin // Execute thread pool tasks join_task_ans(&ans); - int n_seg_sum = 0; + // Write output + nbTasks = 0; for (int dlsch_id = 0; dlsch_id < nrLDPC_slot_encoding_parameters->nb_TBs; dlsch_id++) { nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id]; - int n_seg = (nrLDPC_TB_encoding_parameters->C / 8 + ((nrLDPC_TB_encoding_parameters->C & 7) == 0 ? 0 : 1)); - fix_group_borders(&group_tail[n_seg_sum], n_seg, nrLDPC_TB_encoding_parameters->output); - n_seg_sum += n_seg; + uint32_t C = nrLDPC_TB_encoding_parameters->C; + size_t n_seg = (C / 8 + ((C & 7) == 0 ? 0 : 1)); + + time_stats_t *toutput = nrLDPC_slot_encoding_parameters->toutput; + + for (int j = 0; j < n_seg; j++) { + unsigned int macro_segment = j * 8; + unsigned int macro_segment_end = (C > macro_segment + 8) ? macro_segment + 8 : C; + const uint32_t E = nrLDPC_TB_encoding_parameters->segments[macro_segment].E; + uint32_t E2 = E, E2_first_segment = macro_segment_end - macro_segment; + bool Eshift = false; + uint32_t Emax = E; + for (int s = macro_segment; s < macro_segment_end; s++) { + if (nrLDPC_TB_encoding_parameters->segments[s].E != E) { + E2 = nrLDPC_TB_encoding_parameters->segments[s].E; + Eshift = true; + E2_first_segment = s-macro_segment; + if(E2 > Emax) + Emax = E2; + break; + } + } + + if(toutput != NULL) start_meas(toutput); + + uint32_t Eoffset=0; + for (int s=0; ssegments[s].E); + + write_task_output(&f[nbTasks + j][0], + E, + &f2[nbTasks + j][0], + E2, + Eshift, + E2_first_segment, + macro_segment_end - macro_segment, + nrLDPC_TB_encoding_parameters->output, + Eoffset); + + if(toutput != NULL) stop_meas(toutput); + } + nbTasks += n_seg; } return 0; From 4d092495bf5d32e2d84591f2ab5339cfb51664d3 Mon Sep 17 00:00:00 2001 From: Romain Beurdouche Date: Tue, 8 Apr 2025 14:18:34 +0000 Subject: [PATCH 12/17] feat(nr_codeword_scrambling): Use SIMD XOR for scrambling --- openair1/PHY/NR_TRANSPORT/nr_scrambling.c | 44 ++++++++++------------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/openair1/PHY/NR_TRANSPORT/nr_scrambling.c b/openair1/PHY/NR_TRANSPORT/nr_scrambling.c index 821d92a44d..9577f943e4 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_scrambling.c +++ b/openair1/PHY/NR_TRANSPORT/nr_scrambling.c @@ -33,33 +33,27 @@ void nr_codeword_scrambling(uint8_t *in, { const int roundedSz = (size + 31) / 32; uint32_t *seq = gold_cache((n_RNTI << 15) + (q << 14) + Nid, roundedSz); - for (int i = 0; i < roundedSz; i++) - out[i] = ((uint32_t*)in)[i] ^ seq[i]; -#if 0 -//#elif __AVX2__ - for (int i = 0; i < roundedSz; i++) { - __m256i c = ((__m256i*)in)[i]; - uint32_t in32 = _mm256_movemask_epi8(_mm256_slli_epi16(c, 7)); - out[i] = in32 ^ seq[i]; - DEBUG_SCRAMBLING(LOG_D(PHY, "in[%d] %x => %x\n", i, in32, out[i])); - } -//#elif defined (__aarch64__) - const int8_t __attribute__ ((aligned (16))) ucShift[] = {0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7}; - int8x16_t vshift = vld1q_s8(ucShift); - uint8x16_t *c = (uint8x16_t*)in; - uint8x16_t cshift; - uint32_t in32; - for (int i = 0; i < roundedSz; i++) { - cshift = vshlq_u8(*c++, vshift); - in32 = vaddv_u8(vget_low_u8(cshift)); - in32 += (vaddv_u8(vget_high_u8(cshift)) << 8); - cshift = vshlq_u8(*c++, vshift); - in32 += (vaddv_u8(vget_low_u8(cshift)) << 16); - in32 += (vaddv_u8(vget_high_u8(cshift)) << 24); - out[i] = in32 ^ seq[i]; - DEBUG_SCRAMBLING(LOG_D(PHY, "in[%d] %x => %x\n", i, in32, out[i])); + unsigned int i_32 = 0; +#if defined(__AVX512F__) && defined(__AVX512VL__) + for (; i_32 < ((roundedSz >> 3) << 3); i_32 += 8) { + __m256i in_256 = _mm256_load_epi32(&((uint32_t *)in)[i_32]); + __m256i seq_256 = _mm256_load_epi32(&seq[i_32]); + _mm256_storeu_epi32(&out[i_32], _mm256_xor_si256(in_256, seq_256)); } #endif +#if defined(__aarch64__) + for (; i_32 < ((roundedSz >> 2) << 2); i_32 += 4) { + unsigned int i = i_32 >> 2; + uint32x4_t in_32x4 = vld1q_u32(&((uint32_t *)in)[i_32]); + uint32x4_t seq_32x4 = vld1q_u32(&seq[i_32]); + ((uint32x4_t *)out)[i] = veorq_u32(in_32x4, seq_32x4); + } +#endif + for (; i_32 < roundedSz; i_32++) { + unsigned int i = i_32; + out[i] = ((uint32_t *)in)[i] ^ seq[i]; + DEBUG_SCRAMBLING(LOG_D(PHY, "in[%d] %x => %x\n", i, ((uint32_t*)in)[i], out[i])); + } } void nr_codeword_unscrambling(int16_t* llr, uint32_t size, uint8_t q, uint32_t Nid, uint32_t n_RNTI) From b31dd728cb33db60bd6680af21b75e1d76645fdc Mon Sep 17 00:00:00 2001 From: Jaroslava Fiedlerova Date: Fri, 31 Jan 2025 18:11:51 +0100 Subject: [PATCH 13/17] T2: Use 8b packed output of the T2 encoder - modify retrieve_ldpc_enc_op() to store encoded results in a packed format - add timing measurements for T2 encoder - add simde implementation for bit reversal - Introduce new bit reversal implementation using simde_mm_gf2p8affine_epi64_epi8 instruction, based on https://wunkolo.github.io/post/2020/11/gf2p8affineqb-bit-reversal/. This implementation requires GFNI and AVX512VL support on the target machine. --- common/utils/nr/nr_common.c | 17 ++++++++ .../nrLDPC_coding_t2/nrLDPC_coding_t2.c | 41 +++++++++++++++---- 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/common/utils/nr/nr_common.c b/common/utils/nr/nr_common.c index 30e8e6c32e..abf3cdd280 100644 --- a/common/utils/nr/nr_common.c +++ b/common/utils/nr/nr_common.c @@ -37,6 +37,7 @@ #include "nr_common.h" #include #include +#include #define C_SRS_NUMBER (64) #define B_SRS_NUMBER (4) @@ -131,8 +132,24 @@ void reverse_bits_u8(uint8_t const* in, size_t sz, uint8_t* out) DevAssert(in != NULL); DevAssert(out != NULL); +// Bit reversal implementation based on https://wunkolo.github.io/post/2020/11/gf2p8affineqb-bit-reversal/ +#if defined(__GFNI__) && defined(__AVX512VL__) + int simde_sz = 16; + int i = 0; + int simde_bound = sz - simde_sz; + for (; i <= simde_bound; i += simde_sz) { + __m128i input = simde_mm_loadu_si128((__m128i *)&in[i]); + __m128i reversed = simde_mm_gf2p8affine_epi64_epi8(input, simde_mm_set1_epi64x(0x8040201008040201), 0); + simde_mm_storeu_si128((__m128i *)&out[i], reversed); + } + + for (; i < sz; ++i) { + out[i] = bit_reverse_table_256[in[i]]; + } +#else for(size_t i = 0; i < sz; ++i) out[i] = bit_reverse_table_256[in[i]]; +#endif } // Reverse bits implementation based on http://graphics.stanford.edu/~seander/bithacks.html diff --git a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_t2/nrLDPC_coding_t2.c b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_t2/nrLDPC_coding_t2.c index 97792ed1fc..7974979792 100644 --- a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_t2/nrLDPC_coding_t2.c +++ b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_t2/nrLDPC_coding_t2.c @@ -650,25 +650,40 @@ static int retrieve_ldpc_enc_op(struct rte_bbdev_enc_op **ops, nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters) { + uint8_t *p_out = NULL; unsigned int j = 0; for (unsigned int h = 0; h < nrLDPC_slot_encoding_parameters->nb_TBs; ++h){ + int E_sum = 0; + int bit_offset = 0; + int byte_offset = 0; + p_out = nrLDPC_slot_encoding_parameters->TBs[h].output; for (unsigned int i = 0; i < nrLDPC_slot_encoding_parameters->TBs[h].C; ++i) { struct rte_bbdev_op_data *output = &ops[j]->ldpc_enc.output; struct rte_mbuf *m = output->data; uint16_t data_len = rte_pktmbuf_data_len(m) - output->offset; - uint8_t *out = nrLDPC_slot_encoding_parameters->TBs[h].output; const char *data = m->buf_addr + m->data_off; - const char *end = data + data_len; - while (data < end) { - uint8_t byte = *data++; // get the current byte - for (int bit = 7; bit >= 0; --bit) { - *out++ = (byte >> bit) & 1; // extract each bit + if (bit_offset == 0) { + memcpy(&p_out[byte_offset], data, data_len); + } else { + uint8_t carry = 0; + p_out[byte_offset - 1] |= data[0] >> bit_offset; + for (size_t i = 0; i < data_len; i++) { + uint8_t current = *data++; + p_out[byte_offset + i] = (current << (8 - bit_offset)); + if (i != 0) { + carry = current >> bit_offset; + p_out[byte_offset + i - 1] |= carry; + } } } + E_sum += nrLDPC_slot_encoding_parameters->TBs[h].segments[i].E; + byte_offset = (E_sum + 7) / 8; + bit_offset = E_sum % 8; rte_pktmbuf_free(m); rte_pktmbuf_free(ops[j]->ldpc_enc.input.data); ++j; } + reverse_bits_u8(p_out, byte_offset, p_out); } return 0; } @@ -815,6 +830,10 @@ static int pmd_lcore_ldpc_enc(void *arg) AssertFatal(ret == 0, "Allocation failed for %d ops", num_segments); set_ldpc_enc_op(ops_enq, 0, bufs->inputs, bufs->hard_outputs, nrLDPC_slot_encoding_parameters); + if (nrLDPC_slot_encoding_parameters->tprep != NULL) + stop_meas(nrLDPC_slot_encoding_parameters->tprep); + if (nrLDPC_slot_encoding_parameters->tparity != NULL) + start_meas(nrLDPC_slot_encoding_parameters->tparity); for (enq = 0, deq = 0; enq < num_segments;) { num_to_enq = num_segments; if (unlikely(num_segments - enq < num_to_enq)) @@ -828,10 +847,14 @@ static int pmd_lcore_ldpc_enc(void *arg) time_out++; DevAssert(time_out <= TIME_OUT_POLL); } - + if (nrLDPC_slot_encoding_parameters->tparity != NULL) + stop_meas(nrLDPC_slot_encoding_parameters->tparity); + if (nrLDPC_slot_encoding_parameters->toutput != NULL) + start_meas(nrLDPC_slot_encoding_parameters->toutput); ret = retrieve_ldpc_enc_op(ops_deq, nrLDPC_slot_encoding_parameters); AssertFatal(ret == 0, "Failed to retrieve LDPC encoding op!"); - + if (nrLDPC_slot_encoding_parameters->toutput != NULL) + stop_meas(nrLDPC_slot_encoding_parameters->toutput); rte_bbdev_enc_op_free_bulk(ops_enq, num_segments); rte_free(ops_enq); rte_free(ops_deq); @@ -1074,6 +1097,8 @@ int32_t nrLDPC_coding_decoder(nrLDPC_slot_decoding_parameters_t *nrLDPC_slot_dec int32_t nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters) { pthread_mutex_lock(&encode_mutex); + if (nrLDPC_slot_encoding_parameters->tprep != NULL) + start_meas(nrLDPC_slot_encoding_parameters->tprep); // hardcoded to use the first found board struct active_device *ad = active_devs; int ret; From 2feb36407cb144d3af5c630656c6bfb5c3527d65 Mon Sep 17 00:00:00 2001 From: Romain Beurdouche Date: Wed, 9 Apr 2025 07:59:23 +0000 Subject: [PATCH 14/17] feat(reverse_bits_u8): increase vector size in bits reversal Reverse bits by 512 bits vector instead of 128 bits vector Also removed SIMDE as the preprocessor directives were allowing only AVX512 --- common/utils/nr/nr_common.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/common/utils/nr/nr_common.c b/common/utils/nr/nr_common.c index abf3cdd280..078ef027e8 100644 --- a/common/utils/nr/nr_common.c +++ b/common/utils/nr/nr_common.c @@ -133,14 +133,14 @@ void reverse_bits_u8(uint8_t const* in, size_t sz, uint8_t* out) DevAssert(out != NULL); // Bit reversal implementation based on https://wunkolo.github.io/post/2020/11/gf2p8affineqb-bit-reversal/ -#if defined(__GFNI__) && defined(__AVX512VL__) - int simde_sz = 16; +#if defined(__GFNI__) && defined(__AVX512F__) + int simde_sz = 64; int i = 0; int simde_bound = sz - simde_sz; for (; i <= simde_bound; i += simde_sz) { - __m128i input = simde_mm_loadu_si128((__m128i *)&in[i]); - __m128i reversed = simde_mm_gf2p8affine_epi64_epi8(input, simde_mm_set1_epi64x(0x8040201008040201), 0); - simde_mm_storeu_si128((__m128i *)&out[i], reversed); + __m512i input = _mm512_loadu_epi8(&in[i]); + __m512i reversed = _mm512_gf2p8affine_epi64_epi8(input, _mm512_set1_epi64(0x8040201008040201), 0); + _mm512_storeu_epi8(&out[i], reversed); } for (; i < sz; ++i) { From 360f341394bcf560ecca6245da8453acb3b1459f Mon Sep 17 00:00:00 2001 From: Romain Beurdouche Date: Fri, 9 May 2025 13:31:01 +0000 Subject: [PATCH 15/17] feat(nrLDPC_coding_t2): Shift encoder output with 64 bits shift Use 64 bits shift to shift the output segment of the encoding operations to their offset in the coding library output buffer. --- .../nrLDPC_coding_t2/nrLDPC_coding_t2.c | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_t2/nrLDPC_coding_t2.c b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_t2/nrLDPC_coding_t2.c index 7974979792..6eb8f0dacc 100644 --- a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_t2/nrLDPC_coding_t2.c +++ b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_t2/nrLDPC_coding_t2.c @@ -657,33 +657,41 @@ retrieve_ldpc_enc_op(struct rte_bbdev_enc_op **ops, int bit_offset = 0; int byte_offset = 0; p_out = nrLDPC_slot_encoding_parameters->TBs[h].output; - for (unsigned int i = 0; i < nrLDPC_slot_encoding_parameters->TBs[h].C; ++i) { + for (unsigned int r = 0; r < nrLDPC_slot_encoding_parameters->TBs[h].C; ++r) { struct rte_bbdev_op_data *output = &ops[j]->ldpc_enc.output; struct rte_mbuf *m = output->data; uint16_t data_len = rte_pktmbuf_data_len(m) - output->offset; - const char *data = m->buf_addr + m->data_off; + uint8_t *data = m->buf_addr + m->data_off; + reverse_bits_u8(data, data_len, data); if (bit_offset == 0) { memcpy(&p_out[byte_offset], data, data_len); } else { - uint8_t carry = 0; - p_out[byte_offset - 1] |= data[0] >> bit_offset; - for (size_t i = 0; i < data_len; i++) { + size_t i = 0; + for (; i < (data_len & ~0x7); i += 8) { + uint8_t carry = *data << bit_offset; + p_out[byte_offset + i - 1] |= carry; + + simde__m64 current = *((simde__m64 *)data); + data += 8; + current = simde_mm_srli_si64(current, 8 - bit_offset); + *(simde__m64 *)&p_out[byte_offset + i] = current; + } + for (; i < data_len; i++) { uint8_t current = *data++; - p_out[byte_offset + i] = (current << (8 - bit_offset)); - if (i != 0) { - carry = current >> bit_offset; - p_out[byte_offset + i - 1] |= carry; - } + + uint8_t carry = current << bit_offset; + p_out[byte_offset + i - 1] |= carry; + + p_out[byte_offset + i] = (current >> (8 - bit_offset)); } } - E_sum += nrLDPC_slot_encoding_parameters->TBs[h].segments[i].E; + E_sum += nrLDPC_slot_encoding_parameters->TBs[h].segments[r].E; byte_offset = (E_sum + 7) / 8; bit_offset = E_sum % 8; rte_pktmbuf_free(m); rte_pktmbuf_free(ops[j]->ldpc_enc.input.data); ++j; } - reverse_bits_u8(p_out, byte_offset, p_out); } return 0; } From 1734e1916ca1c23971a3110961f6d42b2fbd1322 Mon Sep 17 00:00:00 2001 From: Romain Beurdouche Date: Tue, 15 Apr 2025 13:27:09 +0000 Subject: [PATCH 16/17] fix(nrLDPC_coding_interface): mistaken line forgotten in documentation --- openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_interface.h | 1 - 1 file changed, 1 deletion(-) diff --git a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_interface.h b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_interface.h index e99e1badbc..ca57c9bab3 100644 --- a/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_interface.h +++ b/openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_interface.h @@ -141,7 +141,6 @@ typedef struct nrLDPC_slot_decoding_parameters_s{ * \brief encoding parameter of segments * \var E input llr segment size * \var c Pointers to code blocks before LDPC encoding (38.212 V15.4.0 section 5.2.2) - * flag indicating that the decoding of the segment was successful * IT MUST BE FILLED BY THE IMPLEMENTATION * \var ts_interleave interleaving time stats * \var ts_rate_match rate matching time stats From b1e0bfdda696b6dea72acc0375a67422c949efc7 Mon Sep 17 00:00:00 2001 From: Romain Beurdouche Date: Thu, 24 Apr 2025 07:31:29 +0000 Subject: [PATCH 17/17] fix(CI RAN-gNB-N300-Timing-Phytest-LDPC): update reference timings Update reference `L1 Tx processing` and `DLSCH encoding` timing in `RAN-gNB-N300-Timing-Phytest-LDPC` CI test to reflect performance improvements. --- ci-scripts/datalog_rt_stats.100.2x2.yaml | 4 ++-- ci-scripts/datalog_rt_stats.1x1.60.yaml | 4 ++-- ci-scripts/datalog_rt_stats.60.2x2.yaml | 4 ++-- ci-scripts/datalog_rt_stats.default.yaml | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ci-scripts/datalog_rt_stats.100.2x2.yaml b/ci-scripts/datalog_rt_stats.100.2x2.yaml index c0a7717346..024dfa8e9e 100644 --- a/ci-scripts/datalog_rt_stats.100.2x2.yaml +++ b/ci-scripts/datalog_rt_stats.100.2x2.yaml @@ -11,8 +11,8 @@ Ref : feptx_prec : 0.0 feptx_ofdm : 60.0 feptx_total : 150.0 - L1 Tx processing : 530.0 - DLSCH encoding : 220.0 + L1 Tx processing : 475.0 + DLSCH encoding : 200.0 L1 Rx processing : 530.0 PUSCH inner-receiver : 360.0 Schedule Response : 1.0 diff --git a/ci-scripts/datalog_rt_stats.1x1.60.yaml b/ci-scripts/datalog_rt_stats.1x1.60.yaml index 47021b502b..85075bac97 100644 --- a/ci-scripts/datalog_rt_stats.1x1.60.yaml +++ b/ci-scripts/datalog_rt_stats.1x1.60.yaml @@ -11,8 +11,8 @@ Ref : feptx_prec : 15.0 feptx_ofdm : 30.0 feptx_total : 45.0 - L1 Tx processing : 205.0 - DLSCH encoding : 140.0 + L1 Tx processing : 175.0 + DLSCH encoding : 110.0 L1 Rx processing : 345.0 PUSCH inner-receiver : 150.0 Schedule Response : 1.0 diff --git a/ci-scripts/datalog_rt_stats.60.2x2.yaml b/ci-scripts/datalog_rt_stats.60.2x2.yaml index 064d6374c1..b371356971 100644 --- a/ci-scripts/datalog_rt_stats.60.2x2.yaml +++ b/ci-scripts/datalog_rt_stats.60.2x2.yaml @@ -11,8 +11,8 @@ Ref : feptx_prec : 14.0 feptx_ofdm : 30.0 feptx_total : 80.0 - L1 Tx processing : 315.0 - DLSCH encoding : 155.0 + L1 Tx processing : 280.0 + DLSCH encoding : 140.0 L1 Rx processing : 345.0 PUSCH inner-receiver : 155.0 Schedule Response : 1.0 diff --git a/ci-scripts/datalog_rt_stats.default.yaml b/ci-scripts/datalog_rt_stats.default.yaml index 17e45cf367..32977944a0 100644 --- a/ci-scripts/datalog_rt_stats.default.yaml +++ b/ci-scripts/datalog_rt_stats.default.yaml @@ -11,8 +11,8 @@ Ref : feptx_prec : 13.0 feptx_ofdm : 30.0 feptx_total : 45.0 - L1 Tx processing : 160.0 - DLSCH encoding : 100.0 + L1 Tx processing : 145.0 + DLSCH encoding : 90.0 L1 Rx processing : 290.0 PUSCH inner-receiver : 115.0 Schedule Response : 1.0