diff --git a/CMakeLists.txt b/CMakeLists.txt index ec60df8b35..e1527d227c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1040,6 +1040,7 @@ set(PHY_SRC_UE ${OPENAIR1_DIR}/PHY/NR_REFSIG/dmrs_nr.c ${OPENAIR1_DIR}/PHY/NR_REFSIG/ptrs_nr.c ${OPENAIR1_DIR}/PHY/NR_ESTIMATION/nr_ul_channel_estimation.c + ${OPENAIR1_DIR}/PHY/NR_ESTIMATION/nr_freq_equalization.c ${OPENAIR1_DIR}/PHY/NR_ESTIMATION/nr_measurements_gNB.c ${OPENAIR1_DIR}/PHY/TOOLS/file_output.c ${OPENAIR1_DIR}/PHY/TOOLS/cadd_vv.c diff --git a/executables/nr-gnb.c b/executables/nr-gnb.c index 33fe6b3f2a..d436828294 100644 --- a/executables/nr-gnb.c +++ b/executables/nr-gnb.c @@ -479,6 +479,7 @@ void init_gNB_Tpool(int inst) { // ULSCH decoding threadpool initTpool(get_softmodem_params()->threadPoolConfig, &gNB->threadPool, cpumeas(CPUMEAS_GETSTATE)); // ULSCH decoder result FIFO + initNotifiedFIFO(&gNB->respPuschSymb); initNotifiedFIFO(&gNB->respDecode); // L1 RX result FIFO diff --git a/openair1/PHY/INIT/nr_init.c b/openair1/PHY/INIT/nr_init.c index ed3fd55079..09898d383b 100644 --- a/openair1/PHY/INIT/nr_init.c +++ b/openair1/PHY/INIT/nr_init.c @@ -28,6 +28,7 @@ #include "PHY/CODING/nrPolar_tools/nr_polar_pbch_defs.h" #include "PHY/NR_TRANSPORT/nr_transport_proto.h" #include "PHY/NR_TRANSPORT/nr_transport_common_proto.h" +#include "PHY/NR_ESTIMATION/nr_ul_estimation.h" #include "openair1/PHY/MODULATION/nr_modulation.h" #include "openair1/PHY/defs_RU.h" #include "openair1/PHY/CODING/nrLDPC_extern.h" @@ -525,6 +526,8 @@ int phy_init_nr_gNB(PHY_VARS_gNB *gNB) init_scrambling_luts(); init_pucch2_luts(); + nr_init_fde(); // Init array for frequency equalization of transform precoding of PUSCH + load_nrLDPClib(NULL); if (gNB->ldpc_offload_flag) diff --git a/openair1/PHY/NR_ESTIMATION/nr_freq_equalization.c b/openair1/PHY/NR_ESTIMATION/nr_freq_equalization.c new file mode 100644 index 0000000000..e73ac48fab --- /dev/null +++ b/openair1/PHY/NR_ESTIMATION/nr_freq_equalization.c @@ -0,0 +1,73 @@ +/* + * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The OpenAirInterface Software Alliance licenses this file to You under + * the OAI Public License, Version 1.1 (the "License"); you may not use this file + * except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.openairinterface.org/?page_id=698 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *------------------------------------------------------------------------------- + * For more information about the OpenAirInterface (OAI) Software Alliance: + * contact@openairinterface.org + */ + +#include "PHY/defs_eNB.h" +#include "PHY/sse_intrin.h" +#include "PHY/NR_ESTIMATION/nr_ul_estimation.h" + +// Reference of openair1/PHY/LTE_ESTIMATION/freq_equalization.c + +// This is 4096/(1:4096) in simde__m128i format +simde__m128i nr_inv_ch[4096];/* = {0, 4096/1, 4096/2, 4096/3, 4096/4...}*/ + +void nr_init_fde() +{ + for (int i = 1;i < 4096; i++) + nr_inv_ch[i] = simde_mm_set1_epi16(4096/i); +} + +void nr_freq_equalization (NR_DL_FRAME_PARMS *frame_parms, + int32_t *rxdataF_comp, + int32_t *ul_ch_mag, + int32_t *ul_ch_magb, + uint8_t symbol, + uint16_t Msc_RS, + uint8_t Qm) +{ + simde__m128i *rxdataF_comp128 = (simde__m128i *)rxdataF_comp; + simde__m128i *ul_ch_mag128 = (simde__m128i *)ul_ch_mag; + simde__m128i *ul_ch_magb128 = (simde__m128i *)ul_ch_magb; + + AssertFatal(symbol < frame_parms->symbols_per_slot, "symbol %d >= %d\n", + symbol, frame_parms->symbols_per_slot); + AssertFatal(Msc_RS <= frame_parms->N_RB_UL*12, "Msc_RS %d >= %d\n", + Msc_RS, frame_parms->N_RB_UL*12); + + for (uint16_t re = 0; re < (Msc_RS >> 2); re++) + { + int16_t amp = (*((int16_t*)&ul_ch_mag128[re])); + + if (amp > 4095) + amp = 4095; + + rxdataF_comp128[re] = simde_mm_srai_epi16(simde_mm_mullo_epi16(rxdataF_comp128[re],nr_inv_ch[amp]),3); + + if (Qm == 4) + ul_ch_mag128[re] = simde_mm_set1_epi16(324); // this is 512*2/sqrt(10) + else if (Qm == 6) + { + ul_ch_mag128[re] = simde_mm_set1_epi16(316); // this is 512*4/sqrt(42) + ul_ch_magb128[re] = simde_mm_set1_epi16(158); // this is 512*2/sqrt(42) + } + else if(Qm != 2) + AssertFatal(1, "nr_freq_equalization(), Qm = %d, should be 2, 4 or 6. symbol=%d, Msc_RS=%d\n", Qm, symbol, Msc_RS); + } +} diff --git a/openair1/PHY/NR_ESTIMATION/nr_ul_estimation.h b/openair1/PHY/NR_ESTIMATION/nr_ul_estimation.h index c75b4d4ed7..4934498a4a 100644 --- a/openair1/PHY/NR_ESTIMATION/nr_ul_estimation.h +++ b/openair1/PHY/NR_ESTIMATION/nr_ul_estimation.h @@ -83,4 +83,15 @@ int nr_srs_channel_estimation(const PHY_VARS_gNB *gNB, int32_t srs_estimated_channel_time_shifted[][1<num_ant_ports][gNB->frame_parms.ofdm_symbol_size], int8_t *snr_per_rb, int8_t *snr); + +void nr_freq_equalization(NR_DL_FRAME_PARMS *frame_parms, + int *rxdataF_comp, + int *ul_ch_mag, + int *ul_ch_mag_b, + unsigned char symbol, + unsigned short Msc_RS, + unsigned char Qm); + +void nr_init_fde(void); + #endif diff --git a/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c b/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c index 9c398d1664..3e0b3506b1 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c +++ b/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c @@ -18,10 +18,6 @@ #define INVALID_VALUE 255 -#ifdef __aarch64__ -#define USE_128BIT -#endif - void nr_idft(int32_t *z, uint32_t Msc_PUSCH) { @@ -285,16 +281,16 @@ void nr_idft(int32_t *z, uint32_t Msc_PUSCH) } -void nr_ulsch_extract_rbs0 (c16_t *rxdataF, - int32_t *chF, - int32_t *rxFext, - int32_t *chFext, - int rxoffset, - int choffset, - int aarx, - int is_dmrs_symbol, - nfapi_nr_pusch_pdu_t *pusch_pdu, - NR_DL_FRAME_PARMS *frame_parms) +static void nr_ulsch_extract_rbs0 (c16_t* const rxdataF, + c16_t* const chF, + c16_t *rxFext, + c16_t *chFext, + int rxoffset, + int choffset, + int aarx, + int is_dmrs_symbol, + nfapi_nr_pusch_pdu_t *pusch_pdu, + NR_DL_FRAME_PARMS *frame_parms) { uint8_t delta = 0; @@ -303,206 +299,87 @@ void nr_ulsch_extract_rbs0 (c16_t *rxdataF, int nb_re_pusch = NR_NB_SC_PER_RB * pusch_pdu->rb_size; - int32_t *rxF = (int32_t*)&rxdataF[rxoffset]; - int32_t *rxF_ext = &rxFext[0]; - int32_t *ul_ch0 = &chF[choffset]; - int32_t *ul_ch0_ext = &chFext[0]; - + c16_t *rxF = &rxdataF[rxoffset]; + c16_t *rxF_ext = &rxFext[0]; + c16_t *ul_ch0 = &chF[choffset]; + c16_t *ul_ch0_ext = &chFext[0]; if (is_dmrs_symbol == 0) { - if (start_re + nb_re_pusch <= frame_parms->ofdm_symbol_size) { - memcpy((void*)rxF_ext, - (void*)&rxF[start_re], - nb_re_pusch*sizeof(int32_t)); - } + if (start_re + nb_re_pusch <= frame_parms->ofdm_symbol_size) + memcpy(rxF_ext, &rxF[start_re], nb_re_pusch * sizeof(c16_t)); else { - int neg_length = frame_parms->ofdm_symbol_size-start_re; + int neg_length = frame_parms->ofdm_symbol_size - start_re; int pos_length = nb_re_pusch - neg_length; - memcpy((void*)rxF_ext, (void*)&rxF[start_re], neg_length * sizeof(int32_t)); - memcpy((void*)&rxF_ext[neg_length], (void*)rxF, pos_length * sizeof(int32_t)); + memcpy(rxF_ext, &rxF[start_re], neg_length * sizeof(c16_t)); + memcpy(&rxF_ext[neg_length], rxF, pos_length * sizeof(c16_t)); } - memcpy((void*)ul_ch0_ext,(void*)ul_ch0,nb_re_pusch*sizeof(int32_t)); + memcpy(ul_ch0_ext, ul_ch0, nb_re_pusch * sizeof(c16_t)); } else if (pusch_pdu->dmrs_config_type == pusch_dmrs_type1) // 6 REs / PRB { - AssertFatal(delta==0||delta==1,"Illegal delta %d\n",delta); - int32_t *rxF32 = &rxF[start_re]; - int32_t *rxF_ext32 = rxF_ext; - int32_t *ul_ch032 = ul_ch0; - int32_t *ul_ch0_ext32 = ul_ch0_ext; - int idx,idx2,idx3; + AssertFatal(delta == 0 || delta == 1, "Illegal delta %d\n",delta); + c16_t *rxF32 = &rxF[start_re]; if (start_re + nb_re_pusch < frame_parms->ofdm_symbol_size) { - for (idx=1-delta,idx2=0;idxofdm_symbol_size-start_re; - int pos_length = nb_re_pusch-neg_length; - - for (idx=1-delta,idx2=0;idxofdm_symbol_size - start_re; + int pos_length = nb_re_pusch - neg_length; + int idx, idx2; + for (idx = 1 - delta; idx < neg_length; idx += 2) + { + *rxF_ext++ = rxF32[idx]; + *ul_ch0_ext++= ul_ch0[idx]; } - rxF32=(int32_t*)rxF; - idx3=idx; - for (idx=1-delta;idxdmrs_config_type == pusch_dmrs_type2) // 8 REs / PRB { AssertFatal(delta==0||delta==2||delta==4,"Illegal delta %d\n",delta); - if (start_re + nb_re_pusch < frame_parms->ofdm_symbol_size) { - int64_t *rxF64 = (int64_t*)&rxF[start_re]; - int64_t *rxF_ext64 = (int64_t*)rxF_ext; - int64_t *ul_ch064 = (int64_t*)ul_ch0; - int64_t *ul_ch0_ext64 = (int64_t*)ul_ch0_ext; - if (delta==0) { - for (int idx=0;idx>1;idx+=6) { - rxF_ext64[idx]=rxF64[idx+1]; - rxF_ext64[idx+1]=rxF64[idx+2]; - rxF_ext64[idx+2]=rxF64[idx+4]; - rxF_ext64[idx+3]=rxF64[idx+5]; - ul_ch0_ext64[idx]=ul_ch064[idx+1]; - ul_ch0_ext64[idx+1]=ul_ch064[idx+2]; - ul_ch0_ext64[idx+2]=ul_ch064[idx+4]; - ul_ch0_ext64[idx+3]=ul_ch064[idx+5]; - } - } - else if (delta==2) { - for (int idx=0;idx>1;idx+=6) { - rxF_ext64[idx]=rxF64[idx+0]; - rxF_ext64[idx+1]=rxF64[idx+2]; - rxF_ext64[idx+2]=rxF64[idx+3]; - rxF_ext64[idx+3]=rxF64[idx+5]; - ul_ch0_ext64[idx]=ul_ch064[idx+0]; - ul_ch0_ext64[idx+1]=ul_ch064[idx+2]; - ul_ch0_ext64[idx+2]=ul_ch064[idx+3]; - ul_ch0_ext64[idx+3]=ul_ch064[idx+5]; - } - } - else if (delta==4) { - for (int idx=0;idx>1;idx+=6) { - rxF_ext64[idx]=rxF64[idx+0]; - rxF_ext64[idx+1]=rxF64[idx+1]; - rxF_ext64[idx+2]=rxF64[idx+3]; - rxF_ext64[idx+3]=rxF64[idx+4]; - ul_ch0_ext64[idx]=ul_ch064[idx+0]; - ul_ch0_ext64[idx+1]=ul_ch064[idx+1]; - ul_ch0_ext64[idx+2]=ul_ch064[idx+3]; - ul_ch0_ext64[idx+3]=ul_ch064[idx+4]; - } + if (start_re + nb_re_pusch < frame_parms->ofdm_symbol_size) + { + for (int idx = 0; idx < nb_re_pusch; idx ++) + { + if (idx % 6 == 2 * delta || idx % 6 == 2 * delta + 1) + continue; + *rxF_ext++ = rxF[idx]; + *ul_ch0_ext++ = ul_ch0[idx]; } } - else { - int neg_length = frame_parms->ofdm_symbol_size-start_re; - int pos_length = nb_re_pusch-neg_length; - if ((pos_length%12) > 0 ) pos_length+=12; - int64_t *rxF64 = (int64_t*)&rxF[start_re]; - int64_t *rxF_ext64 = (int64_t*)rxF_ext; - int64_t *ul_ch064 = (int64_t*)ul_ch0; - int64_t *ul_ch0_ext64 = (int64_t*)ul_ch0_ext; - int idx=0; - if (delta==0) { - for (idx=0;idx>1;idx+=6) { - rxF_ext64[idx] =rxF64[idx+1]; - rxF_ext64[idx+1]=rxF64[idx+2]; - rxF_ext64[idx+2]=rxF64[idx+4]; - rxF_ext64[idx+3]=rxF64[idx+5]; - ul_ch0_ext64[idx]=ul_ch064[idx+1]; - ul_ch0_ext64[idx+1]=ul_ch064[idx+2]; - ul_ch0_ext64[idx+2]=ul_ch064[idx+4]; - ul_ch0_ext64[idx+3]=ul_ch064[idx+5]; - } - if ((neg_length%12) > 0) { - rxF_ext64[idx+4]=rxF64[idx+7]; - rxF_ext64[idx+5]=rxF64[idx+8]; - ul_ch0_ext64[idx+4]=ul_ch064[idx+7]; - ul_ch0_ext64[idx+5]=ul_ch064[idx+8]; - } - rxF_ext64+=(neg_length/3); - rxF64=(int64_t*)rxF; - ul_ch0_ext64+=(neg_length/3); - ul_ch064+=(neg_length>>1); - for (idx=0;idx>1;idx+=6) { - rxF_ext64[idx] =rxF64[idx+1]; - rxF_ext64[idx+1]=rxF64[idx+2]; - rxF_ext64[idx+2]=rxF64[idx+4]; - rxF_ext64[idx+3]=rxF64[idx+5]; - ul_ch0_ext64[idx]=ul_ch064[idx+1]; - ul_ch0_ext64[idx+1]=ul_ch064[idx+2]; - ul_ch0_ext64[idx+2]=ul_ch064[idx+4]; - ul_ch0_ext64[idx+3]=ul_ch064[idx+5]; - } + else + { + int neg_length = frame_parms->ofdm_symbol_size - start_re; + int pos_length = nb_re_pusch - neg_length; + c16_t *rxF64 = &rxF[start_re]; + int idx, idx2; + for (idx = 0; idx < neg_length; idx ++) + { + if (idx % 6 == 2 * delta || idx % 6 == 2 * delta + 1) + continue; + *rxF_ext++ = rxF64[idx]; + *ul_ch0_ext++ = ul_ch0[idx]; } - else if (delta==2) { - for (idx=0;idx>1;idx+=6) { - rxF_ext64[idx] =rxF64[idx+0]; - rxF_ext64[idx+1]=rxF64[idx+2]; - rxF_ext64[idx+2]=rxF64[idx+3]; - rxF_ext64[idx+3]=rxF64[idx+5]; - ul_ch0_ext64[idx]=ul_ch064[idx+0]; - ul_ch0_ext64[idx+1]=ul_ch064[idx+2]; - ul_ch0_ext64[idx+2]=ul_ch064[idx+3]; - ul_ch0_ext64[idx+3]=ul_ch064[idx+5]; - } - if ((neg_length%12) > 0) { - rxF_ext64[idx+4]=rxF64[idx+6]; - rxF_ext64[idx+5]=rxF64[idx+8]; - ul_ch0_ext64[idx+4]=ul_ch064[idx+6]; - ul_ch0_ext64[idx+5]=ul_ch064[idx+8]; - } - rxF_ext64+=(neg_length/3); - rxF64=(int64_t*)rxF; - ul_ch0_ext64+=(neg_length/3); - ul_ch064+=(neg_length>>1); - for (idx=0;idx>1;idx+=6) { - rxF_ext64[idx] =rxF64[idx+0]; - rxF_ext64[idx+1]=rxF64[idx+2]; - rxF_ext64[idx+2]=rxF64[idx+3]; - rxF_ext64[idx+3]=rxF64[idx+5]; - ul_ch0_ext64[idx]=ul_ch064[idx+0]; - ul_ch0_ext64[idx+1]=ul_ch064[idx+2]; - ul_ch0_ext64[idx+2]=ul_ch064[idx+3]; - ul_ch0_ext64[idx+3]=ul_ch064[idx+5]; - } - } - else if (delta==4) { - for (idx=0;idx>1;idx+=6) { - rxF_ext64[idx] =rxF64[idx+0]; - rxF_ext64[idx+1]=rxF64[idx+1]; - rxF_ext64[idx+2]=rxF64[idx+3]; - rxF_ext64[idx+3]=rxF64[idx+4]; - ul_ch0_ext64[idx]=ul_ch064[idx+0]; - ul_ch0_ext64[idx+1]=ul_ch064[idx+1]; - ul_ch0_ext64[idx+2]=ul_ch064[idx+3]; - ul_ch0_ext64[idx+3]=ul_ch064[idx+4]; - } - if ((neg_length%12) > 0) { - rxF_ext64[idx+4]=rxF64[idx+6]; - rxF_ext64[idx+5]=rxF64[idx+7]; - ul_ch0_ext64[idx+4]=ul_ch064[idx+6]; - ul_ch0_ext64[idx+5]=ul_ch064[idx+7]; - } - rxF_ext64+=(neg_length/3); - rxF64=(int64_t*)rxF; - ul_ch0_ext64+=(neg_length/3); - ul_ch064+=(neg_length>>1); - for (idx=0;idx>1;idx+=6) { - rxF_ext64[idx] =rxF64[idx+0]; - rxF_ext64[idx+1]=rxF64[idx+1]; - rxF_ext64[idx+2]=rxF64[idx+3]; - rxF_ext64[idx+3]=rxF64[idx+4]; - ul_ch0_ext64[idx]=ul_ch064[idx+0]; - ul_ch0_ext64[idx+1]=ul_ch064[idx+1]; - ul_ch0_ext64[idx+2]=ul_ch064[idx+3]; - ul_ch0_ext64[idx+3]=ul_ch064[idx+4]; - } + rxF64 = rxF; + idx2 = idx; + for (idx = 0; idx < pos_length; idx++, idx2++) + { + if (idx % 6 == 2 * delta || idx % 6 == 2 * delta + 1) + continue; + *rxF_ext++ = rxF64[idx]; + *ul_ch0_ext++ = ul_ch0[idx2]; } } } @@ -662,1423 +539,6 @@ int get_nb_re_pusch (NR_DL_FRAME_PARMS *frame_parms, nfapi_nr_pusch_pdu_t *rel15 } else return(rel15_ul->rb_size * NR_NB_SC_PER_RB); } -void inner_rx_qpsk (int *rxF, - int *ul_ch, - int16_t *llr, - int aarx, - int length, - int output_shift) -{ -#ifndef USE_128BIT - register simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; - register simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); - register simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); - - simde__m256i *rxF256 = (simde__m256i*)rxF; - simde__m256i *ulch256 = (simde__m256i*)ul_ch; - // need to use simde__m64 because llr output is not necessarily aligned to 256 bits, but it is always to 64 bits - simde__m64 *llr64 = (simde__m64 *)llr; - for (int i=0; i<((length>>3)+((length&7)>0?1:0)); i++) { - xmmp0 = simde_mm256_madd_epi16(ulch256[i], rxF256[i]); - // xmmp0 contains real part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm256_shuffle_epi8(ulch256[i], complex_shuffle256); - xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); - xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF256[i]); - // xmmp1 contains imag part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); - xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); - xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); - xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); - xmmp4 = simde_mm256_packs_epi32(xmmp2,xmmp3); - if (aarx == 0) - { - *llr64 = (simde__m64)simde_mm256_extract_epi64(xmmp4,0); llr64++; - *llr64 = (simde__m64)simde_mm256_extract_epi64(xmmp4,1); llr64++; - *llr64 = (simde__m64)simde_mm256_extract_epi64(xmmp4,2); llr64++; - *llr64 = (simde__m64)simde_mm256_extract_epi64(xmmp4,3); llr64++; - } - else - { - *llr64 = simde_mm_adds_pi16(*llr64,(simde__m64)(simde_mm256_extract_epi64(xmmp4,0))); llr64++; - *llr64 = simde_mm_adds_pi16(*llr64,(simde__m64)(simde_mm256_extract_epi64(xmmp4,1))); llr64++; - *llr64 = simde_mm_adds_pi16(*llr64,(simde__m64)(simde_mm256_extract_epi64(xmmp4,2))); llr64++; - *llr64 = simde_mm_adds_pi16(*llr64,(simde__m64)(simde_mm256_extract_epi64(xmmp4,3))); llr64++; - } - } -#else - simde__m128i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; - register simde__m128i complex_shuffle128 = simde_mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); - register simde__m128i conj128 = simde_mm_set_epi16(1, -1, 1, -1, 1, -1, 1, -1); - - simde__m128i *rxF128 = (simde__m128i*)rxF; - simde__m128i *ulch128 = (simde__m128i*)ul_ch; - simde__m128i *llr128 = (simde__m128i*)llr; - for (int i = 0; i < (length >> 2); i++) { - xmmp0 = simde_mm_madd_epi16(ulch128[i], rxF128[i]); - // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm_shuffle_epi8(ulch128[i], complex_shuffle128); - xmmp1 = simde_mm_sign_epi16(xmmp1, conj128); - xmmp1 = simde_mm_madd_epi16(xmmp1, rxF128[i]); - // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm_srai_epi32(xmmp0, output_shift); - xmmp1 = simde_mm_srai_epi32(xmmp1, output_shift); - xmmp2 = simde_mm_unpacklo_epi32(xmmp0, xmmp1); - xmmp3 = simde_mm_unpackhi_epi32(xmmp0, xmmp1); - xmmp4 = simde_mm_packs_epi32(xmmp2, xmmp3); - - if (aarx == 0) - *llr128 = xmmp4; - else - *llr128 = simde_mm_add_epi16(*llr128, xmmp4); - llr128++; - } - if (length & 3) - { - int i = (length>>1) - 1; - simde__m64* llr64 = (simde__m64*)llr128; - simde__m64 xmm0, xmm1, xmm2, xmm3, xmm4; - simde__m64 complex_shuffle64 = simde_mm_set_pi8(5, 4, 7, 6, 1, 0, 3, 2); - simde__m64 conj64 = simde_mm_set_pi16(1, -1, 1, -1); - simde__m64 *rxF64 = (simde__m64*)rxF; - simde__m64 *ulch64 = (simde__m64*)ul_ch; - - xmm0 = simde_mm_madd_pi16(ulch64[i], rxF64[i]); - // xmm0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmm1 = simde_mm_shuffle_pi8(ulch64[i], complex_shuffle64); - xmm1 = simde_mm_sign_pi16(xmm1, conj64); - xmm1 = simde_mm_madd_pi16(xmm1, rxF64[i]); - // xmm1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmm0 = simde_mm_srai_pi32(xmm0, output_shift); - xmm1 = simde_mm_srai_pi32(xmm1, output_shift); - xmm2 = simde_mm_unpacklo_pi32(xmm0, xmm1); - xmm3 = simde_mm_unpackhi_pi32(xmm0, xmm1); - xmm4 = simde_mm_packs_pi32(xmm2, xmm3); - - if (aarx == 0) - *llr64 = xmm4; - else - *llr64 = simde_mm_add_pi16(*llr64, xmm4); - } -#endif -} - -void inner_rx_16qam (int *rxF, - int *ul_ch, - int16_t *llr, - int aarx, - int length, - int output_shift) -{ -#ifndef USE_128BIT - register simde__m256i xmmp0,xmmp1,xmmp2,xmmp3,xmmp4,xmmp5; - register simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); - register simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); - - register simde__m256i QAM_amp256 = simde_mm256_set1_epi16(QAM16_n1); // 2/sqrt(10) - simde__m256i *rxF256 = (simde__m256i*)rxF; - simde__m256i *ulch256 = (simde__m256i*)ul_ch; - // need to use simde__m64 because llr output is not necessarily aligned to 256 bits, but it is always to 64 bits - simde__m64 *llr64 = (simde__m64 *)llr; - - for (int i = 0; i < ((length >> 3) + ((length & 7) > 0 ? 1 : 0)); i++) - { - xmmp0 = simde_mm256_madd_epi16(ulch256[i], rxF256[i]); - // xmmp0 contains real part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm256_shuffle_epi8(ulch256[i], complex_shuffle256); - xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); - xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF256[i]); - // xmmp1 contains imag part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); - xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); - xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); - xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); - xmmp4 = simde_mm256_packs_epi32(xmmp2, xmmp3); - - // compute channel amplitude for LLR - xmmp0 = simde_mm256_madd_epi16(ulch256[i], ulch256[i]); - xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); - xmmp0 = simde_mm256_packs_epi32(xmmp0, xmmp0); - xmmp1 = simde_mm256_unpacklo_epi16(xmmp0, xmmp0); - xmmp1 = simde_mm256_mulhrs_epi16(xmmp1, QAM_amp256); - - xmmp2 = simde_mm256_abs_epi16(xmmp4); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| - xmmp2 = simde_mm256_subs_epi16(xmmp1,xmmp2); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 - - xmmp3 = simde_mm256_unpacklo_epi32(xmmp4,xmmp2); // llr128[0] contains the llrs of the 1st,2nd,5th and 6th REs - xmmp5 = simde_mm256_unpackhi_epi32(xmmp4,xmmp2); // llr128[1] contains the llrs of the 3rd, 4th, 7th and 8th REs - if (aarx == 0) - { - // 1st/2nd RE - llr64[0] = (simde__m64)simde_mm256_extract_epi64(xmmp3,0); // llr32[0] low 16 bits-> y_R , high 16 bits-> y_I - // 3rd/4th RE - llr64[1] = (simde__m64)simde_mm256_extract_epi64(xmmp3,1); // llr32[2] low 16 bits-> y_R , high 16 bits-> y_I - // 5th/6th RE - llr64[2] = (simde__m64)simde_mm256_extract_epi64(xmmp5,0); // llr32[4] low 16 bits-> y_R , high 16 bits-> y_I - // 7Rh/8th RE - llr64[3] = (simde__m64)simde_mm256_extract_epi64(xmmp5,1); // llr32[6] low 16 bits-> y_R , high 16 bits-> y_I - // 9th/10th RE - llr64[4] = (simde__m64)simde_mm256_extract_epi64(xmmp3,2); // llr32[8] low 16 bits-> y_R , high 16 bits-> y_I - // 11th/12th RE - llr64[5] = (simde__m64)simde_mm256_extract_epi64(xmmp3,3); // llr32[10] low 16 bits-> y_R , high 16 bits-> y_I - // 13th/14th RE - llr64[6] = (simde__m64)simde_mm256_extract_epi64(xmmp5,2); // llr32[12] low 16 bits-> y_R , high 16 bits-> y_I - // 15th/16th RE - llr64[7] = (simde__m64)simde_mm256_extract_epi64(xmmp5,3); // llr32[14] low 16 bits-> y_R , high 16 bits-> y_I - llr64+=8; - } - else - { - llr64[0] = simde_mm_adds_pi16(llr64[0],(simde__m64)simde_mm256_extract_epi64(xmmp3,0)); - llr64[1] = simde_mm_adds_pi16(llr64[1],(simde__m64)simde_mm256_extract_epi64(xmmp3,1)); - llr64[2] = simde_mm_adds_pi16(llr64[2],(simde__m64)simde_mm256_extract_epi64(xmmp5,0)); - llr64[3] = simde_mm_adds_pi16(llr64[3],(simde__m64)simde_mm256_extract_epi64(xmmp5,1)); - llr64[4] = simde_mm_adds_pi16(llr64[4],(simde__m64)simde_mm256_extract_epi64(xmmp3,2)); - llr64[5] = simde_mm_adds_pi16(llr64[5],(simde__m64)simde_mm256_extract_epi64(xmmp3,3)); - llr64[6] = simde_mm_adds_pi16(llr64[6],(simde__m64)simde_mm256_extract_epi64(xmmp5,2)); - llr64[7] = simde_mm_adds_pi16(llr64[7],(simde__m64)simde_mm256_extract_epi64(xmmp5,3)); - llr64 += 8; - } - } -#else - simde__m128i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4, xmmp5; - register simde__m128i complex_shuffle128 = simde_mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); - register simde__m128i conj128 = simde_mm_set_epi16(1, -1, 1, -1, 1, -1, 1, -1); - - register simde__m128i QAM_amp128 = simde_mm_set1_epi16(QAM16_n1); // 2/sqrt(10) - simde__m128i *rxF128 = (simde__m128i*)rxF; - simde__m128i *ulch128 = (simde__m128i*)ul_ch; - // need to use simde__m64 because llr output is not necessarily aligned to 256 bits, but it is always to 64 bits - simde__m64 *llr64 = (simde__m64 *)llr; - - for (int i = 0; i < (length >> 2); i++) - { - xmmp0 = simde_mm_madd_epi16(ulch128[i], rxF128[i]); - // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm_shuffle_epi8(ulch128[i], complex_shuffle128); - xmmp1 = simde_mm_sign_epi16(xmmp1, conj128); - xmmp1 = simde_mm_madd_epi16(xmmp1, rxF128[i]); - // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm_srai_epi32(xmmp0, output_shift); - xmmp1 = simde_mm_srai_epi32(xmmp1, output_shift); - xmmp2 = simde_mm_unpacklo_epi32(xmmp0, xmmp1); - xmmp3 = simde_mm_unpackhi_epi32(xmmp0, xmmp1); - xmmp4 = simde_mm_packs_epi32(xmmp2, xmmp3); - - // compute channel amplitude for LLR - xmmp0 = simde_mm_madd_epi16(ulch128[i], ulch128[i]); // |h|^2 - xmmp0 = simde_mm_srai_epi32(xmmp0, output_shift); - xmmp0 = simde_mm_packs_epi32(xmmp0, xmmp0); - xmmp1 = simde_mm_unpacklo_epi16(xmmp0, xmmp0); - xmmp1 = simde_mm_mulhrs_epi16(xmmp1, QAM_amp128); - - xmmp2 = simde_mm_abs_epi16(xmmp4); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| - xmmp2 = simde_mm_subs_epi16(xmmp1, xmmp2); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 - - xmmp3 = simde_mm_unpacklo_epi32(xmmp4, xmmp2); // llr128[0] contains the llrs of the 1st,2nd,5th and 6th REs - xmmp5 = simde_mm_unpackhi_epi32(xmmp4, xmmp2); // llr128[1] contains the llrs of the 3rd, 4th, 7th and 8th REs - if (aarx == 0) - { - llr64[0] = (simde__m64)simde_mm_extract_epi64(xmmp3, 0); // llr32[0] low 16 bits-> y_R, high 16 bits-> y_I - llr64[1] = (simde__m64)simde_mm_extract_epi64(xmmp3, 1); // llr32[2] low 16 bits-> y_R, high 16 bits-> y_I - llr64[2] = (simde__m64)simde_mm_extract_epi64(xmmp5, 0); // llr32[4] low 16 bits-> y_R, high 16 bits-> y_I - llr64[3] = (simde__m64)simde_mm_extract_epi64(xmmp5, 1); // llr32[6] low 16 bits-> y_R, high 16 bits-> y_I - } - else - { - llr64[0] = simde_mm_adds_pi16(llr64[0], (simde__m64)simde_mm_extract_epi64(xmmp3, 0)); - llr64[1] = simde_mm_adds_pi16(llr64[1], (simde__m64)simde_mm_extract_epi64(xmmp3, 1)); - llr64[2] = simde_mm_adds_pi16(llr64[2], (simde__m64)simde_mm_extract_epi64(xmmp5, 0)); - llr64[3] = simde_mm_adds_pi16(llr64[3], (simde__m64)simde_mm_extract_epi64(xmmp5, 1)); - } - llr64 += 4; - } - if (length & 3) - { - int i = (length>>1) - 1; - simde__m64 xmm0, xmm1, xmm2, xmm3, xmm4; - simde__m64 complex_shuffle64 = simde_mm_set_pi8(5,4,7,6,1,0,3,2); - simde__m64 conj64 = simde_mm_set_pi16(1, -1, 1, -1); - simde__m64 *rxF64 = (simde__m64*)rxF; - simde__m64 *ulch64 = (simde__m64*)ul_ch; - simde__m64 QAM_amp = simde_mm_set1_pi16(QAM16_n1); - - xmm0 = simde_mm_madd_pi16(ulch64[i], rxF64[i]); - // xmm0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmm1 = simde_mm_shuffle_pi8(ulch64[i], complex_shuffle64); - xmm1 = simde_mm_sign_pi16(xmm1, conj64); - xmm1 = simde_mm_madd_pi16(xmm1, rxF64[i]); - // xmm1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmm0 = simde_mm_srai_pi32(xmm0, output_shift); - xmm1 = simde_mm_srai_pi32(xmm1, output_shift); - xmm2 = simde_mm_unpacklo_pi32(xmm0, xmm1); - xmm3 = simde_mm_unpackhi_pi32(xmm0, xmm1); - xmm4 = simde_mm_packs_pi32(xmm2, xmm3); - - // compute channel amplitude for LLR - xmm0 = simde_mm_madd_pi16(ulch64[i], ulch64[i]); // |h|^2 - xmm0 = simde_mm_srai_pi32(xmm0, output_shift); - xmm0 = simde_mm_packs_pi32(xmm0, xmm0); - xmm2 = simde_mm_unpacklo_pi16(xmm0, xmm0); - - xmm1 = simde_mm_mulhrs_pi16(xmm2, QAM_amp); - - xmm0 = simde_mm_abs_pi16(xmm4); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| - xmm0 = simde_mm_subs_pi16(xmm1, xmm0); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 - if (aarx == 0) - { - llr64[0] = simde_mm_set_pi32(simde_mm_extract_pi16(xmm0, 0), simde_mm_extract_pi16(xmm4, 0)); - llr64[1] = simde_mm_set_pi32(simde_mm_extract_pi16(xmm0, 1), simde_mm_extract_pi16(xmm4, 1)); - } - else - { - llr64[0] = simde_mm_adds_pi16(llr64[0], simde_mm_set_pi32(simde_mm_extract_pi16(xmm0, 0),simde_mm_extract_pi16(xmm4, 0))); - llr64[1] = simde_mm_adds_pi16(llr64[1], simde_mm_set_pi32(simde_mm_extract_pi16(xmm4, 1),simde_mm_extract_pi16(xmm1, 0))); - } - } - -#endif -} - -void inner_rx_64qam (int *restrict rxF, - int *restrict ul_ch, - int16_t *restrict llr, - int aarx, - int length, - int output_shift) -{ -#ifndef USE_128BIT - register simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4, xmmp6, xmmp7; - register simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29, 28, 31, 30, 25, 24, 27, 26, 21, 20, 23, 22, 17, 16, 19, 18, 13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); - register simde__m256i conj256 = simde_mm256_set_epi16(1, -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1); - - register simde__m256i QAM_amp256 = simde_mm256_set1_epi16(QAM64_n1); // 2/sqrt(10) - register simde__m256i QAM_amp256b = simde_mm256_set1_epi16(QAM64_n2); - simde__m256i *rxF256 = (simde__m256i*)rxF; - simde__m256i *ulch256 = (simde__m256i*)ul_ch; - // need to use simde__m64 because llr output is not necessarily aligned to 256 bits, but it is always to 64 bits - - simde__m64 *llr64 = (simde__m64 *)llr; - for (int i=0;i<((length>>3)+((length&7)>0?1:0));i++) { - xmmp0 = simde_mm256_madd_epi16(ulch256[i],rxF256[i]); - // xmmp0 contains real part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm256_shuffle_epi8(ulch256[i],complex_shuffle256); - xmmp1 = simde_mm256_sign_epi16(xmmp1,conj256); - xmmp1 = simde_mm256_madd_epi16(xmmp1,rxF256[i]); - // xmmp1 contains imag part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm256_srai_epi32(xmmp0,output_shift); - xmmp1 = simde_mm256_srai_epi32(xmmp1,output_shift); - xmmp2 = simde_mm256_unpacklo_epi32(xmmp0,xmmp1); - xmmp3 = simde_mm256_unpackhi_epi32(xmmp0,xmmp1); - xmmp4 = simde_mm256_packs_epi32(xmmp2,xmmp3); - - // compute channel amplitude for LLR - xmmp0 = simde_mm256_madd_epi16(ulch256[i],ulch256[i]); - xmmp0 = simde_mm256_srai_epi32(xmmp0,output_shift); - xmmp0 = simde_mm256_packs_epi32(xmmp0,xmmp0); - xmmp2 = simde_mm256_unpacklo_epi16(xmmp0,xmmp0); - xmmp1 = simde_mm256_mulhrs_epi16(xmmp2,QAM_amp256); - xmmp6 = simde_mm256_mulhrs_epi16(xmmp2,QAM_amp256b); - - xmmp2 = simde_mm256_abs_epi16(xmmp4); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| - xmmp2 = simde_mm256_subs_epi16(xmmp1,xmmp2); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 - xmmp7 = simde_mm256_abs_epi16(xmmp2); - xmmp7 = simde_mm256_subs_epi16(xmmp6,xmmp7); - - if (aarx == 0) - { - llr64[0] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp2,0),simde_mm256_extract_epi32(xmmp4,0)); - llr64[1] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp4,1),simde_mm256_extract_epi32(xmmp7,0)); - llr64[2] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp7,1),simde_mm256_extract_epi32(xmmp2,1)); - llr64[3] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp2,2),simde_mm256_extract_epi32(xmmp4,2)); - llr64[4] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp4,3),simde_mm256_extract_epi32(xmmp7,2)); - llr64[5] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp7,3),simde_mm256_extract_epi32(xmmp2,3)); - llr64[6] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp2,4),simde_mm256_extract_epi32(xmmp4,4)); - llr64[7] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp4,5),simde_mm256_extract_epi32(xmmp7,4)); - llr64[8] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp7,5),simde_mm256_extract_epi32(xmmp2,5)); - llr64[9] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp2,6),simde_mm256_extract_epi32(xmmp4,6)); - llr64[10] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp4,7),simde_mm256_extract_epi32(xmmp7,6)); - llr64[11] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp7,7),simde_mm256_extract_epi32(xmmp2,7)); - llr64+=12; - } - else - { - llr64[0] = simde_mm_adds_pi16(llr64[0],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp2,0),simde_mm256_extract_epi32(xmmp4,0))); - llr64[1] = simde_mm_adds_pi16(llr64[1],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp4,1),simde_mm256_extract_epi32(xmmp7,0))); - llr64[2] = simde_mm_adds_pi16(llr64[2],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp7,1),simde_mm256_extract_epi32(xmmp2,1))); - llr64[3] = simde_mm_adds_pi16(llr64[3],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp2,2),simde_mm256_extract_epi32(xmmp4,2))); - llr64[4] = simde_mm_adds_pi16(llr64[4],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp4,3),simde_mm256_extract_epi32(xmmp7,2))); - llr64[5] = simde_mm_adds_pi16(llr64[5],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp7,3),simde_mm256_extract_epi32(xmmp2,3))); - llr64[6] = simde_mm_adds_pi16(llr64[6],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp2,4),simde_mm256_extract_epi32(xmmp4,4))); - llr64[7] = simde_mm_adds_pi16(llr64[7],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp4,5),simde_mm256_extract_epi32(xmmp7,4))); - llr64[8] = simde_mm_adds_pi16(llr64[8],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp7,5),simde_mm256_extract_epi32(xmmp2,5))); - llr64[9] = simde_mm_adds_pi16(llr64[9],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp2,6),simde_mm256_extract_epi32(xmmp4,6))); - llr64[10] = simde_mm_adds_pi16(llr64[10],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp4,7),simde_mm256_extract_epi32(xmmp7,6))); - llr64[11] = simde_mm_adds_pi16(llr64[11],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp7,7),simde_mm256_extract_epi32(xmmp2,7))); - llr64+=12; - } - } -#else - register simde__m128i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4, xmmp6, xmmp7; - register simde__m128i complex_shuffle128 = simde_mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); - register simde__m128i conj128 = simde_mm_set_epi16(1, -1, 1, -1, 1, -1, 1, -1); - // register simde__m128i conj128 = simde_mm_set_epi16(1, -1, 1, -1, 1, -1, 1, -1); - register simde__m128i QAM_amp128 = simde_mm_set1_epi16(QAM64_n1); // 4/sqrt(42) - register simde__m128i QAM_amp128b = simde_mm_set1_epi16(QAM64_n2); // 2/sqrt(42) - simde__m128i *rxF128 = (simde__m128i*) rxF; - simde__m128i *ulch128 = (simde__m128i*) ul_ch; - // need to use simde__m64 because llr output is not necessarily aligned to 256 bits, but it is always to 64 bits - simde__m64 *llr64 = (simde__m64 *)llr; - for (int i = 0; i < (length>>2); i++) - { - xmmp0 = simde_mm_madd_epi16(ulch128[i], rxF128[i]); - // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm_shuffle_epi8(ulch128[i], complex_shuffle128); - xmmp1 = simde_mm_sign_epi16(xmmp1, conj128); - xmmp1 = simde_mm_madd_epi16(xmmp1, rxF128[i]); - // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm_srai_epi32(xmmp0, output_shift); - xmmp1 = simde_mm_srai_epi32(xmmp1, output_shift); - xmmp2 = simde_mm_unpacklo_epi32(xmmp0, xmmp1); - xmmp3 = simde_mm_unpackhi_epi32(xmmp0, xmmp1); - xmmp4 = simde_mm_packs_epi32(xmmp2, xmmp3); - - // compute channel amplitude for LLR - xmmp0 = simde_mm_madd_epi16(ulch128[i], ulch128[i]); - xmmp0 = simde_mm_srai_epi32(xmmp0, output_shift); - xmmp0 = simde_mm_packs_epi32(xmmp0, xmmp0); - xmmp2 = simde_mm_unpacklo_epi16(xmmp0, xmmp0); - xmmp1 = simde_mm_mulhrs_epi16(xmmp2, QAM_amp128); - xmmp6 = simde_mm_mulhrs_epi16(xmmp2, QAM_amp128b); - - xmmp2 = simde_mm_abs_epi16(xmmp4); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| - xmmp2 = simde_mm_subs_epi16(xmmp1, xmmp2); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 - xmmp7 = simde_mm_abs_epi16(xmmp2); - xmmp7 = simde_mm_subs_epi16(xmmp6, xmmp7); - - if (aarx == 0) - { - llr64[0] = simde_mm_set_pi32(simde_mm_extract_epi32(xmmp2, 0), simde_mm_extract_epi32(xmmp4, 0)); - llr64[1] = simde_mm_set_pi32(simde_mm_extract_epi32(xmmp4, 1), simde_mm_extract_epi32(xmmp7, 0)); - llr64[2] = simde_mm_set_pi32(simde_mm_extract_epi32(xmmp7, 1), simde_mm_extract_epi32(xmmp2, 1)); - llr64[3] = simde_mm_set_pi32(simde_mm_extract_epi32(xmmp2, 2), simde_mm_extract_epi32(xmmp4, 2)); - llr64[4] = simde_mm_set_pi32(simde_mm_extract_epi32(xmmp4, 3), simde_mm_extract_epi32(xmmp7, 2)); - llr64[5] = simde_mm_set_pi32(simde_mm_extract_epi32(xmmp7, 3), simde_mm_extract_epi32(xmmp2, 3)); - llr64 += 6; - } - else - { - llr64[0] = simde_mm_adds_pi16(llr64[0], simde_mm_set_pi32(simde_mm_extract_epi32(xmmp2, 0),simde_mm_extract_epi32(xmmp4, 0))); - llr64[1] = simde_mm_adds_pi16(llr64[1], simde_mm_set_pi32(simde_mm_extract_epi32(xmmp4, 1),simde_mm_extract_epi32(xmmp7, 0))); - llr64[2] = simde_mm_adds_pi16(llr64[2], simde_mm_set_pi32(simde_mm_extract_epi32(xmmp7, 1),simde_mm_extract_epi32(xmmp2, 1))); - llr64[3] = simde_mm_adds_pi16(llr64[3], simde_mm_set_pi32(simde_mm_extract_epi32(xmmp2, 2),simde_mm_extract_epi32(xmmp4, 2))); - llr64[4] = simde_mm_adds_pi16(llr64[4], simde_mm_set_pi32(simde_mm_extract_epi32(xmmp4, 3),simde_mm_extract_epi32(xmmp7, 2))); - llr64[5] = simde_mm_adds_pi16(llr64[5], simde_mm_set_pi32(simde_mm_extract_epi32(xmmp7, 3),simde_mm_extract_epi32(xmmp2, 3))); - llr64 += 6; - } - } - if (length & 3) - { - int i = (length>>1) - 1; - simde__m64 xmm0, xmm1, xmm2, xmm3, xmm4, xmm5; - simde__m64 complex_shuffle64 = simde_mm_set_pi8(5,4,7,6,1,0,3,2); - simde__m64 conj64 = simde_mm_set_pi16(1, -1, 1, -1); - simde__m64 *rxF64 = (simde__m64*)rxF; - simde__m64 *ulch64 = (simde__m64*)ul_ch; - simde__m64 QAM_amp = simde_mm_set1_pi16(QAM64_n1); - simde__m64 QAM_ampb = simde_mm_set1_pi16(QAM64_n2); - - xmm0 = simde_mm_madd_pi16(ulch64[i], rxF64[i]); - // xmm0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmm1 = simde_mm_shuffle_pi8(ulch64[i], complex_shuffle64); - xmm1 = simde_mm_sign_pi16(xmm1, conj64); - xmm1 = simde_mm_madd_pi16(xmm1, rxF64[i]); - // xmm1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmm0 = simde_mm_srai_pi32(xmm0, output_shift); - xmm1 = simde_mm_srai_pi32(xmm1, output_shift); - xmm2 = simde_mm_unpacklo_pi32(xmm0, xmm1); - xmm3 = simde_mm_unpackhi_pi32(xmm0, xmm1); - xmm4 = simde_mm_packs_pi32(xmm2, xmm3); - - // compute channel amplitude for LLR - xmm0 = simde_mm_madd_pi16(ulch64[i], ulch64[i]); // |h|^2 - xmm0 = simde_mm_srai_pi32(xmm0, output_shift); - xmm0 = simde_mm_packs_pi32(xmm0, xmm0); - xmm2 = simde_mm_unpacklo_pi16(xmm0, xmm0); - - xmm1 = simde_mm_mulhrs_pi16(xmm2, QAM_amp); - xmm5 = simde_mm_mulhrs_pi16(xmm2, QAM_ampb); - - xmm0 = simde_mm_abs_pi16(xmm4); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| - xmm0 = simde_mm_subs_pi16(xmm1, xmm0); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 - xmm1 = simde_mm_abs_pi16(xmm0); - xmm1 = simde_mm_subs_pi16(xmm5, xmm1); // contains 8 LLRs - if (aarx == 0) - { - llr64[0] = simde_mm_set_pi32(simde_mm_extract_pi16(xmm0, 0), simde_mm_extract_pi16(xmm4, 0)); - llr64[1] = simde_mm_set_pi32(simde_mm_extract_pi16(xmm4, 1), simde_mm_extract_pi16(xmm1, 0)); - llr64[2] = simde_mm_set_pi32(simde_mm_extract_pi16(xmm1, 1), simde_mm_extract_pi16(xmm0, 1)); - } - else - { - llr64[0] = simde_mm_adds_pi16(llr64[0], simde_mm_set_pi32(simde_mm_extract_pi16(xmm0, 0),simde_mm_extract_pi16(xmm4, 0))); - llr64[1] = simde_mm_adds_pi16(llr64[1], simde_mm_set_pi32(simde_mm_extract_pi16(xmm4, 1),simde_mm_extract_pi16(xmm1, 0))); - llr64[2] = simde_mm_adds_pi16(llr64[2], simde_mm_set_pi32(simde_mm_extract_pi16(xmm1, 1),simde_mm_extract_pi16(xmm0, 1))); - } - } -#endif -} - -void inner_rx_256qam (int *rxF, - int *ul_ch, - int16_t *llr, - int aarx, - int length, - int output_shift) -{ -#ifndef USE_128BIT - register simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4, xmmp5, xmmp6, xmmp7, xmmp8, xmmp9; - register simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); - register simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); - - register simde__m256i QAM_amp256 = simde_mm256_set1_epi16(QAM256_n1); - register simde__m256i QAM_amp256b = simde_mm256_set1_epi16(QAM256_n2); - register simde__m256i QAM_amp256c = simde_mm256_set1_epi16(QAM256_n3); - simde__m256i *rxF256 = (simde__m256i*)rxF; - simde__m256i *ulch256 = (simde__m256i*)ul_ch; - simde__m256i *llr256 = (simde__m256i *)llr; - for (int i = 0; i < ((length >> 3) + (( length & 7) > 0 ? 1 : 0)); i++) - { - xmmp0 = simde_mm256_madd_epi16(ulch256[i],rxF256[i]); - // xmmp0 contains real part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm256_shuffle_epi8(ulch256[i],complex_shuffle256); - xmmp1 = simde_mm256_sign_epi16(xmmp1,conj256); - xmmp1 = simde_mm256_madd_epi16(xmmp1,rxF256[i]); - // xmmp1 contains imag part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm256_srai_epi32(xmmp0,output_shift); - xmmp1 = simde_mm256_srai_epi32(xmmp1,output_shift); - xmmp2 = simde_mm256_unpacklo_epi32(xmmp0,xmmp1); - xmmp3 = simde_mm256_unpackhi_epi32(xmmp0,xmmp1); - xmmp4 = simde_mm256_packs_epi32(xmmp2,xmmp3); - - // compute channel amplitude for LLR - xmmp0 = simde_mm256_madd_epi16(ulch256[i],ulch256[i]); - xmmp0 = simde_mm256_srai_epi32(xmmp0,output_shift); - xmmp0 = simde_mm256_packs_epi32(xmmp0,xmmp0); // contains 16 LLRs - xmmp2 = simde_mm256_unpacklo_epi16(xmmp0,xmmp0); - xmmp1 = simde_mm256_mulhrs_epi16(xmmp2,QAM_amp256); - xmmp6 = simde_mm256_mulhrs_epi16(xmmp2,QAM_amp256b); - xmmp8 = simde_mm256_mulhrs_epi16(xmmp2,QAM_amp256c); - - xmmp2 = simde_mm256_abs_epi16(xmmp4); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| - xmmp2 = simde_mm256_subs_epi16(xmmp1,xmmp2); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 - // xmmp2 contains 16 LLRs - xmmp7 = simde_mm256_abs_epi16(xmmp2); - xmmp7 = simde_mm256_subs_epi16(xmmp6,xmmp7); // contains 16 LLRs - xmmp9 = simde_mm256_abs_epi16(xmmp7); - xmmp9 = simde_mm256_subs_epi16(xmmp8,xmmp9); // contains 16 LLRs - - // xmmp4 A0 A1 A2 A3 A4 A5 A6 A7 - // xmmp2 B0 B1 B2 B3 B4 B5 B6 B7 - // xmmp7 C0 C1 C2 C3 C4 C5 C6 C7 - // xmmp9 D0 D1 D2 D3 D4 D5 D6 D7 - xmmp1 = simde_mm256_unpacklo_epi32(xmmp4,xmmp2); // A0 B0 A1 B1 A4 B4 A5 B5 - xmmp3 = simde_mm256_unpackhi_epi32(xmmp4,xmmp2); // A2 B2 A3 B3 A6 B6 A7 B7 - xmmp5 = simde_mm256_unpacklo_epi32(xmmp7,xmmp9); // C0 D0 C1 D1 C4 D4 C5 D5 - xmmp6 = simde_mm256_unpackhi_epi32(xmmp7,xmmp9); // C2 D2 C3 D3 C6 D6 C7 D7 - - xmmp2 = simde_mm256_unpacklo_epi64(xmmp1,xmmp5); // A0 B0 C0 D0 A4 B4 C4 D4 - xmmp4 = simde_mm256_unpackhi_epi64(xmmp1,xmmp5); // A1 B1 C1 D1 A5 B5 C5 D5 - xmmp1 = simde_mm256_unpacklo_epi64(xmmp3,xmmp6); // A2 B2 C2 D2 A6 B6 C6 D6 - xmmp5 = simde_mm256_unpackhi_epi64(xmmp3,xmmp6); // A3 B3 C3 D3 A7 B7 C7 D7 - if (aarx == 0) - { - llr256[0] = simde_mm256_permute2x128_si256(xmmp2, xmmp4, 0x20); // A0 B0 C0 D0 A1 B1 C1 D1 - llr256[1] = simde_mm256_permute2x128_si256(xmmp1, xmmp5, 0x20); // A2 B2 C2 D2 A3 B3 C3 D3 - llr256[2] = simde_mm256_permute2x128_si256(xmmp2, xmmp4, 0x31); // A4 B4 C4 D4 A5 B5 C5 D5 - llr256[3] = simde_mm256_permute2x128_si256(xmmp1, xmmp5, 0x31); // A6 B6 C6 D6 A7 B7 C7 D7 - llr256+=4; - } - else - { - llr256[0] = simde_mm256_adds_epi16(llr256[0],simde_mm256_permute2x128_si256(xmmp2, xmmp4, 0x20)); // A0 B0 C0 D0 A1 B1 C1 D1 - llr256[1] = simde_mm256_adds_epi16(llr256[1],simde_mm256_permute2x128_si256(xmmp1, xmmp5, 0x20)); // A2 B2 C2 D2 A3 B3 C3 D3 - llr256[2] = simde_mm256_adds_epi16(llr256[2],simde_mm256_permute2x128_si256(xmmp2, xmmp4, 0x31)); // A4 B4 C4 D4 A5 B5 C5 D5 - llr256[3] = simde_mm256_adds_epi16(llr256[3],simde_mm256_permute2x128_si256(xmmp1, xmmp5, 0x31)); // A6 B6 C6 D6 A7 B7 C7 D7 - llr256+=4; - } - } - simde__m128i *llr128 = (simde__m128i*)llr256; - if ((length&7) >= 4) { //there is a single 128-bit input element remaining - int nb_re128 = length>>2; - simde__m128i xmm0,xmm1,xmm2,xmm3,xmm4,xmm5,xmm6; - simde__m128i complex_shuffle128 = simde_mm_set_epi8(13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); - simde__m128i conj128 = simde_mm_set_epi16(1,-1,1,-1,1,-1,1,-1); - simde__m128i *rxF128 = (simde__m128i*)rxF; - simde__m128i *ulch128 = (simde__m128i*)ul_ch; - simde__m128i QAM_amp = simde_mm_set1_epi16(QAM256_n1); // 2/sqrt(10) - simde__m128i QAM_ampb = simde_mm_set1_epi16(QAM256_n2); - simde__m128i QAM_ampc = simde_mm_set1_epi16(QAM256_n3); - - xmm0 = simde_mm_madd_epi16(ulch128[nb_re128-1],rxF128[nb_re128-1]); - // xmm0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmm1 = simde_mm_shuffle_epi8(ulch128[nb_re128-1],complex_shuffle128); - xmm1 = simde_mm_sign_epi16(xmm1,conj128); - xmm1 = simde_mm_madd_epi16(xmm1,rxF128[nb_re128-1]); - // xmm1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmm0 = simde_mm_srai_epi32(xmm0,output_shift); - xmm1 = simde_mm_srai_epi32(xmm1,output_shift); - xmm2 = simde_mm_unpacklo_epi32(xmm0,xmm1); - xmm3 = simde_mm_unpackhi_epi32(xmm0,xmm1); - xmm4 = simde_mm_packs_epi32(xmm2,xmm3); - - // compute channel amplitude for LLR - xmm0 = simde_mm_madd_epi16(ulch128[nb_re128-1],ulch128[nb_re128-1]); - xmm0 = simde_mm_srai_epi32(xmm0,output_shift); - xmm0 = simde_mm_packs_epi32(xmm0,xmm0); // contains 16 LLRs - xmm2 = simde_mm_unpacklo_epi16(xmm0,xmm0); - xmm1 = simde_mm_mulhrs_epi16(xmm2,QAM_amp); - xmm5 = simde_mm_mulhrs_epi16(xmm2,QAM_ampb); - xmm6 = simde_mm_mulhrs_epi16(xmm2,QAM_ampc); - xmm0 = simde_mm_abs_epi16(xmm4); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| - xmm0 = simde_mm_subs_epi16(xmm1,xmm0); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 - // xmmp2 contains 8 LLRs - xmm1 = simde_mm_abs_epi16(xmm0); - xmm1 = simde_mm_subs_epi16(xmm5,xmm1); // contains 8 LLRs - xmm2 = simde_mm_abs_epi16(xmm1); - xmm2 = simde_mm_subs_epi16(xmm6,xmm2); // contains 8 LLRs - // rxF[i] A0 A1 A2 A3 - // xmm0 B0 B1 B2 B3 - // xmm1 C0 C1 C2 C3 - // xmm2 D0 D1 D2 D3 - xmm3 = simde_mm_unpacklo_epi32(rxF128[nb_re128-1],xmm0); // A0 B0 A1 B1 - xmm4 = simde_mm_unpackhi_epi32(rxF128[nb_re128-1],xmm0); // A2 B2 A3 B3 - xmm5 = simde_mm_unpacklo_epi32(xmm1,xmm2); // C0 D0 C1 D1 - xmm6 = simde_mm_unpackhi_epi32(xmm1,xmm2); // C2 D2 C3 D3 - if (aarx == 0) { - llr128[0] = simde_mm_unpacklo_epi64(xmm3,xmm5); // A0 B0 C0 D0 - llr128[1] = simde_mm_unpackhi_epi64(xmm3,xmm5); // A1 B1 C1 D1 - llr128[2] = simde_mm_unpacklo_epi64(xmm4,xmm6); // A2 B2 C2 D2 - llr128[3] = simde_mm_unpackhi_epi64(xmm4,xmm6); // A3 B3 C3 D3 - llr128+=4; - } - else - { - llr128[0] = simde_mm_adds_epi16(llr128[0],simde_mm_unpacklo_epi64(xmm3,xmm5)); // A0 B0 C0 D0 - llr128[1] = simde_mm_adds_epi16(llr128[1],simde_mm_unpackhi_epi64(xmm3,xmm5)); // A1 B1 C1 D1 - llr128[2] = simde_mm_adds_epi16(llr128[2],simde_mm_unpacklo_epi64(xmm4,xmm6)); // A2 B2 C2 D2 - llr128[3] = simde_mm_adds_epi16(llr128[3],simde_mm_unpackhi_epi64(xmm4,xmm6)); // A3 B3 C3 D3 - llr128+=4; - } - } -#else - simde__m128i xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6; - simde__m128i complex_shuffle128 = simde_mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); - simde__m128i conj128 = simde_mm_set_epi16(1, -1, 1, -1, 1, -1, 1, -1); - simde__m128i *rxF128 = (simde__m128i*)rxF; - simde__m128i *ulch128 = (simde__m128i*)ul_ch; - simde__m128i QAM_amp = simde_mm_set1_epi16(QAM256_n1); - simde__m128i QAM_ampb = simde_mm_set1_epi16(QAM256_n2); - simde__m128i QAM_ampc = simde_mm_set1_epi16(QAM256_n3); - simde__m128i *llr128 = (simde__m128i*)llr; - for (int i = 0; i < (length >> 2); i++) - { - xmm0 = simde_mm_madd_epi16(ulch128[i], rxF128[i]); - // xmm0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmm1 = simde_mm_shuffle_epi8(ulch128[i], complex_shuffle128); - xmm1 = simde_mm_sign_epi16(xmm1, conj128); - xmm1 = simde_mm_madd_epi16(xmm1, rxF128[i]); - // xmm1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmm0 = simde_mm_srai_epi32(xmm0, output_shift); - xmm1 = simde_mm_srai_epi32(xmm1, output_shift); - xmm2 = simde_mm_unpacklo_epi32(xmm0, xmm1); - xmm3 = simde_mm_unpackhi_epi32(xmm0, xmm1); - xmm4 = simde_mm_packs_epi32(xmm2, xmm3); - - // compute channel amplitude for LLR - xmm0 = simde_mm_madd_epi16(ulch128[i], ulch128[i]); // |h|^2 - xmm0 = simde_mm_srai_epi32(xmm0, output_shift); - xmm0 = simde_mm_packs_epi32(xmm0, xmm0); - xmm2 = simde_mm_unpacklo_epi16(xmm0, xmm0); - - xmm1 = simde_mm_mulhrs_epi16(xmm2, QAM_amp); - xmm5 = simde_mm_mulhrs_epi16(xmm2, QAM_ampb); - xmm6 = simde_mm_mulhrs_epi16(xmm2, QAM_ampc); - - xmm0 = simde_mm_abs_epi16(xmm4); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| - xmm0 = simde_mm_subs_epi16(xmm1, xmm0); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 - xmm1 = simde_mm_abs_epi16(xmm0); - xmm1 = simde_mm_subs_epi16(xmm5, xmm1); // contains 8 LLRs - xmm2 = simde_mm_abs_epi16(xmm1); - xmm2 = simde_mm_subs_epi16(xmm6, xmm2); // contains 8 LLRs - // rxF[i] A0 A1 A2 A3 - // xmm0 B0 B1 B2 B3 - // xmm1 C0 C1 C2 C3 - // xmm2 D0 D1 D2 D3 - xmm3 = simde_mm_unpacklo_epi32(xmm4, xmm0); // A0 B0 A1 B1 - xmm4 = simde_mm_unpackhi_epi32(xmm4, xmm0); // A2 B2 A3 B3 - xmm5 = simde_mm_unpacklo_epi32(xmm1, xmm2); // C0 D0 C1 D1 - xmm6 = simde_mm_unpackhi_epi32(xmm1, xmm2); // C2 D2 C3 D3 - if (aarx == 0) { - llr128[0] = simde_mm_unpacklo_epi64(xmm3, xmm5); // A0 B0 C0 D0 - llr128[1] = simde_mm_unpackhi_epi64(xmm3, xmm5); // A1 B1 C1 D1 - llr128[2] = simde_mm_unpacklo_epi64(xmm4, xmm6); // A2 B2 C2 D2 - llr128[3] = simde_mm_unpackhi_epi64(xmm4, xmm6); // A3 B3 C3 D3 - } - else { - llr128[0] = simde_mm_adds_epi16(llr128[0], simde_mm_unpacklo_epi64(xmm3, xmm5)); // A0 B0 C0 D0 - llr128[1] = simde_mm_adds_epi16(llr128[1], simde_mm_unpackhi_epi64(xmm3, xmm5)); // A1 B1 C1 D1 - llr128[2] = simde_mm_adds_epi16(llr128[2], simde_mm_unpacklo_epi64(xmm4, xmm6)); // A2 B2 C2 D2 - llr128[3] = simde_mm_adds_epi16(llr128[3], simde_mm_unpackhi_epi64(xmm4, xmm6)); // A3 B3 C3 D3 - } - llr128+=4; - } - if (length & 3) - { - simde__m64 *llr64 = (simde__m64*) llr128; - int i = (length>>1) - 1; - simde__m64 xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6; - simde__m64 complex_shuffle64 = simde_mm_set_pi8(5,4,7,6,1,0,3,2); - simde__m64 conj64 = simde_mm_set_pi16(1, -1, 1, -1); - simde__m64 *rxF64 = (simde__m64*)rxF; - simde__m64 *ulch64 = (simde__m64*)ul_ch; - simde__m64 QAM_amp = simde_mm_set1_pi16(QAM256_n1); - simde__m64 QAM_ampb = simde_mm_set1_pi16(QAM256_n2); - simde__m64 QAM_ampc = simde_mm_set1_pi16(QAM256_n3); - - xmm0 = simde_mm_madd_pi16(ulch64[i], rxF64[i]); - // xmm0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmm1 = simde_mm_shuffle_pi8(ulch64[i], complex_shuffle64); - xmm1 = simde_mm_sign_pi16(xmm1, conj64); - xmm1 = simde_mm_madd_pi16(xmm1, rxF64[i]); - // xmm1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmm0 = simde_mm_srai_pi32(xmm0, output_shift); - xmm1 = simde_mm_srai_pi32(xmm1, output_shift); - xmm2 = simde_mm_unpacklo_pi32(xmm0, xmm1); - xmm3 = simde_mm_unpackhi_pi32(xmm0, xmm1); - xmm4 = simde_mm_packs_pi32(xmm2, xmm3); - - // compute channel amplitude for LLR - xmm0 = simde_mm_madd_pi16(ulch64[i], ulch64[i]); // |h|^2 - xmm0 = simde_mm_srai_pi32(xmm0, output_shift); - xmm0 = simde_mm_packs_pi32(xmm0, xmm0); - xmm2 = simde_mm_unpacklo_pi16(xmm0, xmm0); - - xmm1 = simde_mm_mulhrs_pi16(xmm2, QAM_amp); - xmm5 = simde_mm_mulhrs_pi16(xmm2, QAM_ampb); - xmm6 = simde_mm_mulhrs_pi16(xmm2, QAM_ampc); - - xmm0 = simde_mm_abs_pi16(xmm4); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| - xmm0 = simde_mm_subs_pi16(xmm1, xmm0); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 - xmm1 = simde_mm_abs_pi16(xmm0); - xmm1 = simde_mm_subs_pi16(xmm5, xmm1); // contains 8 LLRs - xmm2 = simde_mm_abs_pi16(xmm1); - xmm2 = simde_mm_subs_pi16(xmm6, xmm2); // contains 8 LLRs - - xmm3 = simde_mm_unpacklo_pi32(xmm4, xmm0); - xmm4 = simde_mm_unpackhi_pi32(xmm4, xmm0); - xmm5 = simde_mm_unpacklo_pi32(xmm1, xmm2); - xmm6 = simde_mm_unpackhi_pi32(xmm1, xmm2); - if (aarx == 0) { - llr64[0] = simde_m_punpckldq(xmm3, xmm5); - llr64[1] = simde_m_punpckhdq(xmm3, xmm5); - llr64[2] = simde_m_punpckldq(xmm4, xmm6); - llr64[3] = simde_m_punpckhdq(xmm4, xmm6); - } - else - { - llr64[0] = simde_mm_adds_pi16(llr64[0], simde_m_punpckldq(xmm3, xmm5)); - llr64[1] = simde_mm_adds_pi16(llr64[1], simde_m_punpckhdq(xmm3, xmm5)); - llr64[2] = simde_mm_adds_pi16(llr64[2], simde_m_punpckldq(xmm4, xmm6)); - llr64[3] = simde_mm_adds_pi16(llr64[3], simde_m_punpckhdq(xmm4, xmm6)); - } - } -#endif -} - -void inner_rx_qpsk_2layer (NR_DL_FRAME_PARMS *frame_parms, - NR_gNB_PUSCH *pusch_vars, - nfapi_nr_pusch_pdu_t *rel15_ul, - int **rxF, - int **ul_ch, - int16_t **llr, - int nb_layer, - int nb_rx_ant, - int soffset, - int length, - int symbol, - int short nb_rb, - int dmrs_symbol_flag, - int output_shift) -{ - int add_shift = 0; - if (length % 8) - add_shift = 8 - length % 8; - int32_t rxFext[nb_rx_ant][length + add_shift] __attribute__((aligned(32))); - int32_t chFext[nb_layer*nb_rx_ant][length + add_shift] __attribute__((aligned(32))); - for (int aarx = 0; aarx < nb_rx_ant; aarx++) - { - for (int aatx = 0; aatx < nb_layer; aatx++) - { - nr_ulsch_extract_rbs0((c16_t *)rxF[aarx], - pusch_vars->ul_ch_estimates[aatx * nb_rx_ant + aarx], - rxFext[aarx], - chFext[aatx * nb_rx_ant + aarx], - soffset+(symbol * frame_parms->ofdm_symbol_size), - pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, - aarx, - dmrs_symbol_flag, - rel15_ul, - frame_parms); - } - } - int32_t rho[nb_layer*nb_layer][length + add_shift] __attribute__((aligned(32))); - int32_t rxFext_comp[nb_layer][length + add_shift] __attribute__((aligned(32))); - for (int aarx = 0; aarx < nb_rx_ant; aarx++) - { - for (int aatx = 0; aatx < nb_layer; aatx++) - { - for (int atx = 0; atx < nb_layer; atx++) - { -#ifdef USE_128BIT - simde__m128i mmtmpD0, mmtmpD1, mmtmpD2, mmtmpD3, mmtmpD4; - simde__m128i *rho128 = (simde__m128i *)rho[aatx*nb_layer+atx]; - simde__m128i *ul_ch128 = (simde__m128i *)chFext[aatx * nb_rx_ant + aarx]; - simde__m128i *ul_ch128_2 = (simde__m128i *)chFext[atx * nb_rx_ant + aarx]; - for (int i = 0; i < (length >> 2)+((length&3)?1:0); i++) - { - // multiply by conjugated channel - mmtmpD0 = simde_mm_madd_epi16(ul_ch128[i], ul_ch128_2[i]); - // mmtmpD0 contains real part of 4 consecutive outputs (32-bit) - mmtmpD1 = simde_mm_shufflelo_epi16(ul_ch128[i], SIMDE_MM_SHUFFLE(2,3,0,1)); - mmtmpD1 = simde_mm_shufflehi_epi16(mmtmpD1, SIMDE_MM_SHUFFLE(2,3,0,1)); - mmtmpD1 = simde_mm_sign_epi16(mmtmpD1, *(simde__m128i*)&conjugate[0]); - mmtmpD1 = simde_mm_madd_epi16(mmtmpD1, ul_ch128_2[i]); - // mmtmpD1 contains imag part of 4 consecutive outputs (32-bit) - mmtmpD0 = simde_mm_srai_epi32(mmtmpD0, output_shift); - mmtmpD1 = simde_mm_srai_epi32(mmtmpD1, output_shift); - mmtmpD2 = simde_mm_unpacklo_epi32(mmtmpD0, mmtmpD1); - mmtmpD3 = simde_mm_unpackhi_epi32(mmtmpD0, mmtmpD1); - mmtmpD4 = simde_mm_packs_epi32(mmtmpD2, mmtmpD3); - if (aarx == 0) - rho128[i] = mmtmpD4; - else - rho128[i] = simde_mm_adds_epi16(rho128[i], mmtmpD4); - } -#else - simde__m256i mmtmpD0, mmtmpD1, mmtmpD2, mmtmpD3, mmtmpD4; - simde__m256i *rho256 = (simde__m256i *)rho[aatx*nb_layer+atx]; - simde__m256i *ul_ch256 = (simde__m256i *)chFext[aatx * nb_rx_ant + aarx]; - simde__m256i *ul_ch256_2 = (simde__m256i *)chFext[atx * nb_rx_ant + aarx]; - register simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); - register simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); - for (int i = 0; i < ((length >> 3)+((length&7)?1:0)); i++) - { - // multiply by conjugated channel - mmtmpD0 = simde_mm256_madd_epi16(ul_ch256[i], ul_ch256_2[i]); - // mmtmpD0 contains real part of 4 consecutive outputs (32-bit) - mmtmpD1 = simde_mm256_shuffle_epi8(ul_ch256[i], complex_shuffle256); - mmtmpD1 = simde_mm256_sign_epi16(mmtmpD1, conj256); - mmtmpD1 = simde_mm256_madd_epi16(mmtmpD1, ul_ch256_2[i]); - // mmtmpD1 contains imag part of 4 consecutive outputs (32-bit) - mmtmpD0 = simde_mm256_srai_epi32(mmtmpD0, output_shift); - mmtmpD1 = simde_mm256_srai_epi32(mmtmpD1, output_shift); - mmtmpD2 = simde_mm256_unpacklo_epi32(mmtmpD0, mmtmpD1); - mmtmpD3 = simde_mm256_unpackhi_epi32(mmtmpD0, mmtmpD1); - mmtmpD4 = simde_mm256_packs_epi32(mmtmpD2, mmtmpD3); - if (aarx == 0) - rho256[i] = mmtmpD4; - else - rho256[i] = simde_mm256_adds_epi16(rho256[i], mmtmpD4); - } -#endif - } - // compensation -#ifdef USE_128BIT - simde__m128i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; - register simde__m128i complex_shuffle128 = simde_mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); - register simde__m128i conj128 = simde_mm_set_epi16(1, -1, 1, -1, 1, -1, 1, -1); - simde__m128i *rxF128 = (simde__m128i*)rxFext[aarx]; - simde__m128i *ulch128 = (simde__m128i*)chFext[aatx * nb_rx_ant + aarx]; - simde__m128i *rxF_comp128 = (simde__m128i*)rxFext_comp[aatx]; - for (int i = 0; i < (length>>2) + ((length&3)?1:0); i++) - { - xmmp0 = simde_mm_madd_epi16(ulch128[i], rxF128[i]); - // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm_shuffle_epi8(ulch128[i], complex_shuffle128); - xmmp1 = simde_mm_sign_epi16(xmmp1, conj128); - xmmp1 = simde_mm_madd_epi16(xmmp1, rxF128[i]); - // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm_srai_epi32(xmmp0, output_shift); - xmmp1 = simde_mm_srai_epi32(xmmp1, output_shift); - xmmp2 = simde_mm_unpacklo_epi32(xmmp0, xmmp1); - xmmp3 = simde_mm_unpackhi_epi32(xmmp0, xmmp1); - xmmp4 = simde_mm_packs_epi32(xmmp2, xmmp3); - - if (aarx == 0) - *rxF_comp128 = xmmp4; - else - *rxF_comp128 = simde_mm_adds_epi16(*rxF_comp128, xmmp4); - - rxF_comp128++; - } -#else - simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; - register simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); - register simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); - simde__m256i *rxF256 = (simde__m256i*)rxFext[aarx]; - simde__m256i *ulch256 = (simde__m256i*)chFext[aatx * nb_rx_ant + aarx]; - simde__m256i *rxF_comp256 = (simde__m256i*)rxFext_comp[aatx]; - for (int i = 0; i < (length>>3) + ((length&7)?1:0); i++) - { - xmmp0 = simde_mm256_madd_epi16(ulch256[i], rxF256[i]); - // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm256_shuffle_epi8(ulch256[i], complex_shuffle256); - xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); - xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF256[i]); - // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); - xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); - xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); - xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); - xmmp4 = simde_mm256_packs_epi32(xmmp2, xmmp3); - - if (aarx == 0) - *rxF_comp256 = xmmp4; - else - *rxF_comp256 = simde_mm256_adds_epi16(*rxF_comp256, xmmp4); - - rxF_comp256++; - } -#endif - } - } - c16_t *rho0 = (c16_t *)rho[1]; - c16_t *rho1 = (c16_t *)rho[2]; - c16_t *llr_0 = (c16_t *)&llr[0][pusch_vars->llr_offset[symbol]]; - c16_t *llr_1 = (c16_t *)&llr[1][pusch_vars->llr_offset[symbol]]; - - nr_ulsch_qpsk_qpsk((c16_t *)rxFext_comp[0], (c16_t *)rxFext_comp[1], llr_0, rho0, length); - nr_ulsch_qpsk_qpsk((c16_t *)rxFext_comp[1], (c16_t *)rxFext_comp[0], llr_1, rho1, length); - - nr_ulsch_shift_llr(pusch_vars->llr_layers, length, pusch_vars->llr_offset[symbol] >> 1, 2, 4); - -} - - -void inner_rx_16qam_2layer (NR_DL_FRAME_PARMS *frame_parms, - NR_gNB_PUSCH *pusch_vars, - nfapi_nr_pusch_pdu_t *rel15_ul, - int **rxF, - int **ul_ch, - int16_t **llr, - int nb_layer, - int nb_rx_ant, - int soffset, - int length, - int symbol, - int short nb_rb, - int dmrs_symbol_flag, - int output_shift) -{ - int add_shift = 0; - if (length % 8) - add_shift = 8 - length % 8; - - int32_t rxFext[nb_rx_ant][length + add_shift] __attribute__((aligned(32))); - int32_t chFext[nb_layer*nb_rx_ant][length + add_shift] __attribute__((aligned(32))); - for (int aarx = 0; aarx < nb_rx_ant; aarx++) - { - for (int aatx = 0; aatx < nb_layer; aatx++) - { - nr_ulsch_extract_rbs0((c16_t *)rxF[aarx], - pusch_vars->ul_ch_estimates[aatx * nb_rx_ant + aarx], - rxFext[aarx], - chFext[aatx * nb_rx_ant + aarx], - soffset+(symbol * frame_parms->ofdm_symbol_size), - pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, - aarx, - dmrs_symbol_flag, - rel15_ul, - frame_parms); - } - } - - int32_t rho[nb_layer*nb_layer][length + add_shift] __attribute__((aligned(32))); - int32_t rxFext_comp[nb_layer][length + add_shift] __attribute__((aligned(32))); - int32_t ul_ch_mag[nb_layer][length + add_shift] __attribute__((aligned(32))); - for (int aatx = 0; aatx < nb_layer; aatx++) - { - for (int aarx = 0; aarx < nb_rx_ant; aarx++) - { - for (int atx = 0; atx < nb_layer; atx++) - { -#ifdef USE_128BIT - simde__m128i mmtmpD0, mmtmpD1, mmtmpD2, mmtmpD3; - simde__m128i *rho128 = (simde__m128i *)rho[aatx*nb_layer+atx]; - simde__m128i *ul_ch128 = (simde__m128i *)chFext[aatx * nb_rx_ant + aarx]; - simde__m128i *ul_ch128_2 = (simde__m128i *)chFext[atx * nb_rx_ant + aarx]; - for (int i = 0; i < (length>>2)+((length&3)?1:0); i++) - { - // multiply by conjugated channel - mmtmpD0 = simde_mm_madd_epi16(ul_ch128[i], ul_ch128_2[i]); - // mmtmpD0 contains real part of 4 consecutive outputs (32-bit) - mmtmpD1 = simde_mm_shufflelo_epi16(ul_ch128[i], SIMDE_MM_SHUFFLE(2,3,0,1)); - mmtmpD1 = simde_mm_shufflehi_epi16(mmtmpD1, SIMDE_MM_SHUFFLE(2,3,0,1)); - mmtmpD1 = simde_mm_sign_epi16(mmtmpD1, *(simde__m128i*)&conjugate[0]); - mmtmpD1 = simde_mm_madd_epi16(mmtmpD1, ul_ch128_2[i]); - // mmtmpD1 contains imag part of 4 consecutive outputs (32-bit) - mmtmpD0 = simde_mm_srai_epi32(mmtmpD0, output_shift); - mmtmpD1 = simde_mm_srai_epi32(mmtmpD1, output_shift); - mmtmpD2 = simde_mm_unpacklo_epi32(mmtmpD0, mmtmpD1); - mmtmpD3 = simde_mm_unpackhi_epi32(mmtmpD0, mmtmpD1); - if (aarx == 0) - rho128[i] = simde_mm_packs_epi32(mmtmpD2, mmtmpD3); - else - rho128[i] = simde_mm_adds_epi16(rho128[i], simde_mm_packs_epi32(mmtmpD2, mmtmpD3)); - } -#else - simde__m256i mmtmpD0, mmtmpD1, mmtmpD2, mmtmpD3; - simde__m256i *rho256 = (simde__m256i *)rho[aatx*nb_layer+atx]; - simde__m256i *ul_ch256 = (simde__m256i *)chFext[aatx * nb_rx_ant + aarx]; - simde__m256i *ul_ch256_2 = (simde__m256i *)chFext[atx * nb_rx_ant + aarx]; - register simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); - register simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); - for (int i = 0; i < (length >> 3)+((length&7)?1:0); i++) - { - // multiply by conjugated channel - mmtmpD0 = simde_mm256_madd_epi16(ul_ch256[i], ul_ch256_2[i]); - // mmtmpD0 contains real part of 4 consecutive outputs (32-bit) - mmtmpD1 = simde_mm256_shuffle_epi8(ul_ch256[i], complex_shuffle256); - mmtmpD1 = simde_mm256_sign_epi16(mmtmpD1, conj256); - mmtmpD1 = simde_mm256_madd_epi16(mmtmpD1, ul_ch256_2[i]); - // mmtmpD1 contains imag part of 4 consecutive outputs (32-bit) - mmtmpD0 = simde_mm256_srai_epi32(mmtmpD0, output_shift); - mmtmpD1 = simde_mm256_srai_epi32(mmtmpD1, output_shift); - mmtmpD2 = simde_mm256_unpacklo_epi32(mmtmpD0, mmtmpD1); - mmtmpD3 = simde_mm256_unpackhi_epi32(mmtmpD0, mmtmpD1); - if (aarx == 0) - rho256[i] = simde_mm256_packs_epi32(mmtmpD2, mmtmpD3); - else - rho256[i] = simde_mm256_adds_epi16(rho256[i], simde_mm256_packs_epi32(mmtmpD2, mmtmpD3)); - } -#endif - } - // compensation -#ifdef USE_128BIT - simde__m128i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; - register simde__m128i complex_shuffle128 = simde_mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); - register simde__m128i conj128 = simde_mm_set_epi16(1, -1, 1, -1, 1, -1, 1, -1); - register simde__m128i QAM_amp128 = simde_mm_set1_epi16(QAM16_n1); // 2/sqrt(10) - simde__m128i *rxF128 = (simde__m128i*)rxFext[aarx]; - simde__m128i *ulch128 = (simde__m128i*)chFext[aatx * nb_rx_ant + aarx]; - simde__m128i *rxF_comp128 = (simde__m128i*)rxFext_comp[aatx]; - simde__m128i *ul_ch_mag128 = (simde__m128i*)ul_ch_mag[aatx]; - for (int i = 0; i < (length>>2)+((length&3)?1:0); i++) - { - xmmp0 = simde_mm_madd_epi16(ulch128[i], rxF128[i]); - // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm_shuffle_epi8(ulch128[i], complex_shuffle128); - xmmp1 = simde_mm_sign_epi16(xmmp1, conj128); - xmmp1 = simde_mm_madd_epi16(xmmp1, rxF128[i]); - // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm_srai_epi32(xmmp0, output_shift); - xmmp1 = simde_mm_srai_epi32(xmmp1, output_shift); - xmmp2 = simde_mm_unpacklo_epi32(xmmp0, xmmp1); - xmmp3 = simde_mm_unpackhi_epi32(xmmp0, xmmp1); - xmmp4 = simde_mm_packs_epi32(xmmp2, xmmp3); - - xmmp0 = simde_mm_madd_epi16(ulch128[i], ulch128[i]); // |h|^2 - xmmp0 = simde_mm_srai_epi32(xmmp0, output_shift); - xmmp0 = simde_mm_packs_epi32(xmmp0, xmmp0); - xmmp1 = simde_mm_unpacklo_epi16(xmmp0, xmmp0); - xmmp1 = simde_mm_mulhrs_epi16(xmmp1, QAM_amp128); - - if (aarx == 0) - { - *rxF_comp128 = xmmp4; - *ul_ch_mag128 = xmmp1; - } - else - { - *rxF_comp128 = simde_mm_adds_epi16(*rxF_comp128, xmmp4); - *ul_ch_mag128 = simde_mm_adds_epi16(*ul_ch_mag128, xmmp1); - } - rxF_comp128++; - ul_ch_mag128++; - } -#else - simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; - register simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); - register simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); - register simde__m256i QAM_amp256 = simde_mm256_set1_epi16(QAM16_n1); // 2/sqrt(10) - simde__m256i *rxF256 = (simde__m256i*)rxFext[aarx]; - simde__m256i *ulch256 = (simde__m256i*)chFext[aatx * nb_rx_ant + aarx]; - simde__m256i *rxF_comp256 = (simde__m256i*)rxFext_comp[aatx]; - simde__m256i *ul_ch_mag256 = (simde__m256i*)ul_ch_mag[aatx]; - for (int i = 0; i < (length>>3) + ((length&7)?1:0); i++) - { - xmmp0 = simde_mm256_madd_epi16(ulch256[i], rxF256[i]); - // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm256_shuffle_epi8(ulch256[i], complex_shuffle256); - xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); - xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF256[i]); - // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); - xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); - xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); - xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); - xmmp4 = simde_mm256_packs_epi32(xmmp2, xmmp3); - - xmmp0 = simde_mm256_madd_epi16(ulch256[i], ulch256[i]); // |h|^2 - xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); - xmmp0 = simde_mm256_packs_epi32(xmmp0, xmmp0); - xmmp1 = simde_mm256_unpacklo_epi16(xmmp0, xmmp0); - xmmp1 = simde_mm256_mulhrs_epi16(xmmp1, QAM_amp256); - - if (aarx == 0) - { - *rxF_comp256 = xmmp4; - *ul_ch_mag256 = xmmp1; - } - else - { - *rxF_comp256 = simde_mm256_adds_epi16(*rxF_comp256, xmmp4); - *ul_ch_mag256 = simde_mm256_adds_epi16(*ul_ch_mag256, xmmp1); - } - rxF_comp256++; - ul_ch_mag256++; - } -#endif - } - } - c16_t *rho0 = (c16_t *)rho[1]; - c16_t *rho1 = (c16_t *)rho[2]; - c16_t *llr_0 = (c16_t *)&llr[0][pusch_vars->llr_offset[symbol]]; - c16_t *llr_1 = (c16_t *)&llr[1][pusch_vars->llr_offset[symbol]]; - c16_t *ul_ch_mag0 = (c16_t *)ul_ch_mag[0]; - c16_t *ul_ch_mag1 = (c16_t *)ul_ch_mag[1]; - nr_ulsch_qam16_qam16((c16_t *)rxFext_comp[0], (c16_t *)rxFext_comp[1], ul_ch_mag0, ul_ch_mag1, llr_0, rho0, length); - nr_ulsch_qam16_qam16((c16_t *)rxFext_comp[1], (c16_t *)rxFext_comp[0], ul_ch_mag1, ul_ch_mag0, llr_1, rho1, length); -} - -void inner_rx_64qam_2layer (NR_DL_FRAME_PARMS *frame_parms, - NR_gNB_PUSCH *pusch_vars, - nfapi_nr_pusch_pdu_t *rel15_ul, - int **rxF, - int **ul_ch, - int16_t **llr, - int nb_layer, - int nb_rx_ant, - int soffset, - int length, - int symbol, - int short nb_rb, - int dmrs_symbol_flag, - int output_shift) -{ - int add_shift = 0; - if (length % 8) - add_shift = 8 - length % 8; - - int32_t rxFext[nb_rx_ant][length + add_shift] __attribute__((aligned(32))); - int32_t chFext[nb_layer*nb_rx_ant][length + add_shift] __attribute__((aligned(32))); - for (int aarx = 0; aarx < nb_rx_ant; aarx++) - { - for (int aatx = 0; aatx < nb_layer; aatx++) - { - nr_ulsch_extract_rbs0((c16_t *)rxF[aarx], - pusch_vars->ul_ch_estimates[aatx * nb_rx_ant + aarx], - rxFext[aarx], - chFext[aatx * nb_rx_ant + aarx], - soffset+(symbol * frame_parms->ofdm_symbol_size), - pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, - aarx, - dmrs_symbol_flag, - rel15_ul, - frame_parms); - } - } - int32_t rho[nb_layer*nb_layer][length + add_shift] __attribute__((aligned(32))); - int32_t rxFext_comp[nb_layer][length + add_shift] __attribute__((aligned(32))); - int32_t ul_ch_mag[nb_layer][length + add_shift] __attribute__((aligned(32))); - for (int aatx = 0; aatx < nb_layer; aatx++) - { - for (int aarx = 0; aarx < nb_rx_ant; aarx++) - { - for (int atx = 0; atx < nb_layer; atx++) - { -#ifdef USE_128BIT - simde__m128i mmtmpD0, mmtmpD1, mmtmpD2, mmtmpD3; - simde__m128i *rho128 = (simde__m128i *)rho[aatx*nb_layer+atx]; - simde__m128i *ul_ch128 = (simde__m128i *)chFext[aatx * nb_rx_ant + aarx]; - simde__m128i *ul_ch128_2 = (simde__m128i *)chFext[atx * nb_rx_ant + aarx]; - for (int i = 0; i < (length>>2)+((length&3)?1:0); i++) - { - // multiply by conjugated channel - mmtmpD0 = simde_mm_madd_epi16(ul_ch128[i], ul_ch128_2[i]); - // mmtmpD0 contains real part of 4 consecutive outputs (32-bit) - mmtmpD1 = simde_mm_shufflelo_epi16(ul_ch128[i], SIMDE_MM_SHUFFLE(2,3,0,1)); - mmtmpD1 = simde_mm_shufflehi_epi16(mmtmpD1, SIMDE_MM_SHUFFLE(2,3,0,1)); - mmtmpD1 = simde_mm_sign_epi16(mmtmpD1, *(simde__m128i*)&conjugate[0]); - mmtmpD1 = simde_mm_madd_epi16(mmtmpD1, ul_ch128_2[i]); - // mmtmpD1 contains imag part of 4 consecutive outputs (32-bit) - mmtmpD0 = simde_mm_srai_epi32(mmtmpD0, output_shift); - mmtmpD1 = simde_mm_srai_epi32(mmtmpD1, output_shift); - mmtmpD2 = simde_mm_unpacklo_epi32(mmtmpD0, mmtmpD1); - mmtmpD3 = simde_mm_unpackhi_epi32(mmtmpD0, mmtmpD1); - if (aarx == 0) - rho128[i] = simde_mm_packs_epi32(mmtmpD2, mmtmpD3); - else - rho128[i] = simde_mm_adds_epi16(rho128[i], simde_mm_packs_epi32(mmtmpD2, mmtmpD3)); - } -#else - simde__m256i mmtmpD0, mmtmpD1, mmtmpD2, mmtmpD3; - simde__m256i *rho256 = (simde__m256i *)rho[aatx*nb_layer+atx]; - simde__m256i *ul_ch256 = (simde__m256i *)chFext[aatx * nb_rx_ant + aarx]; - simde__m256i *ul_ch256_2 = (simde__m256i *)chFext[atx * nb_rx_ant + aarx]; - register simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); - register simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); - for (int i = 0; i < (length >> 3)+((length&7)?1:0); i++) - { - // multiply by conjugated channel - mmtmpD0 = simde_mm256_madd_epi16(ul_ch256[i], ul_ch256_2[i]); - // mmtmpD0 contains real part of 4 consecutive outputs (32-bit) - mmtmpD1 = simde_mm256_shuffle_epi8(ul_ch256[i], complex_shuffle256); - mmtmpD1 = simde_mm256_sign_epi16(mmtmpD1, conj256); - mmtmpD1 = simde_mm256_madd_epi16(mmtmpD1, ul_ch256_2[i]); - // mmtmpD1 contains imag part of 4 consecutive outputs (32-bit) - mmtmpD0 = simde_mm256_srai_epi32(mmtmpD0, output_shift); - mmtmpD1 = simde_mm256_srai_epi32(mmtmpD1, output_shift); - mmtmpD2 = simde_mm256_unpacklo_epi32(mmtmpD0, mmtmpD1); - mmtmpD3 = simde_mm256_unpackhi_epi32(mmtmpD0, mmtmpD1); - if (aarx == 0) - rho256[i] = simde_mm256_packs_epi32(mmtmpD2, mmtmpD3); - else - rho256[i] = simde_mm256_adds_epi16(rho256[i], simde_mm256_packs_epi32(mmtmpD2, mmtmpD3)); - } -#endif - } - // compensation -#ifdef USE_128BIT - simde__m128i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; - register simde__m128i complex_shuffle128 = simde_mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); - register simde__m128i conj128 = simde_mm_set_epi16(1, -1, 1, -1, 1, -1, 1, -1); - register simde__m128i QAM_amp128 = simde_mm_set1_epi16(QAM64_n1); // 2/sqrt(10) - simde__m128i *rxF128 = (simde__m128i*)rxFext[aarx]; - simde__m128i *ulch128 = (simde__m128i*)chFext[aatx * nb_rx_ant + aarx]; - simde__m128i *rxF_comp128 = (simde__m128i*)rxFext_comp[aatx]; - simde__m128i *ul_ch_mag128 = (simde__m128i*)ul_ch_mag[aatx]; - for (int i = 0; i < (length>>2)+((length&3)?1:0); i++) - { - xmmp0 = simde_mm_madd_epi16(ulch128[i], rxF128[i]); - // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm_shuffle_epi8(ulch128[i], complex_shuffle128); - xmmp1 = simde_mm_sign_epi16(xmmp1, conj128); - xmmp1 = simde_mm_madd_epi16(xmmp1, rxF128[i]); - // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm_srai_epi32(xmmp0, output_shift); - xmmp1 = simde_mm_srai_epi32(xmmp1, output_shift); - xmmp2 = simde_mm_unpacklo_epi32(xmmp0, xmmp1); - xmmp3 = simde_mm_unpackhi_epi32(xmmp0, xmmp1); - xmmp4 = simde_mm_packs_epi32(xmmp2, xmmp3); - - xmmp0 = simde_mm_madd_epi16(ulch128[i], ulch128[i]); // |h|^2 - xmmp0 = simde_mm_srai_epi32(xmmp0, output_shift); - xmmp0 = simde_mm_packs_epi32(xmmp0, xmmp0); - xmmp1 = simde_mm_unpacklo_epi16(xmmp0, xmmp0); - xmmp1 = simde_mm_mulhrs_epi16(xmmp1, QAM_amp128); - - if (aarx == 0) - { - *rxF_comp128 = xmmp4; - *ul_ch_mag128 = xmmp1; - } - else - { - *rxF_comp128 = simde_mm_adds_epi16(*rxF_comp128, xmmp4); - *ul_ch_mag128 = simde_mm_adds_epi16(*ul_ch_mag128, xmmp1); - } - rxF_comp128++; - ul_ch_mag128++; - } -#else - simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; - register simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); - register simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); - register simde__m256i QAM_amp256 = simde_mm256_set1_epi16(QAM64_n1); // 2/sqrt(10) - simde__m256i *rxF256 = (simde__m256i*)rxFext[aarx]; - simde__m256i *ulch256 = (simde__m256i*)chFext[aatx * nb_rx_ant + aarx]; - simde__m256i *rxF_comp256 = (simde__m256i*)rxFext_comp[aatx]; - simde__m256i *ul_ch_mag256 = (simde__m256i*)ul_ch_mag[aatx]; - for (int i = 0; i < (length>>3) + ((length&7)?1:0); i++) - { - xmmp0 = simde_mm256_madd_epi16(ulch256[i], rxF256[i]); - // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm256_shuffle_epi8(ulch256[i], complex_shuffle256); - xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); - xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF256[i]); - // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); - xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); - xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); - xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); - xmmp4 = simde_mm256_packs_epi32(xmmp2, xmmp3); - - xmmp0 = simde_mm256_madd_epi16(ulch256[i], ulch256[i]); // |h|^2 - xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); - xmmp0 = simde_mm256_packs_epi32(xmmp0, xmmp0); - xmmp1 = simde_mm256_unpacklo_epi16(xmmp0, xmmp0); - xmmp1 = simde_mm256_mulhrs_epi16(xmmp1, QAM_amp256); - - if (aarx == 0) - { - *rxF_comp256 = xmmp4; - *ul_ch_mag256 = xmmp1; - } - else - { - *rxF_comp256 = simde_mm256_adds_epi16(*rxF_comp256, xmmp4); - *ul_ch_mag256 = simde_mm256_adds_epi16(*ul_ch_mag256, xmmp1); - } - rxF_comp256++; - ul_ch_mag256++; - } -#endif - } - } - c16_t *rho0 = (c16_t *)rho[1]; - c16_t *rho1 = (c16_t *)rho[2]; - c16_t *llr_0 = (c16_t *)&llr[0][pusch_vars->llr_offset[symbol]]; - c16_t *llr_1 = (c16_t *)&llr[1][pusch_vars->llr_offset[symbol]]; - c16_t *ul_ch_mag0 = (c16_t *)ul_ch_mag[0]; - c16_t *ul_ch_mag1 = (c16_t *)ul_ch_mag[1]; - nr_ulsch_qam64_qam64((c16_t *)rxFext_comp[0], (c16_t *)rxFext_comp[1], ul_ch_mag0, ul_ch_mag1, llr_0, rho0, length); - nr_ulsch_qam64_qam64((c16_t *)rxFext_comp[1], (c16_t *)rxFext_comp[0], ul_ch_mag1, ul_ch_mag0, llr_1, rho1, length); -} - -void nr_pusch_symbol_processing_noprecoding(void *arg) -{ - - puschSymbolProc_t *rdata=(puschSymbolProc_t*)arg; - - PHY_VARS_gNB *gNB = rdata->gNB; - NR_DL_FRAME_PARMS *frame_parms = rdata->frame_parms; - nfapi_nr_pusch_pdu_t *rel15_ul = rdata->rel15_ul; - int ulsch_id = rdata->ulsch_id; - int slot = rdata->slot; - NR_gNB_PUSCH *pusch_vars = &gNB->pusch_vars[ulsch_id]; - int16_t *s = rdata->s; - for (int symbol = rdata->startSymbol; symbol < rdata->startSymbol+rdata->numSymbols; symbol++) - { - int dmrs_symbol_flag = (rel15_ul->ul_dmrs_symb_pos >> symbol) & 0x01; - int nb_re_pusch = gNB->pusch_vars[ulsch_id].ul_valid_re_per_slot[symbol]; - // this needs to be reworded for parrellization, we need a table which give dmrs symbol location - // used for chennel estimate, they are being run in parallel! - if (dmrs_symbol_flag == 1) - { - if ((rel15_ul->ul_dmrs_symb_pos >> ((symbol + 1) % frame_parms->symbols_per_slot)) & 0x01) - AssertFatal(1==0,"Double DMRS configuration is not yet supported\n"); - - gNB->pusch_vars[ulsch_id].dmrs_symbol = symbol; - } - - LOG_I(PHY,"symbol %d: nb_re_pusch %d, DMRS symbl used for Chest :%d \n", symbol, nb_re_pusch, gNB->pusch_vars[ulsch_id].dmrs_symbol); - - if (nb_re_pusch == 0) continue; - if (rel15_ul->nrOfLayers == 1) - { - int16_t *llr = &rdata->llr[pusch_vars->llr_offset[symbol]]; - void (*inner_rx)(int *,int *,int16_t *,int,int,int); - if (rel15_ul->qam_mod_order == 2) inner_rx = inner_rx_qpsk; - else if (rel15_ul->qam_mod_order == 4) inner_rx = inner_rx_16qam; - else if (rel15_ul->qam_mod_order == 6) inner_rx = inner_rx_64qam; - else if (rel15_ul->qam_mod_order == 8) inner_rx = inner_rx_256qam; - else AssertFatal(1==0,"rel15_ul->qam_mod_order %d, pusch_pdu->dmrs_config_type %d\n", - rel15_ul->qam_mod_order,rel15_ul->dmrs_config_type); - - int soffset = (slot&3)*frame_parms->symbols_per_slot*frame_parms->ofdm_symbol_size; - int32_t rxFext[nb_re_pusch+8] __attribute__((aligned(32))); - int32_t chFext[nb_re_pusch+8] __attribute__((aligned(32))); - int16_t llr_temp[(nb_re_pusch*rel15_ul->qam_mod_order)+16] __attribute__((aligned(32))); - for (int aa = 0; aa < frame_parms->nb_antennas_rx; aa++) - { - nr_ulsch_extract_rbs0(gNB->common_vars.rxdataF[aa], - gNB->pusch_vars[ulsch_id].ul_ch_estimates[aa], - rxFext, - chFext, - soffset+(symbol * frame_parms->ofdm_symbol_size), - gNB->pusch_vars[ulsch_id].dmrs_symbol*frame_parms->ofdm_symbol_size, - aa, - dmrs_symbol_flag, - rel15_ul, - frame_parms); - // demodulation - inner_rx(rxFext, chFext, llr_temp, aa, nb_re_pusch, gNB->pusch_vars[ulsch_id].log2_maxh); - } - - // unscrambling - simde__m64 *llr64 = (simde__m64 *) llr; - for (int i=0;i<(nb_re_pusch*rel15_ul->qam_mod_order)>>2;i++) - llr64[i] = simde_mm_mullo_pi16(((simde__m64 *)llr_temp)[i],((simde__m64 *)s)[i]); - - s += nb_re_pusch * rel15_ul->qam_mod_order; - llr += nb_re_pusch * rel15_ul->qam_mod_order; - } - else // MIMO for 2x2 - { - int soffset = (slot&3)*frame_parms->symbols_per_slot*frame_parms->ofdm_symbol_size; - void (*inner_rx)(NR_DL_FRAME_PARMS *, - NR_gNB_PUSCH *, - nfapi_nr_pusch_pdu_t *, - int32_t **, - int32_t **, - int16_t **, - int32_t, - int32_t, - int32_t, - int32_t, - int32_t, - int16_t, - int32_t, - int32_t); - if (rel15_ul->qam_mod_order == 2) inner_rx = inner_rx_qpsk_2layer; - else if (rel15_ul->qam_mod_order == 4) inner_rx = inner_rx_16qam_2layer; - else if (rel15_ul->qam_mod_order == 6) inner_rx = inner_rx_64qam_2layer; - else AssertFatal(1==0,"rel15_ul->qam_mod_order %d, pusch_pdu->dmrs_config_type %d\n", - rel15_ul->qam_mod_order,rel15_ul->dmrs_config_type); - - inner_rx(frame_parms, - pusch_vars, - rel15_ul, - (int32_t**)gNB->common_vars.rxdataF, - gNB->pusch_vars[ulsch_id].ul_ch_estimates, - rdata->llr_layers, - rel15_ul->nrOfLayers, - frame_parms->nb_antennas_rx, - soffset, - nb_re_pusch, // length - symbol, // symbol index - rel15_ul->rb_size, // ofdm size - dmrs_symbol_flag, - gNB->pusch_vars[ulsch_id].log2_maxh); - - // layer de-mapping - int16_t* llr_cw = &rdata->llr[pusch_vars->llr_offset[symbol] * rel15_ul->nrOfLayers]; - for (int i = 0; i < (nb_re_pusch); i++) - for (int l = 0; l < rel15_ul->nrOfLayers; l++) - for (int m = 0; m < rel15_ul->qam_mod_order; m++) - llr_cw[i*rel15_ul->nrOfLayers*rel15_ul->qam_mod_order+l*rel15_ul->qam_mod_order+m] = rdata->llr_layers[l][pusch_vars->llr_offset[symbol] + i*rel15_ul->qam_mod_order+m]; - - // unscrambling - simde__m64 *llr64 = (simde__m64 *) &rdata->llr[pusch_vars->llr_offset[symbol] * rel15_ul->nrOfLayers]; - for (int i = 0; i < (nb_re_pusch*rel15_ul->qam_mod_order*rel15_ul->nrOfLayers)>>2; i++) - llr64[i] = simde_mm_mullo_pi16(((simde__m64 *)llr64)[i], ((simde__m64 *)s)[i]); - - s += (nb_re_pusch*rel15_ul->qam_mod_order*rel15_ul->nrOfLayers); - } - } -} - //compute average channel_level on each (TX,RX) antenna pair void nr_ulsch_channel_level(int **ul_ch_estimates_ext, NR_DL_FRAME_PARMS *frame_parms, @@ -2088,8 +548,6 @@ void nr_ulsch_channel_level(int **ul_ch_estimates_ext, uint8_t nrOfLayers, unsigned short nb_rb) { - - short rb; unsigned char aatx, aarx; simde__m128i *ul_ch128, avg128U; @@ -2124,10 +582,9 @@ void nr_ulsch_channel_level(int **ul_ch_estimates_ext, simde_mm_empty(); simde_m_empty(); - } -static simde__m128i a_mult_conjb(simde__m128i a, simde__m128i b, unsigned char output_shift) +static simde__m128i a_mult_conjb(__m128i a, __m128i b, unsigned char output_shift) { simde__m128i mmtmpD0 = simde_mm_madd_epi16(b, a); simde__m128i mmtmpD1 = simde_mm_shufflelo_epi16(b, SIMDE_MM_SHUFFLE(2, 3, 0, 1)); @@ -2138,7 +595,7 @@ static simde__m128i a_mult_conjb(simde__m128i a, simde__m128i b, unsigned char o mmtmpD1 = simde_mm_srai_epi32(mmtmpD1, output_shift); simde__m128i mmtmpD2 = simde_mm_unpacklo_epi32(mmtmpD0, mmtmpD1); simde__m128i mmtmpD3 = simde_mm_unpackhi_epi32(mmtmpD0, mmtmpD1); - return simde_mm_packs_epi32(mmtmpD2, mmtmpD3); + return _mm_packs_epi32(mmtmpD2, mmtmpD3); } //============================================================================================== @@ -2202,29 +659,30 @@ void nr_ulsch_channel_compensation(int **rxdataF_ext, #endif +#if defined(__i386) || defined(__x86_64__) unsigned short rb; unsigned char aatx,aarx; - simde__m128i *ul_ch128,*ul_ch128_2,*ul_ch_mag128,*ul_ch_mag128b,*ul_ch_mag128c,*rxdataF128,*rxdataF_comp128,*rho128; - simde__m128i mmtmpD0,mmtmpD1,mmtmpD2,mmtmpD3,QAM_amp128={0},QAM_amp128b={0},QAM_amp128c={0}; - QAM_amp128b = simde_mm_setzero_si128(); + __m128i *ul_ch128,*ul_ch128_2,*ul_ch_mag128,*ul_ch_mag128b,*ul_ch_mag128c,*rxdataF128,*rxdataF_comp128,*rho128; + __m128i mmtmpD0,mmtmpD1,mmtmpD2,mmtmpD3,QAM_amp128={0},QAM_amp128b={0},QAM_amp128c={0}; + QAM_amp128b = _mm_setzero_si128(); uint32_t nb_rb_0 = length/12 + ((length%12)?1:0); for (aatx=0; aatxnb_antennas_rx][symbol * (off + nb_rb * 12)]; - ul_ch_mag128b_1 = (simde__m128i *)&ul_ch_magb[frame_parms->nb_antennas_rx][symbol * (off + nb_rb * 12)]; - ul_ch_mag128c_1 = (simde__m128i *)&ul_ch_magc[frame_parms->nb_antennas_rx][symbol * (off + nb_rb * 12)]; + ul_ch_mag128_1 = (simde__m128i *) &((int *)ul_ch_mag)[1 * (length + length_aligned)]; + ul_ch_mag128b_1 = (simde__m128i *)&((int *)ul_ch_magb)[1 * (length + length_aligned)]; + ul_ch_mag128c_1 = (simde__m128i *)&((int *)ul_ch_magc)[1 * (length + length_aligned)]; } for (int rb = 0; rb < 3 * nb_rb_0; rb++) { @@ -3245,340 +1696,981 @@ uint8_t nr_ulsch_mmse_2layers(NR_DL_FRAME_PARMS *frame_parms, return(0); } -//============================================================================================== +void inner_rx_qpsk (NR_DL_FRAME_PARMS *frame_parms, + NR_gNB_PUSCH *pusch_vars, + nfapi_nr_pusch_pdu_t *rel15_ul, + c16_t **rxF, + c16_t **ul_ch, + int16_t *llr, + int nb_layer, + int nb_rx_ant, + int soffset, + int length, + int symbol, + int short nb_rb, + int dmrs_symbol_flag, + int output_shift) +{ + int length_aligned = (length % 8) ? (8 - length % 8) : 0; -/* Main Function */ -void nr_rx_pusch(PHY_VARS_gNB *gNB, - uint8_t ulsch_id, - uint32_t frame, - uint8_t slot, - unsigned char harq_pid) + c16_t rxFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + c16_t chFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + for (int aarx = 0; aarx < nb_rx_ant; aarx++) + { + nr_ulsch_extract_rbs0(rxF[aarx], + (c16_t*)pusch_vars->ul_ch_estimates[aarx], + rxFext[aarx], + chFext[aarx], + soffset+(symbol * frame_parms->ofdm_symbol_size), + pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, + aarx, + dmrs_symbol_flag, + rel15_ul, + frame_parms); + simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; + simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); + + simde__m256i *rxF256 = (simde__m256i*)rxFext[aarx]; + simde__m256i *ulch256 = (simde__m256i*)chFext[aarx]; + simde__m256i *llr256 = (simde__m256i *)llr; + for (int i = 0; i < ((length>>3) + ((length&7) > 0 ? 1:0)); i++) + { + xmmp0 = simde_mm256_madd_epi16(ulch256[i], rxF256[i]); + // xmmp0 contains real part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp1 = simde_mm256_shuffle_epi8(ulch256[i], complex_shuffle256); + xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); + xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF256[i]); + // xmmp1 contains imag part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); + xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); + xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); + xmmp4 = simde_mm256_packs_epi32(xmmp2,xmmp3); + + // MRC & LLR + llr256[i] = simde_mm256_add_epi16(llr256[i], xmmp4); + } + } + // Perform IDFT if transform precoding is enabled + if(rel15_ul->transform_precoding == transformPrecoder_enabled) + nr_idft((int32_t *)llr, length); +} + +void inner_rx_16qam (NR_DL_FRAME_PARMS *frame_parms, + NR_gNB_PUSCH *pusch_vars, + nfapi_nr_pusch_pdu_t *rel15_ul, + c16_t **rxF, + c16_t **ul_ch, + int16_t *llr, + int nb_layer, + int nb_rx_ant, + int soffset, + int length, + int symbol, + int short nb_rb, + int dmrs_symbol_flag, + int output_shift) +{ + int length_aligned = (length % 8) ? (8 - length % 8) : 0; + + c16_t rxFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + c16_t chFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + c16_t rxComp[length + length_aligned] __attribute__((aligned(32))); + c16_t rxF_ch_maga[length + length_aligned] __attribute__((aligned(32))); + + memset(rxComp, 0, sizeof(c16_t) * (length + length_aligned)); + memset(rxF_ch_maga, 0, sizeof(c16_t) * (length + length_aligned)); + + simde__m256i *rxComp_256 = (simde__m256i *)rxComp; + simde__m256i *rxF_ch_maga_256 = (simde__m256i *)rxF_ch_maga; + + for (int aarx = 0; aarx < nb_rx_ant; aarx++) + { + nr_ulsch_extract_rbs0(rxF[aarx], + (c16_t*)pusch_vars->ul_ch_estimates[aarx], + rxFext[aarx], + chFext[aarx], + soffset+(symbol * frame_parms->ofdm_symbol_size), + pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, + aarx, + dmrs_symbol_flag, + rel15_ul, + frame_parms); + simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; + simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); + simde__m256i QAM_amp256 = simde_mm256_set1_epi16(QAM16_n1); + + simde__m256i *rxF_256 = (simde__m256i*)rxFext[aarx]; + simde__m256i *chF_256 = (simde__m256i*)chFext[aarx]; + for (int i = 0; i < ((length>>3) + ((length&7) > 0 ? 1:0)); i++) + { + xmmp0 = simde_mm256_madd_epi16(chF_256[i], rxF_256[i]); + // xmmp0 contains real part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp1 = simde_mm256_shuffle_epi8(chF_256[i], complex_shuffle256); + xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); + xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF_256[i]); + // xmmp1 contains imag part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); + xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); + xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); + xmmp4 = simde_mm256_packs_epi32(xmmp2,xmmp3); + + xmmp0 = simde_mm256_madd_epi16(chF_256[i], chF_256[i]); // |h|^2 + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp0 = simde_mm256_packs_epi32(xmmp0, xmmp0); + xmmp0 = simde_mm256_unpacklo_epi16(xmmp0, xmmp0); + + xmmp1 = simde_mm256_mulhrs_epi16(xmmp0, QAM_amp256); + + // MRC + rxF_ch_maga_256[i] = simde_mm256_add_epi16(rxF_ch_maga_256[i], xmmp1); + rxComp_256[i] = simde_mm256_add_epi16(rxComp_256[i], xmmp4); + } + } + + if (rel15_ul->transform_precoding == transformPrecoder_enabled) + { + nr_freq_equalization(frame_parms, + (int *)rxComp, + (int *)rxF_ch_maga, + (int *)rxF_ch_maga, + symbol, + length, + rel15_ul->qam_mod_order); + nr_idft((int32_t*)rxComp, length); + } + nr_ulsch_compute_llr((int32_t *)rxComp, + (int32_t *)rxF_ch_maga, + NULL, + NULL, + llr, + rel15_ul->rb_size, + length, + symbol, + rel15_ul->qam_mod_order); +} + +void inner_rx_64qam (NR_DL_FRAME_PARMS *frame_parms, + NR_gNB_PUSCH *pusch_vars, + nfapi_nr_pusch_pdu_t *rel15_ul, + c16_t **rxF, + c16_t **ul_ch, + int16_t *llr, + int nb_layer, + int nb_rx_ant, + int soffset, + int length, + int symbol, + int short nb_rb, + int dmrs_symbol_flag, + int output_shift) +{ + int length_aligned = (length % 8) ? (8 - length % 8) : 0; + + c16_t rxFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + c16_t chFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + + c16_t rxComp[length + length_aligned] __attribute__((aligned(32))); + c16_t rxF_ch_maga[length + length_aligned] __attribute__((aligned(32))); + c16_t rxF_ch_magb[length + length_aligned] __attribute__((aligned(32))); + + memset(rxComp, 0, sizeof(c16_t) * (length + length_aligned)); + memset(rxF_ch_maga, 0, sizeof(c16_t) * (length + length_aligned)); + memset(rxF_ch_magb, 0, sizeof(c16_t) * (length + length_aligned)); + + simde__m256i *rxComp_256 = (simde__m256i *)rxComp; + simde__m256i *rxF_ch_maga_256 = (simde__m256i *)rxF_ch_maga; + simde__m256i *rxF_ch_magb_256 = (simde__m256i *)rxF_ch_magb; + + for (int aarx = 0; aarx < nb_rx_ant; aarx++) + { + nr_ulsch_extract_rbs0(rxF[aarx], + (c16_t*)pusch_vars->ul_ch_estimates[aarx], + rxFext[aarx], + chFext[aarx], + soffset+(symbol * frame_parms->ofdm_symbol_size), + pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, + aarx, + dmrs_symbol_flag, + rel15_ul, + frame_parms); + simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; + simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); + simde__m256i QAM_amp256 = simde_mm256_set1_epi16(QAM64_n1); // 2/sqrt(10) + simde__m256i QAM_amp256b = simde_mm256_set1_epi16(QAM64_n2); + + simde__m256i *rxF_256 = (simde__m256i*)rxFext[aarx]; + simde__m256i *chF_256 = (simde__m256i*)chFext[aarx]; + for (int i = 0; i < ((length>>3) + ((length&7) > 0 ? 1:0)); i++) + { + xmmp0 = simde_mm256_madd_epi16(chF_256[i], rxF_256[i]); + // xmmp0 contains real part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp1 = simde_mm256_shuffle_epi8(chF_256[i], complex_shuffle256); + xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); + xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF_256[i]); + // xmmp1 contains imag part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); + xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); + xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); + xmmp4 = simde_mm256_packs_epi32(xmmp2,xmmp3); + + xmmp0 = simde_mm256_madd_epi16(chF_256[i], chF_256[i]); // |h|^2 + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp0 = simde_mm256_packs_epi32(xmmp0, xmmp0); + xmmp0 = simde_mm256_unpacklo_epi16(xmmp0, xmmp0); + + xmmp1 = simde_mm256_mulhrs_epi16(xmmp0, QAM_amp256); + xmmp2 = simde_mm256_mulhrs_epi16(xmmp0, QAM_amp256b); + + // MRC + rxComp_256[i] = simde_mm256_add_epi16(rxComp_256[i], xmmp4); + rxF_ch_maga_256[i] = simde_mm256_add_epi16(rxF_ch_maga_256[i], xmmp1); + rxF_ch_magb_256[i] = simde_mm256_add_epi16(rxF_ch_magb_256[i], xmmp2); + } + } + + if (rel15_ul->transform_precoding == transformPrecoder_enabled) { + nr_freq_equalization(frame_parms, + (int *)rxComp, + (int *)rxF_ch_maga, + (int *)rxF_ch_magb, + symbol, + length, + rel15_ul->qam_mod_order); + nr_idft((int32_t*)rxComp, length); + } + nr_ulsch_compute_llr((int32_t*)rxComp, + (int32_t*)rxF_ch_maga, + (int32_t*)rxF_ch_magb, + NULL, + llr, + rel15_ul->rb_size, + length, + symbol, + rel15_ul->qam_mod_order); +} + +void inner_rx_256qam (NR_DL_FRAME_PARMS *frame_parms, + NR_gNB_PUSCH *pusch_vars, + nfapi_nr_pusch_pdu_t *rel15_ul, + c16_t **rxF, + c16_t **ul_ch, + int16_t *llr, + int nb_layer, + int nb_rx_ant, + int soffset, + int length, + int symbol, + int short nb_rb, + int dmrs_symbol_flag, + int output_shift) +{ + int length_aligned = (length % 8) ? (8 - length % 8) : 0; + + c16_t rxFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + c16_t chFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + c16_t rxComp[length + length_aligned] __attribute__((aligned(32))); + c16_t rxF_ch_maga[length + length_aligned] __attribute__((aligned(32))); + c16_t rxF_ch_magb[length + length_aligned] __attribute__((aligned(32))); + c16_t rxF_ch_magc[length + length_aligned] __attribute__((aligned(32))); + + memset(rxComp, 0, sizeof(c16_t) * (length + length_aligned)); + memset(rxF_ch_maga, 0, sizeof(c16_t) * (length + length_aligned)); + memset(rxF_ch_magb, 0, sizeof(c16_t) * (length + length_aligned)); + memset(rxF_ch_magc, 0, sizeof(c16_t) * (length + length_aligned)); + + simde__m256i *rxComp_256 = (simde__m256i *)rxComp; + simde__m256i *rxF_ch_maga_256 = (simde__m256i *)rxF_ch_maga; + simde__m256i *rxF_ch_magb_256 = (simde__m256i *)rxF_ch_magb; + simde__m256i *rxF_ch_magc_256 = (simde__m256i *)rxF_ch_magc; + + for (int aarx = 0; aarx < nb_rx_ant; aarx++) + { + nr_ulsch_extract_rbs0(rxF[aarx], + (c16_t*)pusch_vars->ul_ch_estimates[aarx], + rxFext[aarx], + chFext[aarx], + soffset+(symbol * frame_parms->ofdm_symbol_size), + pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, + aarx, + dmrs_symbol_flag, + rel15_ul, + frame_parms); + simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; + simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); + + simde__m256i QAM_amp256 = simde_mm256_set1_epi16(QAM256_n1); + simde__m256i QAM_amp256b = simde_mm256_set1_epi16(QAM256_n2); + simde__m256i QAM_amp256c = simde_mm256_set1_epi16(QAM256_n3); + + simde__m256i *rxF256 = (simde__m256i*)rxFext[aarx]; + simde__m256i *ulch256 = (simde__m256i*)chFext[aarx]; + for (int i = 0; i < ((length >> 3) + (( length & 7) > 0 ? 1 : 0)); i++) + { + xmmp0 = simde_mm256_madd_epi16(ulch256[i], rxF256[i]); + // xmmp0 contains real part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp1 = simde_mm256_shuffle_epi8(ulch256[i], complex_shuffle256); + xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); + xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF256[i]); + // xmmp1 contains imag part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); + xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); + xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); + xmmp4 = simde_mm256_packs_epi32(xmmp2, xmmp3); + + // compute channel amplitude for LLR + xmmp0 = simde_mm256_madd_epi16(ulch256[i], ulch256[i]); + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp0 = simde_mm256_packs_epi32(xmmp0, xmmp0); // contains 16 LLRs + xmmp2 = simde_mm256_unpacklo_epi16(xmmp0, xmmp0); + xmmp0 = simde_mm256_mulhrs_epi16(xmmp2, QAM_amp256); + xmmp1 = simde_mm256_mulhrs_epi16(xmmp2, QAM_amp256b); + xmmp2 = simde_mm256_mulhrs_epi16(xmmp2, QAM_amp256c); + + // MRC + rxComp_256[i] = simde_mm256_add_epi16(rxComp_256[i], xmmp4); + rxF_ch_maga_256[i] = simde_mm256_add_epi16(rxF_ch_maga_256[i], xmmp0); + rxF_ch_magb_256[i] = simde_mm256_add_epi16(rxF_ch_magb_256[i], xmmp1); + rxF_ch_magc_256[i] = simde_mm256_add_epi16(rxF_ch_magc_256[i], xmmp2); + } + } + nr_ulsch_compute_llr((int32_t*)rxComp, + (int32_t*)rxF_ch_maga, + (int32_t*)rxF_ch_magb, + (int32_t*)rxF_ch_magc, + llr, + rel15_ul->rb_size, + length, + symbol, + rel15_ul->qam_mod_order); +} + +void inner_rx_qpsk_2layer (NR_DL_FRAME_PARMS *frame_parms, + NR_gNB_PUSCH *pusch_vars, + nfapi_nr_pusch_pdu_t *rel15_ul, + c16_t **rxF, + c16_t **ul_ch, + int16_t **llr, + int nb_layer, + int nb_rx_ant, + int soffset, + int length, + int symbol, + int short nb_rb, + int dmrs_symbol_flag, + int output_shift, + uint32_t nvar) +{ + int length_aligned = (length % 8) ? (8 - length % 8) : 0; + + c16_t rxFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + c16_t chFext[nb_layer][nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + for (int aarx = 0; aarx < nb_rx_ant; aarx++) + { + for (int aatx = 0; aatx < nb_layer; aatx++) + { + nr_ulsch_extract_rbs0(rxF[aarx], + (c16_t*)pusch_vars->ul_ch_estimates[aatx * nb_rx_ant + aarx], + rxFext[aarx], + chFext[aatx][aarx], + soffset+(symbol * frame_parms->ofdm_symbol_size), + pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, + aarx, + dmrs_symbol_flag, + rel15_ul, + frame_parms); + } + } + c16_t rxComp[nb_layer][length + length_aligned] __attribute__((aligned(32))); + c16_t rho[nb_layer][nb_layer][length + length_aligned] __attribute__((aligned(32))); + + memset(rxComp, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); + memset(rho, 0, sizeof(c16_t) * nb_layer * nb_layer * (length + length_aligned)); + + for (int aarx = 0; aarx < nb_rx_ant; aarx++) { + for (int aatx = 0; aatx < nb_layer; aatx++) { + for (int atx = 0; atx < nb_layer; atx++) { + simde__m256i mmtmpD0, mmtmpD1, mmtmpD2, mmtmpD3, mmtmpD4; + simde__m256i *rho_256 = (simde__m256i *)rho[aatx][atx]; + simde__m256i *chF_256 = (simde__m256i *)chFext[aatx][aarx]; + simde__m256i *chF2_256 = (simde__m256i *)chFext[atx][aarx]; + simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); + for (int i = 0; i < ((length >> 3)+((length&7)?1:0)); i++) + { + // multiply by conjugated channel + mmtmpD0 = simde_mm256_madd_epi16(chF_256[i], chF2_256[i]); + // mmtmpD0 contains real part of 4 consecutive outputs (32-bit) + mmtmpD1 = simde_mm256_shuffle_epi8(chF_256[i], complex_shuffle256); + mmtmpD1 = simde_mm256_sign_epi16(mmtmpD1, conj256); + mmtmpD1 = simde_mm256_madd_epi16(mmtmpD1, chF2_256[i]); + // mmtmpD1 contains imag part of 4 consecutive outputs (32-bit) + mmtmpD0 = simde_mm256_srai_epi32(mmtmpD0, output_shift); + mmtmpD1 = simde_mm256_srai_epi32(mmtmpD1, output_shift); + mmtmpD2 = simde_mm256_unpacklo_epi32(mmtmpD0, mmtmpD1); + mmtmpD3 = simde_mm256_unpackhi_epi32(mmtmpD0, mmtmpD1); + mmtmpD4 = simde_mm256_packs_epi32(mmtmpD2, mmtmpD3); + + rho_256[i] = simde_mm256_adds_epi16(rho_256[i], mmtmpD4); + } + } + // compensation + simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; + simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); + simde__m256i *rxF_256 = (simde__m256i*)rxFext[aarx]; + simde__m256i *chF_256 = (simde__m256i*)chFext[aatx][aarx]; + simde__m256i *rxComp_256 = (simde__m256i*)rxComp[aatx]; + for (int i = 0; i < (length>>3) + ((length&7)?1:0); i++) + { + xmmp0 = simde_mm256_madd_epi16(chF_256[i], rxF_256[i]); + // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp1 = simde_mm256_shuffle_epi8(chF_256[i], complex_shuffle256); + xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); + xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF_256[i]); + // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); + xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); + xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); + xmmp4 = simde_mm256_packs_epi32(xmmp2, xmmp3); + + rxComp_256[i] = simde_mm256_adds_epi16(rxComp_256[i], xmmp4); + } + } + } + c16_t *rho0 = rho[0][1]; + c16_t *rho1 = rho[1][0]; + c16_t *llr_0 = (c16_t *)&llr[0][pusch_vars->llr_offset[symbol]]; + c16_t *llr_1 = (c16_t *)&llr[1][pusch_vars->llr_offset[symbol]]; + + nr_ulsch_qpsk_qpsk(rxComp[0], rxComp[1], llr_0, rho0, length); + nr_ulsch_qpsk_qpsk(rxComp[1], rxComp[0], llr_1, rho1, length); + + nr_ulsch_shift_llr(pusch_vars->llr_layers, length, pusch_vars->llr_offset[symbol] >> 1, 2, 4); +} + + +void inner_rx_16qam_2layer (NR_DL_FRAME_PARMS *frame_parms, + NR_gNB_PUSCH *pusch_vars, + nfapi_nr_pusch_pdu_t *rel15_ul, + c16_t **rxF, + c16_t **ul_ch, + int16_t **llr, + int nb_layer, + int nb_rx_ant, + int soffset, + int length, + int symbol, + int short nb_rb, + int dmrs_symbol_flag, + int output_shift, + uint32_t nvar) +{ + int length_aligned = (length % 8) ? (8 - length % 8) : 0; + + c16_t rxFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + c16_t chFext[nb_layer][nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + for (int aarx = 0; aarx < nb_rx_ant; aarx++) { + for (int aatx = 0; aatx < nb_layer; aatx++) { + nr_ulsch_extract_rbs0(rxF[aarx], + (c16_t*)pusch_vars->ul_ch_estimates[aatx * nb_rx_ant + aarx], + rxFext[aarx], + chFext[aatx][aarx], + soffset+(symbol * frame_parms->ofdm_symbol_size), + pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, + aarx, + dmrs_symbol_flag, + rel15_ul, + frame_parms); + } + } + + c16_t rxComp[nb_layer][length + length_aligned] __attribute__((aligned(32))); + c16_t rho[nb_layer][nb_layer][length + length_aligned] __attribute__((aligned(32))); + c16_t ul_ch_maga[nb_layer][length + length_aligned] __attribute__((aligned(32))); + + memset(rxComp, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); + memset(rho, 0, sizeof(c16_t) * nb_layer * nb_layer * (length + length_aligned)); + memset(ul_ch_maga, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); + + for (int aatx = 0; aatx < nb_layer; aatx++) { + for (int aarx = 0; aarx < nb_rx_ant; aarx++) { + for (int atx = 0; atx < nb_layer; atx++) { + simde__m256i mmtmpD0, mmtmpD1, mmtmpD2, mmtmpD3; + simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); + simde__m256i *chF_256 = (simde__m256i *)chFext[aatx][aarx]; + simde__m256i *chF2_256 = (simde__m256i *)chFext[atx][aarx]; + simde__m256i *rho_256 = (simde__m256i *)rho[aatx][atx]; + for (int i = 0; i < (length >> 3)+((length&7)?1:0); i++) + { + // multiply by conjugated channel + mmtmpD0 = simde_mm256_madd_epi16(chF_256[i], chF2_256[i]); + // mmtmpD0 contains real part of 4 consecutive outputs (32-bit) + mmtmpD1 = simde_mm256_shuffle_epi8(chF_256[i], complex_shuffle256); + mmtmpD1 = simde_mm256_sign_epi16(mmtmpD1, conj256); + mmtmpD1 = simde_mm256_madd_epi16(mmtmpD1, chF2_256[i]); + // mmtmpD1 contains imag part of 4 consecutive outputs (32-bit) + mmtmpD0 = simde_mm256_srai_epi32(mmtmpD0, output_shift); + mmtmpD1 = simde_mm256_srai_epi32(mmtmpD1, output_shift); + mmtmpD2 = simde_mm256_unpacklo_epi32(mmtmpD0, mmtmpD1); + mmtmpD3 = simde_mm256_unpackhi_epi32(mmtmpD0, mmtmpD1); + + rho_256[i] = simde_mm256_adds_epi16(rho_256[i], simde_mm256_packs_epi32(mmtmpD2, mmtmpD3)); + } + } + // compensation + simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; + simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); + simde__m256i QAM_amp256 = simde_mm256_set1_epi16(QAM16_n1); // 2/sqrt(10) + simde__m256i *rxF_256 = (simde__m256i*)rxFext[aarx]; + simde__m256i *chF_256 = (simde__m256i*)chFext[aatx][aarx]; + simde__m256i *rxComp_256 = (simde__m256i*)rxComp[aatx]; + simde__m256i *ul_ch_maga_256 = (simde__m256i*)ul_ch_maga[aatx]; + for (int i = 0; i < (length>>3) + ((length&7)?1:0); i++) + { + xmmp0 = simde_mm256_madd_epi16(chF_256[i], rxF_256[i]); + // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp1 = simde_mm256_shuffle_epi8(chF_256[i], complex_shuffle256); + xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); + xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF_256[i]); + // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); + xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); + xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); + xmmp4 = simde_mm256_packs_epi32(xmmp2, xmmp3); + + xmmp0 = simde_mm256_madd_epi16(chF_256[i], chF_256[i]); // |h|^2 + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp0 = simde_mm256_packs_epi32(xmmp0, xmmp0); + xmmp1 = simde_mm256_unpacklo_epi16(xmmp0, xmmp0); + xmmp1 = simde_mm256_mulhrs_epi16(xmmp1, QAM_amp256); + + rxComp_256[i] = simde_mm256_adds_epi16(rxComp_256[i], xmmp4); + ul_ch_maga_256[i] = simde_mm256_adds_epi16(ul_ch_maga_256[i], xmmp1); + } + } + } + c16_t *rho0 = rho[0][1]; + c16_t *rho1 = rho[1][0]; + c16_t *llr_0 = (c16_t *)&llr[0][pusch_vars->llr_offset[symbol]]; + c16_t *llr_1 = (c16_t *)&llr[1][pusch_vars->llr_offset[symbol]]; + c16_t *ul_ch_mag0 = ul_ch_maga[0]; + c16_t *ul_ch_mag1 = ul_ch_maga[1]; + nr_ulsch_qam16_qam16(rxComp[0], rxComp[1], ul_ch_mag0, ul_ch_mag1, llr_0, rho0, length); + nr_ulsch_qam16_qam16(rxComp[1], rxComp[0], ul_ch_mag1, ul_ch_mag0, llr_1, rho1, length); +} + +void inner_rx_64qam_2layer (NR_DL_FRAME_PARMS *frame_parms, + NR_gNB_PUSCH *pusch_vars, + nfapi_nr_pusch_pdu_t *rel15_ul, + c16_t **rxF, + c16_t **ul_ch, + int16_t **llr, + int nb_layer, + int nb_rx_ant, + int soffset, + int length, + int symbol, + int short nb_rb, + int dmrs_symbol_flag, + int output_shift, + uint32_t nvar) +{ + int length_aligned = (length % 8) ? (8 - length % 8) : 0; + + c16_t rxFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + c16_t chFext[nb_layer][nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + for (int aarx = 0; aarx < nb_rx_ant; aarx++) { + for (int aatx = 0; aatx < nb_layer; aatx++) { + nr_ulsch_extract_rbs0(rxF[aarx], + (c16_t *)pusch_vars->ul_ch_estimates[aatx * nb_rx_ant + aarx], + rxFext[aarx], + chFext[aatx][aarx], + soffset+(symbol * frame_parms->ofdm_symbol_size), + pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, + aarx, + dmrs_symbol_flag, + rel15_ul, + frame_parms); + } + } + c16_t rho[nb_layer][nb_layer][length + length_aligned] __attribute__((aligned(32))); + c16_t rxComp[nb_layer][length + length_aligned] __attribute__((aligned(32))); + c16_t ul_ch_maga[nb_layer][length + length_aligned] __attribute__((aligned(32))); + c16_t ul_ch_magb[nb_layer][length + length_aligned] __attribute__((aligned(32))); + c16_t ul_ch_magc[nb_layer][length + length_aligned] __attribute__((aligned(32))); + + memset(rho, 0, sizeof(c16_t) * nb_layer * nb_layer* (length + length_aligned)); + memset(rxComp, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); + memset(ul_ch_maga, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); + memset(ul_ch_magb, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); + memset(ul_ch_magc, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); + + for (int aatx = 0; aatx < nb_layer; aatx++) { + for (int aarx = 0; aarx < nb_rx_ant; aarx++) { +#ifdef USE_ML + for (int atx = 0; atx < nb_layer; atx++) { + simde__m256i mmtmpD0, mmtmpD1, mmtmpD2, mmtmpD3; + simde__m256i *rho_256 = (simde__m256i *)rho[aatx][atx]; + simde__m256i *chF_256 = (simde__m256i *)chFext[aatx][aarx]; + simde__m256i *chF2_256 = (simde__m256i *)chFext[atx][aarx]; + simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); + for (int i = 0; i < (length >> 3)+((length&7)?1:0); i++) + { + // multiply by conjugated channel + mmtmpD0 = simde_mm256_madd_epi16(chF_256[i], chF2_256[i]); + // mmtmpD0 contains real part of 4 consecutive outputs (32-bit) + mmtmpD1 = simde_mm256_shuffle_epi8(chF_256[i], complex_shuffle256); + mmtmpD1 = simde_mm256_sign_epi16(mmtmpD1, conj256); + mmtmpD1 = simde_mm256_madd_epi16(mmtmpD1, chF2_256[i]); + // mmtmpD1 contains imag part of 4 consecutive outputs (32-bit) + mmtmpD0 = simde_mm256_srai_epi32(mmtmpD0, output_shift); + mmtmpD1 = simde_mm256_srai_epi32(mmtmpD1, output_shift); + mmtmpD2 = simde_mm256_unpacklo_epi32(mmtmpD0, mmtmpD1); + mmtmpD3 = simde_mm256_unpackhi_epi32(mmtmpD0, mmtmpD1); + + rho_256[i] = simde_mm256_adds_epi16(rho_256[i], simde_mm256_packs_epi32(mmtmpD2, mmtmpD3)); + } + } +#endif + // compensation + simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; + simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); + simde__m256i QAM_ampa_256 = simde_mm256_set1_epi16(QAM64_n1); // 2/sqrt(10) + simde__m256i QAM_ampb_256 = simde_mm256_set1_epi16(QAM64_n2); // 2/sqrt(10) + simde__m256i *rxF_256 = (simde__m256i*)rxFext[aarx]; + simde__m256i *chF_256 = (simde__m256i*)chFext[aatx][aarx]; + simde__m256i *rxComp_256 = (simde__m256i*)rxComp[aatx]; + simde__m256i *ul_ch_maga_256 = (simde__m256i*)ul_ch_maga[aatx]; + simde__m256i *ul_ch_magb_256 = (simde__m256i*)ul_ch_magb[aatx]; + for (int i = 0; i < (length>>3) + ((length&7)?1:0); i++) + { + xmmp0 = simde_mm256_madd_epi16(chF_256[i], rxF_256[i]); + // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp1 = simde_mm256_shuffle_epi8(chF_256[i], complex_shuffle256); + xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); + xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF_256[i]); + // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); + xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); + xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); + xmmp4 = simde_mm256_packs_epi32(xmmp2, xmmp3); + + xmmp0 = simde_mm256_madd_epi16(chF_256[i], chF_256[i]); // |h|^2 + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp0 = simde_mm256_packs_epi32(xmmp0, xmmp0); + xmmp1 = simde_mm256_unpacklo_epi16(xmmp0, xmmp0); + + xmmp2 = simde_mm256_mulhrs_epi16(xmmp1, QAM_ampa_256); + xmmp3 = simde_mm256_mulhrs_epi16(xmmp1, QAM_ampb_256); + + rxComp_256[i] = simde_mm256_adds_epi16(rxComp_256[i], xmmp4); + ul_ch_maga_256[i] = simde_mm256_adds_epi16(ul_ch_maga_256[i], xmmp2); + ul_ch_magb_256[i] = simde_mm256_adds_epi16(ul_ch_magb_256[i], xmmp3); + } + } + } +#ifdef USE_ML + c16_t *rho0 = rho[0][1]; + c16_t *rho1 = rho[1][0]; + c16_t *llr_0 = (c16_t *)&llr[0][pusch_vars->llr_offset[symbol]]; + c16_t *llr_1 = (c16_t *)&llr[1][pusch_vars->llr_offset[symbol]]; + c16_t *ul_ch_mag0 = ul_ch_maga[0]; + c16_t *ul_ch_mag1 = ul_ch_maga[1]; + nr_ulsch_qam64_qam64(rxComp[0], rxComp[1], ul_ch_mag0, ul_ch_mag1, llr_0, rho0, length); + nr_ulsch_qam64_qam64(rxComp[1], rxComp[0], ul_ch_mag1, ul_ch_mag0, llr_1, rho1, length); +#else + nr_ulsch_mmse_2layers(frame_parms, + (int **)rxComp, + (int **)ul_ch_maga, + (int **)ul_ch_magb, + (int **)ul_ch_magc, + (int **)chFext, + rel15_ul->rb_size, + frame_parms->nb_antennas_rx, + rel15_ul->qam_mod_order, + pusch_vars->log2_maxh, + symbol, + length, + nvar); + for (int aatx = 0; aatx < nb_layer; aatx++) + nr_ulsch_compute_llr((int32_t*)rxComp[aatx], + (int32_t*)ul_ch_maga[aatx], + (int32_t*)ul_ch_magb[aatx], + (int32_t*)ul_ch_magc[aatx], + &llr[aatx][pusch_vars->llr_offset[symbol]], + rel15_ul->rb_size, + length, + symbol, + rel15_ul->qam_mod_order); +#endif +} + +void inner_rx_256qam_2layer (NR_DL_FRAME_PARMS *frame_parms, + NR_gNB_PUSCH *pusch_vars, + nfapi_nr_pusch_pdu_t *rel15_ul, + c16_t **rxF, + c16_t **ul_ch, + int16_t **llr, + int nb_layer, + int nb_rx_ant, + int soffset, + int length, + int symbol, + int short nb_rb, + int dmrs_symbol_flag, + int output_shift, + uint32_t nvar) +{ + int length_aligned = (length % 8) ? (8 - length % 8) : 0; + + c16_t rxFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + c16_t chFext[nb_layer][nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + for (int aarx = 0; aarx < nb_rx_ant; aarx++) { + for (int aatx = 0; aatx < nb_layer; aatx++) { + nr_ulsch_extract_rbs0(rxF[aarx], + (c16_t *)pusch_vars->ul_ch_estimates[aatx * nb_rx_ant + aarx], + rxFext[aarx], + chFext[aatx][aarx], + soffset+(symbol * frame_parms->ofdm_symbol_size), + pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, + aarx, + dmrs_symbol_flag, + rel15_ul, + frame_parms); + } + } + c16_t rho[nb_layer][nb_layer][length + length_aligned] __attribute__((aligned(32))); + c16_t rxComp[nb_layer][length + length_aligned] __attribute__((aligned(32))); + c16_t ul_ch_maga[nb_layer][length + length_aligned] __attribute__((aligned(32))); + c16_t ul_ch_magb[nb_layer][length + length_aligned] __attribute__((aligned(32))); + c16_t ul_ch_magc[nb_layer][length + length_aligned] __attribute__((aligned(32))); + + memset(rho, 0, sizeof(c16_t) * nb_layer * nb_layer* (length + length_aligned)); + memset(rxComp, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); + memset(ul_ch_maga, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); + memset(ul_ch_magb, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); + memset(ul_ch_magc, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); + + // compensation + for (int aatx = 0; aatx < nb_layer; aatx++) { + for (int aarx = 0; aarx < nb_rx_ant; aarx++) { + simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; + simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); + simde__m256i QAM_ampa_256 = simde_mm256_set1_epi16(QAM256_n1); // 2/sqrt(10) + simde__m256i QAM_ampb_256 = simde_mm256_set1_epi16(QAM256_n2); // 2/sqrt(10) + simde__m256i QAM_ampc_256 = simde_mm256_set1_epi16(QAM256_n3); // 2/sqrt(10) + simde__m256i *rxF_256 = (simde__m256i*)rxFext[aarx]; + simde__m256i *chF_256 = (simde__m256i*)chFext[aatx][aarx]; + simde__m256i *rxComp_256 = (simde__m256i*)rxComp[aatx]; + simde__m256i *ul_ch_maga_256 = (simde__m256i*)ul_ch_maga[aatx]; + simde__m256i *ul_ch_magb_256 = (simde__m256i*)ul_ch_magb[aatx]; + simde__m256i *ul_ch_magc_256 = (simde__m256i*)ul_ch_magc[aatx]; + for (int i = 0; i < (length>>3) + ((length&7)?1:0); i++) + { + xmmp0 = simde_mm256_madd_epi16(chF_256[i], rxF_256[i]); + // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp1 = simde_mm256_shuffle_epi8(chF_256[i], complex_shuffle256); + xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); + xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF_256[i]); + // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); + xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); + xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); + xmmp4 = simde_mm256_packs_epi32(xmmp2, xmmp3); + + xmmp0 = simde_mm256_madd_epi16(chF_256[i], chF_256[i]); // |h|^2 + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp0 = simde_mm256_packs_epi32(xmmp0, xmmp0); + xmmp1 = simde_mm256_unpacklo_epi16(xmmp0, xmmp0); + + xmmp2 = simde_mm256_mulhrs_epi16(xmmp1, QAM_ampa_256); + xmmp3 = simde_mm256_mulhrs_epi16(xmmp1, QAM_ampb_256); + xmmp1 = simde_mm256_mulhrs_epi16(xmmp1, QAM_ampc_256); + + rxComp_256[i] = simde_mm256_adds_epi16(rxComp_256[i], xmmp4); + ul_ch_maga_256[i] = simde_mm256_adds_epi16(ul_ch_maga_256[i], xmmp2); + ul_ch_magb_256[i] = simde_mm256_adds_epi16(ul_ch_magb_256[i], xmmp3); + ul_ch_magc_256[i] = simde_mm256_adds_epi16(ul_ch_magc_256[i], xmmp1); + } + } + } + + nr_ulsch_mmse_2layers(frame_parms, + (int **)rxComp, + (int **)ul_ch_maga, + (int **)ul_ch_magb, + (int **)ul_ch_magc, + (int **)chFext, + rel15_ul->rb_size, + frame_parms->nb_antennas_rx, + rel15_ul->qam_mod_order, + pusch_vars->log2_maxh, + symbol, + length, + nvar); + for (int aatx = 0; aatx < nb_layer; aatx++) + nr_ulsch_compute_llr((int32_t*)rxComp[aatx], + (int32_t*)ul_ch_maga[aatx], + (int32_t*)ul_ch_magb[aatx], + (int32_t*)ul_ch_magc[aatx], + &llr[aatx][pusch_vars->llr_offset[symbol]], + rel15_ul->rb_size, + length, + symbol, + rel15_ul->qam_mod_order); +} + +void nr_pusch_symbol_processing_noprecoding(void *arg) { - uint8_t aarx, aatx; - uint32_t nb_re_pusch, bwp_start_subcarrier; - int avgs = 0; - - NR_DL_FRAME_PARMS *frame_parms = &gNB->frame_parms; - NR_gNB_ULSCH_t *ulsch = &gNB->ulsch[ulsch_id]; - nfapi_nr_pusch_pdu_t *rel15_ul = &ulsch->harq_process->ulsch_pdu; - int avg[frame_parms->nb_antennas_rx*rel15_ul->nrOfLayers]; + puschSymbolProc_t *rdata=(puschSymbolProc_t*)arg; + PHY_VARS_gNB *gNB = rdata->gNB; + NR_DL_FRAME_PARMS *frame_parms = rdata->frame_parms; + nfapi_nr_pusch_pdu_t *rel15_ul = rdata->rel15_ul; + int ulsch_id = rdata->ulsch_id; + int slot = rdata->slot; NR_gNB_PUSCH *pusch_vars = &gNB->pusch_vars[ulsch_id]; - pusch_vars->dmrs_symbol = INVALID_VALUE; - pusch_vars->cl_done = 0; - - bwp_start_subcarrier = ((rel15_ul->rb_start + rel15_ul->bwp_start)*NR_NB_SC_PER_RB + frame_parms->first_carrier_offset) % frame_parms->ofdm_symbol_size; - LOG_D(PHY,"pusch %d.%d : bwp_start_subcarrier %d, rb_start %d, first_carrier_offset %d\n", frame,slot,bwp_start_subcarrier, rel15_ul->rb_start, frame_parms->first_carrier_offset); - LOG_D(PHY,"pusch %d.%d : ul_dmrs_symb_pos %x\n",frame,slot,rel15_ul->ul_dmrs_symb_pos); - LOG_D(PHY,"ulsch RX %x : start_rb %d nb_rb %d mcs %d Nl %d Tpmi %d bwp_start %d start_sc %d start_symbol %d num_symbols %d cdmgrpsnodata %d num_dmrs %d dmrs_ports %d\n", - rel15_ul->rnti,rel15_ul->rb_start,rel15_ul->rb_size,rel15_ul->mcs_index, - rel15_ul->nrOfLayers,0,rel15_ul->bwp_start,0,rel15_ul->start_symbol_index,rel15_ul->nr_of_symbols, - rel15_ul->num_dmrs_cdm_grps_no_data,rel15_ul->ul_dmrs_symb_pos,rel15_ul->dmrs_ports); - //---------------------------------------------------------- - //--------------------- Channel estimation --------------------- - //---------------------------------------------------------- - start_meas(&gNB->ulsch_channel_estimation_stats); - int max_ch = 0; - uint32_t nvar = 0; - for(uint8_t symbol = rel15_ul->start_symbol_index; symbol < (rel15_ul->start_symbol_index + rel15_ul->nr_of_symbols); symbol++) { - uint8_t dmrs_symbol_flag = (rel15_ul->ul_dmrs_symb_pos >> symbol) & 0x01; - LOG_D(PHY, "symbol %d, dmrs_symbol_flag :%d\n", symbol, dmrs_symbol_flag); - - if (dmrs_symbol_flag == 1) { - if (pusch_vars->dmrs_symbol == INVALID_VALUE) - pusch_vars->dmrs_symbol = symbol; - - for (int nl=0; nlnrOfLayers; nl++) { - uint32_t nvar_tmp = 0; - nr_pusch_channel_estimation(gNB, - slot, - get_dmrs_port(nl,rel15_ul->dmrs_ports), - symbol, - ulsch_id, - bwp_start_subcarrier, - rel15_ul, - &max_ch, - &nvar_tmp); - nvar += nvar_tmp; - } - - nr_gnb_measurements(gNB, ulsch, pusch_vars, symbol, rel15_ul->nrOfLayers); - allocCast2D(n0_subband_power, - unsigned int, - gNB->measurements.n0_subband_power, - frame_parms->nb_antennas_rx, - frame_parms->N_RB_UL, - false); - for (aarx = 0; aarx < frame_parms->nb_antennas_rx; aarx++) { - if (symbol == rel15_ul->start_symbol_index) { - pusch_vars->ulsch_power[aarx] = 0; - pusch_vars->ulsch_noise_power[aarx] = 0; - } - for (aatx = 0; aatx < rel15_ul->nrOfLayers; aatx++) { - pusch_vars->ulsch_power[aarx] += signal_energy_nodc( - &pusch_vars->ul_ch_estimates[aatx * gNB->frame_parms.nb_antennas_rx + aarx][symbol * frame_parms->ofdm_symbol_size], - rel15_ul->rb_size * 12); - } - for (int rb = 0; rb < rel15_ul->rb_size; rb++) { - pusch_vars->ulsch_noise_power[aarx] += - n0_subband_power[aarx][rel15_ul->bwp_start + rel15_ul->rb_start + rb] / rel15_ul->rb_size; - } - LOG_D(PHY, - "aa %d, bwp_start%d, rb_start %d, rb_size %d: ulsch_power %d, ulsch_noise_power %d\n", - aarx, - rel15_ul->bwp_start, - rel15_ul->rb_start, - rel15_ul->rb_size, - pusch_vars->ulsch_power[aarx], - pusch_vars->ulsch_noise_power[aarx]); - } - } - } - - nvar /= (rel15_ul->nr_of_symbols * rel15_ul->nrOfLayers * frame_parms->nb_antennas_rx); - - if (gNB->chest_time == 1) { // averaging time domain channel estimates - nr_chest_time_domain_avg(frame_parms, - pusch_vars->ul_ch_estimates, - rel15_ul->nr_of_symbols, - rel15_ul->start_symbol_index, - rel15_ul->ul_dmrs_symb_pos, - rel15_ul->rb_size); - - pusch_vars->dmrs_symbol = - get_next_dmrs_symbol_in_slot(rel15_ul->ul_dmrs_symb_pos, rel15_ul->start_symbol_index, rel15_ul->nr_of_symbols); - } - stop_meas(&gNB->ulsch_channel_estimation_stats); - - int off = ((rel15_ul->rb_size&1) == 1)? 4:0; - uint32_t rxdataF_ext_offset = 0; - uint8_t shift_ch_ext = rel15_ul->nrOfLayers > 1 ? log2_approx(max_ch >> 11) : 0; - - // Flag to select the receiver: (true) Nonlinear ML receiver, (false) Linear MMSE receiver - // By default, we are using the Nonlinear ML receiver, except - // - for 256QAM as Nonlinear ML receiver is not implemented for 256QAM - // - for 64QAM as Nonlinear ML receiver requires more processing time than MMSE, and many machines are not powerful enough - bool ml_rx = true; - if (rel15_ul->nrOfLayers != 2 || rel15_ul->qam_mod_order >= 6) { - ml_rx = false; - } - - int ad_shift = 0; - if (rel15_ul->nrOfLayers == 1) { - ad_shift = 1 + log2_approx(frame_parms->nb_antennas_rx >> 2); - } else if (ml_rx == false) { - ad_shift = -3; // For 2-layers, we are already doing a bit shift in the nr_ulsch_mmse_2layers() function, so we can use more bits - } - - int num_re_total = 0; - for(uint8_t symbol = rel15_ul->start_symbol_index; symbol < (rel15_ul->start_symbol_index + rel15_ul->nr_of_symbols); symbol++) { - uint8_t dmrs_symbol_flag = (rel15_ul->ul_dmrs_symb_pos >> symbol) & 0x01; - if (dmrs_symbol_flag == 1) { + simde__m64 *s = (simde__m64 *)rdata->s; + for (int symbol = rdata->startSymbol; symbol < rdata->startSymbol+rdata->numSymbols; symbol++) + { + int dmrs_symbol_flag = (rel15_ul->ul_dmrs_symb_pos >> symbol) & 0x01; + int nb_re_pusch = gNB->pusch_vars[ulsch_id].ul_valid_re_per_slot[symbol]; + // this needs to be reworded for parrellization, we need a table which give dmrs symbol location + // used for chennel estimate, they are being run in parallel! + if (dmrs_symbol_flag == 1) + { if ((rel15_ul->ul_dmrs_symb_pos >> ((symbol + 1) % frame_parms->symbols_per_slot)) & 0x01) AssertFatal(1==0,"Double DMRS configuration is not yet supported\n"); - - if (gNB->chest_time == 0) // Non averaging time domain channel estimates - pusch_vars->dmrs_symbol = symbol; - - if (rel15_ul->dmrs_config_type == 0) { - // if no data in dmrs cdm group is 1 only even REs have no data - // if no data in dmrs cdm group is 2 both odd and even REs have no data - nb_re_pusch = rel15_ul->rb_size *(12 - (rel15_ul->num_dmrs_cdm_grps_no_data*6)); - } - else { - nb_re_pusch = rel15_ul->rb_size *(12 - (rel15_ul->num_dmrs_cdm_grps_no_data*4)); - } - } - else { - nb_re_pusch = rel15_ul->rb_size * NR_NB_SC_PER_RB; + gNB->pusch_vars[ulsch_id].dmrs_symbol = symbol; } - num_re_total += nb_re_pusch; - pusch_vars->ul_valid_re_per_slot[symbol] = nb_re_pusch; - LOG_D(PHY, "symbol %d: nb_re_pusch %d, DMRS symbl used for Chest :%d \n", symbol, nb_re_pusch, pusch_vars->dmrs_symbol); + LOG_I(PHY,"symbol %d: nb_re_pusch %d, DMRS symbl used for Chest :%d \n", symbol, nb_re_pusch, gNB->pusch_vars[ulsch_id].dmrs_symbol); - //---------------------------------------------------------- - //--------------------- RBs extraction --------------------- - //---------------------------------------------------------- - if (nb_re_pusch > 0) { - start_meas(&gNB->ulsch_rbs_extraction_stats); - nr_ulsch_extract_rbs(gNB->common_vars.rxdataF, pusch_vars, slot, symbol, dmrs_symbol_flag, rel15_ul, frame_parms); - stop_meas(&gNB->ulsch_rbs_extraction_stats); + if (nb_re_pusch == 0) continue; + if (rel15_ul->nrOfLayers == 1) + { + void (*inner_rx)(NR_DL_FRAME_PARMS *, + NR_gNB_PUSCH *, + nfapi_nr_pusch_pdu_t *, + c16_t **, + c16_t **, + int16_t *, + int32_t, + int32_t, + int32_t, + int32_t, + int32_t, + int16_t, + int32_t, + int32_t); + + if (rel15_ul->qam_mod_order == 2) inner_rx = inner_rx_qpsk; + else if (rel15_ul->qam_mod_order == 4) inner_rx = inner_rx_16qam; + else if (rel15_ul->qam_mod_order == 6) inner_rx = inner_rx_64qam; + else if (rel15_ul->qam_mod_order == 8) inner_rx = inner_rx_256qam; + else AssertFatal(1==0,"rel15_ul->qam_mod_order %d, pusch_pdu->dmrs_config_type %d\n", + rel15_ul->qam_mod_order,rel15_ul->dmrs_config_type); - //---------------------------------------------------------- - //--------------------- Channel Scaling -------------------- - //---------------------------------------------------------- - nr_ulsch_scale_channel(pusch_vars->ul_ch_estimates_ext, - frame_parms, - ulsch, - symbol, - dmrs_symbol_flag, - nb_re_pusch, - rel15_ul->nrOfLayers, - rel15_ul->rb_size, - shift_ch_ext); + int soffset = (slot&3)*frame_parms->symbols_per_slot*frame_parms->ofdm_symbol_size; + int16_t llr_tmp[(nb_re_pusch + 8) * rel15_ul->qam_mod_order] __attribute__((aligned(32))); + memset(llr_tmp, 0, sizeof(int16_t) * (nb_re_pusch + 8) * rel15_ul->qam_mod_order); + inner_rx(frame_parms, + pusch_vars, + rel15_ul, + gNB->common_vars.rxdataF, + (c16_t**)gNB->pusch_vars[ulsch_id].ul_ch_estimates, + llr_tmp, + rel15_ul->nrOfLayers, + frame_parms->nb_antennas_rx, + soffset, + nb_re_pusch, // length + symbol, // symbol index + rel15_ul->rb_size, // ofdm size + dmrs_symbol_flag, + gNB->pusch_vars[ulsch_id].log2_maxh); + + // unscrambling + simde__m64 *llr64 = (simde__m64 *) &rdata->llr[pusch_vars->llr_offset[symbol]]; + for (int i = 0; i < nb_re_pusch*rel15_ul->qam_mod_order >> 2; i++) + llr64[i] = simde_mm_mullo_pi16(((simde__m64 *)llr_tmp)[i], s[i]); + + } + else // MIMO for 2x2 + { + int soffset = (slot&3)*frame_parms->symbols_per_slot*frame_parms->ofdm_symbol_size; + void (*inner_rx)(NR_DL_FRAME_PARMS *, + NR_gNB_PUSCH *, + nfapi_nr_pusch_pdu_t *, + c16_t **, + c16_t **, + int16_t **, + int32_t, + int32_t, + int32_t, + int32_t, + int32_t, + int16_t, + int32_t, + int32_t, + uint32_t); + if (rel15_ul->qam_mod_order == 2) inner_rx = inner_rx_qpsk_2layer; + else if (rel15_ul->qam_mod_order == 4) inner_rx = inner_rx_16qam_2layer; + else if (rel15_ul->qam_mod_order == 6) inner_rx = inner_rx_64qam_2layer; + else if (rel15_ul->qam_mod_order == 8) inner_rx = inner_rx_256qam_2layer; + else AssertFatal(1==0,"rel15_ul->qam_mod_order %d, pusch_pdu->dmrs_config_type %d\n", + rel15_ul->qam_mod_order,rel15_ul->dmrs_config_type); + + inner_rx(frame_parms, + pusch_vars, + rel15_ul, + gNB->common_vars.rxdataF, + (c16_t**)gNB->pusch_vars[ulsch_id].ul_ch_estimates, + rdata->llr_layers, + rel15_ul->nrOfLayers, + frame_parms->nb_antennas_rx, + soffset, + nb_re_pusch, // length + symbol, // symbol index + rel15_ul->rb_size, // ofdm size + dmrs_symbol_flag, + gNB->pusch_vars[ulsch_id].log2_maxh, + rdata->nvar); + + // layer de-mapping + int16_t* llr_cw = &rdata->llr[pusch_vars->llr_offset[symbol] * rel15_ul->nrOfLayers]; + for (int i = 0; i < (nb_re_pusch); i++) + for (int l = 0; l < rel15_ul->nrOfLayers; l++) + for (int m = 0; m < rel15_ul->qam_mod_order; m++) + llr_cw[i*rel15_ul->nrOfLayers*rel15_ul->qam_mod_order+l*rel15_ul->qam_mod_order+m] = rdata->llr_layers[l][pusch_vars->llr_offset[symbol] + i*rel15_ul->qam_mod_order+m]; + + // unscrambling + simde__m64 *llr64 = (simde__m64 *) &rdata->llr[pusch_vars->llr_offset[symbol] * rel15_ul->nrOfLayers]; + for (int i = 0; i < (nb_re_pusch*rel15_ul->qam_mod_order * rel15_ul->nrOfLayers) >> 2; i++) + llr64[i] = simde_mm_mullo_pi16(llr64[i], s[i]); - if (pusch_vars->cl_done == 0) { - nr_ulsch_channel_level(pusch_vars->ul_ch_estimates_ext, - frame_parms, - avg, - symbol, - nb_re_pusch, - rel15_ul->nrOfLayers, - rel15_ul->rb_size); - - avgs = 0; - - for (aatx=0;aatxnrOfLayers;aatx++) - for (aarx=0;aarxnb_antennas_rx;aarx++) - avgs = cmax(avgs,avg[aatx*frame_parms->nb_antennas_rx+aarx]); - - pusch_vars->log2_maxh = (log2_approx(avgs) >> 1) + ad_shift; - if (pusch_vars->log2_maxh < 0) { - pusch_vars->log2_maxh = 0; - } - pusch_vars->cl_done = 1; - } - - //---------------------------------------------------------- - //--------------------- Channel Compensation --------------- - //---------------------------------------------------------- - start_meas(&gNB->ulsch_channel_compensation_stats); - LOG_D(PHY, "Doing channel compensations log2_maxh %d, avgs %d (%d,%d)\n" ,pusch_vars->log2_maxh, avgs,avg[0], avg[1]); - nr_ulsch_channel_compensation(pusch_vars->rxdataF_ext, - pusch_vars->ul_ch_estimates_ext, - pusch_vars->ul_ch_mag0, - pusch_vars->ul_ch_magb0, - pusch_vars->ul_ch_magc0, - pusch_vars->rxdataF_comp, - (rel15_ul->nrOfLayers > 1) ? pusch_vars->rho : NULL, - frame_parms, - symbol, - nb_re_pusch, - dmrs_symbol_flag, - rel15_ul->qam_mod_order, - rel15_ul->nrOfLayers, - rel15_ul->rb_size, - pusch_vars->log2_maxh); - stop_meas(&gNB->ulsch_channel_compensation_stats); - - start_meas(&gNB->ulsch_mrc_stats); - nr_ulsch_detection_mrc(frame_parms, - pusch_vars->rxdataF_comp, - pusch_vars->ul_ch_mag0, - pusch_vars->ul_ch_magb0, - pusch_vars->ul_ch_magc0, - (rel15_ul->nrOfLayers > 1) ? pusch_vars->rho : NULL, - rel15_ul->nrOfLayers, - symbol, - rel15_ul->rb_size, - nb_re_pusch); - - // Apply MMSE for 2 Tx layers - if (ml_rx == false && rel15_ul->nrOfLayers == 2) { - nr_ulsch_mmse_2layers(frame_parms, - pusch_vars->rxdataF_comp, - pusch_vars->ul_ch_mag0, - pusch_vars->ul_ch_magb0, - pusch_vars->ul_ch_magc0, - pusch_vars->ul_ch_estimates_ext, - rel15_ul->rb_size, - frame_parms->nb_antennas_rx, - rel15_ul->qam_mod_order, - pusch_vars->log2_maxh, - symbol, - nb_re_pusch, - nvar); - } - - stop_meas(&gNB->ulsch_mrc_stats); - - if (rel15_ul->transform_precoding == transformPrecoder_enabled) { - // For odd number of resource blocks need byte alignment to multiple of 8 - int nb_re_pusch2 = nb_re_pusch + (nb_re_pusch&7); - - // perform IDFT operation on the compensated rxdata if transform precoding is enabled - nr_idft(&pusch_vars->rxdataF_comp[0][symbol * nb_re_pusch2], nb_re_pusch); - LOG_D(PHY,"Transform precoding being done on data- symbol: %d, nb_re_pusch: %d\n", symbol, nb_re_pusch); - } - - //---------------------------------------------------------- - //--------------------- PTRS Processing -------------------- - //---------------------------------------------------------- - /* In case PTRS is enabled then LLR will be calculated after PTRS symbols are processed * - * otherwise LLR are calculated for each symbol based upon DMRS channel estimates only. */ - if (rel15_ul->pdu_bit_map & PUSCH_PDU_BITMAP_PUSCH_PTRS) { - start_meas(&gNB->ulsch_ptrs_processing_stats); - nr_pusch_ptrs_processing(gNB, - frame_parms, - rel15_ul, - ulsch_id, - slot, - symbol, - nb_re_pusch); - stop_meas(&gNB->ulsch_ptrs_processing_stats); - - /* Subtract total PTRS RE's in the symbol from PUSCH RE's */ - pusch_vars->ul_valid_re_per_slot[symbol] -= pusch_vars->ptrs_re_per_slot; - } - - /*---------------------------------------------------------------------------------------------------- */ - /*-------------------- LLRs computation -------------------------------------------------------------*/ - /*-----------------------------------------------------------------------------------------------------*/ - start_meas(&gNB->ulsch_llr_stats); - if (ml_rx == false || rel15_ul->nrOfLayers == 1) { - for (aatx=0; aatx < rel15_ul->nrOfLayers; aatx++) { - nr_ulsch_compute_llr(&pusch_vars->rxdataF_comp[aatx * frame_parms->nb_antennas_rx][symbol * (off + rel15_ul->rb_size * NR_NB_SC_PER_RB)], - pusch_vars->ul_ch_mag0[aatx * frame_parms->nb_antennas_rx], - pusch_vars->ul_ch_magb0[aatx * frame_parms->nb_antennas_rx], - pusch_vars->ul_ch_magc0[aatx * frame_parms->nb_antennas_rx], - &pusch_vars->llr_layers[aatx][rxdataF_ext_offset * rel15_ul->qam_mod_order], - rel15_ul->rb_size, - pusch_vars->ul_valid_re_per_slot[symbol], - symbol, - rel15_ul->qam_mod_order); - } - } else { - nr_ulsch_compute_ML_llr(pusch_vars->rxdataF_comp, - pusch_vars->ul_ch_mag0, - pusch_vars->rho, - pusch_vars->llr_layers, - frame_parms->nb_antennas_rx, - rel15_ul->rb_size, - nb_re_pusch, - symbol, - rxdataF_ext_offset, - rel15_ul->qam_mod_order); - - if (rel15_ul->qam_mod_order == 2) { - nr_ulsch_shift_llr(pusch_vars->llr_layers, nb_re_pusch, rxdataF_ext_offset, rel15_ul->qam_mod_order, 4); - } - -#ifdef ML_DEBUG - c16_t *llr_layers0 = (c16_t *)&pusch_vars->llr_layers[0][rxdataF_ext_offset * rel15_ul->qam_mod_order]; - c16_t *llr_layers1 = (c16_t *)&pusch_vars->llr_layers[1][rxdataF_ext_offset * rel15_ul->qam_mod_order]; - printf("===============================\n"); - printf("AFTER nr_ulsch_compute_ML_llr()\n"); - printf("===============================\n"); - for (int k = 0; k < nb_re_pusch; k++) { - printf("[%3i] llr_layers0 = (%6i, %6i), llr_layers1 = (%6i, %6i)\n", - k, llr_layers0[k].r, llr_layers0[k].i, llr_layers1[k].r, llr_layers1[k].i); - } - printf("\n"); -#endif - } - stop_meas(&gNB->ulsch_llr_stats); - rxdataF_ext_offset += pusch_vars->ul_valid_re_per_slot[symbol]; } - } // symbol loop - if (!(frame % 128)) { - int num_llr = num_re_total*rel15_ul->qam_mod_order; - GnbScopeUpdate(gNB, puschLLRe, num_llr); - GnbScopeUpdate(gNB, puschIQe, num_re_total); } } + int nr_rx_pusch_tp(PHY_VARS_gNB *gNB, uint8_t ulsch_id, uint32_t frame, @@ -3605,16 +2697,17 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB, //---------------------------------------------------------- start_meas(&gNB->ulsch_channel_estimation_stats); int max_ch = 0; - for (uint8_t symbol = rel15_ul->start_symbol_index; symbol < (rel15_ul->start_symbol_index + rel15_ul->nr_of_symbols); symbol++) - { + uint32_t nvar = 0; + for(uint8_t symbol = rel15_ul->start_symbol_index; symbol < (rel15_ul->start_symbol_index + rel15_ul->nr_of_symbols); symbol++) { uint8_t dmrs_symbol_flag = (rel15_ul->ul_dmrs_symb_pos >> symbol) & 0x01; LOG_D(PHY, "symbol %d, dmrs_symbol_flag :%d\n", symbol, dmrs_symbol_flag); - if (dmrs_symbol_flag == 1) - { + + if (dmrs_symbol_flag == 1) { if (pusch_vars->dmrs_symbol == INVALID_VALUE) pusch_vars->dmrs_symbol = symbol; - for (int nl=0; nlnrOfLayers; nl++) - { + + for (int nl=0; nlnrOfLayers; nl++) { + uint32_t nvar_tmp = 0; nr_pusch_channel_estimation(gNB, slot, get_dmrs_port(nl,rel15_ul->dmrs_ports), @@ -3623,7 +2716,8 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB, bwp_start_subcarrier, rel15_ul, &max_ch, - 0 /* nvar*/); + &nvar_tmp); + nvar += nvar_tmp; } // measure the SNR from the channel estimation nr_gnb_measurements(gNB, @@ -3656,6 +2750,8 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB, } } + nvar /= (rel15_ul->nr_of_symbols * rel15_ul->nrOfLayers * frame_parms->nb_antennas_rx); + // averaging time domain channel estimates if (gNB->chest_time == 1) { @@ -3778,7 +2874,7 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB, { union puschSymbolReqUnion id = {.s={ulsch_id,frame,slot,0}}; id.p=1+symbol; - notifiedFIFO_elt_t *req = newNotifiedFIFO_elt(sizeof(puschSymbolProc_t), id.p, gNB->respPuschSymb, &nr_pusch_symbol_processing_noprecoding); // create a job for Tpool + notifiedFIFO_elt_t *req = newNotifiedFIFO_elt(sizeof(puschSymbolProc_t), id.p, &gNB->respPuschSymb, &nr_pusch_symbol_processing_noprecoding); // create a job for Tpool puschSymbolProc_t *rdata = (puschSymbolProc_t*)NotifiedFifoData(req); // data for the job rdata->gNB = gNB; @@ -3791,6 +2887,7 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB, rdata->llr = pusch_vars->llr; rdata->llr_layers = pusch_vars->llr_layers; rdata->s = &s[pusch_vars->llr_offset[symbol]*rel15_ul->nrOfLayers]; + rdata->nvar = nvar; pushTpool(&gNB->threadPool, req); gNB->nbSymb++; @@ -3801,7 +2898,7 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB, while (gNB->nbSymb > 0) { - notifiedFIFO_elt_t *req=pullTpool(gNB->respPuschSymb, &gNB->threadPool); + notifiedFIFO_elt_t *req=pullTpool(&gNB->respPuschSymb, &gNB->threadPool); gNB->nbSymb--; delNotifiedFIFO_elt(req); } diff --git a/openair1/PHY/NR_TRANSPORT/nr_ulsch_llr_computation.c b/openair1/PHY/NR_TRANSPORT/nr_ulsch_llr_computation.c index f4d526960e..ddbec269b3 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_ulsch_llr_computation.c +++ b/openair1/PHY/NR_TRANSPORT/nr_ulsch_llr_computation.c @@ -70,68 +70,28 @@ void nr_ulsch_16qam_llr(int32_t *rxdataF_comp, uint32_t nb_re, uint8_t symbol) { + simde__m256i *rxF_256 = (simde__m256i*)rxdataF_comp; + simde__m256i *ch_mag = (simde__m256i*)ul_ch_mag; + simde__m64 *llr_64 = (simde__m64*)ulsch_llr; - simde__m256i *rxF = (simde__m256i*)rxdataF_comp; - simde__m256i *ch_mag; - simde__m256i llr256[2]; - register simde__m256i xmm0; - uint32_t *llr32; - int i; + simde__m256i xmm0, xmm1, xmm2; - int off = ((nb_rb&1) == 1)? 4:0; - - llr32 = (uint32_t*)ulsch_llr; - - ch_mag = (simde__m256i*)&ul_ch_mag[(symbol*(off+(nb_rb*12)))]; - unsigned char len_mod8 = nb_re&7; - nb_re >>= 3; // length in quad words (4 REs) - nb_re += (len_mod8 == 0 ? 0 : 1); - - for (i=0; i |y_R|, registers of odd index in xmm0-> |y_I| - xmm0 = simde_mm256_subs_epi16(ch_mag[i],xmm0); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 + for (int i = 0; i < ((nb_re>>3) + ((nb_re&7) ? 1 : 0)); i++) { + xmm0 = simde_mm256_abs_epi16(rxF_256[i]); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| + xmm0 = simde_mm256_subs_epi16(ch_mag[i], xmm0); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 - llr256[0] = simde_mm256_unpacklo_epi32(rxF[i],xmm0); // llr128[0] contains the llrs of the 1st,2nd,5th and 6th REs - llr256[1] = simde_mm256_unpackhi_epi32(rxF[i],xmm0); // llr128[1] contains the llrs of the 3rd, 4th, 7th and 8th REs + xmm1 = simde_mm256_unpacklo_epi32(rxF_256[i], xmm0); // llr128[0] contains the llrs of the 1st,2nd,5th and 6th REs + xmm2 = simde_mm256_unpackhi_epi32(rxF_256[i], xmm0); // llr128[1] contains the llrs of the 3rd, 4th, 7th and 8th REs - // 1st RE - llr32[0] = simde_mm256_extract_epi32(llr256[0],0); // llr32[0] low 16 bits-> y_R , high 16 bits-> y_I - llr32[1] = simde_mm256_extract_epi32(llr256[0],1); // llr32[1] low 16 bits-> |h|-|y_R|^2, high 16 bits-> |h|-|y_I|^2 - - // 2nd RE - llr32[2] = simde_mm256_extract_epi32(llr256[0],2); // llr32[2] low 16 bits-> y_R , high 16 bits-> y_I - llr32[3] = simde_mm256_extract_epi32(llr256[0],3); // llr32[3] low 16 bits-> |h|-|y_R|^2, high 16 bits-> |h|-|y_I|^2 - - // 3rd RE - llr32[4] = simde_mm256_extract_epi32(llr256[1],0); // llr32[4] low 16 bits-> y_R , high 16 bits-> y_I - llr32[5] = simde_mm256_extract_epi32(llr256[1],1); // llr32[5] low 16 bits-> |h|-|y_R|^2, high 16 bits-> |h|-|y_I|^2 - - // 4th RE - llr32[6] = simde_mm256_extract_epi32(llr256[1],2); // llr32[6] low 16 bits-> y_R , high 16 bits-> y_I - llr32[7] = simde_mm256_extract_epi32(llr256[1],3); // llr32[7] low 16 bits-> |h|-|y_R|^2, high 16 bits-> |h|-|y_I|^2 - - // 5th RE - llr32[8] = simde_mm256_extract_epi32(llr256[0],4); // llr32[8] low 16 bits-> y_R , high 16 bits-> y_I - llr32[9] = simde_mm256_extract_epi32(llr256[0],5); // llr32[9] low 16 bits-> |h|-|y_R|^2, high 16 bits-> |h|-|y_I|^2 - - // 6th RE - llr32[10] = simde_mm256_extract_epi32(llr256[0],6); // llr32[10] low 16 bits-> y_R , high 16 bits-> y_I - llr32[11] = simde_mm256_extract_epi32(llr256[0],7); // llr32[11] low 16 bits-> |h|-|y_R|^2, high 16 bits-> |h|-|y_I|^2 - - // 7th RE - llr32[12] = simde_mm256_extract_epi32(llr256[1],4); // llr32[12] low 16 bits-> y_R , high 16 bits-> y_I - llr32[13] = simde_mm256_extract_epi32(llr256[1],5); // llr32[13] low 16 bits-> |h|-|y_R|^2, high 16 bits-> |h|-|y_I|^2 - - // 8th RE - llr32[14] = simde_mm256_extract_epi32(llr256[1],6); // llr32[14] low 16 bits-> y_R , high 16 bits-> y_I - llr32[15] = simde_mm256_extract_epi32(llr256[1],7); // llr32[15] low 16 bits-> |h|-|y_R|^2, high 16 bits-> |h|-|y_I|^2 - - llr32+=16; - + *llr_64++ = (simde__m64)simde_mm256_extract_epi64(xmm1, 0); + *llr_64++ = (simde__m64)simde_mm256_extract_epi64(xmm1, 1); + *llr_64++ = (simde__m64)simde_mm256_extract_epi64(xmm2, 0); + *llr_64++ = (simde__m64)simde_mm256_extract_epi64(xmm2, 1); + *llr_64++ = (simde__m64)simde_mm256_extract_epi64(xmm1, 2); + *llr_64++ = (simde__m64)simde_mm256_extract_epi64(xmm1, 3); + *llr_64++ = (simde__m64)simde_mm256_extract_epi64(xmm2, 2); + *llr_64++ = (simde__m64)simde_mm256_extract_epi64(xmm2, 3); } - - simde_mm_empty(); - simde_m_empty(); } //---------------------------------------------------------------------------------------------- @@ -146,112 +106,53 @@ void nr_ulsch_64qam_llr(int32_t *rxdataF_comp, uint32_t nb_re, uint8_t symbol) { - int off = ((nb_rb&1) == 1)? 4:0; - simde__m256i *rxF = (simde__m256i*)rxdataF_comp; - simde__m256i *ch_mag,*ch_magb; - register simde__m256i xmm0,xmm1,xmm2; + simde__m256i xmm0, xmm1, xmm2; - int i; + simde__m256i *ch_maga = (simde__m256i*)ul_ch_mag; + simde__m256i *ch_magb = (simde__m256i*)ul_ch_magb; - ch_mag = (simde__m256i*)&ul_ch_mag[(symbol*(off+(nb_rb*12)))]; - ch_magb = (simde__m256i*)&ul_ch_magb[(symbol*(off+(nb_rb*12)))]; + int32_t *llr_32 = (int32_t *)ulsch_llr; - int len_mod8 = nb_re&7; - nb_re = nb_re>>3; // length in quad words (4 REs) - nb_re += ((len_mod8 == 0) ? 0 : 1); - - for (i=0; i>3) + ((nb_re&7) ? 1 : 0)); i++) { xmm0 = rxF[i]; xmm1 = simde_mm256_abs_epi16(xmm0); - xmm1 = simde_mm256_subs_epi16(ch_mag[i],xmm1); + xmm1 = simde_mm256_subs_epi16(ch_maga[i], xmm1); xmm2 = simde_mm256_abs_epi16(xmm1); - xmm2 = simde_mm256_subs_epi16(ch_magb[i],xmm2); + xmm2 = simde_mm256_subs_epi16(ch_magb[i], xmm2); - // --------------------------------------- - // 1st RE - // --------------------------------------- - ulsch_llr[0] = simde_mm256_extract_epi16(xmm0,0); - ulsch_llr[1] = simde_mm256_extract_epi16(xmm0,1); - ulsch_llr[2] = simde_mm256_extract_epi16(xmm1,0); - ulsch_llr[3] = simde_mm256_extract_epi16(xmm1,1); - ulsch_llr[4] = simde_mm256_extract_epi16(xmm2,0); - ulsch_llr[5] = simde_mm256_extract_epi16(xmm2,1); - // --------------------------------------- + *llr_32++ = simde_mm256_extract_epi32(xmm0,0); + *llr_32++ = simde_mm256_extract_epi32(xmm1,0); + *llr_32++ = simde_mm256_extract_epi32(xmm2,0); - ulsch_llr+=6; - - // --------------------------------------- - // 2nd RE - // --------------------------------------- - ulsch_llr[0] = simde_mm256_extract_epi16(xmm0,2); - ulsch_llr[1] = simde_mm256_extract_epi16(xmm0,3); - ulsch_llr[2] = simde_mm256_extract_epi16(xmm1,2); - ulsch_llr[3] = simde_mm256_extract_epi16(xmm1,3); - ulsch_llr[4] = simde_mm256_extract_epi16(xmm2,2); - ulsch_llr[5] = simde_mm256_extract_epi16(xmm2,3); - // --------------------------------------- + *llr_32++ = simde_mm256_extract_epi32(xmm0,1); + *llr_32++ = simde_mm256_extract_epi32(xmm1,1); + *llr_32++ = simde_mm256_extract_epi32(xmm2,1); - ulsch_llr+=6; - - // --------------------------------------- - // 3rd RE - // --------------------------------------- - ulsch_llr[0] = simde_mm256_extract_epi16(xmm0,4); - ulsch_llr[1] = simde_mm256_extract_epi16(xmm0,5); - ulsch_llr[2] = simde_mm256_extract_epi16(xmm1,4); - ulsch_llr[3] = simde_mm256_extract_epi16(xmm1,5); - ulsch_llr[4] = simde_mm256_extract_epi16(xmm2,4); - ulsch_llr[5] = simde_mm256_extract_epi16(xmm2,5); - // --------------------------------------- + *llr_32++ = simde_mm256_extract_epi32(xmm0,2); + *llr_32++ = simde_mm256_extract_epi32(xmm1,2); + *llr_32++ = simde_mm256_extract_epi32(xmm2,2); - ulsch_llr+=6; - - // --------------------------------------- - // 4th RE - // --------------------------------------- - ulsch_llr[0] = simde_mm256_extract_epi16(xmm0,6); - ulsch_llr[1] = simde_mm256_extract_epi16(xmm0,7); - ulsch_llr[2] = simde_mm256_extract_epi16(xmm1,6); - ulsch_llr[3] = simde_mm256_extract_epi16(xmm1,7); - ulsch_llr[4] = simde_mm256_extract_epi16(xmm2,6); - ulsch_llr[5] = simde_mm256_extract_epi16(xmm2,7); - // --------------------------------------- + *llr_32++ = simde_mm256_extract_epi32(xmm0,3); + *llr_32++ = simde_mm256_extract_epi32(xmm1,3); + *llr_32++ = simde_mm256_extract_epi32(xmm2,3); - ulsch_llr+=6; - ulsch_llr[0] = simde_mm256_extract_epi16(xmm0,8); - ulsch_llr[1] = simde_mm256_extract_epi16(xmm0,9); - ulsch_llr[2] = simde_mm256_extract_epi16(xmm1,8); - ulsch_llr[3] = simde_mm256_extract_epi16(xmm1,9); - ulsch_llr[4] = simde_mm256_extract_epi16(xmm2,8); - ulsch_llr[5] = simde_mm256_extract_epi16(xmm2,9); + *llr_32++ = simde_mm256_extract_epi32(xmm0,4); + *llr_32++ = simde_mm256_extract_epi32(xmm1,4); + *llr_32++ = simde_mm256_extract_epi32(xmm2,4); - ulsch_llr[6] = simde_mm256_extract_epi16(xmm0,10); - ulsch_llr[7] = simde_mm256_extract_epi16(xmm0,11); - ulsch_llr[8] = simde_mm256_extract_epi16(xmm1,10); - ulsch_llr[9] = simde_mm256_extract_epi16(xmm1,11); - ulsch_llr[10] = simde_mm256_extract_epi16(xmm2,10); - ulsch_llr[11] = simde_mm256_extract_epi16(xmm2,11); + *llr_32++ = simde_mm256_extract_epi32(xmm0,5); + *llr_32++ = simde_mm256_extract_epi32(xmm1,5); + *llr_32++ = simde_mm256_extract_epi32(xmm2,5); - ulsch_llr[12] = simde_mm256_extract_epi16(xmm0,12); - ulsch_llr[13] = simde_mm256_extract_epi16(xmm0,13); - ulsch_llr[14] = simde_mm256_extract_epi16(xmm1,12); - ulsch_llr[15] = simde_mm256_extract_epi16(xmm1,13); - ulsch_llr[16] = simde_mm256_extract_epi16(xmm2,12); - ulsch_llr[17] = simde_mm256_extract_epi16(xmm2,13); + *llr_32++ = simde_mm256_extract_epi32(xmm0,6); + *llr_32++ = simde_mm256_extract_epi32(xmm1,6); + *llr_32++ = simde_mm256_extract_epi32(xmm2,6); - ulsch_llr[18] = simde_mm256_extract_epi16(xmm0,14); - ulsch_llr[19] = simde_mm256_extract_epi16(xmm0,15); - ulsch_llr[20] = simde_mm256_extract_epi16(xmm1,14); - ulsch_llr[21] = simde_mm256_extract_epi16(xmm1,15); - ulsch_llr[22] = simde_mm256_extract_epi16(xmm2,14); - ulsch_llr[23] = simde_mm256_extract_epi16(xmm2,15); - - ulsch_llr+=24; + *llr_32++ = simde_mm256_extract_epi32(xmm0,7); + *llr_32++ = simde_mm256_extract_epi32(xmm1,7); + *llr_32++ = simde_mm256_extract_epi32(xmm2,7); } - - simde_mm_empty(); - simde_m_empty(); } void nr_ulsch_256qam_llr(int32_t *rxdataF_comp, @@ -263,105 +164,42 @@ void nr_ulsch_256qam_llr(int32_t *rxdataF_comp, uint32_t nb_re, uint8_t symbol) { - int off = ((nb_rb&1) == 1)? 4:0; - simde__m256i *rxF = (simde__m256i*)rxdataF_comp; - simde__m256i *ch_mag,*ch_magb,*ch_magc; - register simde__m256i xmm0,xmm1,xmm2,xmm3,xmm4,xmm5,xmm6; + simde__m256i xmm0,xmm1,xmm2,xmm3,xmm4,xmm5,xmm6; simde__m256i *llr256=(simde__m256i*)ulsch_llr; - ch_mag = (simde__m256i*)&ul_ch_mag[(symbol*(off+(nb_rb*12)))]; - ch_magb = (simde__m256i*)&ul_ch_magb[(symbol*(off+(nb_rb*12)))]; - ch_magc = (simde__m256i*)&ul_ch_magc[(symbol*(off+(nb_rb*12)))]; - int len_mod8 = nb_re&7; - int nb_re256 = nb_re>>3; // length in 256-bit words (8 REs) + simde__m256i* ch_maga = (simde__m256i*)ul_ch_mag; + simde__m256i* ch_magb = (simde__m256i*)ul_ch_magb; + simde__m256i* ch_magc = (simde__m256i*)ul_ch_magc; - for (int i=0; i |y_R|, registers of odd index in xmm0-> |y_I| - xmm0 = simde_mm256_subs_epi16(ch_mag[i],xmm0); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 - // xmmtmpD2 contains 16 LLRs - xmm1 = simde_mm256_abs_epi16(xmm0); - xmm1 = simde_mm256_subs_epi16(ch_magb[i],xmm1); // contains 16 LLRs - xmm2 = simde_mm256_abs_epi16(xmm1); - xmm2 = simde_mm256_subs_epi16(ch_magc[i],xmm2); // contains 16 LLRs - // rxF[i] A0 A1 A2 A3 A4 A5 A6 A7 bits 7,6 - // xmm0 B0 B1 B2 B3 B4 B5 B6 B7 bits 5,4 - // xmm1 C0 C1 C2 C3 C4 C5 C6 C7 bits 3,2 - // xmm2 D0 D1 D2 D3 D4 D5 D6 D7 bits 1,0 - xmm3 = simde_mm256_unpacklo_epi32(rxF[i],xmm0); // A0 B0 A1 B1 A4 B4 A5 B5 - xmm4 = simde_mm256_unpackhi_epi32(rxF[i],xmm0); // A2 B2 A3 B3 A6 B6 A7 B7 - xmm5 = simde_mm256_unpacklo_epi32(xmm1,xmm2); // C0 D0 C1 D1 C4 D4 C5 D5 - xmm6 = simde_mm256_unpackhi_epi32(xmm1,xmm2); // C2 D2 C3 D3 C6 D6 C7 D7 - - xmm0 = simde_mm256_unpacklo_epi64(xmm3,xmm5); // A0 B0 C0 D0 A4 B4 C4 D4 - xmm1 = simde_mm256_unpackhi_epi64(xmm3,xmm5); // A1 B1 C1 D1 A5 B5 C5 D5 - xmm2 = simde_mm256_unpacklo_epi64(xmm4,xmm6); // A2 B2 C2 D2 A6 B6 C6 D6 - xmm3 = simde_mm256_unpackhi_epi64(xmm4,xmm6); // A3 B3 C3 D3 A7 B7 C7 D7 - llr256[0] = simde_mm256_permute2x128_si256(xmm0, xmm1, 0x20); // A0 B0 C0 D0 A1 B1 C1 D1 - llr256[1] = simde_mm256_permute2x128_si256(xmm2, xmm3, 0x20); // A2 B2 C2 D2 A3 B3 C3 D3 - llr256[2] = simde_mm256_permute2x128_si256(xmm0, xmm1, 0x31); // A4 B4 C4 D4 A5 B5 C5 D5 - llr256[3] = simde_mm256_permute2x128_si256(xmm2, xmm3, 0x31); // A6 B6 C6 D6 A7 B7 C7 D7 - llr256+=4; + for (int i = 0; i < ((nb_re>>3) + ((nb_re&7) ? 1 : 0)); i++) { + xmm0 = simde_mm256_abs_epi16(rxF[i]); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| + xmm0 = simde_mm256_subs_epi16(ch_maga[i], xmm0); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 + // xmmtmpD2 contains 16 LLRs + xmm1 = simde_mm256_abs_epi16(xmm0); + xmm1 = simde_mm256_subs_epi16(ch_magb[i], xmm1); // contains 16 LLRs + xmm2 = simde_mm256_abs_epi16(xmm1); + xmm2 = simde_mm256_subs_epi16(ch_magc[i], xmm2); // contains 16 LLRs + // rxF[i] A0 A1 A2 A3 A4 A5 A6 A7 bits 7,6 + // xmm0 B0 B1 B2 B3 B4 B5 B6 B7 bits 5,4 + // xmm1 C0 C1 C2 C3 C4 C5 C6 C7 bits 3,2 + // xmm2 D0 D1 D2 D3 D4 D5 D6 D7 bits 1,0 + xmm3 = simde_mm256_unpacklo_epi32(rxF[i], xmm0); // A0 B0 A1 B1 A4 B4 A5 B5 + xmm4 = simde_mm256_unpackhi_epi32(rxF[i], xmm0); // A2 B2 A3 B3 A6 B6 A7 B7 + xmm5 = simde_mm256_unpacklo_epi32(xmm1, xmm2); // C0 D0 C1 D1 C4 D4 C5 D5 + xmm6 = simde_mm256_unpackhi_epi32(xmm1, xmm2); // C2 D2 C3 D3 C6 D6 C7 D7 + xmm0 = simde_mm256_unpacklo_epi64(xmm3, xmm5); // A0 B0 C0 D0 A4 B4 C4 D4 + xmm1 = simde_mm256_unpackhi_epi64(xmm3, xmm5); // A1 B1 C1 D1 A5 B5 C5 D5 + xmm2 = simde_mm256_unpacklo_epi64(xmm4, xmm6); // A2 B2 C2 D2 A6 B6 C6 D6 + xmm3 = simde_mm256_unpackhi_epi64(xmm4, xmm6); // A3 B3 C3 D3 A7 B7 C7 D7 + *llr256++ = simde_mm256_permute2x128_si256(xmm0, xmm1, 0x20); // A0 B0 C0 D0 A1 B1 C1 D1 + *llr256++ = simde_mm256_permute2x128_si256(xmm2, xmm3, 0x20); // A2 B2 C2 D2 A3 B3 C3 D3 + *llr256++ = simde_mm256_permute2x128_si256(xmm0, xmm1, 0x31); // A4 B4 C4 D4 A5 B5 C5 D5 + *llr256++ = simde_mm256_permute2x128_si256(xmm2, xmm3, 0x31); // A6 B6 C6 D6 A7 B7 C7 D7 } - simde__m128i *llr128 = (simde__m128i*)llr256; - if (len_mod8 >= 4) { - int nb_re128 = nb_re>>2; - simde__m128i xmm0,xmm1,xmm2,xmm3,xmm4,xmm5,xmm6; - simde__m128i *rxF = (simde__m128i*)rxdataF_comp; - simde__m128i *ch_mag = (simde__m128i*)&ul_ch_mag[(symbol*(off+(nb_rb*12)))]; - simde__m128i *ch_magb = (simde__m128i*)&ul_ch_magb[(symbol*(off+(nb_rb*12)))]; - simde__m128i *ch_magc = (simde__m128i*)&ul_ch_magc[(symbol*(off+(nb_rb*12)))]; - - xmm0 = simde_mm_abs_epi16(rxF[nb_re128-1]); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| - xmm0 = simde_mm_subs_epi16(ch_mag[nb_re128-1],xmm0); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 - // xmmtmpD2 contains 8 LLRs - xmm1 = simde_mm_abs_epi16(xmm0); - xmm1 = simde_mm_subs_epi16(ch_magb[nb_re128-1],xmm1); // contains 8 LLRs - xmm2 = simde_mm_abs_epi16(xmm1); - xmm2 = simde_mm_subs_epi16(ch_magc[nb_re128-1],xmm2); // contains 8 LLRs - // rxF[i] A0 A1 A2 A3 - // xmm0 B0 B1 B2 B3 - // xmm1 C0 C1 C2 C3 - // xmm2 D0 D1 D2 D3 - xmm3 = simde_mm_unpacklo_epi32(rxF[nb_re128-1],xmm0); // A0 B0 A1 B1 - xmm4 = simde_mm_unpackhi_epi32(rxF[nb_re128-1],xmm0); // A2 B2 A3 B3 - xmm5 = simde_mm_unpacklo_epi32(xmm1,xmm2); // C0 D0 C1 D1 - xmm6 = simde_mm_unpackhi_epi32(xmm1,xmm2); // C2 D2 C3 D3 - - llr128[0] = simde_mm_unpacklo_epi64(xmm3,xmm5); // A0 B0 C0 D0 - llr128[1] = simde_mm_unpackhi_epi64(xmm3,xmm5); // A1 B1 C1 D1 - llr128[2] = simde_mm_unpacklo_epi64(xmm4,xmm6); // A2 B2 C2 D2 - llr128[3] = simde_mm_unpackhi_epi64(xmm4,xmm6); // A3 B3 C3 D3 - llr128+=4; - } - if (len_mod8 == 6) { - int nb_re64 = nb_re>>1; - simde__m64 *llr64 = (simde__m64 *)llr128; - simde__m64 xmm0,xmm1,xmm2; - simde__m64 *rxF = (simde__m64*)rxdataF_comp; - simde__m64 *ch_mag = (simde__m64*)&ul_ch_mag[(symbol*(off+(nb_rb*12)))]; - simde__m64 *ch_magb = (simde__m64*)&ul_ch_magb[(symbol*(off+(nb_rb*12)))]; - simde__m64 *ch_magc = (simde__m64*)&ul_ch_magc[(symbol*(off+(nb_rb*12)))]; - - xmm0 = simde_mm_abs_pi16(rxF[nb_re64-1]); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| - xmm0 = simde_mm_subs_pi16(ch_mag[nb_re-1],xmm0); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 - // xmmtmpD2 contains 4 LLRs - xmm1 = simde_mm_abs_pi16(xmm0); - xmm1 = simde_mm_subs_pi16(ch_magb[nb_re64-1],xmm1); // contains 4 LLRs - xmm2 = simde_mm_abs_pi16(xmm1); - xmm2 = simde_mm_subs_pi16(ch_magc[nb_re64-1],xmm2); // contains 4 LLRs - // rxF[i] A0 A1 - // xmm0 B0 B1 - // xmm1 C0 C1 - // xmm2 D0 D1 - llr64[0] = simde_m_punpckldq(rxF[nb_re64-1],xmm0); // A0 B0 - llr64[2] = simde_m_punpckhdq(rxF[nb_re64-1],xmm0); // A1 B1 - llr64[1] = simde_m_punpckldq(xmm1,xmm2); // C0 D0 - llr64[3] = simde_m_punpckhdq(xmm1,xmm2); // C1 D1 - } - } + void nr_ulsch_compute_llr(int32_t *rxdataF_comp, int32_t *ul_ch_mag, int32_t *ul_ch_magb, diff --git a/openair1/PHY/defs_gNB.h b/openair1/PHY/defs_gNB.h index f1f931e1f4..88367741d4 100644 --- a/openair1/PHY/defs_gNB.h +++ b/openair1/PHY/defs_gNB.h @@ -760,7 +760,7 @@ typedef struct PHY_VARS_gNB_s { time_stats_t rx_dft_stats; time_stats_t ulsch_freq_offset_estimation_stats; */ - notifiedFIFO_t *respPuschSymb; + notifiedFIFO_t respPuschSymb; notifiedFIFO_t respDecode; notifiedFIFO_t resp_L1; notifiedFIFO_t L1_tx_free; @@ -769,7 +769,6 @@ typedef struct PHY_VARS_gNB_s { notifiedFIFO_t resp_RU_tx; tpool_t threadPool; int nbSymb; - int use_pusch_tp; int num_pusch_symbols_per_thread; pthread_t L1_rx_thread; int L1_rx_thread_core; @@ -792,6 +791,7 @@ typedef struct puschSymbolProc_s { int16_t *llr; int16_t **llr_layers; int16_t *s; + uint32_t nvar; } puschSymbolProc_t; struct puschSymbolReqId { diff --git a/openair1/SCHED_NR/phy_procedures_nr_gNB.c b/openair1/SCHED_NR/phy_procedures_nr_gNB.c index 35afb44fe1..64eae8422c 100644 --- a/openair1/SCHED_NR/phy_procedures_nr_gNB.c +++ b/openair1/SCHED_NR/phy_procedures_nr_gNB.c @@ -398,20 +398,20 @@ static int nr_ulsch_procedures(PHY_VARS_gNB *gNB, int frame_rx, int slot_rx, int pusch_pdu->qam_mod_order, pusch_pdu->nrOfLayers); - if (gNB->use_pusch_tp == 0 ) - { - nr_ulsch_layer_demapping(gNB->pusch_vars[ULSCH_id].llr, - pusch_pdu->nrOfLayers, - pusch_pdu->qam_mod_order, - G, - gNB->pusch_vars[ULSCH_id].llr_layers); - //---------------------------------------------------------- - //------------------- ULSCH unscrambling ------------------- - //---------------------------------------------------------- - start_meas(&gNB->ulsch_unscrambling_stats); - nr_ulsch_unscrambling(gNB->pusch_vars[ULSCH_id].llr, G, pusch_pdu->data_scrambling_id, pusch_pdu->rnti); - stop_meas(&gNB->ulsch_unscrambling_stats); - } + + + // nr_ulsch_layer_demapping(gNB->pusch_vars[ULSCH_id].llr, + // pusch_pdu->nrOfLayers, + // pusch_pdu->qam_mod_order, + // G, + // gNB->pusch_vars[ULSCH_id].llr_layers); + // //---------------------------------------------------------- + // //------------------- ULSCH unscrambling ------------------- + // //---------------------------------------------------------- + // start_meas(&gNB->ulsch_unscrambling_stats); + // nr_ulsch_unscrambling(gNB->pusch_vars[ULSCH_id].llr, G, pusch_pdu->data_scrambling_id, pusch_pdu->rnti); + // stop_meas(&gNB->ulsch_unscrambling_stats); + //---------------------------------------------------------- //--------------------- ULSCH decoding --------------------- @@ -902,8 +902,9 @@ int phy_procedures_gNB_uespec_RX(PHY_VARS_gNB *gNB, int frame_rx, int slot_rx) VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_NR_RX_PUSCH, 1); start_meas(&gNB->rx_pusch_stats); - if (gNB->use_pusch_tp) nr_rx_pusch_tp(gNB, ULSCH_id, frame_rx, slot_rx, ulsch->harq_pid); - else nr_rx_pusch(gNB, ULSCH_id, frame_rx, slot_rx, ulsch->harq_pid); + // if (gNB->use_pusch_tp) nr_rx_pusch_tp(gNB, ULSCH_id, frame_rx, slot_rx, ulsch->harq_pid); + // else nr_rx_pusch(gNB, ULSCH_id, frame_rx, slot_rx, ulsch->harq_pid); + nr_rx_pusch_tp(gNB, ULSCH_id, frame_rx, slot_rx, ulsch->harq_pid); NR_gNB_PUSCH *pusch_vars = &gNB->pusch_vars[ULSCH_id]; pusch_vars->ulsch_power_tot = 0; pusch_vars->ulsch_noise_power_tot = 0; diff --git a/openair1/SIMULATION/NR_PHY/ulsim.c b/openair1/SIMULATION/NR_PHY/ulsim.c index 879d6deda1..5c3abe09be 100644 --- a/openair1/SIMULATION/NR_PHY/ulsim.c +++ b/openair1/SIMULATION/NR_PHY/ulsim.c @@ -222,7 +222,6 @@ int main(int argc, char *argv[]) int ibwp_rboffset=41; int params_from_file = 0; int threadCnt=0; - int use_tpool = 0; int max_ldpc_iterations = 5; if ( load_configmodule(argc,argv,CONFIG_ENABLECMDLINEONLY) == 0 ) { exit_fun("[NR_ULSIM] Error, configuration module init failed\n"); @@ -364,7 +363,6 @@ int main(int argc, char *argv[]) case 'C': threadCnt = atoi(optarg); - use_tpool = 1; break; case 'u': @@ -563,14 +561,12 @@ int main(int argc, char *argv[]) RC.gNB[0] = calloc(1,sizeof(PHY_VARS_gNB)); gNB = RC.gNB[0]; gNB->ofdm_offset_divisor = UINT_MAX; - initNotifiedFIFO(&gNB->respDecode); - gNB->use_pusch_tp = use_tpool; gNB->num_pusch_symbols_per_thread = 1; initFloatingCoresTpool(threadCnt, &gNB->threadPool, false, "gNB-tpool"); initNotifiedFIFO(&gNB->respDecode); - gNB->respPuschSymb = (notifiedFIFO_t*) malloc(sizeof(notifiedFIFO_t)); - initNotifiedFIFO(gNB->respPuschSymb); + + initNotifiedFIFO(&gNB->respPuschSymb); initNotifiedFIFO(&gNB->L1_tx_free); initNotifiedFIFO(&gNB->L1_tx_filled); initNotifiedFIFO(&gNB->L1_tx_out); @@ -1602,20 +1598,16 @@ int main(int argc, char *argv[]) printDistribution(&gNB->phy_proc_rx,table_rx,"Total PHY proc rx"); printStatIndent(&gNB->rx_pusch_stats,"RX PUSCH time"); printStatIndent2(&gNB->ulsch_channel_estimation_stats,"ULSCH channel estimation time"); - if (use_tpool == 1) - { - printStatIndent2(&gNB->rx_pusch_init_stats,"RX PUSCH Initialization time"); - printStatIndent2(&gNB->rx_pusch_symbol_processing_stats,"RX PUSCH Symbol Processing time"); - } - else - { - printStatIndent2(&gNB->ulsch_ptrs_processing_stats,"ULSCH PTRS Processing time"); - printStatIndent2(&gNB->ulsch_rbs_extraction_stats,"ULSCH rbs extraction time"); - printStatIndent2(&gNB->ulsch_channel_compensation_stats,"ULSCH channel compensation time"); - printStatIndent2(&gNB->ulsch_mrc_stats,"ULSCH mrc computation"); - printStatIndent2(&gNB->ulsch_llr_stats,"ULSCH llr computation"); - printStatIndent(&gNB->ulsch_unscrambling_stats,"ULSCH unscrambling"); - } + printStatIndent2(&gNB->rx_pusch_init_stats,"RX PUSCH Initialization time"); + printStatIndent2(&gNB->rx_pusch_symbol_processing_stats,"RX PUSCH Symbol Processing time"); + + // printStatIndent2(&gNB->ulsch_ptrs_processing_stats,"ULSCH PTRS Processing time"); + // printStatIndent2(&gNB->ulsch_rbs_extraction_stats,"ULSCH rbs extraction time"); + // printStatIndent2(&gNB->ulsch_channel_compensation_stats,"ULSCH channel compensation time"); + // printStatIndent2(&gNB->ulsch_mrc_stats,"ULSCH mrc computation"); + // printStatIndent2(&gNB->ulsch_llr_stats,"ULSCH llr computation"); + // printStatIndent(&gNB->ulsch_unscrambling_stats,"ULSCH unscrambling"); + printStatIndent(&gNB->ulsch_decoding_stats,"ULSCH total decoding time"); // printStatIndent2(&gNB->ulsch_deinterleaving_stats,"ULSCH deinterleaving"); // printStatIndent2(&gNB->ulsch_rate_unmatching_stats,"ULSCH rate matching rx");