From 1a42fa5326599d9c5e090558c17cb1c2f731c7b9 Mon Sep 17 00:00:00 2001 From: Tsung-Yu Chan Date: Thu, 20 Jul 2023 12:04:11 +0200 Subject: [PATCH 01/13] feat / parallelize LLR by symbol --- openair1/PHY/NR_TRANSPORT/nr_scrambling.c | 19 + .../NR_TRANSPORT/nr_transport_common_proto.h | 1 + .../PHY/NR_TRANSPORT/nr_transport_proto.h | 24 + .../PHY/NR_TRANSPORT/nr_ulsch_demodulation.c | 1890 +++++++++++++++++ openair1/PHY/defs_gNB.h | 35 + openair1/SCHED_NR/phy_procedures_nr_gNB.c | 33 +- openair1/SIMULATION/NR_PHY/ulsim.c | 39 +- 7 files changed, 2017 insertions(+), 24 deletions(-) diff --git a/openair1/PHY/NR_TRANSPORT/nr_scrambling.c b/openair1/PHY/NR_TRANSPORT/nr_scrambling.c index 9663b1f3d4..fd0da127c8 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_scrambling.c +++ b/openair1/PHY/NR_TRANSPORT/nr_scrambling.c @@ -76,3 +76,22 @@ void nr_codeword_unscrambling(int16_t* llr, uint32_t size, uint8_t q, uint32_t N } #endif } + +void nr_codeword_unscrambling_init(int16_t *s2, uint32_t size, uint8_t q, uint32_t Nid, uint32_t n_RNTI) +{ + uint32_t x1; + uint32_t x2 = (n_RNTI << 15) + (q << 14) + Nid; + uint32_t s = 0; + + uint8_t *s8=(uint8_t *)&s; + s = lte_gold_generic(&x1, &x2, 1); + simde__m128i *s128=(simde__m128i *)s2; + + for (int i = 0, j = 0; i < ((size >> 5) + ((size & 0x1f) > 0 ? 1 : 0)); i++, j += 4) { + s128[j] = byte2m128i[s8[0]]; + s128[j+1] = byte2m128i[s8[1]]; + s128[j+2] = byte2m128i[s8[2]]; + s128[j+3] = byte2m128i[s8[3]]; + s = lte_gold_generic(&x1, &x2, 0); + } +} diff --git a/openair1/PHY/NR_TRANSPORT/nr_transport_common_proto.h b/openair1/PHY/NR_TRANSPORT/nr_transport_common_proto.h index 5437e2e815..64ffc9a36a 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_transport_common_proto.h +++ b/openair1/PHY/NR_TRANSPORT/nr_transport_common_proto.h @@ -73,6 +73,7 @@ void nr_codeword_scrambling(uint8_t *in, uint32_t* out); void nr_codeword_unscrambling(int16_t* llr, uint32_t size, uint8_t q, uint32_t Nid, uint32_t n_RNTI); +void nr_codeword_unscrambling_init(int16_t *s, uint32_t size, uint8_t q, uint32_t Nid, uint32_t n_RNTI); /**@}*/ diff --git a/openair1/PHY/NR_TRANSPORT/nr_transport_proto.h b/openair1/PHY/NR_TRANSPORT/nr_transport_proto.h index 2478748688..97d32232cf 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_transport_proto.h +++ b/openair1/PHY/NR_TRANSPORT/nr_transport_proto.h @@ -131,6 +131,12 @@ void nr_rx_pusch(PHY_VARS_gNB *gNB, uint8_t slot, unsigned char harq_pid); +int nr_rx_pusch_tp(PHY_VARS_gNB *gNB, + uint8_t ulsch_id, + uint32_t frame, + uint8_t slot, + unsigned char harq_pid); + /** \brief This function performs RB extraction (signal and channel estimates) (currently signal only until channel estimation and compensation are implemented) @param rxdataF pointer to the received frequency domain signal @param rxdataF_ext pointer to the extracted frequency domain signal @@ -209,6 +215,24 @@ void nr_ulsch_channel_compensation(int **rxdataF_ext, */ void nr_idft(int32_t *z, uint32_t Msc_PUSCH); +void nr_ulsch_qpsk_qpsk(c16_t *stream0_in, c16_t *stream1_in, c16_t *stream0_out, c16_t *rho01, uint32_t length); + +void nr_ulsch_qam16_qam16(c16_t *stream0_in, + c16_t *stream1_in, + c16_t *ch_mag, + c16_t *ch_mag_i, + c16_t *stream0_out, + c16_t *rho01, + uint32_t length); + +void nr_ulsch_qam64_qam64(c16_t *stream0_in, + c16_t *stream1_in, + c16_t *ch_mag, + c16_t *ch_mag_i, + c16_t *stream0_out, + c16_t *rho01, + uint32_t length); + /** \brief This function generates log-likelihood ratios (decoder input) for single-stream QPSK received waveforms. @param rxdataF_comp Compensated channel output @param ulsch_llr llr output diff --git a/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c b/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c index 6f491ffafa..9c398d1664 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c +++ b/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c @@ -18,6 +18,10 @@ #define INVALID_VALUE 255 +#ifdef __aarch64__ +#define USE_128BIT +#endif + void nr_idft(int32_t *z, uint32_t Msc_PUSCH) { @@ -281,6 +285,228 @@ void nr_idft(int32_t *z, uint32_t Msc_PUSCH) } +void nr_ulsch_extract_rbs0 (c16_t *rxdataF, + int32_t *chF, + int32_t *rxFext, + int32_t *chFext, + int rxoffset, + int choffset, + int aarx, + int is_dmrs_symbol, + nfapi_nr_pusch_pdu_t *pusch_pdu, + NR_DL_FRAME_PARMS *frame_parms) +{ + + uint8_t delta = 0; + + int start_re = (frame_parms->first_carrier_offset + (pusch_pdu->rb_start + pusch_pdu->bwp_start) * NR_NB_SC_PER_RB)%frame_parms->ofdm_symbol_size; + int nb_re_pusch = NR_NB_SC_PER_RB * pusch_pdu->rb_size; + + + int32_t *rxF = (int32_t*)&rxdataF[rxoffset]; + int32_t *rxF_ext = &rxFext[0]; + int32_t *ul_ch0 = &chF[choffset]; + int32_t *ul_ch0_ext = &chFext[0]; + + + if (is_dmrs_symbol == 0) { + if (start_re + nb_re_pusch <= frame_parms->ofdm_symbol_size) { + memcpy((void*)rxF_ext, + (void*)&rxF[start_re], + nb_re_pusch*sizeof(int32_t)); + } + else + { + int neg_length = frame_parms->ofdm_symbol_size-start_re; + int pos_length = nb_re_pusch - neg_length; + memcpy((void*)rxF_ext, (void*)&rxF[start_re], neg_length * sizeof(int32_t)); + memcpy((void*)&rxF_ext[neg_length], (void*)rxF, pos_length * sizeof(int32_t)); + } + memcpy((void*)ul_ch0_ext,(void*)ul_ch0,nb_re_pusch*sizeof(int32_t)); + } + else if (pusch_pdu->dmrs_config_type == pusch_dmrs_type1) // 6 REs / PRB + { + AssertFatal(delta==0||delta==1,"Illegal delta %d\n",delta); + int32_t *rxF32 = &rxF[start_re]; + int32_t *rxF_ext32 = rxF_ext; + int32_t *ul_ch032 = ul_ch0; + int32_t *ul_ch0_ext32 = ul_ch0_ext; + int idx,idx2,idx3; + if (start_re + nb_re_pusch < frame_parms->ofdm_symbol_size) { + for (idx=1-delta,idx2=0;idxofdm_symbol_size-start_re; + int pos_length = nb_re_pusch-neg_length; + + for (idx=1-delta,idx2=0;idxdmrs_config_type == pusch_dmrs_type2) // 8 REs / PRB + { + AssertFatal(delta==0||delta==2||delta==4,"Illegal delta %d\n",delta); + if (start_re + nb_re_pusch < frame_parms->ofdm_symbol_size) { + int64_t *rxF64 = (int64_t*)&rxF[start_re]; + int64_t *rxF_ext64 = (int64_t*)rxF_ext; + int64_t *ul_ch064 = (int64_t*)ul_ch0; + int64_t *ul_ch0_ext64 = (int64_t*)ul_ch0_ext; + if (delta==0) { + for (int idx=0;idx>1;idx+=6) { + rxF_ext64[idx]=rxF64[idx+1]; + rxF_ext64[idx+1]=rxF64[idx+2]; + rxF_ext64[idx+2]=rxF64[idx+4]; + rxF_ext64[idx+3]=rxF64[idx+5]; + ul_ch0_ext64[idx]=ul_ch064[idx+1]; + ul_ch0_ext64[idx+1]=ul_ch064[idx+2]; + ul_ch0_ext64[idx+2]=ul_ch064[idx+4]; + ul_ch0_ext64[idx+3]=ul_ch064[idx+5]; + } + } + else if (delta==2) { + for (int idx=0;idx>1;idx+=6) { + rxF_ext64[idx]=rxF64[idx+0]; + rxF_ext64[idx+1]=rxF64[idx+2]; + rxF_ext64[idx+2]=rxF64[idx+3]; + rxF_ext64[idx+3]=rxF64[idx+5]; + ul_ch0_ext64[idx]=ul_ch064[idx+0]; + ul_ch0_ext64[idx+1]=ul_ch064[idx+2]; + ul_ch0_ext64[idx+2]=ul_ch064[idx+3]; + ul_ch0_ext64[idx+3]=ul_ch064[idx+5]; + } + } + else if (delta==4) { + for (int idx=0;idx>1;idx+=6) { + rxF_ext64[idx]=rxF64[idx+0]; + rxF_ext64[idx+1]=rxF64[idx+1]; + rxF_ext64[idx+2]=rxF64[idx+3]; + rxF_ext64[idx+3]=rxF64[idx+4]; + ul_ch0_ext64[idx]=ul_ch064[idx+0]; + ul_ch0_ext64[idx+1]=ul_ch064[idx+1]; + ul_ch0_ext64[idx+2]=ul_ch064[idx+3]; + ul_ch0_ext64[idx+3]=ul_ch064[idx+4]; + } + } + } + else { + int neg_length = frame_parms->ofdm_symbol_size-start_re; + int pos_length = nb_re_pusch-neg_length; + if ((pos_length%12) > 0 ) pos_length+=12; + int64_t *rxF64 = (int64_t*)&rxF[start_re]; + int64_t *rxF_ext64 = (int64_t*)rxF_ext; + int64_t *ul_ch064 = (int64_t*)ul_ch0; + int64_t *ul_ch0_ext64 = (int64_t*)ul_ch0_ext; + int idx=0; + if (delta==0) { + for (idx=0;idx>1;idx+=6) { + rxF_ext64[idx] =rxF64[idx+1]; + rxF_ext64[idx+1]=rxF64[idx+2]; + rxF_ext64[idx+2]=rxF64[idx+4]; + rxF_ext64[idx+3]=rxF64[idx+5]; + ul_ch0_ext64[idx]=ul_ch064[idx+1]; + ul_ch0_ext64[idx+1]=ul_ch064[idx+2]; + ul_ch0_ext64[idx+2]=ul_ch064[idx+4]; + ul_ch0_ext64[idx+3]=ul_ch064[idx+5]; + } + if ((neg_length%12) > 0) { + rxF_ext64[idx+4]=rxF64[idx+7]; + rxF_ext64[idx+5]=rxF64[idx+8]; + ul_ch0_ext64[idx+4]=ul_ch064[idx+7]; + ul_ch0_ext64[idx+5]=ul_ch064[idx+8]; + } + rxF_ext64+=(neg_length/3); + rxF64=(int64_t*)rxF; + ul_ch0_ext64+=(neg_length/3); + ul_ch064+=(neg_length>>1); + for (idx=0;idx>1;idx+=6) { + rxF_ext64[idx] =rxF64[idx+1]; + rxF_ext64[idx+1]=rxF64[idx+2]; + rxF_ext64[idx+2]=rxF64[idx+4]; + rxF_ext64[idx+3]=rxF64[idx+5]; + ul_ch0_ext64[idx]=ul_ch064[idx+1]; + ul_ch0_ext64[idx+1]=ul_ch064[idx+2]; + ul_ch0_ext64[idx+2]=ul_ch064[idx+4]; + ul_ch0_ext64[idx+3]=ul_ch064[idx+5]; + } + } + else if (delta==2) { + for (idx=0;idx>1;idx+=6) { + rxF_ext64[idx] =rxF64[idx+0]; + rxF_ext64[idx+1]=rxF64[idx+2]; + rxF_ext64[idx+2]=rxF64[idx+3]; + rxF_ext64[idx+3]=rxF64[idx+5]; + ul_ch0_ext64[idx]=ul_ch064[idx+0]; + ul_ch0_ext64[idx+1]=ul_ch064[idx+2]; + ul_ch0_ext64[idx+2]=ul_ch064[idx+3]; + ul_ch0_ext64[idx+3]=ul_ch064[idx+5]; + } + if ((neg_length%12) > 0) { + rxF_ext64[idx+4]=rxF64[idx+6]; + rxF_ext64[idx+5]=rxF64[idx+8]; + ul_ch0_ext64[idx+4]=ul_ch064[idx+6]; + ul_ch0_ext64[idx+5]=ul_ch064[idx+8]; + } + rxF_ext64+=(neg_length/3); + rxF64=(int64_t*)rxF; + ul_ch0_ext64+=(neg_length/3); + ul_ch064+=(neg_length>>1); + for (idx=0;idx>1;idx+=6) { + rxF_ext64[idx] =rxF64[idx+0]; + rxF_ext64[idx+1]=rxF64[idx+2]; + rxF_ext64[idx+2]=rxF64[idx+3]; + rxF_ext64[idx+3]=rxF64[idx+5]; + ul_ch0_ext64[idx]=ul_ch064[idx+0]; + ul_ch0_ext64[idx+1]=ul_ch064[idx+2]; + ul_ch0_ext64[idx+2]=ul_ch064[idx+3]; + ul_ch0_ext64[idx+3]=ul_ch064[idx+5]; + } + } + else if (delta==4) { + for (idx=0;idx>1;idx+=6) { + rxF_ext64[idx] =rxF64[idx+0]; + rxF_ext64[idx+1]=rxF64[idx+1]; + rxF_ext64[idx+2]=rxF64[idx+3]; + rxF_ext64[idx+3]=rxF64[idx+4]; + ul_ch0_ext64[idx]=ul_ch064[idx+0]; + ul_ch0_ext64[idx+1]=ul_ch064[idx+1]; + ul_ch0_ext64[idx+2]=ul_ch064[idx+3]; + ul_ch0_ext64[idx+3]=ul_ch064[idx+4]; + } + if ((neg_length%12) > 0) { + rxF_ext64[idx+4]=rxF64[idx+6]; + rxF_ext64[idx+5]=rxF64[idx+7]; + ul_ch0_ext64[idx+4]=ul_ch064[idx+6]; + ul_ch0_ext64[idx+5]=ul_ch064[idx+7]; + } + rxF_ext64+=(neg_length/3); + rxF64=(int64_t*)rxF; + ul_ch0_ext64+=(neg_length/3); + ul_ch064+=(neg_length>>1); + for (idx=0;idx>1;idx+=6) { + rxF_ext64[idx] =rxF64[idx+0]; + rxF_ext64[idx+1]=rxF64[idx+1]; + rxF_ext64[idx+2]=rxF64[idx+3]; + rxF_ext64[idx+3]=rxF64[idx+4]; + ul_ch0_ext64[idx]=ul_ch064[idx+0]; + ul_ch0_ext64[idx+1]=ul_ch064[idx+1]; + ul_ch0_ext64[idx+2]=ul_ch064[idx+3]; + ul_ch0_ext64[idx+3]=ul_ch064[idx+4]; + } + } + } + } +} void nr_ulsch_extract_rbs(c16_t **rxdataF, NR_gNB_PUSCH *pusch_vars, @@ -420,6 +646,1439 @@ void nr_ulsch_scale_channel(int **ul_ch_estimates_ext, } } +int get_nb_re_pusch (NR_DL_FRAME_PARMS *frame_parms, nfapi_nr_pusch_pdu_t *rel15_ul,int symbol) +{ + uint8_t dmrs_symbol_flag = (rel15_ul->ul_dmrs_symb_pos >> symbol) & 0x01; + if (dmrs_symbol_flag == 1) { + if ((rel15_ul->ul_dmrs_symb_pos >> ((symbol + 1) % frame_parms->symbols_per_slot)) & 0x01) + AssertFatal(1==0,"Double DMRS configuration is not yet supported\n"); + + if (rel15_ul->dmrs_config_type == 0) { + // if no data in dmrs cdm group is 1 only even REs have no data + // if no data in dmrs cdm group is 2 both odd and even REs have no data + return(rel15_ul->rb_size *(12 - (rel15_ul->num_dmrs_cdm_grps_no_data*6))); + } + else return(rel15_ul->rb_size *(12 - (rel15_ul->num_dmrs_cdm_grps_no_data*4))); + } else return(rel15_ul->rb_size * NR_NB_SC_PER_RB); +} + +void inner_rx_qpsk (int *rxF, + int *ul_ch, + int16_t *llr, + int aarx, + int length, + int output_shift) +{ +#ifndef USE_128BIT + register simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; + register simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + register simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); + + simde__m256i *rxF256 = (simde__m256i*)rxF; + simde__m256i *ulch256 = (simde__m256i*)ul_ch; + // need to use simde__m64 because llr output is not necessarily aligned to 256 bits, but it is always to 64 bits + simde__m64 *llr64 = (simde__m64 *)llr; + for (int i=0; i<((length>>3)+((length&7)>0?1:0)); i++) { + xmmp0 = simde_mm256_madd_epi16(ulch256[i], rxF256[i]); + // xmmp0 contains real part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp1 = simde_mm256_shuffle_epi8(ulch256[i], complex_shuffle256); + xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); + xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF256[i]); + // xmmp1 contains imag part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); + xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); + xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); + xmmp4 = simde_mm256_packs_epi32(xmmp2,xmmp3); + if (aarx == 0) + { + *llr64 = (simde__m64)simde_mm256_extract_epi64(xmmp4,0); llr64++; + *llr64 = (simde__m64)simde_mm256_extract_epi64(xmmp4,1); llr64++; + *llr64 = (simde__m64)simde_mm256_extract_epi64(xmmp4,2); llr64++; + *llr64 = (simde__m64)simde_mm256_extract_epi64(xmmp4,3); llr64++; + } + else + { + *llr64 = simde_mm_adds_pi16(*llr64,(simde__m64)(simde_mm256_extract_epi64(xmmp4,0))); llr64++; + *llr64 = simde_mm_adds_pi16(*llr64,(simde__m64)(simde_mm256_extract_epi64(xmmp4,1))); llr64++; + *llr64 = simde_mm_adds_pi16(*llr64,(simde__m64)(simde_mm256_extract_epi64(xmmp4,2))); llr64++; + *llr64 = simde_mm_adds_pi16(*llr64,(simde__m64)(simde_mm256_extract_epi64(xmmp4,3))); llr64++; + } + } +#else + simde__m128i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; + register simde__m128i complex_shuffle128 = simde_mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); + register simde__m128i conj128 = simde_mm_set_epi16(1, -1, 1, -1, 1, -1, 1, -1); + + simde__m128i *rxF128 = (simde__m128i*)rxF; + simde__m128i *ulch128 = (simde__m128i*)ul_ch; + simde__m128i *llr128 = (simde__m128i*)llr; + for (int i = 0; i < (length >> 2); i++) { + xmmp0 = simde_mm_madd_epi16(ulch128[i], rxF128[i]); + // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp1 = simde_mm_shuffle_epi8(ulch128[i], complex_shuffle128); + xmmp1 = simde_mm_sign_epi16(xmmp1, conj128); + xmmp1 = simde_mm_madd_epi16(xmmp1, rxF128[i]); + // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp0 = simde_mm_srai_epi32(xmmp0, output_shift); + xmmp1 = simde_mm_srai_epi32(xmmp1, output_shift); + xmmp2 = simde_mm_unpacklo_epi32(xmmp0, xmmp1); + xmmp3 = simde_mm_unpackhi_epi32(xmmp0, xmmp1); + xmmp4 = simde_mm_packs_epi32(xmmp2, xmmp3); + + if (aarx == 0) + *llr128 = xmmp4; + else + *llr128 = simde_mm_add_epi16(*llr128, xmmp4); + llr128++; + } + if (length & 3) + { + int i = (length>>1) - 1; + simde__m64* llr64 = (simde__m64*)llr128; + simde__m64 xmm0, xmm1, xmm2, xmm3, xmm4; + simde__m64 complex_shuffle64 = simde_mm_set_pi8(5, 4, 7, 6, 1, 0, 3, 2); + simde__m64 conj64 = simde_mm_set_pi16(1, -1, 1, -1); + simde__m64 *rxF64 = (simde__m64*)rxF; + simde__m64 *ulch64 = (simde__m64*)ul_ch; + + xmm0 = simde_mm_madd_pi16(ulch64[i], rxF64[i]); + // xmm0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmm1 = simde_mm_shuffle_pi8(ulch64[i], complex_shuffle64); + xmm1 = simde_mm_sign_pi16(xmm1, conj64); + xmm1 = simde_mm_madd_pi16(xmm1, rxF64[i]); + // xmm1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmm0 = simde_mm_srai_pi32(xmm0, output_shift); + xmm1 = simde_mm_srai_pi32(xmm1, output_shift); + xmm2 = simde_mm_unpacklo_pi32(xmm0, xmm1); + xmm3 = simde_mm_unpackhi_pi32(xmm0, xmm1); + xmm4 = simde_mm_packs_pi32(xmm2, xmm3); + + if (aarx == 0) + *llr64 = xmm4; + else + *llr64 = simde_mm_add_pi16(*llr64, xmm4); + } +#endif +} + +void inner_rx_16qam (int *rxF, + int *ul_ch, + int16_t *llr, + int aarx, + int length, + int output_shift) +{ +#ifndef USE_128BIT + register simde__m256i xmmp0,xmmp1,xmmp2,xmmp3,xmmp4,xmmp5; + register simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + register simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); + + register simde__m256i QAM_amp256 = simde_mm256_set1_epi16(QAM16_n1); // 2/sqrt(10) + simde__m256i *rxF256 = (simde__m256i*)rxF; + simde__m256i *ulch256 = (simde__m256i*)ul_ch; + // need to use simde__m64 because llr output is not necessarily aligned to 256 bits, but it is always to 64 bits + simde__m64 *llr64 = (simde__m64 *)llr; + + for (int i = 0; i < ((length >> 3) + ((length & 7) > 0 ? 1 : 0)); i++) + { + xmmp0 = simde_mm256_madd_epi16(ulch256[i], rxF256[i]); + // xmmp0 contains real part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp1 = simde_mm256_shuffle_epi8(ulch256[i], complex_shuffle256); + xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); + xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF256[i]); + // xmmp1 contains imag part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); + xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); + xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); + xmmp4 = simde_mm256_packs_epi32(xmmp2, xmmp3); + + // compute channel amplitude for LLR + xmmp0 = simde_mm256_madd_epi16(ulch256[i], ulch256[i]); + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp0 = simde_mm256_packs_epi32(xmmp0, xmmp0); + xmmp1 = simde_mm256_unpacklo_epi16(xmmp0, xmmp0); + xmmp1 = simde_mm256_mulhrs_epi16(xmmp1, QAM_amp256); + + xmmp2 = simde_mm256_abs_epi16(xmmp4); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| + xmmp2 = simde_mm256_subs_epi16(xmmp1,xmmp2); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 + + xmmp3 = simde_mm256_unpacklo_epi32(xmmp4,xmmp2); // llr128[0] contains the llrs of the 1st,2nd,5th and 6th REs + xmmp5 = simde_mm256_unpackhi_epi32(xmmp4,xmmp2); // llr128[1] contains the llrs of the 3rd, 4th, 7th and 8th REs + if (aarx == 0) + { + // 1st/2nd RE + llr64[0] = (simde__m64)simde_mm256_extract_epi64(xmmp3,0); // llr32[0] low 16 bits-> y_R , high 16 bits-> y_I + // 3rd/4th RE + llr64[1] = (simde__m64)simde_mm256_extract_epi64(xmmp3,1); // llr32[2] low 16 bits-> y_R , high 16 bits-> y_I + // 5th/6th RE + llr64[2] = (simde__m64)simde_mm256_extract_epi64(xmmp5,0); // llr32[4] low 16 bits-> y_R , high 16 bits-> y_I + // 7Rh/8th RE + llr64[3] = (simde__m64)simde_mm256_extract_epi64(xmmp5,1); // llr32[6] low 16 bits-> y_R , high 16 bits-> y_I + // 9th/10th RE + llr64[4] = (simde__m64)simde_mm256_extract_epi64(xmmp3,2); // llr32[8] low 16 bits-> y_R , high 16 bits-> y_I + // 11th/12th RE + llr64[5] = (simde__m64)simde_mm256_extract_epi64(xmmp3,3); // llr32[10] low 16 bits-> y_R , high 16 bits-> y_I + // 13th/14th RE + llr64[6] = (simde__m64)simde_mm256_extract_epi64(xmmp5,2); // llr32[12] low 16 bits-> y_R , high 16 bits-> y_I + // 15th/16th RE + llr64[7] = (simde__m64)simde_mm256_extract_epi64(xmmp5,3); // llr32[14] low 16 bits-> y_R , high 16 bits-> y_I + llr64+=8; + } + else + { + llr64[0] = simde_mm_adds_pi16(llr64[0],(simde__m64)simde_mm256_extract_epi64(xmmp3,0)); + llr64[1] = simde_mm_adds_pi16(llr64[1],(simde__m64)simde_mm256_extract_epi64(xmmp3,1)); + llr64[2] = simde_mm_adds_pi16(llr64[2],(simde__m64)simde_mm256_extract_epi64(xmmp5,0)); + llr64[3] = simde_mm_adds_pi16(llr64[3],(simde__m64)simde_mm256_extract_epi64(xmmp5,1)); + llr64[4] = simde_mm_adds_pi16(llr64[4],(simde__m64)simde_mm256_extract_epi64(xmmp3,2)); + llr64[5] = simde_mm_adds_pi16(llr64[5],(simde__m64)simde_mm256_extract_epi64(xmmp3,3)); + llr64[6] = simde_mm_adds_pi16(llr64[6],(simde__m64)simde_mm256_extract_epi64(xmmp5,2)); + llr64[7] = simde_mm_adds_pi16(llr64[7],(simde__m64)simde_mm256_extract_epi64(xmmp5,3)); + llr64 += 8; + } + } +#else + simde__m128i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4, xmmp5; + register simde__m128i complex_shuffle128 = simde_mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); + register simde__m128i conj128 = simde_mm_set_epi16(1, -1, 1, -1, 1, -1, 1, -1); + + register simde__m128i QAM_amp128 = simde_mm_set1_epi16(QAM16_n1); // 2/sqrt(10) + simde__m128i *rxF128 = (simde__m128i*)rxF; + simde__m128i *ulch128 = (simde__m128i*)ul_ch; + // need to use simde__m64 because llr output is not necessarily aligned to 256 bits, but it is always to 64 bits + simde__m64 *llr64 = (simde__m64 *)llr; + + for (int i = 0; i < (length >> 2); i++) + { + xmmp0 = simde_mm_madd_epi16(ulch128[i], rxF128[i]); + // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp1 = simde_mm_shuffle_epi8(ulch128[i], complex_shuffle128); + xmmp1 = simde_mm_sign_epi16(xmmp1, conj128); + xmmp1 = simde_mm_madd_epi16(xmmp1, rxF128[i]); + // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp0 = simde_mm_srai_epi32(xmmp0, output_shift); + xmmp1 = simde_mm_srai_epi32(xmmp1, output_shift); + xmmp2 = simde_mm_unpacklo_epi32(xmmp0, xmmp1); + xmmp3 = simde_mm_unpackhi_epi32(xmmp0, xmmp1); + xmmp4 = simde_mm_packs_epi32(xmmp2, xmmp3); + + // compute channel amplitude for LLR + xmmp0 = simde_mm_madd_epi16(ulch128[i], ulch128[i]); // |h|^2 + xmmp0 = simde_mm_srai_epi32(xmmp0, output_shift); + xmmp0 = simde_mm_packs_epi32(xmmp0, xmmp0); + xmmp1 = simde_mm_unpacklo_epi16(xmmp0, xmmp0); + xmmp1 = simde_mm_mulhrs_epi16(xmmp1, QAM_amp128); + + xmmp2 = simde_mm_abs_epi16(xmmp4); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| + xmmp2 = simde_mm_subs_epi16(xmmp1, xmmp2); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 + + xmmp3 = simde_mm_unpacklo_epi32(xmmp4, xmmp2); // llr128[0] contains the llrs of the 1st,2nd,5th and 6th REs + xmmp5 = simde_mm_unpackhi_epi32(xmmp4, xmmp2); // llr128[1] contains the llrs of the 3rd, 4th, 7th and 8th REs + if (aarx == 0) + { + llr64[0] = (simde__m64)simde_mm_extract_epi64(xmmp3, 0); // llr32[0] low 16 bits-> y_R, high 16 bits-> y_I + llr64[1] = (simde__m64)simde_mm_extract_epi64(xmmp3, 1); // llr32[2] low 16 bits-> y_R, high 16 bits-> y_I + llr64[2] = (simde__m64)simde_mm_extract_epi64(xmmp5, 0); // llr32[4] low 16 bits-> y_R, high 16 bits-> y_I + llr64[3] = (simde__m64)simde_mm_extract_epi64(xmmp5, 1); // llr32[6] low 16 bits-> y_R, high 16 bits-> y_I + } + else + { + llr64[0] = simde_mm_adds_pi16(llr64[0], (simde__m64)simde_mm_extract_epi64(xmmp3, 0)); + llr64[1] = simde_mm_adds_pi16(llr64[1], (simde__m64)simde_mm_extract_epi64(xmmp3, 1)); + llr64[2] = simde_mm_adds_pi16(llr64[2], (simde__m64)simde_mm_extract_epi64(xmmp5, 0)); + llr64[3] = simde_mm_adds_pi16(llr64[3], (simde__m64)simde_mm_extract_epi64(xmmp5, 1)); + } + llr64 += 4; + } + if (length & 3) + { + int i = (length>>1) - 1; + simde__m64 xmm0, xmm1, xmm2, xmm3, xmm4; + simde__m64 complex_shuffle64 = simde_mm_set_pi8(5,4,7,6,1,0,3,2); + simde__m64 conj64 = simde_mm_set_pi16(1, -1, 1, -1); + simde__m64 *rxF64 = (simde__m64*)rxF; + simde__m64 *ulch64 = (simde__m64*)ul_ch; + simde__m64 QAM_amp = simde_mm_set1_pi16(QAM16_n1); + + xmm0 = simde_mm_madd_pi16(ulch64[i], rxF64[i]); + // xmm0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmm1 = simde_mm_shuffle_pi8(ulch64[i], complex_shuffle64); + xmm1 = simde_mm_sign_pi16(xmm1, conj64); + xmm1 = simde_mm_madd_pi16(xmm1, rxF64[i]); + // xmm1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmm0 = simde_mm_srai_pi32(xmm0, output_shift); + xmm1 = simde_mm_srai_pi32(xmm1, output_shift); + xmm2 = simde_mm_unpacklo_pi32(xmm0, xmm1); + xmm3 = simde_mm_unpackhi_pi32(xmm0, xmm1); + xmm4 = simde_mm_packs_pi32(xmm2, xmm3); + + // compute channel amplitude for LLR + xmm0 = simde_mm_madd_pi16(ulch64[i], ulch64[i]); // |h|^2 + xmm0 = simde_mm_srai_pi32(xmm0, output_shift); + xmm0 = simde_mm_packs_pi32(xmm0, xmm0); + xmm2 = simde_mm_unpacklo_pi16(xmm0, xmm0); + + xmm1 = simde_mm_mulhrs_pi16(xmm2, QAM_amp); + + xmm0 = simde_mm_abs_pi16(xmm4); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| + xmm0 = simde_mm_subs_pi16(xmm1, xmm0); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 + if (aarx == 0) + { + llr64[0] = simde_mm_set_pi32(simde_mm_extract_pi16(xmm0, 0), simde_mm_extract_pi16(xmm4, 0)); + llr64[1] = simde_mm_set_pi32(simde_mm_extract_pi16(xmm0, 1), simde_mm_extract_pi16(xmm4, 1)); + } + else + { + llr64[0] = simde_mm_adds_pi16(llr64[0], simde_mm_set_pi32(simde_mm_extract_pi16(xmm0, 0),simde_mm_extract_pi16(xmm4, 0))); + llr64[1] = simde_mm_adds_pi16(llr64[1], simde_mm_set_pi32(simde_mm_extract_pi16(xmm4, 1),simde_mm_extract_pi16(xmm1, 0))); + } + } + +#endif +} + +void inner_rx_64qam (int *restrict rxF, + int *restrict ul_ch, + int16_t *restrict llr, + int aarx, + int length, + int output_shift) +{ +#ifndef USE_128BIT + register simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4, xmmp6, xmmp7; + register simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29, 28, 31, 30, 25, 24, 27, 26, 21, 20, 23, 22, 17, 16, 19, 18, 13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); + register simde__m256i conj256 = simde_mm256_set_epi16(1, -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1); + + register simde__m256i QAM_amp256 = simde_mm256_set1_epi16(QAM64_n1); // 2/sqrt(10) + register simde__m256i QAM_amp256b = simde_mm256_set1_epi16(QAM64_n2); + simde__m256i *rxF256 = (simde__m256i*)rxF; + simde__m256i *ulch256 = (simde__m256i*)ul_ch; + // need to use simde__m64 because llr output is not necessarily aligned to 256 bits, but it is always to 64 bits + + simde__m64 *llr64 = (simde__m64 *)llr; + for (int i=0;i<((length>>3)+((length&7)>0?1:0));i++) { + xmmp0 = simde_mm256_madd_epi16(ulch256[i],rxF256[i]); + // xmmp0 contains real part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp1 = simde_mm256_shuffle_epi8(ulch256[i],complex_shuffle256); + xmmp1 = simde_mm256_sign_epi16(xmmp1,conj256); + xmmp1 = simde_mm256_madd_epi16(xmmp1,rxF256[i]); + // xmmp1 contains imag part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp0 = simde_mm256_srai_epi32(xmmp0,output_shift); + xmmp1 = simde_mm256_srai_epi32(xmmp1,output_shift); + xmmp2 = simde_mm256_unpacklo_epi32(xmmp0,xmmp1); + xmmp3 = simde_mm256_unpackhi_epi32(xmmp0,xmmp1); + xmmp4 = simde_mm256_packs_epi32(xmmp2,xmmp3); + + // compute channel amplitude for LLR + xmmp0 = simde_mm256_madd_epi16(ulch256[i],ulch256[i]); + xmmp0 = simde_mm256_srai_epi32(xmmp0,output_shift); + xmmp0 = simde_mm256_packs_epi32(xmmp0,xmmp0); + xmmp2 = simde_mm256_unpacklo_epi16(xmmp0,xmmp0); + xmmp1 = simde_mm256_mulhrs_epi16(xmmp2,QAM_amp256); + xmmp6 = simde_mm256_mulhrs_epi16(xmmp2,QAM_amp256b); + + xmmp2 = simde_mm256_abs_epi16(xmmp4); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| + xmmp2 = simde_mm256_subs_epi16(xmmp1,xmmp2); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 + xmmp7 = simde_mm256_abs_epi16(xmmp2); + xmmp7 = simde_mm256_subs_epi16(xmmp6,xmmp7); + + if (aarx == 0) + { + llr64[0] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp2,0),simde_mm256_extract_epi32(xmmp4,0)); + llr64[1] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp4,1),simde_mm256_extract_epi32(xmmp7,0)); + llr64[2] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp7,1),simde_mm256_extract_epi32(xmmp2,1)); + llr64[3] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp2,2),simde_mm256_extract_epi32(xmmp4,2)); + llr64[4] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp4,3),simde_mm256_extract_epi32(xmmp7,2)); + llr64[5] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp7,3),simde_mm256_extract_epi32(xmmp2,3)); + llr64[6] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp2,4),simde_mm256_extract_epi32(xmmp4,4)); + llr64[7] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp4,5),simde_mm256_extract_epi32(xmmp7,4)); + llr64[8] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp7,5),simde_mm256_extract_epi32(xmmp2,5)); + llr64[9] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp2,6),simde_mm256_extract_epi32(xmmp4,6)); + llr64[10] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp4,7),simde_mm256_extract_epi32(xmmp7,6)); + llr64[11] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp7,7),simde_mm256_extract_epi32(xmmp2,7)); + llr64+=12; + } + else + { + llr64[0] = simde_mm_adds_pi16(llr64[0],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp2,0),simde_mm256_extract_epi32(xmmp4,0))); + llr64[1] = simde_mm_adds_pi16(llr64[1],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp4,1),simde_mm256_extract_epi32(xmmp7,0))); + llr64[2] = simde_mm_adds_pi16(llr64[2],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp7,1),simde_mm256_extract_epi32(xmmp2,1))); + llr64[3] = simde_mm_adds_pi16(llr64[3],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp2,2),simde_mm256_extract_epi32(xmmp4,2))); + llr64[4] = simde_mm_adds_pi16(llr64[4],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp4,3),simde_mm256_extract_epi32(xmmp7,2))); + llr64[5] = simde_mm_adds_pi16(llr64[5],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp7,3),simde_mm256_extract_epi32(xmmp2,3))); + llr64[6] = simde_mm_adds_pi16(llr64[6],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp2,4),simde_mm256_extract_epi32(xmmp4,4))); + llr64[7] = simde_mm_adds_pi16(llr64[7],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp4,5),simde_mm256_extract_epi32(xmmp7,4))); + llr64[8] = simde_mm_adds_pi16(llr64[8],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp7,5),simde_mm256_extract_epi32(xmmp2,5))); + llr64[9] = simde_mm_adds_pi16(llr64[9],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp2,6),simde_mm256_extract_epi32(xmmp4,6))); + llr64[10] = simde_mm_adds_pi16(llr64[10],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp4,7),simde_mm256_extract_epi32(xmmp7,6))); + llr64[11] = simde_mm_adds_pi16(llr64[11],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp7,7),simde_mm256_extract_epi32(xmmp2,7))); + llr64+=12; + } + } +#else + register simde__m128i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4, xmmp6, xmmp7; + register simde__m128i complex_shuffle128 = simde_mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); + register simde__m128i conj128 = simde_mm_set_epi16(1, -1, 1, -1, 1, -1, 1, -1); + // register simde__m128i conj128 = simde_mm_set_epi16(1, -1, 1, -1, 1, -1, 1, -1); + register simde__m128i QAM_amp128 = simde_mm_set1_epi16(QAM64_n1); // 4/sqrt(42) + register simde__m128i QAM_amp128b = simde_mm_set1_epi16(QAM64_n2); // 2/sqrt(42) + simde__m128i *rxF128 = (simde__m128i*) rxF; + simde__m128i *ulch128 = (simde__m128i*) ul_ch; + // need to use simde__m64 because llr output is not necessarily aligned to 256 bits, but it is always to 64 bits + simde__m64 *llr64 = (simde__m64 *)llr; + for (int i = 0; i < (length>>2); i++) + { + xmmp0 = simde_mm_madd_epi16(ulch128[i], rxF128[i]); + // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp1 = simde_mm_shuffle_epi8(ulch128[i], complex_shuffle128); + xmmp1 = simde_mm_sign_epi16(xmmp1, conj128); + xmmp1 = simde_mm_madd_epi16(xmmp1, rxF128[i]); + // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp0 = simde_mm_srai_epi32(xmmp0, output_shift); + xmmp1 = simde_mm_srai_epi32(xmmp1, output_shift); + xmmp2 = simde_mm_unpacklo_epi32(xmmp0, xmmp1); + xmmp3 = simde_mm_unpackhi_epi32(xmmp0, xmmp1); + xmmp4 = simde_mm_packs_epi32(xmmp2, xmmp3); + + // compute channel amplitude for LLR + xmmp0 = simde_mm_madd_epi16(ulch128[i], ulch128[i]); + xmmp0 = simde_mm_srai_epi32(xmmp0, output_shift); + xmmp0 = simde_mm_packs_epi32(xmmp0, xmmp0); + xmmp2 = simde_mm_unpacklo_epi16(xmmp0, xmmp0); + xmmp1 = simde_mm_mulhrs_epi16(xmmp2, QAM_amp128); + xmmp6 = simde_mm_mulhrs_epi16(xmmp2, QAM_amp128b); + + xmmp2 = simde_mm_abs_epi16(xmmp4); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| + xmmp2 = simde_mm_subs_epi16(xmmp1, xmmp2); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 + xmmp7 = simde_mm_abs_epi16(xmmp2); + xmmp7 = simde_mm_subs_epi16(xmmp6, xmmp7); + + if (aarx == 0) + { + llr64[0] = simde_mm_set_pi32(simde_mm_extract_epi32(xmmp2, 0), simde_mm_extract_epi32(xmmp4, 0)); + llr64[1] = simde_mm_set_pi32(simde_mm_extract_epi32(xmmp4, 1), simde_mm_extract_epi32(xmmp7, 0)); + llr64[2] = simde_mm_set_pi32(simde_mm_extract_epi32(xmmp7, 1), simde_mm_extract_epi32(xmmp2, 1)); + llr64[3] = simde_mm_set_pi32(simde_mm_extract_epi32(xmmp2, 2), simde_mm_extract_epi32(xmmp4, 2)); + llr64[4] = simde_mm_set_pi32(simde_mm_extract_epi32(xmmp4, 3), simde_mm_extract_epi32(xmmp7, 2)); + llr64[5] = simde_mm_set_pi32(simde_mm_extract_epi32(xmmp7, 3), simde_mm_extract_epi32(xmmp2, 3)); + llr64 += 6; + } + else + { + llr64[0] = simde_mm_adds_pi16(llr64[0], simde_mm_set_pi32(simde_mm_extract_epi32(xmmp2, 0),simde_mm_extract_epi32(xmmp4, 0))); + llr64[1] = simde_mm_adds_pi16(llr64[1], simde_mm_set_pi32(simde_mm_extract_epi32(xmmp4, 1),simde_mm_extract_epi32(xmmp7, 0))); + llr64[2] = simde_mm_adds_pi16(llr64[2], simde_mm_set_pi32(simde_mm_extract_epi32(xmmp7, 1),simde_mm_extract_epi32(xmmp2, 1))); + llr64[3] = simde_mm_adds_pi16(llr64[3], simde_mm_set_pi32(simde_mm_extract_epi32(xmmp2, 2),simde_mm_extract_epi32(xmmp4, 2))); + llr64[4] = simde_mm_adds_pi16(llr64[4], simde_mm_set_pi32(simde_mm_extract_epi32(xmmp4, 3),simde_mm_extract_epi32(xmmp7, 2))); + llr64[5] = simde_mm_adds_pi16(llr64[5], simde_mm_set_pi32(simde_mm_extract_epi32(xmmp7, 3),simde_mm_extract_epi32(xmmp2, 3))); + llr64 += 6; + } + } + if (length & 3) + { + int i = (length>>1) - 1; + simde__m64 xmm0, xmm1, xmm2, xmm3, xmm4, xmm5; + simde__m64 complex_shuffle64 = simde_mm_set_pi8(5,4,7,6,1,0,3,2); + simde__m64 conj64 = simde_mm_set_pi16(1, -1, 1, -1); + simde__m64 *rxF64 = (simde__m64*)rxF; + simde__m64 *ulch64 = (simde__m64*)ul_ch; + simde__m64 QAM_amp = simde_mm_set1_pi16(QAM64_n1); + simde__m64 QAM_ampb = simde_mm_set1_pi16(QAM64_n2); + + xmm0 = simde_mm_madd_pi16(ulch64[i], rxF64[i]); + // xmm0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmm1 = simde_mm_shuffle_pi8(ulch64[i], complex_shuffle64); + xmm1 = simde_mm_sign_pi16(xmm1, conj64); + xmm1 = simde_mm_madd_pi16(xmm1, rxF64[i]); + // xmm1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmm0 = simde_mm_srai_pi32(xmm0, output_shift); + xmm1 = simde_mm_srai_pi32(xmm1, output_shift); + xmm2 = simde_mm_unpacklo_pi32(xmm0, xmm1); + xmm3 = simde_mm_unpackhi_pi32(xmm0, xmm1); + xmm4 = simde_mm_packs_pi32(xmm2, xmm3); + + // compute channel amplitude for LLR + xmm0 = simde_mm_madd_pi16(ulch64[i], ulch64[i]); // |h|^2 + xmm0 = simde_mm_srai_pi32(xmm0, output_shift); + xmm0 = simde_mm_packs_pi32(xmm0, xmm0); + xmm2 = simde_mm_unpacklo_pi16(xmm0, xmm0); + + xmm1 = simde_mm_mulhrs_pi16(xmm2, QAM_amp); + xmm5 = simde_mm_mulhrs_pi16(xmm2, QAM_ampb); + + xmm0 = simde_mm_abs_pi16(xmm4); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| + xmm0 = simde_mm_subs_pi16(xmm1, xmm0); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 + xmm1 = simde_mm_abs_pi16(xmm0); + xmm1 = simde_mm_subs_pi16(xmm5, xmm1); // contains 8 LLRs + if (aarx == 0) + { + llr64[0] = simde_mm_set_pi32(simde_mm_extract_pi16(xmm0, 0), simde_mm_extract_pi16(xmm4, 0)); + llr64[1] = simde_mm_set_pi32(simde_mm_extract_pi16(xmm4, 1), simde_mm_extract_pi16(xmm1, 0)); + llr64[2] = simde_mm_set_pi32(simde_mm_extract_pi16(xmm1, 1), simde_mm_extract_pi16(xmm0, 1)); + } + else + { + llr64[0] = simde_mm_adds_pi16(llr64[0], simde_mm_set_pi32(simde_mm_extract_pi16(xmm0, 0),simde_mm_extract_pi16(xmm4, 0))); + llr64[1] = simde_mm_adds_pi16(llr64[1], simde_mm_set_pi32(simde_mm_extract_pi16(xmm4, 1),simde_mm_extract_pi16(xmm1, 0))); + llr64[2] = simde_mm_adds_pi16(llr64[2], simde_mm_set_pi32(simde_mm_extract_pi16(xmm1, 1),simde_mm_extract_pi16(xmm0, 1))); + } + } +#endif +} + +void inner_rx_256qam (int *rxF, + int *ul_ch, + int16_t *llr, + int aarx, + int length, + int output_shift) +{ +#ifndef USE_128BIT + register simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4, xmmp5, xmmp6, xmmp7, xmmp8, xmmp9; + register simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + register simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); + + register simde__m256i QAM_amp256 = simde_mm256_set1_epi16(QAM256_n1); + register simde__m256i QAM_amp256b = simde_mm256_set1_epi16(QAM256_n2); + register simde__m256i QAM_amp256c = simde_mm256_set1_epi16(QAM256_n3); + simde__m256i *rxF256 = (simde__m256i*)rxF; + simde__m256i *ulch256 = (simde__m256i*)ul_ch; + simde__m256i *llr256 = (simde__m256i *)llr; + for (int i = 0; i < ((length >> 3) + (( length & 7) > 0 ? 1 : 0)); i++) + { + xmmp0 = simde_mm256_madd_epi16(ulch256[i],rxF256[i]); + // xmmp0 contains real part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp1 = simde_mm256_shuffle_epi8(ulch256[i],complex_shuffle256); + xmmp1 = simde_mm256_sign_epi16(xmmp1,conj256); + xmmp1 = simde_mm256_madd_epi16(xmmp1,rxF256[i]); + // xmmp1 contains imag part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp0 = simde_mm256_srai_epi32(xmmp0,output_shift); + xmmp1 = simde_mm256_srai_epi32(xmmp1,output_shift); + xmmp2 = simde_mm256_unpacklo_epi32(xmmp0,xmmp1); + xmmp3 = simde_mm256_unpackhi_epi32(xmmp0,xmmp1); + xmmp4 = simde_mm256_packs_epi32(xmmp2,xmmp3); + + // compute channel amplitude for LLR + xmmp0 = simde_mm256_madd_epi16(ulch256[i],ulch256[i]); + xmmp0 = simde_mm256_srai_epi32(xmmp0,output_shift); + xmmp0 = simde_mm256_packs_epi32(xmmp0,xmmp0); // contains 16 LLRs + xmmp2 = simde_mm256_unpacklo_epi16(xmmp0,xmmp0); + xmmp1 = simde_mm256_mulhrs_epi16(xmmp2,QAM_amp256); + xmmp6 = simde_mm256_mulhrs_epi16(xmmp2,QAM_amp256b); + xmmp8 = simde_mm256_mulhrs_epi16(xmmp2,QAM_amp256c); + + xmmp2 = simde_mm256_abs_epi16(xmmp4); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| + xmmp2 = simde_mm256_subs_epi16(xmmp1,xmmp2); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 + // xmmp2 contains 16 LLRs + xmmp7 = simde_mm256_abs_epi16(xmmp2); + xmmp7 = simde_mm256_subs_epi16(xmmp6,xmmp7); // contains 16 LLRs + xmmp9 = simde_mm256_abs_epi16(xmmp7); + xmmp9 = simde_mm256_subs_epi16(xmmp8,xmmp9); // contains 16 LLRs + + // xmmp4 A0 A1 A2 A3 A4 A5 A6 A7 + // xmmp2 B0 B1 B2 B3 B4 B5 B6 B7 + // xmmp7 C0 C1 C2 C3 C4 C5 C6 C7 + // xmmp9 D0 D1 D2 D3 D4 D5 D6 D7 + xmmp1 = simde_mm256_unpacklo_epi32(xmmp4,xmmp2); // A0 B0 A1 B1 A4 B4 A5 B5 + xmmp3 = simde_mm256_unpackhi_epi32(xmmp4,xmmp2); // A2 B2 A3 B3 A6 B6 A7 B7 + xmmp5 = simde_mm256_unpacklo_epi32(xmmp7,xmmp9); // C0 D0 C1 D1 C4 D4 C5 D5 + xmmp6 = simde_mm256_unpackhi_epi32(xmmp7,xmmp9); // C2 D2 C3 D3 C6 D6 C7 D7 + + xmmp2 = simde_mm256_unpacklo_epi64(xmmp1,xmmp5); // A0 B0 C0 D0 A4 B4 C4 D4 + xmmp4 = simde_mm256_unpackhi_epi64(xmmp1,xmmp5); // A1 B1 C1 D1 A5 B5 C5 D5 + xmmp1 = simde_mm256_unpacklo_epi64(xmmp3,xmmp6); // A2 B2 C2 D2 A6 B6 C6 D6 + xmmp5 = simde_mm256_unpackhi_epi64(xmmp3,xmmp6); // A3 B3 C3 D3 A7 B7 C7 D7 + if (aarx == 0) + { + llr256[0] = simde_mm256_permute2x128_si256(xmmp2, xmmp4, 0x20); // A0 B0 C0 D0 A1 B1 C1 D1 + llr256[1] = simde_mm256_permute2x128_si256(xmmp1, xmmp5, 0x20); // A2 B2 C2 D2 A3 B3 C3 D3 + llr256[2] = simde_mm256_permute2x128_si256(xmmp2, xmmp4, 0x31); // A4 B4 C4 D4 A5 B5 C5 D5 + llr256[3] = simde_mm256_permute2x128_si256(xmmp1, xmmp5, 0x31); // A6 B6 C6 D6 A7 B7 C7 D7 + llr256+=4; + } + else + { + llr256[0] = simde_mm256_adds_epi16(llr256[0],simde_mm256_permute2x128_si256(xmmp2, xmmp4, 0x20)); // A0 B0 C0 D0 A1 B1 C1 D1 + llr256[1] = simde_mm256_adds_epi16(llr256[1],simde_mm256_permute2x128_si256(xmmp1, xmmp5, 0x20)); // A2 B2 C2 D2 A3 B3 C3 D3 + llr256[2] = simde_mm256_adds_epi16(llr256[2],simde_mm256_permute2x128_si256(xmmp2, xmmp4, 0x31)); // A4 B4 C4 D4 A5 B5 C5 D5 + llr256[3] = simde_mm256_adds_epi16(llr256[3],simde_mm256_permute2x128_si256(xmmp1, xmmp5, 0x31)); // A6 B6 C6 D6 A7 B7 C7 D7 + llr256+=4; + } + } + simde__m128i *llr128 = (simde__m128i*)llr256; + if ((length&7) >= 4) { //there is a single 128-bit input element remaining + int nb_re128 = length>>2; + simde__m128i xmm0,xmm1,xmm2,xmm3,xmm4,xmm5,xmm6; + simde__m128i complex_shuffle128 = simde_mm_set_epi8(13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + simde__m128i conj128 = simde_mm_set_epi16(1,-1,1,-1,1,-1,1,-1); + simde__m128i *rxF128 = (simde__m128i*)rxF; + simde__m128i *ulch128 = (simde__m128i*)ul_ch; + simde__m128i QAM_amp = simde_mm_set1_epi16(QAM256_n1); // 2/sqrt(10) + simde__m128i QAM_ampb = simde_mm_set1_epi16(QAM256_n2); + simde__m128i QAM_ampc = simde_mm_set1_epi16(QAM256_n3); + + xmm0 = simde_mm_madd_epi16(ulch128[nb_re128-1],rxF128[nb_re128-1]); + // xmm0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmm1 = simde_mm_shuffle_epi8(ulch128[nb_re128-1],complex_shuffle128); + xmm1 = simde_mm_sign_epi16(xmm1,conj128); + xmm1 = simde_mm_madd_epi16(xmm1,rxF128[nb_re128-1]); + // xmm1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmm0 = simde_mm_srai_epi32(xmm0,output_shift); + xmm1 = simde_mm_srai_epi32(xmm1,output_shift); + xmm2 = simde_mm_unpacklo_epi32(xmm0,xmm1); + xmm3 = simde_mm_unpackhi_epi32(xmm0,xmm1); + xmm4 = simde_mm_packs_epi32(xmm2,xmm3); + + // compute channel amplitude for LLR + xmm0 = simde_mm_madd_epi16(ulch128[nb_re128-1],ulch128[nb_re128-1]); + xmm0 = simde_mm_srai_epi32(xmm0,output_shift); + xmm0 = simde_mm_packs_epi32(xmm0,xmm0); // contains 16 LLRs + xmm2 = simde_mm_unpacklo_epi16(xmm0,xmm0); + xmm1 = simde_mm_mulhrs_epi16(xmm2,QAM_amp); + xmm5 = simde_mm_mulhrs_epi16(xmm2,QAM_ampb); + xmm6 = simde_mm_mulhrs_epi16(xmm2,QAM_ampc); + xmm0 = simde_mm_abs_epi16(xmm4); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| + xmm0 = simde_mm_subs_epi16(xmm1,xmm0); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 + // xmmp2 contains 8 LLRs + xmm1 = simde_mm_abs_epi16(xmm0); + xmm1 = simde_mm_subs_epi16(xmm5,xmm1); // contains 8 LLRs + xmm2 = simde_mm_abs_epi16(xmm1); + xmm2 = simde_mm_subs_epi16(xmm6,xmm2); // contains 8 LLRs + // rxF[i] A0 A1 A2 A3 + // xmm0 B0 B1 B2 B3 + // xmm1 C0 C1 C2 C3 + // xmm2 D0 D1 D2 D3 + xmm3 = simde_mm_unpacklo_epi32(rxF128[nb_re128-1],xmm0); // A0 B0 A1 B1 + xmm4 = simde_mm_unpackhi_epi32(rxF128[nb_re128-1],xmm0); // A2 B2 A3 B3 + xmm5 = simde_mm_unpacklo_epi32(xmm1,xmm2); // C0 D0 C1 D1 + xmm6 = simde_mm_unpackhi_epi32(xmm1,xmm2); // C2 D2 C3 D3 + if (aarx == 0) { + llr128[0] = simde_mm_unpacklo_epi64(xmm3,xmm5); // A0 B0 C0 D0 + llr128[1] = simde_mm_unpackhi_epi64(xmm3,xmm5); // A1 B1 C1 D1 + llr128[2] = simde_mm_unpacklo_epi64(xmm4,xmm6); // A2 B2 C2 D2 + llr128[3] = simde_mm_unpackhi_epi64(xmm4,xmm6); // A3 B3 C3 D3 + llr128+=4; + } + else + { + llr128[0] = simde_mm_adds_epi16(llr128[0],simde_mm_unpacklo_epi64(xmm3,xmm5)); // A0 B0 C0 D0 + llr128[1] = simde_mm_adds_epi16(llr128[1],simde_mm_unpackhi_epi64(xmm3,xmm5)); // A1 B1 C1 D1 + llr128[2] = simde_mm_adds_epi16(llr128[2],simde_mm_unpacklo_epi64(xmm4,xmm6)); // A2 B2 C2 D2 + llr128[3] = simde_mm_adds_epi16(llr128[3],simde_mm_unpackhi_epi64(xmm4,xmm6)); // A3 B3 C3 D3 + llr128+=4; + } + } +#else + simde__m128i xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6; + simde__m128i complex_shuffle128 = simde_mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); + simde__m128i conj128 = simde_mm_set_epi16(1, -1, 1, -1, 1, -1, 1, -1); + simde__m128i *rxF128 = (simde__m128i*)rxF; + simde__m128i *ulch128 = (simde__m128i*)ul_ch; + simde__m128i QAM_amp = simde_mm_set1_epi16(QAM256_n1); + simde__m128i QAM_ampb = simde_mm_set1_epi16(QAM256_n2); + simde__m128i QAM_ampc = simde_mm_set1_epi16(QAM256_n3); + simde__m128i *llr128 = (simde__m128i*)llr; + for (int i = 0; i < (length >> 2); i++) + { + xmm0 = simde_mm_madd_epi16(ulch128[i], rxF128[i]); + // xmm0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmm1 = simde_mm_shuffle_epi8(ulch128[i], complex_shuffle128); + xmm1 = simde_mm_sign_epi16(xmm1, conj128); + xmm1 = simde_mm_madd_epi16(xmm1, rxF128[i]); + // xmm1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmm0 = simde_mm_srai_epi32(xmm0, output_shift); + xmm1 = simde_mm_srai_epi32(xmm1, output_shift); + xmm2 = simde_mm_unpacklo_epi32(xmm0, xmm1); + xmm3 = simde_mm_unpackhi_epi32(xmm0, xmm1); + xmm4 = simde_mm_packs_epi32(xmm2, xmm3); + + // compute channel amplitude for LLR + xmm0 = simde_mm_madd_epi16(ulch128[i], ulch128[i]); // |h|^2 + xmm0 = simde_mm_srai_epi32(xmm0, output_shift); + xmm0 = simde_mm_packs_epi32(xmm0, xmm0); + xmm2 = simde_mm_unpacklo_epi16(xmm0, xmm0); + + xmm1 = simde_mm_mulhrs_epi16(xmm2, QAM_amp); + xmm5 = simde_mm_mulhrs_epi16(xmm2, QAM_ampb); + xmm6 = simde_mm_mulhrs_epi16(xmm2, QAM_ampc); + + xmm0 = simde_mm_abs_epi16(xmm4); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| + xmm0 = simde_mm_subs_epi16(xmm1, xmm0); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 + xmm1 = simde_mm_abs_epi16(xmm0); + xmm1 = simde_mm_subs_epi16(xmm5, xmm1); // contains 8 LLRs + xmm2 = simde_mm_abs_epi16(xmm1); + xmm2 = simde_mm_subs_epi16(xmm6, xmm2); // contains 8 LLRs + // rxF[i] A0 A1 A2 A3 + // xmm0 B0 B1 B2 B3 + // xmm1 C0 C1 C2 C3 + // xmm2 D0 D1 D2 D3 + xmm3 = simde_mm_unpacklo_epi32(xmm4, xmm0); // A0 B0 A1 B1 + xmm4 = simde_mm_unpackhi_epi32(xmm4, xmm0); // A2 B2 A3 B3 + xmm5 = simde_mm_unpacklo_epi32(xmm1, xmm2); // C0 D0 C1 D1 + xmm6 = simde_mm_unpackhi_epi32(xmm1, xmm2); // C2 D2 C3 D3 + if (aarx == 0) { + llr128[0] = simde_mm_unpacklo_epi64(xmm3, xmm5); // A0 B0 C0 D0 + llr128[1] = simde_mm_unpackhi_epi64(xmm3, xmm5); // A1 B1 C1 D1 + llr128[2] = simde_mm_unpacklo_epi64(xmm4, xmm6); // A2 B2 C2 D2 + llr128[3] = simde_mm_unpackhi_epi64(xmm4, xmm6); // A3 B3 C3 D3 + } + else { + llr128[0] = simde_mm_adds_epi16(llr128[0], simde_mm_unpacklo_epi64(xmm3, xmm5)); // A0 B0 C0 D0 + llr128[1] = simde_mm_adds_epi16(llr128[1], simde_mm_unpackhi_epi64(xmm3, xmm5)); // A1 B1 C1 D1 + llr128[2] = simde_mm_adds_epi16(llr128[2], simde_mm_unpacklo_epi64(xmm4, xmm6)); // A2 B2 C2 D2 + llr128[3] = simde_mm_adds_epi16(llr128[3], simde_mm_unpackhi_epi64(xmm4, xmm6)); // A3 B3 C3 D3 + } + llr128+=4; + } + if (length & 3) + { + simde__m64 *llr64 = (simde__m64*) llr128; + int i = (length>>1) - 1; + simde__m64 xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6; + simde__m64 complex_shuffle64 = simde_mm_set_pi8(5,4,7,6,1,0,3,2); + simde__m64 conj64 = simde_mm_set_pi16(1, -1, 1, -1); + simde__m64 *rxF64 = (simde__m64*)rxF; + simde__m64 *ulch64 = (simde__m64*)ul_ch; + simde__m64 QAM_amp = simde_mm_set1_pi16(QAM256_n1); + simde__m64 QAM_ampb = simde_mm_set1_pi16(QAM256_n2); + simde__m64 QAM_ampc = simde_mm_set1_pi16(QAM256_n3); + + xmm0 = simde_mm_madd_pi16(ulch64[i], rxF64[i]); + // xmm0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmm1 = simde_mm_shuffle_pi8(ulch64[i], complex_shuffle64); + xmm1 = simde_mm_sign_pi16(xmm1, conj64); + xmm1 = simde_mm_madd_pi16(xmm1, rxF64[i]); + // xmm1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmm0 = simde_mm_srai_pi32(xmm0, output_shift); + xmm1 = simde_mm_srai_pi32(xmm1, output_shift); + xmm2 = simde_mm_unpacklo_pi32(xmm0, xmm1); + xmm3 = simde_mm_unpackhi_pi32(xmm0, xmm1); + xmm4 = simde_mm_packs_pi32(xmm2, xmm3); + + // compute channel amplitude for LLR + xmm0 = simde_mm_madd_pi16(ulch64[i], ulch64[i]); // |h|^2 + xmm0 = simde_mm_srai_pi32(xmm0, output_shift); + xmm0 = simde_mm_packs_pi32(xmm0, xmm0); + xmm2 = simde_mm_unpacklo_pi16(xmm0, xmm0); + + xmm1 = simde_mm_mulhrs_pi16(xmm2, QAM_amp); + xmm5 = simde_mm_mulhrs_pi16(xmm2, QAM_ampb); + xmm6 = simde_mm_mulhrs_pi16(xmm2, QAM_ampc); + + xmm0 = simde_mm_abs_pi16(xmm4); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| + xmm0 = simde_mm_subs_pi16(xmm1, xmm0); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 + xmm1 = simde_mm_abs_pi16(xmm0); + xmm1 = simde_mm_subs_pi16(xmm5, xmm1); // contains 8 LLRs + xmm2 = simde_mm_abs_pi16(xmm1); + xmm2 = simde_mm_subs_pi16(xmm6, xmm2); // contains 8 LLRs + + xmm3 = simde_mm_unpacklo_pi32(xmm4, xmm0); + xmm4 = simde_mm_unpackhi_pi32(xmm4, xmm0); + xmm5 = simde_mm_unpacklo_pi32(xmm1, xmm2); + xmm6 = simde_mm_unpackhi_pi32(xmm1, xmm2); + if (aarx == 0) { + llr64[0] = simde_m_punpckldq(xmm3, xmm5); + llr64[1] = simde_m_punpckhdq(xmm3, xmm5); + llr64[2] = simde_m_punpckldq(xmm4, xmm6); + llr64[3] = simde_m_punpckhdq(xmm4, xmm6); + } + else + { + llr64[0] = simde_mm_adds_pi16(llr64[0], simde_m_punpckldq(xmm3, xmm5)); + llr64[1] = simde_mm_adds_pi16(llr64[1], simde_m_punpckhdq(xmm3, xmm5)); + llr64[2] = simde_mm_adds_pi16(llr64[2], simde_m_punpckldq(xmm4, xmm6)); + llr64[3] = simde_mm_adds_pi16(llr64[3], simde_m_punpckhdq(xmm4, xmm6)); + } + } +#endif +} + +void inner_rx_qpsk_2layer (NR_DL_FRAME_PARMS *frame_parms, + NR_gNB_PUSCH *pusch_vars, + nfapi_nr_pusch_pdu_t *rel15_ul, + int **rxF, + int **ul_ch, + int16_t **llr, + int nb_layer, + int nb_rx_ant, + int soffset, + int length, + int symbol, + int short nb_rb, + int dmrs_symbol_flag, + int output_shift) +{ + int add_shift = 0; + if (length % 8) + add_shift = 8 - length % 8; + int32_t rxFext[nb_rx_ant][length + add_shift] __attribute__((aligned(32))); + int32_t chFext[nb_layer*nb_rx_ant][length + add_shift] __attribute__((aligned(32))); + for (int aarx = 0; aarx < nb_rx_ant; aarx++) + { + for (int aatx = 0; aatx < nb_layer; aatx++) + { + nr_ulsch_extract_rbs0((c16_t *)rxF[aarx], + pusch_vars->ul_ch_estimates[aatx * nb_rx_ant + aarx], + rxFext[aarx], + chFext[aatx * nb_rx_ant + aarx], + soffset+(symbol * frame_parms->ofdm_symbol_size), + pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, + aarx, + dmrs_symbol_flag, + rel15_ul, + frame_parms); + } + } + int32_t rho[nb_layer*nb_layer][length + add_shift] __attribute__((aligned(32))); + int32_t rxFext_comp[nb_layer][length + add_shift] __attribute__((aligned(32))); + for (int aarx = 0; aarx < nb_rx_ant; aarx++) + { + for (int aatx = 0; aatx < nb_layer; aatx++) + { + for (int atx = 0; atx < nb_layer; atx++) + { +#ifdef USE_128BIT + simde__m128i mmtmpD0, mmtmpD1, mmtmpD2, mmtmpD3, mmtmpD4; + simde__m128i *rho128 = (simde__m128i *)rho[aatx*nb_layer+atx]; + simde__m128i *ul_ch128 = (simde__m128i *)chFext[aatx * nb_rx_ant + aarx]; + simde__m128i *ul_ch128_2 = (simde__m128i *)chFext[atx * nb_rx_ant + aarx]; + for (int i = 0; i < (length >> 2)+((length&3)?1:0); i++) + { + // multiply by conjugated channel + mmtmpD0 = simde_mm_madd_epi16(ul_ch128[i], ul_ch128_2[i]); + // mmtmpD0 contains real part of 4 consecutive outputs (32-bit) + mmtmpD1 = simde_mm_shufflelo_epi16(ul_ch128[i], SIMDE_MM_SHUFFLE(2,3,0,1)); + mmtmpD1 = simde_mm_shufflehi_epi16(mmtmpD1, SIMDE_MM_SHUFFLE(2,3,0,1)); + mmtmpD1 = simde_mm_sign_epi16(mmtmpD1, *(simde__m128i*)&conjugate[0]); + mmtmpD1 = simde_mm_madd_epi16(mmtmpD1, ul_ch128_2[i]); + // mmtmpD1 contains imag part of 4 consecutive outputs (32-bit) + mmtmpD0 = simde_mm_srai_epi32(mmtmpD0, output_shift); + mmtmpD1 = simde_mm_srai_epi32(mmtmpD1, output_shift); + mmtmpD2 = simde_mm_unpacklo_epi32(mmtmpD0, mmtmpD1); + mmtmpD3 = simde_mm_unpackhi_epi32(mmtmpD0, mmtmpD1); + mmtmpD4 = simde_mm_packs_epi32(mmtmpD2, mmtmpD3); + if (aarx == 0) + rho128[i] = mmtmpD4; + else + rho128[i] = simde_mm_adds_epi16(rho128[i], mmtmpD4); + } +#else + simde__m256i mmtmpD0, mmtmpD1, mmtmpD2, mmtmpD3, mmtmpD4; + simde__m256i *rho256 = (simde__m256i *)rho[aatx*nb_layer+atx]; + simde__m256i *ul_ch256 = (simde__m256i *)chFext[aatx * nb_rx_ant + aarx]; + simde__m256i *ul_ch256_2 = (simde__m256i *)chFext[atx * nb_rx_ant + aarx]; + register simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + register simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); + for (int i = 0; i < ((length >> 3)+((length&7)?1:0)); i++) + { + // multiply by conjugated channel + mmtmpD0 = simde_mm256_madd_epi16(ul_ch256[i], ul_ch256_2[i]); + // mmtmpD0 contains real part of 4 consecutive outputs (32-bit) + mmtmpD1 = simde_mm256_shuffle_epi8(ul_ch256[i], complex_shuffle256); + mmtmpD1 = simde_mm256_sign_epi16(mmtmpD1, conj256); + mmtmpD1 = simde_mm256_madd_epi16(mmtmpD1, ul_ch256_2[i]); + // mmtmpD1 contains imag part of 4 consecutive outputs (32-bit) + mmtmpD0 = simde_mm256_srai_epi32(mmtmpD0, output_shift); + mmtmpD1 = simde_mm256_srai_epi32(mmtmpD1, output_shift); + mmtmpD2 = simde_mm256_unpacklo_epi32(mmtmpD0, mmtmpD1); + mmtmpD3 = simde_mm256_unpackhi_epi32(mmtmpD0, mmtmpD1); + mmtmpD4 = simde_mm256_packs_epi32(mmtmpD2, mmtmpD3); + if (aarx == 0) + rho256[i] = mmtmpD4; + else + rho256[i] = simde_mm256_adds_epi16(rho256[i], mmtmpD4); + } +#endif + } + // compensation +#ifdef USE_128BIT + simde__m128i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; + register simde__m128i complex_shuffle128 = simde_mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); + register simde__m128i conj128 = simde_mm_set_epi16(1, -1, 1, -1, 1, -1, 1, -1); + simde__m128i *rxF128 = (simde__m128i*)rxFext[aarx]; + simde__m128i *ulch128 = (simde__m128i*)chFext[aatx * nb_rx_ant + aarx]; + simde__m128i *rxF_comp128 = (simde__m128i*)rxFext_comp[aatx]; + for (int i = 0; i < (length>>2) + ((length&3)?1:0); i++) + { + xmmp0 = simde_mm_madd_epi16(ulch128[i], rxF128[i]); + // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp1 = simde_mm_shuffle_epi8(ulch128[i], complex_shuffle128); + xmmp1 = simde_mm_sign_epi16(xmmp1, conj128); + xmmp1 = simde_mm_madd_epi16(xmmp1, rxF128[i]); + // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp0 = simde_mm_srai_epi32(xmmp0, output_shift); + xmmp1 = simde_mm_srai_epi32(xmmp1, output_shift); + xmmp2 = simde_mm_unpacklo_epi32(xmmp0, xmmp1); + xmmp3 = simde_mm_unpackhi_epi32(xmmp0, xmmp1); + xmmp4 = simde_mm_packs_epi32(xmmp2, xmmp3); + + if (aarx == 0) + *rxF_comp128 = xmmp4; + else + *rxF_comp128 = simde_mm_adds_epi16(*rxF_comp128, xmmp4); + + rxF_comp128++; + } +#else + simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; + register simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + register simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); + simde__m256i *rxF256 = (simde__m256i*)rxFext[aarx]; + simde__m256i *ulch256 = (simde__m256i*)chFext[aatx * nb_rx_ant + aarx]; + simde__m256i *rxF_comp256 = (simde__m256i*)rxFext_comp[aatx]; + for (int i = 0; i < (length>>3) + ((length&7)?1:0); i++) + { + xmmp0 = simde_mm256_madd_epi16(ulch256[i], rxF256[i]); + // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp1 = simde_mm256_shuffle_epi8(ulch256[i], complex_shuffle256); + xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); + xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF256[i]); + // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); + xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); + xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); + xmmp4 = simde_mm256_packs_epi32(xmmp2, xmmp3); + + if (aarx == 0) + *rxF_comp256 = xmmp4; + else + *rxF_comp256 = simde_mm256_adds_epi16(*rxF_comp256, xmmp4); + + rxF_comp256++; + } +#endif + } + } + c16_t *rho0 = (c16_t *)rho[1]; + c16_t *rho1 = (c16_t *)rho[2]; + c16_t *llr_0 = (c16_t *)&llr[0][pusch_vars->llr_offset[symbol]]; + c16_t *llr_1 = (c16_t *)&llr[1][pusch_vars->llr_offset[symbol]]; + + nr_ulsch_qpsk_qpsk((c16_t *)rxFext_comp[0], (c16_t *)rxFext_comp[1], llr_0, rho0, length); + nr_ulsch_qpsk_qpsk((c16_t *)rxFext_comp[1], (c16_t *)rxFext_comp[0], llr_1, rho1, length); + + nr_ulsch_shift_llr(pusch_vars->llr_layers, length, pusch_vars->llr_offset[symbol] >> 1, 2, 4); + +} + + +void inner_rx_16qam_2layer (NR_DL_FRAME_PARMS *frame_parms, + NR_gNB_PUSCH *pusch_vars, + nfapi_nr_pusch_pdu_t *rel15_ul, + int **rxF, + int **ul_ch, + int16_t **llr, + int nb_layer, + int nb_rx_ant, + int soffset, + int length, + int symbol, + int short nb_rb, + int dmrs_symbol_flag, + int output_shift) +{ + int add_shift = 0; + if (length % 8) + add_shift = 8 - length % 8; + + int32_t rxFext[nb_rx_ant][length + add_shift] __attribute__((aligned(32))); + int32_t chFext[nb_layer*nb_rx_ant][length + add_shift] __attribute__((aligned(32))); + for (int aarx = 0; aarx < nb_rx_ant; aarx++) + { + for (int aatx = 0; aatx < nb_layer; aatx++) + { + nr_ulsch_extract_rbs0((c16_t *)rxF[aarx], + pusch_vars->ul_ch_estimates[aatx * nb_rx_ant + aarx], + rxFext[aarx], + chFext[aatx * nb_rx_ant + aarx], + soffset+(symbol * frame_parms->ofdm_symbol_size), + pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, + aarx, + dmrs_symbol_flag, + rel15_ul, + frame_parms); + } + } + + int32_t rho[nb_layer*nb_layer][length + add_shift] __attribute__((aligned(32))); + int32_t rxFext_comp[nb_layer][length + add_shift] __attribute__((aligned(32))); + int32_t ul_ch_mag[nb_layer][length + add_shift] __attribute__((aligned(32))); + for (int aatx = 0; aatx < nb_layer; aatx++) + { + for (int aarx = 0; aarx < nb_rx_ant; aarx++) + { + for (int atx = 0; atx < nb_layer; atx++) + { +#ifdef USE_128BIT + simde__m128i mmtmpD0, mmtmpD1, mmtmpD2, mmtmpD3; + simde__m128i *rho128 = (simde__m128i *)rho[aatx*nb_layer+atx]; + simde__m128i *ul_ch128 = (simde__m128i *)chFext[aatx * nb_rx_ant + aarx]; + simde__m128i *ul_ch128_2 = (simde__m128i *)chFext[atx * nb_rx_ant + aarx]; + for (int i = 0; i < (length>>2)+((length&3)?1:0); i++) + { + // multiply by conjugated channel + mmtmpD0 = simde_mm_madd_epi16(ul_ch128[i], ul_ch128_2[i]); + // mmtmpD0 contains real part of 4 consecutive outputs (32-bit) + mmtmpD1 = simde_mm_shufflelo_epi16(ul_ch128[i], SIMDE_MM_SHUFFLE(2,3,0,1)); + mmtmpD1 = simde_mm_shufflehi_epi16(mmtmpD1, SIMDE_MM_SHUFFLE(2,3,0,1)); + mmtmpD1 = simde_mm_sign_epi16(mmtmpD1, *(simde__m128i*)&conjugate[0]); + mmtmpD1 = simde_mm_madd_epi16(mmtmpD1, ul_ch128_2[i]); + // mmtmpD1 contains imag part of 4 consecutive outputs (32-bit) + mmtmpD0 = simde_mm_srai_epi32(mmtmpD0, output_shift); + mmtmpD1 = simde_mm_srai_epi32(mmtmpD1, output_shift); + mmtmpD2 = simde_mm_unpacklo_epi32(mmtmpD0, mmtmpD1); + mmtmpD3 = simde_mm_unpackhi_epi32(mmtmpD0, mmtmpD1); + if (aarx == 0) + rho128[i] = simde_mm_packs_epi32(mmtmpD2, mmtmpD3); + else + rho128[i] = simde_mm_adds_epi16(rho128[i], simde_mm_packs_epi32(mmtmpD2, mmtmpD3)); + } +#else + simde__m256i mmtmpD0, mmtmpD1, mmtmpD2, mmtmpD3; + simde__m256i *rho256 = (simde__m256i *)rho[aatx*nb_layer+atx]; + simde__m256i *ul_ch256 = (simde__m256i *)chFext[aatx * nb_rx_ant + aarx]; + simde__m256i *ul_ch256_2 = (simde__m256i *)chFext[atx * nb_rx_ant + aarx]; + register simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + register simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); + for (int i = 0; i < (length >> 3)+((length&7)?1:0); i++) + { + // multiply by conjugated channel + mmtmpD0 = simde_mm256_madd_epi16(ul_ch256[i], ul_ch256_2[i]); + // mmtmpD0 contains real part of 4 consecutive outputs (32-bit) + mmtmpD1 = simde_mm256_shuffle_epi8(ul_ch256[i], complex_shuffle256); + mmtmpD1 = simde_mm256_sign_epi16(mmtmpD1, conj256); + mmtmpD1 = simde_mm256_madd_epi16(mmtmpD1, ul_ch256_2[i]); + // mmtmpD1 contains imag part of 4 consecutive outputs (32-bit) + mmtmpD0 = simde_mm256_srai_epi32(mmtmpD0, output_shift); + mmtmpD1 = simde_mm256_srai_epi32(mmtmpD1, output_shift); + mmtmpD2 = simde_mm256_unpacklo_epi32(mmtmpD0, mmtmpD1); + mmtmpD3 = simde_mm256_unpackhi_epi32(mmtmpD0, mmtmpD1); + if (aarx == 0) + rho256[i] = simde_mm256_packs_epi32(mmtmpD2, mmtmpD3); + else + rho256[i] = simde_mm256_adds_epi16(rho256[i], simde_mm256_packs_epi32(mmtmpD2, mmtmpD3)); + } +#endif + } + // compensation +#ifdef USE_128BIT + simde__m128i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; + register simde__m128i complex_shuffle128 = simde_mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); + register simde__m128i conj128 = simde_mm_set_epi16(1, -1, 1, -1, 1, -1, 1, -1); + register simde__m128i QAM_amp128 = simde_mm_set1_epi16(QAM16_n1); // 2/sqrt(10) + simde__m128i *rxF128 = (simde__m128i*)rxFext[aarx]; + simde__m128i *ulch128 = (simde__m128i*)chFext[aatx * nb_rx_ant + aarx]; + simde__m128i *rxF_comp128 = (simde__m128i*)rxFext_comp[aatx]; + simde__m128i *ul_ch_mag128 = (simde__m128i*)ul_ch_mag[aatx]; + for (int i = 0; i < (length>>2)+((length&3)?1:0); i++) + { + xmmp0 = simde_mm_madd_epi16(ulch128[i], rxF128[i]); + // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp1 = simde_mm_shuffle_epi8(ulch128[i], complex_shuffle128); + xmmp1 = simde_mm_sign_epi16(xmmp1, conj128); + xmmp1 = simde_mm_madd_epi16(xmmp1, rxF128[i]); + // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp0 = simde_mm_srai_epi32(xmmp0, output_shift); + xmmp1 = simde_mm_srai_epi32(xmmp1, output_shift); + xmmp2 = simde_mm_unpacklo_epi32(xmmp0, xmmp1); + xmmp3 = simde_mm_unpackhi_epi32(xmmp0, xmmp1); + xmmp4 = simde_mm_packs_epi32(xmmp2, xmmp3); + + xmmp0 = simde_mm_madd_epi16(ulch128[i], ulch128[i]); // |h|^2 + xmmp0 = simde_mm_srai_epi32(xmmp0, output_shift); + xmmp0 = simde_mm_packs_epi32(xmmp0, xmmp0); + xmmp1 = simde_mm_unpacklo_epi16(xmmp0, xmmp0); + xmmp1 = simde_mm_mulhrs_epi16(xmmp1, QAM_amp128); + + if (aarx == 0) + { + *rxF_comp128 = xmmp4; + *ul_ch_mag128 = xmmp1; + } + else + { + *rxF_comp128 = simde_mm_adds_epi16(*rxF_comp128, xmmp4); + *ul_ch_mag128 = simde_mm_adds_epi16(*ul_ch_mag128, xmmp1); + } + rxF_comp128++; + ul_ch_mag128++; + } +#else + simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; + register simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + register simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); + register simde__m256i QAM_amp256 = simde_mm256_set1_epi16(QAM16_n1); // 2/sqrt(10) + simde__m256i *rxF256 = (simde__m256i*)rxFext[aarx]; + simde__m256i *ulch256 = (simde__m256i*)chFext[aatx * nb_rx_ant + aarx]; + simde__m256i *rxF_comp256 = (simde__m256i*)rxFext_comp[aatx]; + simde__m256i *ul_ch_mag256 = (simde__m256i*)ul_ch_mag[aatx]; + for (int i = 0; i < (length>>3) + ((length&7)?1:0); i++) + { + xmmp0 = simde_mm256_madd_epi16(ulch256[i], rxF256[i]); + // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp1 = simde_mm256_shuffle_epi8(ulch256[i], complex_shuffle256); + xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); + xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF256[i]); + // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); + xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); + xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); + xmmp4 = simde_mm256_packs_epi32(xmmp2, xmmp3); + + xmmp0 = simde_mm256_madd_epi16(ulch256[i], ulch256[i]); // |h|^2 + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp0 = simde_mm256_packs_epi32(xmmp0, xmmp0); + xmmp1 = simde_mm256_unpacklo_epi16(xmmp0, xmmp0); + xmmp1 = simde_mm256_mulhrs_epi16(xmmp1, QAM_amp256); + + if (aarx == 0) + { + *rxF_comp256 = xmmp4; + *ul_ch_mag256 = xmmp1; + } + else + { + *rxF_comp256 = simde_mm256_adds_epi16(*rxF_comp256, xmmp4); + *ul_ch_mag256 = simde_mm256_adds_epi16(*ul_ch_mag256, xmmp1); + } + rxF_comp256++; + ul_ch_mag256++; + } +#endif + } + } + c16_t *rho0 = (c16_t *)rho[1]; + c16_t *rho1 = (c16_t *)rho[2]; + c16_t *llr_0 = (c16_t *)&llr[0][pusch_vars->llr_offset[symbol]]; + c16_t *llr_1 = (c16_t *)&llr[1][pusch_vars->llr_offset[symbol]]; + c16_t *ul_ch_mag0 = (c16_t *)ul_ch_mag[0]; + c16_t *ul_ch_mag1 = (c16_t *)ul_ch_mag[1]; + nr_ulsch_qam16_qam16((c16_t *)rxFext_comp[0], (c16_t *)rxFext_comp[1], ul_ch_mag0, ul_ch_mag1, llr_0, rho0, length); + nr_ulsch_qam16_qam16((c16_t *)rxFext_comp[1], (c16_t *)rxFext_comp[0], ul_ch_mag1, ul_ch_mag0, llr_1, rho1, length); +} + +void inner_rx_64qam_2layer (NR_DL_FRAME_PARMS *frame_parms, + NR_gNB_PUSCH *pusch_vars, + nfapi_nr_pusch_pdu_t *rel15_ul, + int **rxF, + int **ul_ch, + int16_t **llr, + int nb_layer, + int nb_rx_ant, + int soffset, + int length, + int symbol, + int short nb_rb, + int dmrs_symbol_flag, + int output_shift) +{ + int add_shift = 0; + if (length % 8) + add_shift = 8 - length % 8; + + int32_t rxFext[nb_rx_ant][length + add_shift] __attribute__((aligned(32))); + int32_t chFext[nb_layer*nb_rx_ant][length + add_shift] __attribute__((aligned(32))); + for (int aarx = 0; aarx < nb_rx_ant; aarx++) + { + for (int aatx = 0; aatx < nb_layer; aatx++) + { + nr_ulsch_extract_rbs0((c16_t *)rxF[aarx], + pusch_vars->ul_ch_estimates[aatx * nb_rx_ant + aarx], + rxFext[aarx], + chFext[aatx * nb_rx_ant + aarx], + soffset+(symbol * frame_parms->ofdm_symbol_size), + pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, + aarx, + dmrs_symbol_flag, + rel15_ul, + frame_parms); + } + } + int32_t rho[nb_layer*nb_layer][length + add_shift] __attribute__((aligned(32))); + int32_t rxFext_comp[nb_layer][length + add_shift] __attribute__((aligned(32))); + int32_t ul_ch_mag[nb_layer][length + add_shift] __attribute__((aligned(32))); + for (int aatx = 0; aatx < nb_layer; aatx++) + { + for (int aarx = 0; aarx < nb_rx_ant; aarx++) + { + for (int atx = 0; atx < nb_layer; atx++) + { +#ifdef USE_128BIT + simde__m128i mmtmpD0, mmtmpD1, mmtmpD2, mmtmpD3; + simde__m128i *rho128 = (simde__m128i *)rho[aatx*nb_layer+atx]; + simde__m128i *ul_ch128 = (simde__m128i *)chFext[aatx * nb_rx_ant + aarx]; + simde__m128i *ul_ch128_2 = (simde__m128i *)chFext[atx * nb_rx_ant + aarx]; + for (int i = 0; i < (length>>2)+((length&3)?1:0); i++) + { + // multiply by conjugated channel + mmtmpD0 = simde_mm_madd_epi16(ul_ch128[i], ul_ch128_2[i]); + // mmtmpD0 contains real part of 4 consecutive outputs (32-bit) + mmtmpD1 = simde_mm_shufflelo_epi16(ul_ch128[i], SIMDE_MM_SHUFFLE(2,3,0,1)); + mmtmpD1 = simde_mm_shufflehi_epi16(mmtmpD1, SIMDE_MM_SHUFFLE(2,3,0,1)); + mmtmpD1 = simde_mm_sign_epi16(mmtmpD1, *(simde__m128i*)&conjugate[0]); + mmtmpD1 = simde_mm_madd_epi16(mmtmpD1, ul_ch128_2[i]); + // mmtmpD1 contains imag part of 4 consecutive outputs (32-bit) + mmtmpD0 = simde_mm_srai_epi32(mmtmpD0, output_shift); + mmtmpD1 = simde_mm_srai_epi32(mmtmpD1, output_shift); + mmtmpD2 = simde_mm_unpacklo_epi32(mmtmpD0, mmtmpD1); + mmtmpD3 = simde_mm_unpackhi_epi32(mmtmpD0, mmtmpD1); + if (aarx == 0) + rho128[i] = simde_mm_packs_epi32(mmtmpD2, mmtmpD3); + else + rho128[i] = simde_mm_adds_epi16(rho128[i], simde_mm_packs_epi32(mmtmpD2, mmtmpD3)); + } +#else + simde__m256i mmtmpD0, mmtmpD1, mmtmpD2, mmtmpD3; + simde__m256i *rho256 = (simde__m256i *)rho[aatx*nb_layer+atx]; + simde__m256i *ul_ch256 = (simde__m256i *)chFext[aatx * nb_rx_ant + aarx]; + simde__m256i *ul_ch256_2 = (simde__m256i *)chFext[atx * nb_rx_ant + aarx]; + register simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + register simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); + for (int i = 0; i < (length >> 3)+((length&7)?1:0); i++) + { + // multiply by conjugated channel + mmtmpD0 = simde_mm256_madd_epi16(ul_ch256[i], ul_ch256_2[i]); + // mmtmpD0 contains real part of 4 consecutive outputs (32-bit) + mmtmpD1 = simde_mm256_shuffle_epi8(ul_ch256[i], complex_shuffle256); + mmtmpD1 = simde_mm256_sign_epi16(mmtmpD1, conj256); + mmtmpD1 = simde_mm256_madd_epi16(mmtmpD1, ul_ch256_2[i]); + // mmtmpD1 contains imag part of 4 consecutive outputs (32-bit) + mmtmpD0 = simde_mm256_srai_epi32(mmtmpD0, output_shift); + mmtmpD1 = simde_mm256_srai_epi32(mmtmpD1, output_shift); + mmtmpD2 = simde_mm256_unpacklo_epi32(mmtmpD0, mmtmpD1); + mmtmpD3 = simde_mm256_unpackhi_epi32(mmtmpD0, mmtmpD1); + if (aarx == 0) + rho256[i] = simde_mm256_packs_epi32(mmtmpD2, mmtmpD3); + else + rho256[i] = simde_mm256_adds_epi16(rho256[i], simde_mm256_packs_epi32(mmtmpD2, mmtmpD3)); + } +#endif + } + // compensation +#ifdef USE_128BIT + simde__m128i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; + register simde__m128i complex_shuffle128 = simde_mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); + register simde__m128i conj128 = simde_mm_set_epi16(1, -1, 1, -1, 1, -1, 1, -1); + register simde__m128i QAM_amp128 = simde_mm_set1_epi16(QAM64_n1); // 2/sqrt(10) + simde__m128i *rxF128 = (simde__m128i*)rxFext[aarx]; + simde__m128i *ulch128 = (simde__m128i*)chFext[aatx * nb_rx_ant + aarx]; + simde__m128i *rxF_comp128 = (simde__m128i*)rxFext_comp[aatx]; + simde__m128i *ul_ch_mag128 = (simde__m128i*)ul_ch_mag[aatx]; + for (int i = 0; i < (length>>2)+((length&3)?1:0); i++) + { + xmmp0 = simde_mm_madd_epi16(ulch128[i], rxF128[i]); + // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp1 = simde_mm_shuffle_epi8(ulch128[i], complex_shuffle128); + xmmp1 = simde_mm_sign_epi16(xmmp1, conj128); + xmmp1 = simde_mm_madd_epi16(xmmp1, rxF128[i]); + // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp0 = simde_mm_srai_epi32(xmmp0, output_shift); + xmmp1 = simde_mm_srai_epi32(xmmp1, output_shift); + xmmp2 = simde_mm_unpacklo_epi32(xmmp0, xmmp1); + xmmp3 = simde_mm_unpackhi_epi32(xmmp0, xmmp1); + xmmp4 = simde_mm_packs_epi32(xmmp2, xmmp3); + + xmmp0 = simde_mm_madd_epi16(ulch128[i], ulch128[i]); // |h|^2 + xmmp0 = simde_mm_srai_epi32(xmmp0, output_shift); + xmmp0 = simde_mm_packs_epi32(xmmp0, xmmp0); + xmmp1 = simde_mm_unpacklo_epi16(xmmp0, xmmp0); + xmmp1 = simde_mm_mulhrs_epi16(xmmp1, QAM_amp128); + + if (aarx == 0) + { + *rxF_comp128 = xmmp4; + *ul_ch_mag128 = xmmp1; + } + else + { + *rxF_comp128 = simde_mm_adds_epi16(*rxF_comp128, xmmp4); + *ul_ch_mag128 = simde_mm_adds_epi16(*ul_ch_mag128, xmmp1); + } + rxF_comp128++; + ul_ch_mag128++; + } +#else + simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; + register simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + register simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); + register simde__m256i QAM_amp256 = simde_mm256_set1_epi16(QAM64_n1); // 2/sqrt(10) + simde__m256i *rxF256 = (simde__m256i*)rxFext[aarx]; + simde__m256i *ulch256 = (simde__m256i*)chFext[aatx * nb_rx_ant + aarx]; + simde__m256i *rxF_comp256 = (simde__m256i*)rxFext_comp[aatx]; + simde__m256i *ul_ch_mag256 = (simde__m256i*)ul_ch_mag[aatx]; + for (int i = 0; i < (length>>3) + ((length&7)?1:0); i++) + { + xmmp0 = simde_mm256_madd_epi16(ulch256[i], rxF256[i]); + // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp1 = simde_mm256_shuffle_epi8(ulch256[i], complex_shuffle256); + xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); + xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF256[i]); + // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); + xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); + xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); + xmmp4 = simde_mm256_packs_epi32(xmmp2, xmmp3); + + xmmp0 = simde_mm256_madd_epi16(ulch256[i], ulch256[i]); // |h|^2 + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp0 = simde_mm256_packs_epi32(xmmp0, xmmp0); + xmmp1 = simde_mm256_unpacklo_epi16(xmmp0, xmmp0); + xmmp1 = simde_mm256_mulhrs_epi16(xmmp1, QAM_amp256); + + if (aarx == 0) + { + *rxF_comp256 = xmmp4; + *ul_ch_mag256 = xmmp1; + } + else + { + *rxF_comp256 = simde_mm256_adds_epi16(*rxF_comp256, xmmp4); + *ul_ch_mag256 = simde_mm256_adds_epi16(*ul_ch_mag256, xmmp1); + } + rxF_comp256++; + ul_ch_mag256++; + } +#endif + } + } + c16_t *rho0 = (c16_t *)rho[1]; + c16_t *rho1 = (c16_t *)rho[2]; + c16_t *llr_0 = (c16_t *)&llr[0][pusch_vars->llr_offset[symbol]]; + c16_t *llr_1 = (c16_t *)&llr[1][pusch_vars->llr_offset[symbol]]; + c16_t *ul_ch_mag0 = (c16_t *)ul_ch_mag[0]; + c16_t *ul_ch_mag1 = (c16_t *)ul_ch_mag[1]; + nr_ulsch_qam64_qam64((c16_t *)rxFext_comp[0], (c16_t *)rxFext_comp[1], ul_ch_mag0, ul_ch_mag1, llr_0, rho0, length); + nr_ulsch_qam64_qam64((c16_t *)rxFext_comp[1], (c16_t *)rxFext_comp[0], ul_ch_mag1, ul_ch_mag0, llr_1, rho1, length); +} + +void nr_pusch_symbol_processing_noprecoding(void *arg) +{ + + puschSymbolProc_t *rdata=(puschSymbolProc_t*)arg; + + PHY_VARS_gNB *gNB = rdata->gNB; + NR_DL_FRAME_PARMS *frame_parms = rdata->frame_parms; + nfapi_nr_pusch_pdu_t *rel15_ul = rdata->rel15_ul; + int ulsch_id = rdata->ulsch_id; + int slot = rdata->slot; + NR_gNB_PUSCH *pusch_vars = &gNB->pusch_vars[ulsch_id]; + int16_t *s = rdata->s; + for (int symbol = rdata->startSymbol; symbol < rdata->startSymbol+rdata->numSymbols; symbol++) + { + int dmrs_symbol_flag = (rel15_ul->ul_dmrs_symb_pos >> symbol) & 0x01; + int nb_re_pusch = gNB->pusch_vars[ulsch_id].ul_valid_re_per_slot[symbol]; + // this needs to be reworded for parrellization, we need a table which give dmrs symbol location + // used for chennel estimate, they are being run in parallel! + if (dmrs_symbol_flag == 1) + { + if ((rel15_ul->ul_dmrs_symb_pos >> ((symbol + 1) % frame_parms->symbols_per_slot)) & 0x01) + AssertFatal(1==0,"Double DMRS configuration is not yet supported\n"); + + gNB->pusch_vars[ulsch_id].dmrs_symbol = symbol; + } + + LOG_I(PHY,"symbol %d: nb_re_pusch %d, DMRS symbl used for Chest :%d \n", symbol, nb_re_pusch, gNB->pusch_vars[ulsch_id].dmrs_symbol); + + if (nb_re_pusch == 0) continue; + if (rel15_ul->nrOfLayers == 1) + { + int16_t *llr = &rdata->llr[pusch_vars->llr_offset[symbol]]; + void (*inner_rx)(int *,int *,int16_t *,int,int,int); + if (rel15_ul->qam_mod_order == 2) inner_rx = inner_rx_qpsk; + else if (rel15_ul->qam_mod_order == 4) inner_rx = inner_rx_16qam; + else if (rel15_ul->qam_mod_order == 6) inner_rx = inner_rx_64qam; + else if (rel15_ul->qam_mod_order == 8) inner_rx = inner_rx_256qam; + else AssertFatal(1==0,"rel15_ul->qam_mod_order %d, pusch_pdu->dmrs_config_type %d\n", + rel15_ul->qam_mod_order,rel15_ul->dmrs_config_type); + + int soffset = (slot&3)*frame_parms->symbols_per_slot*frame_parms->ofdm_symbol_size; + int32_t rxFext[nb_re_pusch+8] __attribute__((aligned(32))); + int32_t chFext[nb_re_pusch+8] __attribute__((aligned(32))); + int16_t llr_temp[(nb_re_pusch*rel15_ul->qam_mod_order)+16] __attribute__((aligned(32))); + for (int aa = 0; aa < frame_parms->nb_antennas_rx; aa++) + { + nr_ulsch_extract_rbs0(gNB->common_vars.rxdataF[aa], + gNB->pusch_vars[ulsch_id].ul_ch_estimates[aa], + rxFext, + chFext, + soffset+(symbol * frame_parms->ofdm_symbol_size), + gNB->pusch_vars[ulsch_id].dmrs_symbol*frame_parms->ofdm_symbol_size, + aa, + dmrs_symbol_flag, + rel15_ul, + frame_parms); + // demodulation + inner_rx(rxFext, chFext, llr_temp, aa, nb_re_pusch, gNB->pusch_vars[ulsch_id].log2_maxh); + } + + // unscrambling + simde__m64 *llr64 = (simde__m64 *) llr; + for (int i=0;i<(nb_re_pusch*rel15_ul->qam_mod_order)>>2;i++) + llr64[i] = simde_mm_mullo_pi16(((simde__m64 *)llr_temp)[i],((simde__m64 *)s)[i]); + + s += nb_re_pusch * rel15_ul->qam_mod_order; + llr += nb_re_pusch * rel15_ul->qam_mod_order; + } + else // MIMO for 2x2 + { + int soffset = (slot&3)*frame_parms->symbols_per_slot*frame_parms->ofdm_symbol_size; + void (*inner_rx)(NR_DL_FRAME_PARMS *, + NR_gNB_PUSCH *, + nfapi_nr_pusch_pdu_t *, + int32_t **, + int32_t **, + int16_t **, + int32_t, + int32_t, + int32_t, + int32_t, + int32_t, + int16_t, + int32_t, + int32_t); + if (rel15_ul->qam_mod_order == 2) inner_rx = inner_rx_qpsk_2layer; + else if (rel15_ul->qam_mod_order == 4) inner_rx = inner_rx_16qam_2layer; + else if (rel15_ul->qam_mod_order == 6) inner_rx = inner_rx_64qam_2layer; + else AssertFatal(1==0,"rel15_ul->qam_mod_order %d, pusch_pdu->dmrs_config_type %d\n", + rel15_ul->qam_mod_order,rel15_ul->dmrs_config_type); + + inner_rx(frame_parms, + pusch_vars, + rel15_ul, + (int32_t**)gNB->common_vars.rxdataF, + gNB->pusch_vars[ulsch_id].ul_ch_estimates, + rdata->llr_layers, + rel15_ul->nrOfLayers, + frame_parms->nb_antennas_rx, + soffset, + nb_re_pusch, // length + symbol, // symbol index + rel15_ul->rb_size, // ofdm size + dmrs_symbol_flag, + gNB->pusch_vars[ulsch_id].log2_maxh); + + // layer de-mapping + int16_t* llr_cw = &rdata->llr[pusch_vars->llr_offset[symbol] * rel15_ul->nrOfLayers]; + for (int i = 0; i < (nb_re_pusch); i++) + for (int l = 0; l < rel15_ul->nrOfLayers; l++) + for (int m = 0; m < rel15_ul->qam_mod_order; m++) + llr_cw[i*rel15_ul->nrOfLayers*rel15_ul->qam_mod_order+l*rel15_ul->qam_mod_order+m] = rdata->llr_layers[l][pusch_vars->llr_offset[symbol] + i*rel15_ul->qam_mod_order+m]; + + // unscrambling + simde__m64 *llr64 = (simde__m64 *) &rdata->llr[pusch_vars->llr_offset[symbol] * rel15_ul->nrOfLayers]; + for (int i = 0; i < (nb_re_pusch*rel15_ul->qam_mod_order*rel15_ul->nrOfLayers)>>2; i++) + llr64[i] = simde_mm_mullo_pi16(((simde__m64 *)llr64)[i], ((simde__m64 *)s)[i]); + + s += (nb_re_pusch*rel15_ul->qam_mod_order*rel15_ul->nrOfLayers); + } + } +} + //compute average channel_level on each (TX,RX) antenna pair void nr_ulsch_channel_level(int **ul_ch_estimates_ext, NR_DL_FRAME_PARMS *frame_parms, @@ -1919,3 +3578,234 @@ void nr_rx_pusch(PHY_VARS_gNB *gNB, GnbScopeUpdate(gNB, puschIQe, num_re_total); } } + +int nr_rx_pusch_tp(PHY_VARS_gNB *gNB, + uint8_t ulsch_id, + uint32_t frame, + uint8_t slot, + unsigned char harq_pid) +{ + uint8_t aarx; + uint32_t bwp_start_subcarrier; + + NR_DL_FRAME_PARMS *frame_parms = &gNB->frame_parms; + nfapi_nr_pusch_pdu_t *rel15_ul = &gNB->ulsch[ulsch_id].harq_process->ulsch_pdu; + + NR_gNB_PUSCH *pusch_vars = &gNB->pusch_vars[ulsch_id]; + pusch_vars->dmrs_symbol = INVALID_VALUE; + pusch_vars->cl_done = 0; + memset(pusch_vars->extraction_done,0,14*sizeof(int)); + gNB->nbSymb=0; + bwp_start_subcarrier = ((rel15_ul->rb_start + rel15_ul->bwp_start)*NR_NB_SC_PER_RB + frame_parms->first_carrier_offset) % frame_parms->ofdm_symbol_size; + LOG_D(PHY,"pusch %d.%d : bwp_start_subcarrier %d, rb_start %d, first_carrier_offset %d\n", frame,slot,bwp_start_subcarrier, rel15_ul->rb_start, frame_parms->first_carrier_offset); + LOG_D(PHY,"pusch %d.%d : ul_dmrs_symb_pos %x\n",frame,slot,rel15_ul->ul_dmrs_symb_pos); + + //---------------------------------------------------------- + //------------------- Channel estimation ------------------- + //---------------------------------------------------------- + start_meas(&gNB->ulsch_channel_estimation_stats); + int max_ch = 0; + for (uint8_t symbol = rel15_ul->start_symbol_index; symbol < (rel15_ul->start_symbol_index + rel15_ul->nr_of_symbols); symbol++) + { + uint8_t dmrs_symbol_flag = (rel15_ul->ul_dmrs_symb_pos >> symbol) & 0x01; + LOG_D(PHY, "symbol %d, dmrs_symbol_flag :%d\n", symbol, dmrs_symbol_flag); + if (dmrs_symbol_flag == 1) + { + if (pusch_vars->dmrs_symbol == INVALID_VALUE) + pusch_vars->dmrs_symbol = symbol; + for (int nl=0; nlnrOfLayers; nl++) + { + nr_pusch_channel_estimation(gNB, + slot, + get_dmrs_port(nl,rel15_ul->dmrs_ports), + symbol, + ulsch_id, + bwp_start_subcarrier, + rel15_ul, + &max_ch, + 0 /* nvar*/); + } + // measure the SNR from the channel estimation + nr_gnb_measurements(gNB, + &gNB->ulsch[ulsch_id], + pusch_vars, + symbol, + rel15_ul->nrOfLayers); + allocCast2D(n0_subband_power, + unsigned int, + gNB->measurements.n0_subband_power, + frame_parms->nb_antennas_rx, + frame_parms->N_RB_UL, + false); + for (aarx = 0; aarx < frame_parms->nb_antennas_rx; aarx++) + { + if (symbol == rel15_ul->start_symbol_index) + { + pusch_vars->ulsch_power[aarx] = 0; + pusch_vars->ulsch_noise_power[aarx] = 0; + } + for (int aatx = 0; aatx < rel15_ul->nrOfLayers; aatx++) { + pusch_vars->ulsch_power[aarx] += signal_energy_nodc( + &pusch_vars->ul_ch_estimates[aatx * gNB->frame_parms.nb_antennas_rx + aarx][symbol * frame_parms->ofdm_symbol_size], + rel15_ul->rb_size * 12); + } + for (int rb = 0; rb < rel15_ul->rb_size; rb++) + pusch_vars->ulsch_noise_power[aarx] += + n0_subband_power[aarx][rel15_ul->bwp_start + rel15_ul->rb_start + rb] / rel15_ul->rb_size; + } + } + } + + // averaging time domain channel estimates + if (gNB->chest_time == 1) + { + nr_chest_time_domain_avg(frame_parms, + pusch_vars->ul_ch_estimates, + rel15_ul->nr_of_symbols, + rel15_ul->start_symbol_index, + rel15_ul->ul_dmrs_symb_pos, + rel15_ul->rb_size); + pusch_vars->dmrs_symbol = get_next_dmrs_symbol_in_slot(rel15_ul->ul_dmrs_symb_pos, + rel15_ul->start_symbol_index, + rel15_ul->nr_of_symbols); + } + + stop_meas(&gNB->ulsch_channel_estimation_stats); + + start_meas(&gNB->rx_pusch_init_stats); + + // Scrambling initialization + int number_dmrs_symbols = 0; + for (int l = rel15_ul->start_symbol_index; l < rel15_ul->start_symbol_index + rel15_ul->nr_of_symbols; l++) + number_dmrs_symbols += ((rel15_ul->ul_dmrs_symb_pos)>>l) & 0x01; + int nb_re_dmrs; + if (rel15_ul->dmrs_config_type == pusch_dmrs_type1) + nb_re_dmrs = 6*rel15_ul->num_dmrs_cdm_grps_no_data; + else + nb_re_dmrs = 4*rel15_ul->num_dmrs_cdm_grps_no_data; + + // get how many bit in a slot // + int G = nr_get_G(rel15_ul->rb_size, + rel15_ul->nr_of_symbols, + nb_re_dmrs, + number_dmrs_symbols, // number of dmrs symbols irrespective of single or double symbol dmrs + rel15_ul->qam_mod_order, + rel15_ul->nrOfLayers); + // initialize scrambling sequence // + int16_t s[G+96] __attribute__((aligned(32))); + + nr_codeword_unscrambling_init(s, G, 0, rel15_ul->data_scrambling_id, rel15_ul->rnti); + + // first the computation of channel levels + + int nb_re_pusch = 0, meas_symbol = -1; + for(meas_symbol = rel15_ul->start_symbol_index; + meas_symbol < (rel15_ul->start_symbol_index + rel15_ul->nr_of_symbols); + meas_symbol++) + if ((nb_re_pusch = get_nb_re_pusch(frame_parms,rel15_ul,meas_symbol)) > 0) + break; + + AssertFatal(nb_re_pusch>0 && meas_symbol>=0,"nb_re_pusch %d cannot be 0 or meas_symbol %d cannot be negative here\n",nb_re_pusch,meas_symbol); + + start_meas(&gNB->ulsch_rbs_extraction_stats); + // extract the first dmrs for the channel level computation + // extract the data in the OFDM frame, to the start of the array + nr_ulsch_extract_rbs(gNB->common_vars.rxdataF, + pusch_vars, + slot, + meas_symbol, + (rel15_ul->ul_dmrs_symb_pos >> meas_symbol) & 0x01, + rel15_ul, + frame_parms); + stop_meas(&gNB->ulsch_rbs_extraction_stats); + + int avgs = 0; + int avg[frame_parms->nb_antennas_rx*rel15_ul->nrOfLayers]; + uint8_t shift_ch_ext = rel15_ul->nrOfLayers > 1 ? log2_approx(max_ch >> 11) : 0; + + //---------------------------------------------------------- + //--------------------- Channel Scaling -------------------- + //---------------------------------------------------------- + nr_ulsch_scale_channel(pusch_vars->ul_ch_estimates_ext, + frame_parms, + &gNB->ulsch[ulsch_id], + meas_symbol, + (rel15_ul->ul_dmrs_symb_pos >> meas_symbol) & 0x01, + nb_re_pusch, + rel15_ul->nrOfLayers, + rel15_ul->rb_size, + shift_ch_ext); + + nr_ulsch_channel_level(pusch_vars->ul_ch_estimates_ext, + frame_parms, + avg, + meas_symbol, // index of the start symbol + nb_re_pusch, // number of the re in pusch + rel15_ul->nrOfLayers, + rel15_ul->rb_size); + + for (int aatx = 0; aatx < rel15_ul->nrOfLayers; aatx++) + for (int aarx = 0; aarx < frame_parms->nb_antennas_rx; aarx++) + avgs = cmax(avgs, avg[aatx*frame_parms->nb_antennas_rx+aarx]); + + if (rel15_ul->nrOfLayers == 1) + pusch_vars->log2_maxh = (log2_approx(avgs) >> 1) + 2; + else + pusch_vars->log2_maxh = (log2_approx(avgs) >> 1); + + pusch_vars->cl_done = 1; + pusch_vars->extraction_done[meas_symbol] = 1; + + stop_meas(&gNB->rx_pusch_init_stats); + + start_meas(&gNB->rx_pusch_symbol_processing_stats); + int numSymbols = gNB->num_pusch_symbols_per_thread; + + for(uint8_t symbol = rel15_ul->start_symbol_index; + symbol < (rel15_ul->start_symbol_index + rel15_ul->nr_of_symbols); + symbol += numSymbols) + { + int total_res = 0; + for (int s = 0; s < numSymbols;s++) + { + pusch_vars->ul_valid_re_per_slot[symbol+s] = get_nb_re_pusch(frame_parms,rel15_ul,symbol+s); + pusch_vars->llr_offset[symbol+s] = ((symbol+s) == rel15_ul->start_symbol_index) ? + 0 : + pusch_vars->llr_offset[symbol+s-1] + pusch_vars->ul_valid_re_per_slot[symbol+s-1] * rel15_ul->qam_mod_order; + total_res+=pusch_vars->ul_valid_re_per_slot[symbol+s]; + } + if (total_res > 0) + { + union puschSymbolReqUnion id = {.s={ulsch_id,frame,slot,0}}; + id.p=1+symbol; + notifiedFIFO_elt_t *req = newNotifiedFIFO_elt(sizeof(puschSymbolProc_t), id.p, gNB->respPuschSymb, &nr_pusch_symbol_processing_noprecoding); // create a job for Tpool + puschSymbolProc_t *rdata = (puschSymbolProc_t*)NotifiedFifoData(req); // data for the job + + rdata->gNB = gNB; + rdata->frame_parms = frame_parms; + rdata->rel15_ul = rel15_ul; + rdata->slot = slot; + rdata->startSymbol = symbol; + rdata->numSymbols = numSymbols; + rdata->ulsch_id = ulsch_id; + rdata->llr = pusch_vars->llr; + rdata->llr_layers = pusch_vars->llr_layers; + rdata->s = &s[pusch_vars->llr_offset[symbol]*rel15_ul->nrOfLayers]; + + pushTpool(&gNB->threadPool, req); + gNB->nbSymb++; + + LOG_D(PHY,"%d.%d Added symbol %d (count %d) to process, in pipe\n",frame,slot,symbol,gNB->nbSymb); + } + } // symbol loop + + while (gNB->nbSymb > 0) + { + notifiedFIFO_elt_t *req=pullTpool(gNB->respPuschSymb, &gNB->threadPool); + gNB->nbSymb--; + delNotifiedFIFO_elt(req); + } + + stop_meas(&gNB->rx_pusch_symbol_processing_stats); + return 0; +} diff --git a/openair1/PHY/defs_gNB.h b/openair1/PHY/defs_gNB.h index c2e6dfba75..f1f931e1f4 100644 --- a/openair1/PHY/defs_gNB.h +++ b/openair1/PHY/defs_gNB.h @@ -409,8 +409,12 @@ typedef struct { /// \brief Total RE count after DMRS/PTRS RE's are extracted from respective symbol. /// - first index: ? [0...14] smybol per slot int16_t *ul_valid_re_per_slot; + /// \brief offset for llr corresponding to each symbol + int llr_offset[14]; /// flag to verify if channel level computation is done uint8_t cl_done; + /// flag to indicate if channel extraction is done + int extraction_done[14]; /// flag to indicate DTX on reception int DTX; } NR_gNB_PUSCH; @@ -728,6 +732,8 @@ typedef struct PHY_VARS_gNB_s { time_stats_t dlsch_segmentation_stats; time_stats_t rx_pusch_stats; + time_stats_t rx_pusch_init_stats; + time_stats_t rx_pusch_symbol_processing_stats; time_stats_t ul_indication_stats; time_stats_t schedule_response_stats; time_stats_t ulsch_decoding_stats; @@ -754,6 +760,7 @@ typedef struct PHY_VARS_gNB_s { time_stats_t rx_dft_stats; time_stats_t ulsch_freq_offset_estimation_stats; */ + notifiedFIFO_t *respPuschSymb; notifiedFIFO_t respDecode; notifiedFIFO_t resp_L1; notifiedFIFO_t L1_tx_free; @@ -761,6 +768,9 @@ typedef struct PHY_VARS_gNB_s { notifiedFIFO_t L1_tx_out; notifiedFIFO_t resp_RU_tx; tpool_t threadPool; + int nbSymb; + int use_pusch_tp; + int num_pusch_symbols_per_thread; pthread_t L1_rx_thread; int L1_rx_thread_core; pthread_t L1_tx_thread; @@ -771,6 +781,31 @@ typedef struct PHY_VARS_gNB_s { rt_L1_profiling_t rt_L1_profiling; } PHY_VARS_gNB; +typedef struct puschSymbolProc_s { + PHY_VARS_gNB *gNB; + NR_DL_FRAME_PARMS *frame_parms; + nfapi_nr_pusch_pdu_t *rel15_ul; + int ulsch_id; + int slot; + int startSymbol; + int numSymbols; + int16_t *llr; + int16_t **llr_layers; + int16_t *s; +} puschSymbolProc_t; + +struct puschSymbolReqId { + uint16_t ulsch_id; + uint16_t frame; + uint8_t slot; + uint16_t spare; +} __attribute__((packed)); + +union puschSymbolReqUnion { + struct puschSymbolReqId s; + uint64_t p; +}; + typedef struct LDPCDecode_s { PHY_VARS_gNB *gNB; NR_UL_gNB_HARQ_t *ulsch_harq; diff --git a/openair1/SCHED_NR/phy_procedures_nr_gNB.c b/openair1/SCHED_NR/phy_procedures_nr_gNB.c index 184e3ec63b..35afb44fe1 100644 --- a/openair1/SCHED_NR/phy_procedures_nr_gNB.c +++ b/openair1/SCHED_NR/phy_procedures_nr_gNB.c @@ -398,18 +398,21 @@ static int nr_ulsch_procedures(PHY_VARS_gNB *gNB, int frame_rx, int slot_rx, int pusch_pdu->qam_mod_order, pusch_pdu->nrOfLayers); - nr_ulsch_layer_demapping(gNB->pusch_vars[ULSCH_id].llr, - pusch_pdu->nrOfLayers, - pusch_pdu->qam_mod_order, - G, - gNB->pusch_vars[ULSCH_id].llr_layers); - - //---------------------------------------------------------- - //------------------- ULSCH unscrambling ------------------- - //---------------------------------------------------------- - start_meas(&gNB->ulsch_unscrambling_stats); - nr_ulsch_unscrambling(gNB->pusch_vars[ULSCH_id].llr, G, pusch_pdu->data_scrambling_id, pusch_pdu->rnti); - stop_meas(&gNB->ulsch_unscrambling_stats); + if (gNB->use_pusch_tp == 0 ) + { + nr_ulsch_layer_demapping(gNB->pusch_vars[ULSCH_id].llr, + pusch_pdu->nrOfLayers, + pusch_pdu->qam_mod_order, + G, + gNB->pusch_vars[ULSCH_id].llr_layers); + //---------------------------------------------------------- + //------------------- ULSCH unscrambling ------------------- + //---------------------------------------------------------- + start_meas(&gNB->ulsch_unscrambling_stats); + nr_ulsch_unscrambling(gNB->pusch_vars[ULSCH_id].llr, G, pusch_pdu->data_scrambling_id, pusch_pdu->rnti); + stop_meas(&gNB->ulsch_unscrambling_stats); + } + //---------------------------------------------------------- //--------------------- ULSCH decoding --------------------- //---------------------------------------------------------- @@ -417,7 +420,7 @@ static int nr_ulsch_procedures(PHY_VARS_gNB *gNB, int frame_rx, int slot_rx, int start_meas(&gNB->ulsch_decoding_stats); int nbDecode = nr_ulsch_decoding(gNB, ULSCH_id, gNB->pusch_vars[ULSCH_id].llr, frame_parms, pusch_pdu, frame_rx, slot_rx, harq_pid, G); - stop_meas(&gNB->ulsch_decoding_stats); + return nbDecode; } @@ -899,7 +902,8 @@ int phy_procedures_gNB_uespec_RX(PHY_VARS_gNB *gNB, int frame_rx, int slot_rx) VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_NR_RX_PUSCH, 1); start_meas(&gNB->rx_pusch_stats); - nr_rx_pusch(gNB, ULSCH_id, frame_rx, slot_rx, ulsch->harq_pid); + if (gNB->use_pusch_tp) nr_rx_pusch_tp(gNB, ULSCH_id, frame_rx, slot_rx, ulsch->harq_pid); + else nr_rx_pusch(gNB, ULSCH_id, frame_rx, slot_rx, ulsch->harq_pid); NR_gNB_PUSCH *pusch_vars = &gNB->pusch_vars[ULSCH_id]; pusch_vars->ulsch_power_tot = 0; pusch_vars->ulsch_noise_power_tot = 0; @@ -962,6 +966,7 @@ int phy_procedures_gNB_uespec_RX(PHY_VARS_gNB *gNB, int frame_rx, int slot_rx) delNotifiedFIFO_elt(req); totalDecode--; } + stop_meas(&gNB->ulsch_decoding_stats); } for (int i = 0; i < gNB->max_nb_srs; i++) { NR_gNB_SRS_t *srs = &gNB->srs[i]; diff --git a/openair1/SIMULATION/NR_PHY/ulsim.c b/openair1/SIMULATION/NR_PHY/ulsim.c index 5ee660d1e0..879d6deda1 100644 --- a/openair1/SIMULATION/NR_PHY/ulsim.c +++ b/openair1/SIMULATION/NR_PHY/ulsim.c @@ -222,6 +222,7 @@ int main(int argc, char *argv[]) int ibwp_rboffset=41; int params_from_file = 0; int threadCnt=0; + int use_tpool = 0; int max_ldpc_iterations = 5; if ( load_configmodule(argc,argv,CONFIG_ENABLECMDLINEONLY) == 0 ) { exit_fun("[NR_ULSIM] Error, configuration module init failed\n"); @@ -363,6 +364,7 @@ int main(int argc, char *argv[]) case 'C': threadCnt = atoi(optarg); + use_tpool = 1; break; case 'u': @@ -562,9 +564,13 @@ int main(int argc, char *argv[]) gNB = RC.gNB[0]; gNB->ofdm_offset_divisor = UINT_MAX; initNotifiedFIFO(&gNB->respDecode); + gNB->use_pusch_tp = use_tpool; + gNB->num_pusch_symbols_per_thread = 1; initFloatingCoresTpool(threadCnt, &gNB->threadPool, false, "gNB-tpool"); initNotifiedFIFO(&gNB->respDecode); + gNB->respPuschSymb = (notifiedFIFO_t*) malloc(sizeof(notifiedFIFO_t)); + initNotifiedFIFO(gNB->respPuschSymb); initNotifiedFIFO(&gNB->L1_tx_free); initNotifiedFIFO(&gNB->L1_tx_filled); initNotifiedFIFO(&gNB->L1_tx_out); @@ -927,6 +933,8 @@ int main(int argc, char *argv[]) roundStats = 0; reset_meas(&gNB->phy_proc_rx); reset_meas(&gNB->rx_pusch_stats); + reset_meas(&gNB->rx_pusch_init_stats); + reset_meas(&gNB->rx_pusch_symbol_processing_stats); reset_meas(&gNB->ulsch_decoding_stats); reset_meas(&gNB->ulsch_deinterleaving_stats); reset_meas(&gNB->ulsch_rate_unmatching_stats); @@ -1588,25 +1596,36 @@ int main(int argc, char *argv[]) dump_pusch_stats(fd,gNB); fclose(fd); - if (print_perf==1) { + if (print_perf==1) + { + printf("gNB RX\n"); printDistribution(&gNB->phy_proc_rx,table_rx,"Total PHY proc rx"); printStatIndent(&gNB->rx_pusch_stats,"RX PUSCH time"); printStatIndent2(&gNB->ulsch_channel_estimation_stats,"ULSCH channel estimation time"); - printStatIndent2(&gNB->ulsch_ptrs_processing_stats,"ULSCH PTRS Processing time"); - printStatIndent2(&gNB->ulsch_rbs_extraction_stats,"ULSCH rbs extraction time"); - printStatIndent2(&gNB->ulsch_channel_compensation_stats,"ULSCH channel compensation time"); - printStatIndent2(&gNB->ulsch_mrc_stats,"ULSCH mrc computation"); - printStatIndent2(&gNB->ulsch_llr_stats,"ULSCH llr computation"); - printStatIndent(&gNB->ulsch_unscrambling_stats,"ULSCH unscrambling"); + if (use_tpool == 1) + { + printStatIndent2(&gNB->rx_pusch_init_stats,"RX PUSCH Initialization time"); + printStatIndent2(&gNB->rx_pusch_symbol_processing_stats,"RX PUSCH Symbol Processing time"); + } + else + { + printStatIndent2(&gNB->ulsch_ptrs_processing_stats,"ULSCH PTRS Processing time"); + printStatIndent2(&gNB->ulsch_rbs_extraction_stats,"ULSCH rbs extraction time"); + printStatIndent2(&gNB->ulsch_channel_compensation_stats,"ULSCH channel compensation time"); + printStatIndent2(&gNB->ulsch_mrc_stats,"ULSCH mrc computation"); + printStatIndent2(&gNB->ulsch_llr_stats,"ULSCH llr computation"); + printStatIndent(&gNB->ulsch_unscrambling_stats,"ULSCH unscrambling"); + } printStatIndent(&gNB->ulsch_decoding_stats,"ULSCH total decoding time"); + // printStatIndent2(&gNB->ulsch_deinterleaving_stats,"ULSCH deinterleaving"); + // printStatIndent2(&gNB->ulsch_rate_unmatching_stats,"ULSCH rate matching rx"); + // printStatIndent2(&gNB->ulsch_ldpc_decoding_stats,"ULSCH ldpc decoding"); + printf("\nUE TX\n"); printStatIndent(&UE->ulsch_encoding_stats,"ULSCH total encoding time"); printStatIndent2(&UE->ulsch_segmentation_stats,"ULSCH segmentation time"); printStatIndent2(&UE->ulsch_ldpc_encoding_stats,"ULSCH LDPC encoder time"); printStatIndent2(&UE->ulsch_rate_matching_stats,"ULSCH rate-matching time"); printStatIndent2(&UE->ulsch_interleaving_stats,"ULSCH interleaving time"); - //printStatIndent2(&gNB->ulsch_deinterleaving_stats,"ULSCH deinterleaving"); - //printStatIndent2(&gNB->ulsch_rate_unmatching_stats,"ULSCH rate matching rx"); - //printStatIndent2(&gNB->ulsch_ldpc_decoding_stats,"ULSCH ldpc decoding"); printStatIndent(&gNB->rx_srs_stats,"RX SRS time"); printStatIndent2(&gNB->generate_srs_stats,"Generate SRS sequence time"); printStatIndent2(&gNB->get_srs_signal_stats,"Get SRS signal time"); From 04073b03e46225e98ba98940cd18f456f39c0ebd Mon Sep 17 00:00:00 2001 From: Tsung-Yu Chan Date: Tue, 5 Sep 2023 14:31:48 +0200 Subject: [PATCH 02/13] feat / add the MMSE and precoding - avoid the finction duplication - add the transform predoding to the nr_rx_pusch_tp - add the MMSE for 2 layer which modulation greater than 16 qam - can support to 256 qam for 1 layer and 2 layer --- CMakeLists.txt | 1 + executables/nr-gnb.c | 1 + openair1/PHY/INIT/nr_init.c | 3 + .../PHY/NR_ESTIMATION/nr_freq_equalization.c | 73 + openair1/PHY/NR_ESTIMATION/nr_ul_estimation.h | 11 + .../PHY/NR_TRANSPORT/nr_ulsch_demodulation.c | 3061 ++++++----------- .../NR_TRANSPORT/nr_ulsch_llr_computation.c | 316 +- openair1/PHY/defs_gNB.h | 4 +- openair1/SCHED_NR/phy_procedures_nr_gNB.c | 33 +- openair1/SIMULATION/NR_PHY/ulsim.c | 32 +- 10 files changed, 1276 insertions(+), 2259 deletions(-) create mode 100644 openair1/PHY/NR_ESTIMATION/nr_freq_equalization.c diff --git a/CMakeLists.txt b/CMakeLists.txt index ec60df8b35..e1527d227c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1040,6 +1040,7 @@ set(PHY_SRC_UE ${OPENAIR1_DIR}/PHY/NR_REFSIG/dmrs_nr.c ${OPENAIR1_DIR}/PHY/NR_REFSIG/ptrs_nr.c ${OPENAIR1_DIR}/PHY/NR_ESTIMATION/nr_ul_channel_estimation.c + ${OPENAIR1_DIR}/PHY/NR_ESTIMATION/nr_freq_equalization.c ${OPENAIR1_DIR}/PHY/NR_ESTIMATION/nr_measurements_gNB.c ${OPENAIR1_DIR}/PHY/TOOLS/file_output.c ${OPENAIR1_DIR}/PHY/TOOLS/cadd_vv.c diff --git a/executables/nr-gnb.c b/executables/nr-gnb.c index 33fe6b3f2a..d436828294 100644 --- a/executables/nr-gnb.c +++ b/executables/nr-gnb.c @@ -479,6 +479,7 @@ void init_gNB_Tpool(int inst) { // ULSCH decoding threadpool initTpool(get_softmodem_params()->threadPoolConfig, &gNB->threadPool, cpumeas(CPUMEAS_GETSTATE)); // ULSCH decoder result FIFO + initNotifiedFIFO(&gNB->respPuschSymb); initNotifiedFIFO(&gNB->respDecode); // L1 RX result FIFO diff --git a/openair1/PHY/INIT/nr_init.c b/openair1/PHY/INIT/nr_init.c index ed3fd55079..09898d383b 100644 --- a/openair1/PHY/INIT/nr_init.c +++ b/openair1/PHY/INIT/nr_init.c @@ -28,6 +28,7 @@ #include "PHY/CODING/nrPolar_tools/nr_polar_pbch_defs.h" #include "PHY/NR_TRANSPORT/nr_transport_proto.h" #include "PHY/NR_TRANSPORT/nr_transport_common_proto.h" +#include "PHY/NR_ESTIMATION/nr_ul_estimation.h" #include "openair1/PHY/MODULATION/nr_modulation.h" #include "openair1/PHY/defs_RU.h" #include "openair1/PHY/CODING/nrLDPC_extern.h" @@ -525,6 +526,8 @@ int phy_init_nr_gNB(PHY_VARS_gNB *gNB) init_scrambling_luts(); init_pucch2_luts(); + nr_init_fde(); // Init array for frequency equalization of transform precoding of PUSCH + load_nrLDPClib(NULL); if (gNB->ldpc_offload_flag) diff --git a/openair1/PHY/NR_ESTIMATION/nr_freq_equalization.c b/openair1/PHY/NR_ESTIMATION/nr_freq_equalization.c new file mode 100644 index 0000000000..e73ac48fab --- /dev/null +++ b/openair1/PHY/NR_ESTIMATION/nr_freq_equalization.c @@ -0,0 +1,73 @@ +/* + * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The OpenAirInterface Software Alliance licenses this file to You under + * the OAI Public License, Version 1.1 (the "License"); you may not use this file + * except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.openairinterface.org/?page_id=698 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *------------------------------------------------------------------------------- + * For more information about the OpenAirInterface (OAI) Software Alliance: + * contact@openairinterface.org + */ + +#include "PHY/defs_eNB.h" +#include "PHY/sse_intrin.h" +#include "PHY/NR_ESTIMATION/nr_ul_estimation.h" + +// Reference of openair1/PHY/LTE_ESTIMATION/freq_equalization.c + +// This is 4096/(1:4096) in simde__m128i format +simde__m128i nr_inv_ch[4096];/* = {0, 4096/1, 4096/2, 4096/3, 4096/4...}*/ + +void nr_init_fde() +{ + for (int i = 1;i < 4096; i++) + nr_inv_ch[i] = simde_mm_set1_epi16(4096/i); +} + +void nr_freq_equalization (NR_DL_FRAME_PARMS *frame_parms, + int32_t *rxdataF_comp, + int32_t *ul_ch_mag, + int32_t *ul_ch_magb, + uint8_t symbol, + uint16_t Msc_RS, + uint8_t Qm) +{ + simde__m128i *rxdataF_comp128 = (simde__m128i *)rxdataF_comp; + simde__m128i *ul_ch_mag128 = (simde__m128i *)ul_ch_mag; + simde__m128i *ul_ch_magb128 = (simde__m128i *)ul_ch_magb; + + AssertFatal(symbol < frame_parms->symbols_per_slot, "symbol %d >= %d\n", + symbol, frame_parms->symbols_per_slot); + AssertFatal(Msc_RS <= frame_parms->N_RB_UL*12, "Msc_RS %d >= %d\n", + Msc_RS, frame_parms->N_RB_UL*12); + + for (uint16_t re = 0; re < (Msc_RS >> 2); re++) + { + int16_t amp = (*((int16_t*)&ul_ch_mag128[re])); + + if (amp > 4095) + amp = 4095; + + rxdataF_comp128[re] = simde_mm_srai_epi16(simde_mm_mullo_epi16(rxdataF_comp128[re],nr_inv_ch[amp]),3); + + if (Qm == 4) + ul_ch_mag128[re] = simde_mm_set1_epi16(324); // this is 512*2/sqrt(10) + else if (Qm == 6) + { + ul_ch_mag128[re] = simde_mm_set1_epi16(316); // this is 512*4/sqrt(42) + ul_ch_magb128[re] = simde_mm_set1_epi16(158); // this is 512*2/sqrt(42) + } + else if(Qm != 2) + AssertFatal(1, "nr_freq_equalization(), Qm = %d, should be 2, 4 or 6. symbol=%d, Msc_RS=%d\n", Qm, symbol, Msc_RS); + } +} diff --git a/openair1/PHY/NR_ESTIMATION/nr_ul_estimation.h b/openair1/PHY/NR_ESTIMATION/nr_ul_estimation.h index c75b4d4ed7..4934498a4a 100644 --- a/openair1/PHY/NR_ESTIMATION/nr_ul_estimation.h +++ b/openair1/PHY/NR_ESTIMATION/nr_ul_estimation.h @@ -83,4 +83,15 @@ int nr_srs_channel_estimation(const PHY_VARS_gNB *gNB, int32_t srs_estimated_channel_time_shifted[][1<num_ant_ports][gNB->frame_parms.ofdm_symbol_size], int8_t *snr_per_rb, int8_t *snr); + +void nr_freq_equalization(NR_DL_FRAME_PARMS *frame_parms, + int *rxdataF_comp, + int *ul_ch_mag, + int *ul_ch_mag_b, + unsigned char symbol, + unsigned short Msc_RS, + unsigned char Qm); + +void nr_init_fde(void); + #endif diff --git a/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c b/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c index 9c398d1664..3e0b3506b1 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c +++ b/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c @@ -18,10 +18,6 @@ #define INVALID_VALUE 255 -#ifdef __aarch64__ -#define USE_128BIT -#endif - void nr_idft(int32_t *z, uint32_t Msc_PUSCH) { @@ -285,16 +281,16 @@ void nr_idft(int32_t *z, uint32_t Msc_PUSCH) } -void nr_ulsch_extract_rbs0 (c16_t *rxdataF, - int32_t *chF, - int32_t *rxFext, - int32_t *chFext, - int rxoffset, - int choffset, - int aarx, - int is_dmrs_symbol, - nfapi_nr_pusch_pdu_t *pusch_pdu, - NR_DL_FRAME_PARMS *frame_parms) +static void nr_ulsch_extract_rbs0 (c16_t* const rxdataF, + c16_t* const chF, + c16_t *rxFext, + c16_t *chFext, + int rxoffset, + int choffset, + int aarx, + int is_dmrs_symbol, + nfapi_nr_pusch_pdu_t *pusch_pdu, + NR_DL_FRAME_PARMS *frame_parms) { uint8_t delta = 0; @@ -303,206 +299,87 @@ void nr_ulsch_extract_rbs0 (c16_t *rxdataF, int nb_re_pusch = NR_NB_SC_PER_RB * pusch_pdu->rb_size; - int32_t *rxF = (int32_t*)&rxdataF[rxoffset]; - int32_t *rxF_ext = &rxFext[0]; - int32_t *ul_ch0 = &chF[choffset]; - int32_t *ul_ch0_ext = &chFext[0]; - + c16_t *rxF = &rxdataF[rxoffset]; + c16_t *rxF_ext = &rxFext[0]; + c16_t *ul_ch0 = &chF[choffset]; + c16_t *ul_ch0_ext = &chFext[0]; if (is_dmrs_symbol == 0) { - if (start_re + nb_re_pusch <= frame_parms->ofdm_symbol_size) { - memcpy((void*)rxF_ext, - (void*)&rxF[start_re], - nb_re_pusch*sizeof(int32_t)); - } + if (start_re + nb_re_pusch <= frame_parms->ofdm_symbol_size) + memcpy(rxF_ext, &rxF[start_re], nb_re_pusch * sizeof(c16_t)); else { - int neg_length = frame_parms->ofdm_symbol_size-start_re; + int neg_length = frame_parms->ofdm_symbol_size - start_re; int pos_length = nb_re_pusch - neg_length; - memcpy((void*)rxF_ext, (void*)&rxF[start_re], neg_length * sizeof(int32_t)); - memcpy((void*)&rxF_ext[neg_length], (void*)rxF, pos_length * sizeof(int32_t)); + memcpy(rxF_ext, &rxF[start_re], neg_length * sizeof(c16_t)); + memcpy(&rxF_ext[neg_length], rxF, pos_length * sizeof(c16_t)); } - memcpy((void*)ul_ch0_ext,(void*)ul_ch0,nb_re_pusch*sizeof(int32_t)); + memcpy(ul_ch0_ext, ul_ch0, nb_re_pusch * sizeof(c16_t)); } else if (pusch_pdu->dmrs_config_type == pusch_dmrs_type1) // 6 REs / PRB { - AssertFatal(delta==0||delta==1,"Illegal delta %d\n",delta); - int32_t *rxF32 = &rxF[start_re]; - int32_t *rxF_ext32 = rxF_ext; - int32_t *ul_ch032 = ul_ch0; - int32_t *ul_ch0_ext32 = ul_ch0_ext; - int idx,idx2,idx3; + AssertFatal(delta == 0 || delta == 1, "Illegal delta %d\n",delta); + c16_t *rxF32 = &rxF[start_re]; if (start_re + nb_re_pusch < frame_parms->ofdm_symbol_size) { - for (idx=1-delta,idx2=0;idxofdm_symbol_size-start_re; - int pos_length = nb_re_pusch-neg_length; - - for (idx=1-delta,idx2=0;idxofdm_symbol_size - start_re; + int pos_length = nb_re_pusch - neg_length; + int idx, idx2; + for (idx = 1 - delta; idx < neg_length; idx += 2) + { + *rxF_ext++ = rxF32[idx]; + *ul_ch0_ext++= ul_ch0[idx]; } - rxF32=(int32_t*)rxF; - idx3=idx; - for (idx=1-delta;idxdmrs_config_type == pusch_dmrs_type2) // 8 REs / PRB { AssertFatal(delta==0||delta==2||delta==4,"Illegal delta %d\n",delta); - if (start_re + nb_re_pusch < frame_parms->ofdm_symbol_size) { - int64_t *rxF64 = (int64_t*)&rxF[start_re]; - int64_t *rxF_ext64 = (int64_t*)rxF_ext; - int64_t *ul_ch064 = (int64_t*)ul_ch0; - int64_t *ul_ch0_ext64 = (int64_t*)ul_ch0_ext; - if (delta==0) { - for (int idx=0;idx>1;idx+=6) { - rxF_ext64[idx]=rxF64[idx+1]; - rxF_ext64[idx+1]=rxF64[idx+2]; - rxF_ext64[idx+2]=rxF64[idx+4]; - rxF_ext64[idx+3]=rxF64[idx+5]; - ul_ch0_ext64[idx]=ul_ch064[idx+1]; - ul_ch0_ext64[idx+1]=ul_ch064[idx+2]; - ul_ch0_ext64[idx+2]=ul_ch064[idx+4]; - ul_ch0_ext64[idx+3]=ul_ch064[idx+5]; - } - } - else if (delta==2) { - for (int idx=0;idx>1;idx+=6) { - rxF_ext64[idx]=rxF64[idx+0]; - rxF_ext64[idx+1]=rxF64[idx+2]; - rxF_ext64[idx+2]=rxF64[idx+3]; - rxF_ext64[idx+3]=rxF64[idx+5]; - ul_ch0_ext64[idx]=ul_ch064[idx+0]; - ul_ch0_ext64[idx+1]=ul_ch064[idx+2]; - ul_ch0_ext64[idx+2]=ul_ch064[idx+3]; - ul_ch0_ext64[idx+3]=ul_ch064[idx+5]; - } - } - else if (delta==4) { - for (int idx=0;idx>1;idx+=6) { - rxF_ext64[idx]=rxF64[idx+0]; - rxF_ext64[idx+1]=rxF64[idx+1]; - rxF_ext64[idx+2]=rxF64[idx+3]; - rxF_ext64[idx+3]=rxF64[idx+4]; - ul_ch0_ext64[idx]=ul_ch064[idx+0]; - ul_ch0_ext64[idx+1]=ul_ch064[idx+1]; - ul_ch0_ext64[idx+2]=ul_ch064[idx+3]; - ul_ch0_ext64[idx+3]=ul_ch064[idx+4]; - } + if (start_re + nb_re_pusch < frame_parms->ofdm_symbol_size) + { + for (int idx = 0; idx < nb_re_pusch; idx ++) + { + if (idx % 6 == 2 * delta || idx % 6 == 2 * delta + 1) + continue; + *rxF_ext++ = rxF[idx]; + *ul_ch0_ext++ = ul_ch0[idx]; } } - else { - int neg_length = frame_parms->ofdm_symbol_size-start_re; - int pos_length = nb_re_pusch-neg_length; - if ((pos_length%12) > 0 ) pos_length+=12; - int64_t *rxF64 = (int64_t*)&rxF[start_re]; - int64_t *rxF_ext64 = (int64_t*)rxF_ext; - int64_t *ul_ch064 = (int64_t*)ul_ch0; - int64_t *ul_ch0_ext64 = (int64_t*)ul_ch0_ext; - int idx=0; - if (delta==0) { - for (idx=0;idx>1;idx+=6) { - rxF_ext64[idx] =rxF64[idx+1]; - rxF_ext64[idx+1]=rxF64[idx+2]; - rxF_ext64[idx+2]=rxF64[idx+4]; - rxF_ext64[idx+3]=rxF64[idx+5]; - ul_ch0_ext64[idx]=ul_ch064[idx+1]; - ul_ch0_ext64[idx+1]=ul_ch064[idx+2]; - ul_ch0_ext64[idx+2]=ul_ch064[idx+4]; - ul_ch0_ext64[idx+3]=ul_ch064[idx+5]; - } - if ((neg_length%12) > 0) { - rxF_ext64[idx+4]=rxF64[idx+7]; - rxF_ext64[idx+5]=rxF64[idx+8]; - ul_ch0_ext64[idx+4]=ul_ch064[idx+7]; - ul_ch0_ext64[idx+5]=ul_ch064[idx+8]; - } - rxF_ext64+=(neg_length/3); - rxF64=(int64_t*)rxF; - ul_ch0_ext64+=(neg_length/3); - ul_ch064+=(neg_length>>1); - for (idx=0;idx>1;idx+=6) { - rxF_ext64[idx] =rxF64[idx+1]; - rxF_ext64[idx+1]=rxF64[idx+2]; - rxF_ext64[idx+2]=rxF64[idx+4]; - rxF_ext64[idx+3]=rxF64[idx+5]; - ul_ch0_ext64[idx]=ul_ch064[idx+1]; - ul_ch0_ext64[idx+1]=ul_ch064[idx+2]; - ul_ch0_ext64[idx+2]=ul_ch064[idx+4]; - ul_ch0_ext64[idx+3]=ul_ch064[idx+5]; - } + else + { + int neg_length = frame_parms->ofdm_symbol_size - start_re; + int pos_length = nb_re_pusch - neg_length; + c16_t *rxF64 = &rxF[start_re]; + int idx, idx2; + for (idx = 0; idx < neg_length; idx ++) + { + if (idx % 6 == 2 * delta || idx % 6 == 2 * delta + 1) + continue; + *rxF_ext++ = rxF64[idx]; + *ul_ch0_ext++ = ul_ch0[idx]; } - else if (delta==2) { - for (idx=0;idx>1;idx+=6) { - rxF_ext64[idx] =rxF64[idx+0]; - rxF_ext64[idx+1]=rxF64[idx+2]; - rxF_ext64[idx+2]=rxF64[idx+3]; - rxF_ext64[idx+3]=rxF64[idx+5]; - ul_ch0_ext64[idx]=ul_ch064[idx+0]; - ul_ch0_ext64[idx+1]=ul_ch064[idx+2]; - ul_ch0_ext64[idx+2]=ul_ch064[idx+3]; - ul_ch0_ext64[idx+3]=ul_ch064[idx+5]; - } - if ((neg_length%12) > 0) { - rxF_ext64[idx+4]=rxF64[idx+6]; - rxF_ext64[idx+5]=rxF64[idx+8]; - ul_ch0_ext64[idx+4]=ul_ch064[idx+6]; - ul_ch0_ext64[idx+5]=ul_ch064[idx+8]; - } - rxF_ext64+=(neg_length/3); - rxF64=(int64_t*)rxF; - ul_ch0_ext64+=(neg_length/3); - ul_ch064+=(neg_length>>1); - for (idx=0;idx>1;idx+=6) { - rxF_ext64[idx] =rxF64[idx+0]; - rxF_ext64[idx+1]=rxF64[idx+2]; - rxF_ext64[idx+2]=rxF64[idx+3]; - rxF_ext64[idx+3]=rxF64[idx+5]; - ul_ch0_ext64[idx]=ul_ch064[idx+0]; - ul_ch0_ext64[idx+1]=ul_ch064[idx+2]; - ul_ch0_ext64[idx+2]=ul_ch064[idx+3]; - ul_ch0_ext64[idx+3]=ul_ch064[idx+5]; - } - } - else if (delta==4) { - for (idx=0;idx>1;idx+=6) { - rxF_ext64[idx] =rxF64[idx+0]; - rxF_ext64[idx+1]=rxF64[idx+1]; - rxF_ext64[idx+2]=rxF64[idx+3]; - rxF_ext64[idx+3]=rxF64[idx+4]; - ul_ch0_ext64[idx]=ul_ch064[idx+0]; - ul_ch0_ext64[idx+1]=ul_ch064[idx+1]; - ul_ch0_ext64[idx+2]=ul_ch064[idx+3]; - ul_ch0_ext64[idx+3]=ul_ch064[idx+4]; - } - if ((neg_length%12) > 0) { - rxF_ext64[idx+4]=rxF64[idx+6]; - rxF_ext64[idx+5]=rxF64[idx+7]; - ul_ch0_ext64[idx+4]=ul_ch064[idx+6]; - ul_ch0_ext64[idx+5]=ul_ch064[idx+7]; - } - rxF_ext64+=(neg_length/3); - rxF64=(int64_t*)rxF; - ul_ch0_ext64+=(neg_length/3); - ul_ch064+=(neg_length>>1); - for (idx=0;idx>1;idx+=6) { - rxF_ext64[idx] =rxF64[idx+0]; - rxF_ext64[idx+1]=rxF64[idx+1]; - rxF_ext64[idx+2]=rxF64[idx+3]; - rxF_ext64[idx+3]=rxF64[idx+4]; - ul_ch0_ext64[idx]=ul_ch064[idx+0]; - ul_ch0_ext64[idx+1]=ul_ch064[idx+1]; - ul_ch0_ext64[idx+2]=ul_ch064[idx+3]; - ul_ch0_ext64[idx+3]=ul_ch064[idx+4]; - } + rxF64 = rxF; + idx2 = idx; + for (idx = 0; idx < pos_length; idx++, idx2++) + { + if (idx % 6 == 2 * delta || idx % 6 == 2 * delta + 1) + continue; + *rxF_ext++ = rxF64[idx]; + *ul_ch0_ext++ = ul_ch0[idx2]; } } } @@ -662,1423 +539,6 @@ int get_nb_re_pusch (NR_DL_FRAME_PARMS *frame_parms, nfapi_nr_pusch_pdu_t *rel15 } else return(rel15_ul->rb_size * NR_NB_SC_PER_RB); } -void inner_rx_qpsk (int *rxF, - int *ul_ch, - int16_t *llr, - int aarx, - int length, - int output_shift) -{ -#ifndef USE_128BIT - register simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; - register simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); - register simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); - - simde__m256i *rxF256 = (simde__m256i*)rxF; - simde__m256i *ulch256 = (simde__m256i*)ul_ch; - // need to use simde__m64 because llr output is not necessarily aligned to 256 bits, but it is always to 64 bits - simde__m64 *llr64 = (simde__m64 *)llr; - for (int i=0; i<((length>>3)+((length&7)>0?1:0)); i++) { - xmmp0 = simde_mm256_madd_epi16(ulch256[i], rxF256[i]); - // xmmp0 contains real part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm256_shuffle_epi8(ulch256[i], complex_shuffle256); - xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); - xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF256[i]); - // xmmp1 contains imag part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); - xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); - xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); - xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); - xmmp4 = simde_mm256_packs_epi32(xmmp2,xmmp3); - if (aarx == 0) - { - *llr64 = (simde__m64)simde_mm256_extract_epi64(xmmp4,0); llr64++; - *llr64 = (simde__m64)simde_mm256_extract_epi64(xmmp4,1); llr64++; - *llr64 = (simde__m64)simde_mm256_extract_epi64(xmmp4,2); llr64++; - *llr64 = (simde__m64)simde_mm256_extract_epi64(xmmp4,3); llr64++; - } - else - { - *llr64 = simde_mm_adds_pi16(*llr64,(simde__m64)(simde_mm256_extract_epi64(xmmp4,0))); llr64++; - *llr64 = simde_mm_adds_pi16(*llr64,(simde__m64)(simde_mm256_extract_epi64(xmmp4,1))); llr64++; - *llr64 = simde_mm_adds_pi16(*llr64,(simde__m64)(simde_mm256_extract_epi64(xmmp4,2))); llr64++; - *llr64 = simde_mm_adds_pi16(*llr64,(simde__m64)(simde_mm256_extract_epi64(xmmp4,3))); llr64++; - } - } -#else - simde__m128i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; - register simde__m128i complex_shuffle128 = simde_mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); - register simde__m128i conj128 = simde_mm_set_epi16(1, -1, 1, -1, 1, -1, 1, -1); - - simde__m128i *rxF128 = (simde__m128i*)rxF; - simde__m128i *ulch128 = (simde__m128i*)ul_ch; - simde__m128i *llr128 = (simde__m128i*)llr; - for (int i = 0; i < (length >> 2); i++) { - xmmp0 = simde_mm_madd_epi16(ulch128[i], rxF128[i]); - // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm_shuffle_epi8(ulch128[i], complex_shuffle128); - xmmp1 = simde_mm_sign_epi16(xmmp1, conj128); - xmmp1 = simde_mm_madd_epi16(xmmp1, rxF128[i]); - // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm_srai_epi32(xmmp0, output_shift); - xmmp1 = simde_mm_srai_epi32(xmmp1, output_shift); - xmmp2 = simde_mm_unpacklo_epi32(xmmp0, xmmp1); - xmmp3 = simde_mm_unpackhi_epi32(xmmp0, xmmp1); - xmmp4 = simde_mm_packs_epi32(xmmp2, xmmp3); - - if (aarx == 0) - *llr128 = xmmp4; - else - *llr128 = simde_mm_add_epi16(*llr128, xmmp4); - llr128++; - } - if (length & 3) - { - int i = (length>>1) - 1; - simde__m64* llr64 = (simde__m64*)llr128; - simde__m64 xmm0, xmm1, xmm2, xmm3, xmm4; - simde__m64 complex_shuffle64 = simde_mm_set_pi8(5, 4, 7, 6, 1, 0, 3, 2); - simde__m64 conj64 = simde_mm_set_pi16(1, -1, 1, -1); - simde__m64 *rxF64 = (simde__m64*)rxF; - simde__m64 *ulch64 = (simde__m64*)ul_ch; - - xmm0 = simde_mm_madd_pi16(ulch64[i], rxF64[i]); - // xmm0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmm1 = simde_mm_shuffle_pi8(ulch64[i], complex_shuffle64); - xmm1 = simde_mm_sign_pi16(xmm1, conj64); - xmm1 = simde_mm_madd_pi16(xmm1, rxF64[i]); - // xmm1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmm0 = simde_mm_srai_pi32(xmm0, output_shift); - xmm1 = simde_mm_srai_pi32(xmm1, output_shift); - xmm2 = simde_mm_unpacklo_pi32(xmm0, xmm1); - xmm3 = simde_mm_unpackhi_pi32(xmm0, xmm1); - xmm4 = simde_mm_packs_pi32(xmm2, xmm3); - - if (aarx == 0) - *llr64 = xmm4; - else - *llr64 = simde_mm_add_pi16(*llr64, xmm4); - } -#endif -} - -void inner_rx_16qam (int *rxF, - int *ul_ch, - int16_t *llr, - int aarx, - int length, - int output_shift) -{ -#ifndef USE_128BIT - register simde__m256i xmmp0,xmmp1,xmmp2,xmmp3,xmmp4,xmmp5; - register simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); - register simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); - - register simde__m256i QAM_amp256 = simde_mm256_set1_epi16(QAM16_n1); // 2/sqrt(10) - simde__m256i *rxF256 = (simde__m256i*)rxF; - simde__m256i *ulch256 = (simde__m256i*)ul_ch; - // need to use simde__m64 because llr output is not necessarily aligned to 256 bits, but it is always to 64 bits - simde__m64 *llr64 = (simde__m64 *)llr; - - for (int i = 0; i < ((length >> 3) + ((length & 7) > 0 ? 1 : 0)); i++) - { - xmmp0 = simde_mm256_madd_epi16(ulch256[i], rxF256[i]); - // xmmp0 contains real part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm256_shuffle_epi8(ulch256[i], complex_shuffle256); - xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); - xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF256[i]); - // xmmp1 contains imag part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); - xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); - xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); - xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); - xmmp4 = simde_mm256_packs_epi32(xmmp2, xmmp3); - - // compute channel amplitude for LLR - xmmp0 = simde_mm256_madd_epi16(ulch256[i], ulch256[i]); - xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); - xmmp0 = simde_mm256_packs_epi32(xmmp0, xmmp0); - xmmp1 = simde_mm256_unpacklo_epi16(xmmp0, xmmp0); - xmmp1 = simde_mm256_mulhrs_epi16(xmmp1, QAM_amp256); - - xmmp2 = simde_mm256_abs_epi16(xmmp4); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| - xmmp2 = simde_mm256_subs_epi16(xmmp1,xmmp2); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 - - xmmp3 = simde_mm256_unpacklo_epi32(xmmp4,xmmp2); // llr128[0] contains the llrs of the 1st,2nd,5th and 6th REs - xmmp5 = simde_mm256_unpackhi_epi32(xmmp4,xmmp2); // llr128[1] contains the llrs of the 3rd, 4th, 7th and 8th REs - if (aarx == 0) - { - // 1st/2nd RE - llr64[0] = (simde__m64)simde_mm256_extract_epi64(xmmp3,0); // llr32[0] low 16 bits-> y_R , high 16 bits-> y_I - // 3rd/4th RE - llr64[1] = (simde__m64)simde_mm256_extract_epi64(xmmp3,1); // llr32[2] low 16 bits-> y_R , high 16 bits-> y_I - // 5th/6th RE - llr64[2] = (simde__m64)simde_mm256_extract_epi64(xmmp5,0); // llr32[4] low 16 bits-> y_R , high 16 bits-> y_I - // 7Rh/8th RE - llr64[3] = (simde__m64)simde_mm256_extract_epi64(xmmp5,1); // llr32[6] low 16 bits-> y_R , high 16 bits-> y_I - // 9th/10th RE - llr64[4] = (simde__m64)simde_mm256_extract_epi64(xmmp3,2); // llr32[8] low 16 bits-> y_R , high 16 bits-> y_I - // 11th/12th RE - llr64[5] = (simde__m64)simde_mm256_extract_epi64(xmmp3,3); // llr32[10] low 16 bits-> y_R , high 16 bits-> y_I - // 13th/14th RE - llr64[6] = (simde__m64)simde_mm256_extract_epi64(xmmp5,2); // llr32[12] low 16 bits-> y_R , high 16 bits-> y_I - // 15th/16th RE - llr64[7] = (simde__m64)simde_mm256_extract_epi64(xmmp5,3); // llr32[14] low 16 bits-> y_R , high 16 bits-> y_I - llr64+=8; - } - else - { - llr64[0] = simde_mm_adds_pi16(llr64[0],(simde__m64)simde_mm256_extract_epi64(xmmp3,0)); - llr64[1] = simde_mm_adds_pi16(llr64[1],(simde__m64)simde_mm256_extract_epi64(xmmp3,1)); - llr64[2] = simde_mm_adds_pi16(llr64[2],(simde__m64)simde_mm256_extract_epi64(xmmp5,0)); - llr64[3] = simde_mm_adds_pi16(llr64[3],(simde__m64)simde_mm256_extract_epi64(xmmp5,1)); - llr64[4] = simde_mm_adds_pi16(llr64[4],(simde__m64)simde_mm256_extract_epi64(xmmp3,2)); - llr64[5] = simde_mm_adds_pi16(llr64[5],(simde__m64)simde_mm256_extract_epi64(xmmp3,3)); - llr64[6] = simde_mm_adds_pi16(llr64[6],(simde__m64)simde_mm256_extract_epi64(xmmp5,2)); - llr64[7] = simde_mm_adds_pi16(llr64[7],(simde__m64)simde_mm256_extract_epi64(xmmp5,3)); - llr64 += 8; - } - } -#else - simde__m128i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4, xmmp5; - register simde__m128i complex_shuffle128 = simde_mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); - register simde__m128i conj128 = simde_mm_set_epi16(1, -1, 1, -1, 1, -1, 1, -1); - - register simde__m128i QAM_amp128 = simde_mm_set1_epi16(QAM16_n1); // 2/sqrt(10) - simde__m128i *rxF128 = (simde__m128i*)rxF; - simde__m128i *ulch128 = (simde__m128i*)ul_ch; - // need to use simde__m64 because llr output is not necessarily aligned to 256 bits, but it is always to 64 bits - simde__m64 *llr64 = (simde__m64 *)llr; - - for (int i = 0; i < (length >> 2); i++) - { - xmmp0 = simde_mm_madd_epi16(ulch128[i], rxF128[i]); - // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm_shuffle_epi8(ulch128[i], complex_shuffle128); - xmmp1 = simde_mm_sign_epi16(xmmp1, conj128); - xmmp1 = simde_mm_madd_epi16(xmmp1, rxF128[i]); - // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm_srai_epi32(xmmp0, output_shift); - xmmp1 = simde_mm_srai_epi32(xmmp1, output_shift); - xmmp2 = simde_mm_unpacklo_epi32(xmmp0, xmmp1); - xmmp3 = simde_mm_unpackhi_epi32(xmmp0, xmmp1); - xmmp4 = simde_mm_packs_epi32(xmmp2, xmmp3); - - // compute channel amplitude for LLR - xmmp0 = simde_mm_madd_epi16(ulch128[i], ulch128[i]); // |h|^2 - xmmp0 = simde_mm_srai_epi32(xmmp0, output_shift); - xmmp0 = simde_mm_packs_epi32(xmmp0, xmmp0); - xmmp1 = simde_mm_unpacklo_epi16(xmmp0, xmmp0); - xmmp1 = simde_mm_mulhrs_epi16(xmmp1, QAM_amp128); - - xmmp2 = simde_mm_abs_epi16(xmmp4); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| - xmmp2 = simde_mm_subs_epi16(xmmp1, xmmp2); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 - - xmmp3 = simde_mm_unpacklo_epi32(xmmp4, xmmp2); // llr128[0] contains the llrs of the 1st,2nd,5th and 6th REs - xmmp5 = simde_mm_unpackhi_epi32(xmmp4, xmmp2); // llr128[1] contains the llrs of the 3rd, 4th, 7th and 8th REs - if (aarx == 0) - { - llr64[0] = (simde__m64)simde_mm_extract_epi64(xmmp3, 0); // llr32[0] low 16 bits-> y_R, high 16 bits-> y_I - llr64[1] = (simde__m64)simde_mm_extract_epi64(xmmp3, 1); // llr32[2] low 16 bits-> y_R, high 16 bits-> y_I - llr64[2] = (simde__m64)simde_mm_extract_epi64(xmmp5, 0); // llr32[4] low 16 bits-> y_R, high 16 bits-> y_I - llr64[3] = (simde__m64)simde_mm_extract_epi64(xmmp5, 1); // llr32[6] low 16 bits-> y_R, high 16 bits-> y_I - } - else - { - llr64[0] = simde_mm_adds_pi16(llr64[0], (simde__m64)simde_mm_extract_epi64(xmmp3, 0)); - llr64[1] = simde_mm_adds_pi16(llr64[1], (simde__m64)simde_mm_extract_epi64(xmmp3, 1)); - llr64[2] = simde_mm_adds_pi16(llr64[2], (simde__m64)simde_mm_extract_epi64(xmmp5, 0)); - llr64[3] = simde_mm_adds_pi16(llr64[3], (simde__m64)simde_mm_extract_epi64(xmmp5, 1)); - } - llr64 += 4; - } - if (length & 3) - { - int i = (length>>1) - 1; - simde__m64 xmm0, xmm1, xmm2, xmm3, xmm4; - simde__m64 complex_shuffle64 = simde_mm_set_pi8(5,4,7,6,1,0,3,2); - simde__m64 conj64 = simde_mm_set_pi16(1, -1, 1, -1); - simde__m64 *rxF64 = (simde__m64*)rxF; - simde__m64 *ulch64 = (simde__m64*)ul_ch; - simde__m64 QAM_amp = simde_mm_set1_pi16(QAM16_n1); - - xmm0 = simde_mm_madd_pi16(ulch64[i], rxF64[i]); - // xmm0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmm1 = simde_mm_shuffle_pi8(ulch64[i], complex_shuffle64); - xmm1 = simde_mm_sign_pi16(xmm1, conj64); - xmm1 = simde_mm_madd_pi16(xmm1, rxF64[i]); - // xmm1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmm0 = simde_mm_srai_pi32(xmm0, output_shift); - xmm1 = simde_mm_srai_pi32(xmm1, output_shift); - xmm2 = simde_mm_unpacklo_pi32(xmm0, xmm1); - xmm3 = simde_mm_unpackhi_pi32(xmm0, xmm1); - xmm4 = simde_mm_packs_pi32(xmm2, xmm3); - - // compute channel amplitude for LLR - xmm0 = simde_mm_madd_pi16(ulch64[i], ulch64[i]); // |h|^2 - xmm0 = simde_mm_srai_pi32(xmm0, output_shift); - xmm0 = simde_mm_packs_pi32(xmm0, xmm0); - xmm2 = simde_mm_unpacklo_pi16(xmm0, xmm0); - - xmm1 = simde_mm_mulhrs_pi16(xmm2, QAM_amp); - - xmm0 = simde_mm_abs_pi16(xmm4); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| - xmm0 = simde_mm_subs_pi16(xmm1, xmm0); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 - if (aarx == 0) - { - llr64[0] = simde_mm_set_pi32(simde_mm_extract_pi16(xmm0, 0), simde_mm_extract_pi16(xmm4, 0)); - llr64[1] = simde_mm_set_pi32(simde_mm_extract_pi16(xmm0, 1), simde_mm_extract_pi16(xmm4, 1)); - } - else - { - llr64[0] = simde_mm_adds_pi16(llr64[0], simde_mm_set_pi32(simde_mm_extract_pi16(xmm0, 0),simde_mm_extract_pi16(xmm4, 0))); - llr64[1] = simde_mm_adds_pi16(llr64[1], simde_mm_set_pi32(simde_mm_extract_pi16(xmm4, 1),simde_mm_extract_pi16(xmm1, 0))); - } - } - -#endif -} - -void inner_rx_64qam (int *restrict rxF, - int *restrict ul_ch, - int16_t *restrict llr, - int aarx, - int length, - int output_shift) -{ -#ifndef USE_128BIT - register simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4, xmmp6, xmmp7; - register simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29, 28, 31, 30, 25, 24, 27, 26, 21, 20, 23, 22, 17, 16, 19, 18, 13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); - register simde__m256i conj256 = simde_mm256_set_epi16(1, -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1, 1, -1); - - register simde__m256i QAM_amp256 = simde_mm256_set1_epi16(QAM64_n1); // 2/sqrt(10) - register simde__m256i QAM_amp256b = simde_mm256_set1_epi16(QAM64_n2); - simde__m256i *rxF256 = (simde__m256i*)rxF; - simde__m256i *ulch256 = (simde__m256i*)ul_ch; - // need to use simde__m64 because llr output is not necessarily aligned to 256 bits, but it is always to 64 bits - - simde__m64 *llr64 = (simde__m64 *)llr; - for (int i=0;i<((length>>3)+((length&7)>0?1:0));i++) { - xmmp0 = simde_mm256_madd_epi16(ulch256[i],rxF256[i]); - // xmmp0 contains real part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm256_shuffle_epi8(ulch256[i],complex_shuffle256); - xmmp1 = simde_mm256_sign_epi16(xmmp1,conj256); - xmmp1 = simde_mm256_madd_epi16(xmmp1,rxF256[i]); - // xmmp1 contains imag part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm256_srai_epi32(xmmp0,output_shift); - xmmp1 = simde_mm256_srai_epi32(xmmp1,output_shift); - xmmp2 = simde_mm256_unpacklo_epi32(xmmp0,xmmp1); - xmmp3 = simde_mm256_unpackhi_epi32(xmmp0,xmmp1); - xmmp4 = simde_mm256_packs_epi32(xmmp2,xmmp3); - - // compute channel amplitude for LLR - xmmp0 = simde_mm256_madd_epi16(ulch256[i],ulch256[i]); - xmmp0 = simde_mm256_srai_epi32(xmmp0,output_shift); - xmmp0 = simde_mm256_packs_epi32(xmmp0,xmmp0); - xmmp2 = simde_mm256_unpacklo_epi16(xmmp0,xmmp0); - xmmp1 = simde_mm256_mulhrs_epi16(xmmp2,QAM_amp256); - xmmp6 = simde_mm256_mulhrs_epi16(xmmp2,QAM_amp256b); - - xmmp2 = simde_mm256_abs_epi16(xmmp4); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| - xmmp2 = simde_mm256_subs_epi16(xmmp1,xmmp2); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 - xmmp7 = simde_mm256_abs_epi16(xmmp2); - xmmp7 = simde_mm256_subs_epi16(xmmp6,xmmp7); - - if (aarx == 0) - { - llr64[0] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp2,0),simde_mm256_extract_epi32(xmmp4,0)); - llr64[1] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp4,1),simde_mm256_extract_epi32(xmmp7,0)); - llr64[2] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp7,1),simde_mm256_extract_epi32(xmmp2,1)); - llr64[3] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp2,2),simde_mm256_extract_epi32(xmmp4,2)); - llr64[4] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp4,3),simde_mm256_extract_epi32(xmmp7,2)); - llr64[5] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp7,3),simde_mm256_extract_epi32(xmmp2,3)); - llr64[6] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp2,4),simde_mm256_extract_epi32(xmmp4,4)); - llr64[7] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp4,5),simde_mm256_extract_epi32(xmmp7,4)); - llr64[8] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp7,5),simde_mm256_extract_epi32(xmmp2,5)); - llr64[9] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp2,6),simde_mm256_extract_epi32(xmmp4,6)); - llr64[10] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp4,7),simde_mm256_extract_epi32(xmmp7,6)); - llr64[11] = simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp7,7),simde_mm256_extract_epi32(xmmp2,7)); - llr64+=12; - } - else - { - llr64[0] = simde_mm_adds_pi16(llr64[0],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp2,0),simde_mm256_extract_epi32(xmmp4,0))); - llr64[1] = simde_mm_adds_pi16(llr64[1],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp4,1),simde_mm256_extract_epi32(xmmp7,0))); - llr64[2] = simde_mm_adds_pi16(llr64[2],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp7,1),simde_mm256_extract_epi32(xmmp2,1))); - llr64[3] = simde_mm_adds_pi16(llr64[3],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp2,2),simde_mm256_extract_epi32(xmmp4,2))); - llr64[4] = simde_mm_adds_pi16(llr64[4],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp4,3),simde_mm256_extract_epi32(xmmp7,2))); - llr64[5] = simde_mm_adds_pi16(llr64[5],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp7,3),simde_mm256_extract_epi32(xmmp2,3))); - llr64[6] = simde_mm_adds_pi16(llr64[6],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp2,4),simde_mm256_extract_epi32(xmmp4,4))); - llr64[7] = simde_mm_adds_pi16(llr64[7],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp4,5),simde_mm256_extract_epi32(xmmp7,4))); - llr64[8] = simde_mm_adds_pi16(llr64[8],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp7,5),simde_mm256_extract_epi32(xmmp2,5))); - llr64[9] = simde_mm_adds_pi16(llr64[9],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp2,6),simde_mm256_extract_epi32(xmmp4,6))); - llr64[10] = simde_mm_adds_pi16(llr64[10],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp4,7),simde_mm256_extract_epi32(xmmp7,6))); - llr64[11] = simde_mm_adds_pi16(llr64[11],simde_mm_set_pi32(simde_mm256_extract_epi32(xmmp7,7),simde_mm256_extract_epi32(xmmp2,7))); - llr64+=12; - } - } -#else - register simde__m128i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4, xmmp6, xmmp7; - register simde__m128i complex_shuffle128 = simde_mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); - register simde__m128i conj128 = simde_mm_set_epi16(1, -1, 1, -1, 1, -1, 1, -1); - // register simde__m128i conj128 = simde_mm_set_epi16(1, -1, 1, -1, 1, -1, 1, -1); - register simde__m128i QAM_amp128 = simde_mm_set1_epi16(QAM64_n1); // 4/sqrt(42) - register simde__m128i QAM_amp128b = simde_mm_set1_epi16(QAM64_n2); // 2/sqrt(42) - simde__m128i *rxF128 = (simde__m128i*) rxF; - simde__m128i *ulch128 = (simde__m128i*) ul_ch; - // need to use simde__m64 because llr output is not necessarily aligned to 256 bits, but it is always to 64 bits - simde__m64 *llr64 = (simde__m64 *)llr; - for (int i = 0; i < (length>>2); i++) - { - xmmp0 = simde_mm_madd_epi16(ulch128[i], rxF128[i]); - // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm_shuffle_epi8(ulch128[i], complex_shuffle128); - xmmp1 = simde_mm_sign_epi16(xmmp1, conj128); - xmmp1 = simde_mm_madd_epi16(xmmp1, rxF128[i]); - // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm_srai_epi32(xmmp0, output_shift); - xmmp1 = simde_mm_srai_epi32(xmmp1, output_shift); - xmmp2 = simde_mm_unpacklo_epi32(xmmp0, xmmp1); - xmmp3 = simde_mm_unpackhi_epi32(xmmp0, xmmp1); - xmmp4 = simde_mm_packs_epi32(xmmp2, xmmp3); - - // compute channel amplitude for LLR - xmmp0 = simde_mm_madd_epi16(ulch128[i], ulch128[i]); - xmmp0 = simde_mm_srai_epi32(xmmp0, output_shift); - xmmp0 = simde_mm_packs_epi32(xmmp0, xmmp0); - xmmp2 = simde_mm_unpacklo_epi16(xmmp0, xmmp0); - xmmp1 = simde_mm_mulhrs_epi16(xmmp2, QAM_amp128); - xmmp6 = simde_mm_mulhrs_epi16(xmmp2, QAM_amp128b); - - xmmp2 = simde_mm_abs_epi16(xmmp4); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| - xmmp2 = simde_mm_subs_epi16(xmmp1, xmmp2); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 - xmmp7 = simde_mm_abs_epi16(xmmp2); - xmmp7 = simde_mm_subs_epi16(xmmp6, xmmp7); - - if (aarx == 0) - { - llr64[0] = simde_mm_set_pi32(simde_mm_extract_epi32(xmmp2, 0), simde_mm_extract_epi32(xmmp4, 0)); - llr64[1] = simde_mm_set_pi32(simde_mm_extract_epi32(xmmp4, 1), simde_mm_extract_epi32(xmmp7, 0)); - llr64[2] = simde_mm_set_pi32(simde_mm_extract_epi32(xmmp7, 1), simde_mm_extract_epi32(xmmp2, 1)); - llr64[3] = simde_mm_set_pi32(simde_mm_extract_epi32(xmmp2, 2), simde_mm_extract_epi32(xmmp4, 2)); - llr64[4] = simde_mm_set_pi32(simde_mm_extract_epi32(xmmp4, 3), simde_mm_extract_epi32(xmmp7, 2)); - llr64[5] = simde_mm_set_pi32(simde_mm_extract_epi32(xmmp7, 3), simde_mm_extract_epi32(xmmp2, 3)); - llr64 += 6; - } - else - { - llr64[0] = simde_mm_adds_pi16(llr64[0], simde_mm_set_pi32(simde_mm_extract_epi32(xmmp2, 0),simde_mm_extract_epi32(xmmp4, 0))); - llr64[1] = simde_mm_adds_pi16(llr64[1], simde_mm_set_pi32(simde_mm_extract_epi32(xmmp4, 1),simde_mm_extract_epi32(xmmp7, 0))); - llr64[2] = simde_mm_adds_pi16(llr64[2], simde_mm_set_pi32(simde_mm_extract_epi32(xmmp7, 1),simde_mm_extract_epi32(xmmp2, 1))); - llr64[3] = simde_mm_adds_pi16(llr64[3], simde_mm_set_pi32(simde_mm_extract_epi32(xmmp2, 2),simde_mm_extract_epi32(xmmp4, 2))); - llr64[4] = simde_mm_adds_pi16(llr64[4], simde_mm_set_pi32(simde_mm_extract_epi32(xmmp4, 3),simde_mm_extract_epi32(xmmp7, 2))); - llr64[5] = simde_mm_adds_pi16(llr64[5], simde_mm_set_pi32(simde_mm_extract_epi32(xmmp7, 3),simde_mm_extract_epi32(xmmp2, 3))); - llr64 += 6; - } - } - if (length & 3) - { - int i = (length>>1) - 1; - simde__m64 xmm0, xmm1, xmm2, xmm3, xmm4, xmm5; - simde__m64 complex_shuffle64 = simde_mm_set_pi8(5,4,7,6,1,0,3,2); - simde__m64 conj64 = simde_mm_set_pi16(1, -1, 1, -1); - simde__m64 *rxF64 = (simde__m64*)rxF; - simde__m64 *ulch64 = (simde__m64*)ul_ch; - simde__m64 QAM_amp = simde_mm_set1_pi16(QAM64_n1); - simde__m64 QAM_ampb = simde_mm_set1_pi16(QAM64_n2); - - xmm0 = simde_mm_madd_pi16(ulch64[i], rxF64[i]); - // xmm0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmm1 = simde_mm_shuffle_pi8(ulch64[i], complex_shuffle64); - xmm1 = simde_mm_sign_pi16(xmm1, conj64); - xmm1 = simde_mm_madd_pi16(xmm1, rxF64[i]); - // xmm1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmm0 = simde_mm_srai_pi32(xmm0, output_shift); - xmm1 = simde_mm_srai_pi32(xmm1, output_shift); - xmm2 = simde_mm_unpacklo_pi32(xmm0, xmm1); - xmm3 = simde_mm_unpackhi_pi32(xmm0, xmm1); - xmm4 = simde_mm_packs_pi32(xmm2, xmm3); - - // compute channel amplitude for LLR - xmm0 = simde_mm_madd_pi16(ulch64[i], ulch64[i]); // |h|^2 - xmm0 = simde_mm_srai_pi32(xmm0, output_shift); - xmm0 = simde_mm_packs_pi32(xmm0, xmm0); - xmm2 = simde_mm_unpacklo_pi16(xmm0, xmm0); - - xmm1 = simde_mm_mulhrs_pi16(xmm2, QAM_amp); - xmm5 = simde_mm_mulhrs_pi16(xmm2, QAM_ampb); - - xmm0 = simde_mm_abs_pi16(xmm4); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| - xmm0 = simde_mm_subs_pi16(xmm1, xmm0); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 - xmm1 = simde_mm_abs_pi16(xmm0); - xmm1 = simde_mm_subs_pi16(xmm5, xmm1); // contains 8 LLRs - if (aarx == 0) - { - llr64[0] = simde_mm_set_pi32(simde_mm_extract_pi16(xmm0, 0), simde_mm_extract_pi16(xmm4, 0)); - llr64[1] = simde_mm_set_pi32(simde_mm_extract_pi16(xmm4, 1), simde_mm_extract_pi16(xmm1, 0)); - llr64[2] = simde_mm_set_pi32(simde_mm_extract_pi16(xmm1, 1), simde_mm_extract_pi16(xmm0, 1)); - } - else - { - llr64[0] = simde_mm_adds_pi16(llr64[0], simde_mm_set_pi32(simde_mm_extract_pi16(xmm0, 0),simde_mm_extract_pi16(xmm4, 0))); - llr64[1] = simde_mm_adds_pi16(llr64[1], simde_mm_set_pi32(simde_mm_extract_pi16(xmm4, 1),simde_mm_extract_pi16(xmm1, 0))); - llr64[2] = simde_mm_adds_pi16(llr64[2], simde_mm_set_pi32(simde_mm_extract_pi16(xmm1, 1),simde_mm_extract_pi16(xmm0, 1))); - } - } -#endif -} - -void inner_rx_256qam (int *rxF, - int *ul_ch, - int16_t *llr, - int aarx, - int length, - int output_shift) -{ -#ifndef USE_128BIT - register simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4, xmmp5, xmmp6, xmmp7, xmmp8, xmmp9; - register simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); - register simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); - - register simde__m256i QAM_amp256 = simde_mm256_set1_epi16(QAM256_n1); - register simde__m256i QAM_amp256b = simde_mm256_set1_epi16(QAM256_n2); - register simde__m256i QAM_amp256c = simde_mm256_set1_epi16(QAM256_n3); - simde__m256i *rxF256 = (simde__m256i*)rxF; - simde__m256i *ulch256 = (simde__m256i*)ul_ch; - simde__m256i *llr256 = (simde__m256i *)llr; - for (int i = 0; i < ((length >> 3) + (( length & 7) > 0 ? 1 : 0)); i++) - { - xmmp0 = simde_mm256_madd_epi16(ulch256[i],rxF256[i]); - // xmmp0 contains real part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm256_shuffle_epi8(ulch256[i],complex_shuffle256); - xmmp1 = simde_mm256_sign_epi16(xmmp1,conj256); - xmmp1 = simde_mm256_madd_epi16(xmmp1,rxF256[i]); - // xmmp1 contains imag part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm256_srai_epi32(xmmp0,output_shift); - xmmp1 = simde_mm256_srai_epi32(xmmp1,output_shift); - xmmp2 = simde_mm256_unpacklo_epi32(xmmp0,xmmp1); - xmmp3 = simde_mm256_unpackhi_epi32(xmmp0,xmmp1); - xmmp4 = simde_mm256_packs_epi32(xmmp2,xmmp3); - - // compute channel amplitude for LLR - xmmp0 = simde_mm256_madd_epi16(ulch256[i],ulch256[i]); - xmmp0 = simde_mm256_srai_epi32(xmmp0,output_shift); - xmmp0 = simde_mm256_packs_epi32(xmmp0,xmmp0); // contains 16 LLRs - xmmp2 = simde_mm256_unpacklo_epi16(xmmp0,xmmp0); - xmmp1 = simde_mm256_mulhrs_epi16(xmmp2,QAM_amp256); - xmmp6 = simde_mm256_mulhrs_epi16(xmmp2,QAM_amp256b); - xmmp8 = simde_mm256_mulhrs_epi16(xmmp2,QAM_amp256c); - - xmmp2 = simde_mm256_abs_epi16(xmmp4); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| - xmmp2 = simde_mm256_subs_epi16(xmmp1,xmmp2); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 - // xmmp2 contains 16 LLRs - xmmp7 = simde_mm256_abs_epi16(xmmp2); - xmmp7 = simde_mm256_subs_epi16(xmmp6,xmmp7); // contains 16 LLRs - xmmp9 = simde_mm256_abs_epi16(xmmp7); - xmmp9 = simde_mm256_subs_epi16(xmmp8,xmmp9); // contains 16 LLRs - - // xmmp4 A0 A1 A2 A3 A4 A5 A6 A7 - // xmmp2 B0 B1 B2 B3 B4 B5 B6 B7 - // xmmp7 C0 C1 C2 C3 C4 C5 C6 C7 - // xmmp9 D0 D1 D2 D3 D4 D5 D6 D7 - xmmp1 = simde_mm256_unpacklo_epi32(xmmp4,xmmp2); // A0 B0 A1 B1 A4 B4 A5 B5 - xmmp3 = simde_mm256_unpackhi_epi32(xmmp4,xmmp2); // A2 B2 A3 B3 A6 B6 A7 B7 - xmmp5 = simde_mm256_unpacklo_epi32(xmmp7,xmmp9); // C0 D0 C1 D1 C4 D4 C5 D5 - xmmp6 = simde_mm256_unpackhi_epi32(xmmp7,xmmp9); // C2 D2 C3 D3 C6 D6 C7 D7 - - xmmp2 = simde_mm256_unpacklo_epi64(xmmp1,xmmp5); // A0 B0 C0 D0 A4 B4 C4 D4 - xmmp4 = simde_mm256_unpackhi_epi64(xmmp1,xmmp5); // A1 B1 C1 D1 A5 B5 C5 D5 - xmmp1 = simde_mm256_unpacklo_epi64(xmmp3,xmmp6); // A2 B2 C2 D2 A6 B6 C6 D6 - xmmp5 = simde_mm256_unpackhi_epi64(xmmp3,xmmp6); // A3 B3 C3 D3 A7 B7 C7 D7 - if (aarx == 0) - { - llr256[0] = simde_mm256_permute2x128_si256(xmmp2, xmmp4, 0x20); // A0 B0 C0 D0 A1 B1 C1 D1 - llr256[1] = simde_mm256_permute2x128_si256(xmmp1, xmmp5, 0x20); // A2 B2 C2 D2 A3 B3 C3 D3 - llr256[2] = simde_mm256_permute2x128_si256(xmmp2, xmmp4, 0x31); // A4 B4 C4 D4 A5 B5 C5 D5 - llr256[3] = simde_mm256_permute2x128_si256(xmmp1, xmmp5, 0x31); // A6 B6 C6 D6 A7 B7 C7 D7 - llr256+=4; - } - else - { - llr256[0] = simde_mm256_adds_epi16(llr256[0],simde_mm256_permute2x128_si256(xmmp2, xmmp4, 0x20)); // A0 B0 C0 D0 A1 B1 C1 D1 - llr256[1] = simde_mm256_adds_epi16(llr256[1],simde_mm256_permute2x128_si256(xmmp1, xmmp5, 0x20)); // A2 B2 C2 D2 A3 B3 C3 D3 - llr256[2] = simde_mm256_adds_epi16(llr256[2],simde_mm256_permute2x128_si256(xmmp2, xmmp4, 0x31)); // A4 B4 C4 D4 A5 B5 C5 D5 - llr256[3] = simde_mm256_adds_epi16(llr256[3],simde_mm256_permute2x128_si256(xmmp1, xmmp5, 0x31)); // A6 B6 C6 D6 A7 B7 C7 D7 - llr256+=4; - } - } - simde__m128i *llr128 = (simde__m128i*)llr256; - if ((length&7) >= 4) { //there is a single 128-bit input element remaining - int nb_re128 = length>>2; - simde__m128i xmm0,xmm1,xmm2,xmm3,xmm4,xmm5,xmm6; - simde__m128i complex_shuffle128 = simde_mm_set_epi8(13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); - simde__m128i conj128 = simde_mm_set_epi16(1,-1,1,-1,1,-1,1,-1); - simde__m128i *rxF128 = (simde__m128i*)rxF; - simde__m128i *ulch128 = (simde__m128i*)ul_ch; - simde__m128i QAM_amp = simde_mm_set1_epi16(QAM256_n1); // 2/sqrt(10) - simde__m128i QAM_ampb = simde_mm_set1_epi16(QAM256_n2); - simde__m128i QAM_ampc = simde_mm_set1_epi16(QAM256_n3); - - xmm0 = simde_mm_madd_epi16(ulch128[nb_re128-1],rxF128[nb_re128-1]); - // xmm0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmm1 = simde_mm_shuffle_epi8(ulch128[nb_re128-1],complex_shuffle128); - xmm1 = simde_mm_sign_epi16(xmm1,conj128); - xmm1 = simde_mm_madd_epi16(xmm1,rxF128[nb_re128-1]); - // xmm1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmm0 = simde_mm_srai_epi32(xmm0,output_shift); - xmm1 = simde_mm_srai_epi32(xmm1,output_shift); - xmm2 = simde_mm_unpacklo_epi32(xmm0,xmm1); - xmm3 = simde_mm_unpackhi_epi32(xmm0,xmm1); - xmm4 = simde_mm_packs_epi32(xmm2,xmm3); - - // compute channel amplitude for LLR - xmm0 = simde_mm_madd_epi16(ulch128[nb_re128-1],ulch128[nb_re128-1]); - xmm0 = simde_mm_srai_epi32(xmm0,output_shift); - xmm0 = simde_mm_packs_epi32(xmm0,xmm0); // contains 16 LLRs - xmm2 = simde_mm_unpacklo_epi16(xmm0,xmm0); - xmm1 = simde_mm_mulhrs_epi16(xmm2,QAM_amp); - xmm5 = simde_mm_mulhrs_epi16(xmm2,QAM_ampb); - xmm6 = simde_mm_mulhrs_epi16(xmm2,QAM_ampc); - xmm0 = simde_mm_abs_epi16(xmm4); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| - xmm0 = simde_mm_subs_epi16(xmm1,xmm0); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 - // xmmp2 contains 8 LLRs - xmm1 = simde_mm_abs_epi16(xmm0); - xmm1 = simde_mm_subs_epi16(xmm5,xmm1); // contains 8 LLRs - xmm2 = simde_mm_abs_epi16(xmm1); - xmm2 = simde_mm_subs_epi16(xmm6,xmm2); // contains 8 LLRs - // rxF[i] A0 A1 A2 A3 - // xmm0 B0 B1 B2 B3 - // xmm1 C0 C1 C2 C3 - // xmm2 D0 D1 D2 D3 - xmm3 = simde_mm_unpacklo_epi32(rxF128[nb_re128-1],xmm0); // A0 B0 A1 B1 - xmm4 = simde_mm_unpackhi_epi32(rxF128[nb_re128-1],xmm0); // A2 B2 A3 B3 - xmm5 = simde_mm_unpacklo_epi32(xmm1,xmm2); // C0 D0 C1 D1 - xmm6 = simde_mm_unpackhi_epi32(xmm1,xmm2); // C2 D2 C3 D3 - if (aarx == 0) { - llr128[0] = simde_mm_unpacklo_epi64(xmm3,xmm5); // A0 B0 C0 D0 - llr128[1] = simde_mm_unpackhi_epi64(xmm3,xmm5); // A1 B1 C1 D1 - llr128[2] = simde_mm_unpacklo_epi64(xmm4,xmm6); // A2 B2 C2 D2 - llr128[3] = simde_mm_unpackhi_epi64(xmm4,xmm6); // A3 B3 C3 D3 - llr128+=4; - } - else - { - llr128[0] = simde_mm_adds_epi16(llr128[0],simde_mm_unpacklo_epi64(xmm3,xmm5)); // A0 B0 C0 D0 - llr128[1] = simde_mm_adds_epi16(llr128[1],simde_mm_unpackhi_epi64(xmm3,xmm5)); // A1 B1 C1 D1 - llr128[2] = simde_mm_adds_epi16(llr128[2],simde_mm_unpacklo_epi64(xmm4,xmm6)); // A2 B2 C2 D2 - llr128[3] = simde_mm_adds_epi16(llr128[3],simde_mm_unpackhi_epi64(xmm4,xmm6)); // A3 B3 C3 D3 - llr128+=4; - } - } -#else - simde__m128i xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6; - simde__m128i complex_shuffle128 = simde_mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); - simde__m128i conj128 = simde_mm_set_epi16(1, -1, 1, -1, 1, -1, 1, -1); - simde__m128i *rxF128 = (simde__m128i*)rxF; - simde__m128i *ulch128 = (simde__m128i*)ul_ch; - simde__m128i QAM_amp = simde_mm_set1_epi16(QAM256_n1); - simde__m128i QAM_ampb = simde_mm_set1_epi16(QAM256_n2); - simde__m128i QAM_ampc = simde_mm_set1_epi16(QAM256_n3); - simde__m128i *llr128 = (simde__m128i*)llr; - for (int i = 0; i < (length >> 2); i++) - { - xmm0 = simde_mm_madd_epi16(ulch128[i], rxF128[i]); - // xmm0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmm1 = simde_mm_shuffle_epi8(ulch128[i], complex_shuffle128); - xmm1 = simde_mm_sign_epi16(xmm1, conj128); - xmm1 = simde_mm_madd_epi16(xmm1, rxF128[i]); - // xmm1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmm0 = simde_mm_srai_epi32(xmm0, output_shift); - xmm1 = simde_mm_srai_epi32(xmm1, output_shift); - xmm2 = simde_mm_unpacklo_epi32(xmm0, xmm1); - xmm3 = simde_mm_unpackhi_epi32(xmm0, xmm1); - xmm4 = simde_mm_packs_epi32(xmm2, xmm3); - - // compute channel amplitude for LLR - xmm0 = simde_mm_madd_epi16(ulch128[i], ulch128[i]); // |h|^2 - xmm0 = simde_mm_srai_epi32(xmm0, output_shift); - xmm0 = simde_mm_packs_epi32(xmm0, xmm0); - xmm2 = simde_mm_unpacklo_epi16(xmm0, xmm0); - - xmm1 = simde_mm_mulhrs_epi16(xmm2, QAM_amp); - xmm5 = simde_mm_mulhrs_epi16(xmm2, QAM_ampb); - xmm6 = simde_mm_mulhrs_epi16(xmm2, QAM_ampc); - - xmm0 = simde_mm_abs_epi16(xmm4); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| - xmm0 = simde_mm_subs_epi16(xmm1, xmm0); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 - xmm1 = simde_mm_abs_epi16(xmm0); - xmm1 = simde_mm_subs_epi16(xmm5, xmm1); // contains 8 LLRs - xmm2 = simde_mm_abs_epi16(xmm1); - xmm2 = simde_mm_subs_epi16(xmm6, xmm2); // contains 8 LLRs - // rxF[i] A0 A1 A2 A3 - // xmm0 B0 B1 B2 B3 - // xmm1 C0 C1 C2 C3 - // xmm2 D0 D1 D2 D3 - xmm3 = simde_mm_unpacklo_epi32(xmm4, xmm0); // A0 B0 A1 B1 - xmm4 = simde_mm_unpackhi_epi32(xmm4, xmm0); // A2 B2 A3 B3 - xmm5 = simde_mm_unpacklo_epi32(xmm1, xmm2); // C0 D0 C1 D1 - xmm6 = simde_mm_unpackhi_epi32(xmm1, xmm2); // C2 D2 C3 D3 - if (aarx == 0) { - llr128[0] = simde_mm_unpacklo_epi64(xmm3, xmm5); // A0 B0 C0 D0 - llr128[1] = simde_mm_unpackhi_epi64(xmm3, xmm5); // A1 B1 C1 D1 - llr128[2] = simde_mm_unpacklo_epi64(xmm4, xmm6); // A2 B2 C2 D2 - llr128[3] = simde_mm_unpackhi_epi64(xmm4, xmm6); // A3 B3 C3 D3 - } - else { - llr128[0] = simde_mm_adds_epi16(llr128[0], simde_mm_unpacklo_epi64(xmm3, xmm5)); // A0 B0 C0 D0 - llr128[1] = simde_mm_adds_epi16(llr128[1], simde_mm_unpackhi_epi64(xmm3, xmm5)); // A1 B1 C1 D1 - llr128[2] = simde_mm_adds_epi16(llr128[2], simde_mm_unpacklo_epi64(xmm4, xmm6)); // A2 B2 C2 D2 - llr128[3] = simde_mm_adds_epi16(llr128[3], simde_mm_unpackhi_epi64(xmm4, xmm6)); // A3 B3 C3 D3 - } - llr128+=4; - } - if (length & 3) - { - simde__m64 *llr64 = (simde__m64*) llr128; - int i = (length>>1) - 1; - simde__m64 xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6; - simde__m64 complex_shuffle64 = simde_mm_set_pi8(5,4,7,6,1,0,3,2); - simde__m64 conj64 = simde_mm_set_pi16(1, -1, 1, -1); - simde__m64 *rxF64 = (simde__m64*)rxF; - simde__m64 *ulch64 = (simde__m64*)ul_ch; - simde__m64 QAM_amp = simde_mm_set1_pi16(QAM256_n1); - simde__m64 QAM_ampb = simde_mm_set1_pi16(QAM256_n2); - simde__m64 QAM_ampc = simde_mm_set1_pi16(QAM256_n3); - - xmm0 = simde_mm_madd_pi16(ulch64[i], rxF64[i]); - // xmm0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmm1 = simde_mm_shuffle_pi8(ulch64[i], complex_shuffle64); - xmm1 = simde_mm_sign_pi16(xmm1, conj64); - xmm1 = simde_mm_madd_pi16(xmm1, rxF64[i]); - // xmm1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmm0 = simde_mm_srai_pi32(xmm0, output_shift); - xmm1 = simde_mm_srai_pi32(xmm1, output_shift); - xmm2 = simde_mm_unpacklo_pi32(xmm0, xmm1); - xmm3 = simde_mm_unpackhi_pi32(xmm0, xmm1); - xmm4 = simde_mm_packs_pi32(xmm2, xmm3); - - // compute channel amplitude for LLR - xmm0 = simde_mm_madd_pi16(ulch64[i], ulch64[i]); // |h|^2 - xmm0 = simde_mm_srai_pi32(xmm0, output_shift); - xmm0 = simde_mm_packs_pi32(xmm0, xmm0); - xmm2 = simde_mm_unpacklo_pi16(xmm0, xmm0); - - xmm1 = simde_mm_mulhrs_pi16(xmm2, QAM_amp); - xmm5 = simde_mm_mulhrs_pi16(xmm2, QAM_ampb); - xmm6 = simde_mm_mulhrs_pi16(xmm2, QAM_ampc); - - xmm0 = simde_mm_abs_pi16(xmm4); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| - xmm0 = simde_mm_subs_pi16(xmm1, xmm0); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 - xmm1 = simde_mm_abs_pi16(xmm0); - xmm1 = simde_mm_subs_pi16(xmm5, xmm1); // contains 8 LLRs - xmm2 = simde_mm_abs_pi16(xmm1); - xmm2 = simde_mm_subs_pi16(xmm6, xmm2); // contains 8 LLRs - - xmm3 = simde_mm_unpacklo_pi32(xmm4, xmm0); - xmm4 = simde_mm_unpackhi_pi32(xmm4, xmm0); - xmm5 = simde_mm_unpacklo_pi32(xmm1, xmm2); - xmm6 = simde_mm_unpackhi_pi32(xmm1, xmm2); - if (aarx == 0) { - llr64[0] = simde_m_punpckldq(xmm3, xmm5); - llr64[1] = simde_m_punpckhdq(xmm3, xmm5); - llr64[2] = simde_m_punpckldq(xmm4, xmm6); - llr64[3] = simde_m_punpckhdq(xmm4, xmm6); - } - else - { - llr64[0] = simde_mm_adds_pi16(llr64[0], simde_m_punpckldq(xmm3, xmm5)); - llr64[1] = simde_mm_adds_pi16(llr64[1], simde_m_punpckhdq(xmm3, xmm5)); - llr64[2] = simde_mm_adds_pi16(llr64[2], simde_m_punpckldq(xmm4, xmm6)); - llr64[3] = simde_mm_adds_pi16(llr64[3], simde_m_punpckhdq(xmm4, xmm6)); - } - } -#endif -} - -void inner_rx_qpsk_2layer (NR_DL_FRAME_PARMS *frame_parms, - NR_gNB_PUSCH *pusch_vars, - nfapi_nr_pusch_pdu_t *rel15_ul, - int **rxF, - int **ul_ch, - int16_t **llr, - int nb_layer, - int nb_rx_ant, - int soffset, - int length, - int symbol, - int short nb_rb, - int dmrs_symbol_flag, - int output_shift) -{ - int add_shift = 0; - if (length % 8) - add_shift = 8 - length % 8; - int32_t rxFext[nb_rx_ant][length + add_shift] __attribute__((aligned(32))); - int32_t chFext[nb_layer*nb_rx_ant][length + add_shift] __attribute__((aligned(32))); - for (int aarx = 0; aarx < nb_rx_ant; aarx++) - { - for (int aatx = 0; aatx < nb_layer; aatx++) - { - nr_ulsch_extract_rbs0((c16_t *)rxF[aarx], - pusch_vars->ul_ch_estimates[aatx * nb_rx_ant + aarx], - rxFext[aarx], - chFext[aatx * nb_rx_ant + aarx], - soffset+(symbol * frame_parms->ofdm_symbol_size), - pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, - aarx, - dmrs_symbol_flag, - rel15_ul, - frame_parms); - } - } - int32_t rho[nb_layer*nb_layer][length + add_shift] __attribute__((aligned(32))); - int32_t rxFext_comp[nb_layer][length + add_shift] __attribute__((aligned(32))); - for (int aarx = 0; aarx < nb_rx_ant; aarx++) - { - for (int aatx = 0; aatx < nb_layer; aatx++) - { - for (int atx = 0; atx < nb_layer; atx++) - { -#ifdef USE_128BIT - simde__m128i mmtmpD0, mmtmpD1, mmtmpD2, mmtmpD3, mmtmpD4; - simde__m128i *rho128 = (simde__m128i *)rho[aatx*nb_layer+atx]; - simde__m128i *ul_ch128 = (simde__m128i *)chFext[aatx * nb_rx_ant + aarx]; - simde__m128i *ul_ch128_2 = (simde__m128i *)chFext[atx * nb_rx_ant + aarx]; - for (int i = 0; i < (length >> 2)+((length&3)?1:0); i++) - { - // multiply by conjugated channel - mmtmpD0 = simde_mm_madd_epi16(ul_ch128[i], ul_ch128_2[i]); - // mmtmpD0 contains real part of 4 consecutive outputs (32-bit) - mmtmpD1 = simde_mm_shufflelo_epi16(ul_ch128[i], SIMDE_MM_SHUFFLE(2,3,0,1)); - mmtmpD1 = simde_mm_shufflehi_epi16(mmtmpD1, SIMDE_MM_SHUFFLE(2,3,0,1)); - mmtmpD1 = simde_mm_sign_epi16(mmtmpD1, *(simde__m128i*)&conjugate[0]); - mmtmpD1 = simde_mm_madd_epi16(mmtmpD1, ul_ch128_2[i]); - // mmtmpD1 contains imag part of 4 consecutive outputs (32-bit) - mmtmpD0 = simde_mm_srai_epi32(mmtmpD0, output_shift); - mmtmpD1 = simde_mm_srai_epi32(mmtmpD1, output_shift); - mmtmpD2 = simde_mm_unpacklo_epi32(mmtmpD0, mmtmpD1); - mmtmpD3 = simde_mm_unpackhi_epi32(mmtmpD0, mmtmpD1); - mmtmpD4 = simde_mm_packs_epi32(mmtmpD2, mmtmpD3); - if (aarx == 0) - rho128[i] = mmtmpD4; - else - rho128[i] = simde_mm_adds_epi16(rho128[i], mmtmpD4); - } -#else - simde__m256i mmtmpD0, mmtmpD1, mmtmpD2, mmtmpD3, mmtmpD4; - simde__m256i *rho256 = (simde__m256i *)rho[aatx*nb_layer+atx]; - simde__m256i *ul_ch256 = (simde__m256i *)chFext[aatx * nb_rx_ant + aarx]; - simde__m256i *ul_ch256_2 = (simde__m256i *)chFext[atx * nb_rx_ant + aarx]; - register simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); - register simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); - for (int i = 0; i < ((length >> 3)+((length&7)?1:0)); i++) - { - // multiply by conjugated channel - mmtmpD0 = simde_mm256_madd_epi16(ul_ch256[i], ul_ch256_2[i]); - // mmtmpD0 contains real part of 4 consecutive outputs (32-bit) - mmtmpD1 = simde_mm256_shuffle_epi8(ul_ch256[i], complex_shuffle256); - mmtmpD1 = simde_mm256_sign_epi16(mmtmpD1, conj256); - mmtmpD1 = simde_mm256_madd_epi16(mmtmpD1, ul_ch256_2[i]); - // mmtmpD1 contains imag part of 4 consecutive outputs (32-bit) - mmtmpD0 = simde_mm256_srai_epi32(mmtmpD0, output_shift); - mmtmpD1 = simde_mm256_srai_epi32(mmtmpD1, output_shift); - mmtmpD2 = simde_mm256_unpacklo_epi32(mmtmpD0, mmtmpD1); - mmtmpD3 = simde_mm256_unpackhi_epi32(mmtmpD0, mmtmpD1); - mmtmpD4 = simde_mm256_packs_epi32(mmtmpD2, mmtmpD3); - if (aarx == 0) - rho256[i] = mmtmpD4; - else - rho256[i] = simde_mm256_adds_epi16(rho256[i], mmtmpD4); - } -#endif - } - // compensation -#ifdef USE_128BIT - simde__m128i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; - register simde__m128i complex_shuffle128 = simde_mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); - register simde__m128i conj128 = simde_mm_set_epi16(1, -1, 1, -1, 1, -1, 1, -1); - simde__m128i *rxF128 = (simde__m128i*)rxFext[aarx]; - simde__m128i *ulch128 = (simde__m128i*)chFext[aatx * nb_rx_ant + aarx]; - simde__m128i *rxF_comp128 = (simde__m128i*)rxFext_comp[aatx]; - for (int i = 0; i < (length>>2) + ((length&3)?1:0); i++) - { - xmmp0 = simde_mm_madd_epi16(ulch128[i], rxF128[i]); - // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm_shuffle_epi8(ulch128[i], complex_shuffle128); - xmmp1 = simde_mm_sign_epi16(xmmp1, conj128); - xmmp1 = simde_mm_madd_epi16(xmmp1, rxF128[i]); - // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm_srai_epi32(xmmp0, output_shift); - xmmp1 = simde_mm_srai_epi32(xmmp1, output_shift); - xmmp2 = simde_mm_unpacklo_epi32(xmmp0, xmmp1); - xmmp3 = simde_mm_unpackhi_epi32(xmmp0, xmmp1); - xmmp4 = simde_mm_packs_epi32(xmmp2, xmmp3); - - if (aarx == 0) - *rxF_comp128 = xmmp4; - else - *rxF_comp128 = simde_mm_adds_epi16(*rxF_comp128, xmmp4); - - rxF_comp128++; - } -#else - simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; - register simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); - register simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); - simde__m256i *rxF256 = (simde__m256i*)rxFext[aarx]; - simde__m256i *ulch256 = (simde__m256i*)chFext[aatx * nb_rx_ant + aarx]; - simde__m256i *rxF_comp256 = (simde__m256i*)rxFext_comp[aatx]; - for (int i = 0; i < (length>>3) + ((length&7)?1:0); i++) - { - xmmp0 = simde_mm256_madd_epi16(ulch256[i], rxF256[i]); - // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm256_shuffle_epi8(ulch256[i], complex_shuffle256); - xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); - xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF256[i]); - // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); - xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); - xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); - xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); - xmmp4 = simde_mm256_packs_epi32(xmmp2, xmmp3); - - if (aarx == 0) - *rxF_comp256 = xmmp4; - else - *rxF_comp256 = simde_mm256_adds_epi16(*rxF_comp256, xmmp4); - - rxF_comp256++; - } -#endif - } - } - c16_t *rho0 = (c16_t *)rho[1]; - c16_t *rho1 = (c16_t *)rho[2]; - c16_t *llr_0 = (c16_t *)&llr[0][pusch_vars->llr_offset[symbol]]; - c16_t *llr_1 = (c16_t *)&llr[1][pusch_vars->llr_offset[symbol]]; - - nr_ulsch_qpsk_qpsk((c16_t *)rxFext_comp[0], (c16_t *)rxFext_comp[1], llr_0, rho0, length); - nr_ulsch_qpsk_qpsk((c16_t *)rxFext_comp[1], (c16_t *)rxFext_comp[0], llr_1, rho1, length); - - nr_ulsch_shift_llr(pusch_vars->llr_layers, length, pusch_vars->llr_offset[symbol] >> 1, 2, 4); - -} - - -void inner_rx_16qam_2layer (NR_DL_FRAME_PARMS *frame_parms, - NR_gNB_PUSCH *pusch_vars, - nfapi_nr_pusch_pdu_t *rel15_ul, - int **rxF, - int **ul_ch, - int16_t **llr, - int nb_layer, - int nb_rx_ant, - int soffset, - int length, - int symbol, - int short nb_rb, - int dmrs_symbol_flag, - int output_shift) -{ - int add_shift = 0; - if (length % 8) - add_shift = 8 - length % 8; - - int32_t rxFext[nb_rx_ant][length + add_shift] __attribute__((aligned(32))); - int32_t chFext[nb_layer*nb_rx_ant][length + add_shift] __attribute__((aligned(32))); - for (int aarx = 0; aarx < nb_rx_ant; aarx++) - { - for (int aatx = 0; aatx < nb_layer; aatx++) - { - nr_ulsch_extract_rbs0((c16_t *)rxF[aarx], - pusch_vars->ul_ch_estimates[aatx * nb_rx_ant + aarx], - rxFext[aarx], - chFext[aatx * nb_rx_ant + aarx], - soffset+(symbol * frame_parms->ofdm_symbol_size), - pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, - aarx, - dmrs_symbol_flag, - rel15_ul, - frame_parms); - } - } - - int32_t rho[nb_layer*nb_layer][length + add_shift] __attribute__((aligned(32))); - int32_t rxFext_comp[nb_layer][length + add_shift] __attribute__((aligned(32))); - int32_t ul_ch_mag[nb_layer][length + add_shift] __attribute__((aligned(32))); - for (int aatx = 0; aatx < nb_layer; aatx++) - { - for (int aarx = 0; aarx < nb_rx_ant; aarx++) - { - for (int atx = 0; atx < nb_layer; atx++) - { -#ifdef USE_128BIT - simde__m128i mmtmpD0, mmtmpD1, mmtmpD2, mmtmpD3; - simde__m128i *rho128 = (simde__m128i *)rho[aatx*nb_layer+atx]; - simde__m128i *ul_ch128 = (simde__m128i *)chFext[aatx * nb_rx_ant + aarx]; - simde__m128i *ul_ch128_2 = (simde__m128i *)chFext[atx * nb_rx_ant + aarx]; - for (int i = 0; i < (length>>2)+((length&3)?1:0); i++) - { - // multiply by conjugated channel - mmtmpD0 = simde_mm_madd_epi16(ul_ch128[i], ul_ch128_2[i]); - // mmtmpD0 contains real part of 4 consecutive outputs (32-bit) - mmtmpD1 = simde_mm_shufflelo_epi16(ul_ch128[i], SIMDE_MM_SHUFFLE(2,3,0,1)); - mmtmpD1 = simde_mm_shufflehi_epi16(mmtmpD1, SIMDE_MM_SHUFFLE(2,3,0,1)); - mmtmpD1 = simde_mm_sign_epi16(mmtmpD1, *(simde__m128i*)&conjugate[0]); - mmtmpD1 = simde_mm_madd_epi16(mmtmpD1, ul_ch128_2[i]); - // mmtmpD1 contains imag part of 4 consecutive outputs (32-bit) - mmtmpD0 = simde_mm_srai_epi32(mmtmpD0, output_shift); - mmtmpD1 = simde_mm_srai_epi32(mmtmpD1, output_shift); - mmtmpD2 = simde_mm_unpacklo_epi32(mmtmpD0, mmtmpD1); - mmtmpD3 = simde_mm_unpackhi_epi32(mmtmpD0, mmtmpD1); - if (aarx == 0) - rho128[i] = simde_mm_packs_epi32(mmtmpD2, mmtmpD3); - else - rho128[i] = simde_mm_adds_epi16(rho128[i], simde_mm_packs_epi32(mmtmpD2, mmtmpD3)); - } -#else - simde__m256i mmtmpD0, mmtmpD1, mmtmpD2, mmtmpD3; - simde__m256i *rho256 = (simde__m256i *)rho[aatx*nb_layer+atx]; - simde__m256i *ul_ch256 = (simde__m256i *)chFext[aatx * nb_rx_ant + aarx]; - simde__m256i *ul_ch256_2 = (simde__m256i *)chFext[atx * nb_rx_ant + aarx]; - register simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); - register simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); - for (int i = 0; i < (length >> 3)+((length&7)?1:0); i++) - { - // multiply by conjugated channel - mmtmpD0 = simde_mm256_madd_epi16(ul_ch256[i], ul_ch256_2[i]); - // mmtmpD0 contains real part of 4 consecutive outputs (32-bit) - mmtmpD1 = simde_mm256_shuffle_epi8(ul_ch256[i], complex_shuffle256); - mmtmpD1 = simde_mm256_sign_epi16(mmtmpD1, conj256); - mmtmpD1 = simde_mm256_madd_epi16(mmtmpD1, ul_ch256_2[i]); - // mmtmpD1 contains imag part of 4 consecutive outputs (32-bit) - mmtmpD0 = simde_mm256_srai_epi32(mmtmpD0, output_shift); - mmtmpD1 = simde_mm256_srai_epi32(mmtmpD1, output_shift); - mmtmpD2 = simde_mm256_unpacklo_epi32(mmtmpD0, mmtmpD1); - mmtmpD3 = simde_mm256_unpackhi_epi32(mmtmpD0, mmtmpD1); - if (aarx == 0) - rho256[i] = simde_mm256_packs_epi32(mmtmpD2, mmtmpD3); - else - rho256[i] = simde_mm256_adds_epi16(rho256[i], simde_mm256_packs_epi32(mmtmpD2, mmtmpD3)); - } -#endif - } - // compensation -#ifdef USE_128BIT - simde__m128i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; - register simde__m128i complex_shuffle128 = simde_mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); - register simde__m128i conj128 = simde_mm_set_epi16(1, -1, 1, -1, 1, -1, 1, -1); - register simde__m128i QAM_amp128 = simde_mm_set1_epi16(QAM16_n1); // 2/sqrt(10) - simde__m128i *rxF128 = (simde__m128i*)rxFext[aarx]; - simde__m128i *ulch128 = (simde__m128i*)chFext[aatx * nb_rx_ant + aarx]; - simde__m128i *rxF_comp128 = (simde__m128i*)rxFext_comp[aatx]; - simde__m128i *ul_ch_mag128 = (simde__m128i*)ul_ch_mag[aatx]; - for (int i = 0; i < (length>>2)+((length&3)?1:0); i++) - { - xmmp0 = simde_mm_madd_epi16(ulch128[i], rxF128[i]); - // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm_shuffle_epi8(ulch128[i], complex_shuffle128); - xmmp1 = simde_mm_sign_epi16(xmmp1, conj128); - xmmp1 = simde_mm_madd_epi16(xmmp1, rxF128[i]); - // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm_srai_epi32(xmmp0, output_shift); - xmmp1 = simde_mm_srai_epi32(xmmp1, output_shift); - xmmp2 = simde_mm_unpacklo_epi32(xmmp0, xmmp1); - xmmp3 = simde_mm_unpackhi_epi32(xmmp0, xmmp1); - xmmp4 = simde_mm_packs_epi32(xmmp2, xmmp3); - - xmmp0 = simde_mm_madd_epi16(ulch128[i], ulch128[i]); // |h|^2 - xmmp0 = simde_mm_srai_epi32(xmmp0, output_shift); - xmmp0 = simde_mm_packs_epi32(xmmp0, xmmp0); - xmmp1 = simde_mm_unpacklo_epi16(xmmp0, xmmp0); - xmmp1 = simde_mm_mulhrs_epi16(xmmp1, QAM_amp128); - - if (aarx == 0) - { - *rxF_comp128 = xmmp4; - *ul_ch_mag128 = xmmp1; - } - else - { - *rxF_comp128 = simde_mm_adds_epi16(*rxF_comp128, xmmp4); - *ul_ch_mag128 = simde_mm_adds_epi16(*ul_ch_mag128, xmmp1); - } - rxF_comp128++; - ul_ch_mag128++; - } -#else - simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; - register simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); - register simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); - register simde__m256i QAM_amp256 = simde_mm256_set1_epi16(QAM16_n1); // 2/sqrt(10) - simde__m256i *rxF256 = (simde__m256i*)rxFext[aarx]; - simde__m256i *ulch256 = (simde__m256i*)chFext[aatx * nb_rx_ant + aarx]; - simde__m256i *rxF_comp256 = (simde__m256i*)rxFext_comp[aatx]; - simde__m256i *ul_ch_mag256 = (simde__m256i*)ul_ch_mag[aatx]; - for (int i = 0; i < (length>>3) + ((length&7)?1:0); i++) - { - xmmp0 = simde_mm256_madd_epi16(ulch256[i], rxF256[i]); - // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm256_shuffle_epi8(ulch256[i], complex_shuffle256); - xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); - xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF256[i]); - // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); - xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); - xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); - xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); - xmmp4 = simde_mm256_packs_epi32(xmmp2, xmmp3); - - xmmp0 = simde_mm256_madd_epi16(ulch256[i], ulch256[i]); // |h|^2 - xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); - xmmp0 = simde_mm256_packs_epi32(xmmp0, xmmp0); - xmmp1 = simde_mm256_unpacklo_epi16(xmmp0, xmmp0); - xmmp1 = simde_mm256_mulhrs_epi16(xmmp1, QAM_amp256); - - if (aarx == 0) - { - *rxF_comp256 = xmmp4; - *ul_ch_mag256 = xmmp1; - } - else - { - *rxF_comp256 = simde_mm256_adds_epi16(*rxF_comp256, xmmp4); - *ul_ch_mag256 = simde_mm256_adds_epi16(*ul_ch_mag256, xmmp1); - } - rxF_comp256++; - ul_ch_mag256++; - } -#endif - } - } - c16_t *rho0 = (c16_t *)rho[1]; - c16_t *rho1 = (c16_t *)rho[2]; - c16_t *llr_0 = (c16_t *)&llr[0][pusch_vars->llr_offset[symbol]]; - c16_t *llr_1 = (c16_t *)&llr[1][pusch_vars->llr_offset[symbol]]; - c16_t *ul_ch_mag0 = (c16_t *)ul_ch_mag[0]; - c16_t *ul_ch_mag1 = (c16_t *)ul_ch_mag[1]; - nr_ulsch_qam16_qam16((c16_t *)rxFext_comp[0], (c16_t *)rxFext_comp[1], ul_ch_mag0, ul_ch_mag1, llr_0, rho0, length); - nr_ulsch_qam16_qam16((c16_t *)rxFext_comp[1], (c16_t *)rxFext_comp[0], ul_ch_mag1, ul_ch_mag0, llr_1, rho1, length); -} - -void inner_rx_64qam_2layer (NR_DL_FRAME_PARMS *frame_parms, - NR_gNB_PUSCH *pusch_vars, - nfapi_nr_pusch_pdu_t *rel15_ul, - int **rxF, - int **ul_ch, - int16_t **llr, - int nb_layer, - int nb_rx_ant, - int soffset, - int length, - int symbol, - int short nb_rb, - int dmrs_symbol_flag, - int output_shift) -{ - int add_shift = 0; - if (length % 8) - add_shift = 8 - length % 8; - - int32_t rxFext[nb_rx_ant][length + add_shift] __attribute__((aligned(32))); - int32_t chFext[nb_layer*nb_rx_ant][length + add_shift] __attribute__((aligned(32))); - for (int aarx = 0; aarx < nb_rx_ant; aarx++) - { - for (int aatx = 0; aatx < nb_layer; aatx++) - { - nr_ulsch_extract_rbs0((c16_t *)rxF[aarx], - pusch_vars->ul_ch_estimates[aatx * nb_rx_ant + aarx], - rxFext[aarx], - chFext[aatx * nb_rx_ant + aarx], - soffset+(symbol * frame_parms->ofdm_symbol_size), - pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, - aarx, - dmrs_symbol_flag, - rel15_ul, - frame_parms); - } - } - int32_t rho[nb_layer*nb_layer][length + add_shift] __attribute__((aligned(32))); - int32_t rxFext_comp[nb_layer][length + add_shift] __attribute__((aligned(32))); - int32_t ul_ch_mag[nb_layer][length + add_shift] __attribute__((aligned(32))); - for (int aatx = 0; aatx < nb_layer; aatx++) - { - for (int aarx = 0; aarx < nb_rx_ant; aarx++) - { - for (int atx = 0; atx < nb_layer; atx++) - { -#ifdef USE_128BIT - simde__m128i mmtmpD0, mmtmpD1, mmtmpD2, mmtmpD3; - simde__m128i *rho128 = (simde__m128i *)rho[aatx*nb_layer+atx]; - simde__m128i *ul_ch128 = (simde__m128i *)chFext[aatx * nb_rx_ant + aarx]; - simde__m128i *ul_ch128_2 = (simde__m128i *)chFext[atx * nb_rx_ant + aarx]; - for (int i = 0; i < (length>>2)+((length&3)?1:0); i++) - { - // multiply by conjugated channel - mmtmpD0 = simde_mm_madd_epi16(ul_ch128[i], ul_ch128_2[i]); - // mmtmpD0 contains real part of 4 consecutive outputs (32-bit) - mmtmpD1 = simde_mm_shufflelo_epi16(ul_ch128[i], SIMDE_MM_SHUFFLE(2,3,0,1)); - mmtmpD1 = simde_mm_shufflehi_epi16(mmtmpD1, SIMDE_MM_SHUFFLE(2,3,0,1)); - mmtmpD1 = simde_mm_sign_epi16(mmtmpD1, *(simde__m128i*)&conjugate[0]); - mmtmpD1 = simde_mm_madd_epi16(mmtmpD1, ul_ch128_2[i]); - // mmtmpD1 contains imag part of 4 consecutive outputs (32-bit) - mmtmpD0 = simde_mm_srai_epi32(mmtmpD0, output_shift); - mmtmpD1 = simde_mm_srai_epi32(mmtmpD1, output_shift); - mmtmpD2 = simde_mm_unpacklo_epi32(mmtmpD0, mmtmpD1); - mmtmpD3 = simde_mm_unpackhi_epi32(mmtmpD0, mmtmpD1); - if (aarx == 0) - rho128[i] = simde_mm_packs_epi32(mmtmpD2, mmtmpD3); - else - rho128[i] = simde_mm_adds_epi16(rho128[i], simde_mm_packs_epi32(mmtmpD2, mmtmpD3)); - } -#else - simde__m256i mmtmpD0, mmtmpD1, mmtmpD2, mmtmpD3; - simde__m256i *rho256 = (simde__m256i *)rho[aatx*nb_layer+atx]; - simde__m256i *ul_ch256 = (simde__m256i *)chFext[aatx * nb_rx_ant + aarx]; - simde__m256i *ul_ch256_2 = (simde__m256i *)chFext[atx * nb_rx_ant + aarx]; - register simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); - register simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); - for (int i = 0; i < (length >> 3)+((length&7)?1:0); i++) - { - // multiply by conjugated channel - mmtmpD0 = simde_mm256_madd_epi16(ul_ch256[i], ul_ch256_2[i]); - // mmtmpD0 contains real part of 4 consecutive outputs (32-bit) - mmtmpD1 = simde_mm256_shuffle_epi8(ul_ch256[i], complex_shuffle256); - mmtmpD1 = simde_mm256_sign_epi16(mmtmpD1, conj256); - mmtmpD1 = simde_mm256_madd_epi16(mmtmpD1, ul_ch256_2[i]); - // mmtmpD1 contains imag part of 4 consecutive outputs (32-bit) - mmtmpD0 = simde_mm256_srai_epi32(mmtmpD0, output_shift); - mmtmpD1 = simde_mm256_srai_epi32(mmtmpD1, output_shift); - mmtmpD2 = simde_mm256_unpacklo_epi32(mmtmpD0, mmtmpD1); - mmtmpD3 = simde_mm256_unpackhi_epi32(mmtmpD0, mmtmpD1); - if (aarx == 0) - rho256[i] = simde_mm256_packs_epi32(mmtmpD2, mmtmpD3); - else - rho256[i] = simde_mm256_adds_epi16(rho256[i], simde_mm256_packs_epi32(mmtmpD2, mmtmpD3)); - } -#endif - } - // compensation -#ifdef USE_128BIT - simde__m128i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; - register simde__m128i complex_shuffle128 = simde_mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); - register simde__m128i conj128 = simde_mm_set_epi16(1, -1, 1, -1, 1, -1, 1, -1); - register simde__m128i QAM_amp128 = simde_mm_set1_epi16(QAM64_n1); // 2/sqrt(10) - simde__m128i *rxF128 = (simde__m128i*)rxFext[aarx]; - simde__m128i *ulch128 = (simde__m128i*)chFext[aatx * nb_rx_ant + aarx]; - simde__m128i *rxF_comp128 = (simde__m128i*)rxFext_comp[aatx]; - simde__m128i *ul_ch_mag128 = (simde__m128i*)ul_ch_mag[aatx]; - for (int i = 0; i < (length>>2)+((length&3)?1:0); i++) - { - xmmp0 = simde_mm_madd_epi16(ulch128[i], rxF128[i]); - // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm_shuffle_epi8(ulch128[i], complex_shuffle128); - xmmp1 = simde_mm_sign_epi16(xmmp1, conj128); - xmmp1 = simde_mm_madd_epi16(xmmp1, rxF128[i]); - // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm_srai_epi32(xmmp0, output_shift); - xmmp1 = simde_mm_srai_epi32(xmmp1, output_shift); - xmmp2 = simde_mm_unpacklo_epi32(xmmp0, xmmp1); - xmmp3 = simde_mm_unpackhi_epi32(xmmp0, xmmp1); - xmmp4 = simde_mm_packs_epi32(xmmp2, xmmp3); - - xmmp0 = simde_mm_madd_epi16(ulch128[i], ulch128[i]); // |h|^2 - xmmp0 = simde_mm_srai_epi32(xmmp0, output_shift); - xmmp0 = simde_mm_packs_epi32(xmmp0, xmmp0); - xmmp1 = simde_mm_unpacklo_epi16(xmmp0, xmmp0); - xmmp1 = simde_mm_mulhrs_epi16(xmmp1, QAM_amp128); - - if (aarx == 0) - { - *rxF_comp128 = xmmp4; - *ul_ch_mag128 = xmmp1; - } - else - { - *rxF_comp128 = simde_mm_adds_epi16(*rxF_comp128, xmmp4); - *ul_ch_mag128 = simde_mm_adds_epi16(*ul_ch_mag128, xmmp1); - } - rxF_comp128++; - ul_ch_mag128++; - } -#else - simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; - register simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); - register simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); - register simde__m256i QAM_amp256 = simde_mm256_set1_epi16(QAM64_n1); // 2/sqrt(10) - simde__m256i *rxF256 = (simde__m256i*)rxFext[aarx]; - simde__m256i *ulch256 = (simde__m256i*)chFext[aatx * nb_rx_ant + aarx]; - simde__m256i *rxF_comp256 = (simde__m256i*)rxFext_comp[aatx]; - simde__m256i *ul_ch_mag256 = (simde__m256i*)ul_ch_mag[aatx]; - for (int i = 0; i < (length>>3) + ((length&7)?1:0); i++) - { - xmmp0 = simde_mm256_madd_epi16(ulch256[i], rxF256[i]); - // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm256_shuffle_epi8(ulch256[i], complex_shuffle256); - xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); - xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF256[i]); - // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); - xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); - xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); - xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); - xmmp4 = simde_mm256_packs_epi32(xmmp2, xmmp3); - - xmmp0 = simde_mm256_madd_epi16(ulch256[i], ulch256[i]); // |h|^2 - xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); - xmmp0 = simde_mm256_packs_epi32(xmmp0, xmmp0); - xmmp1 = simde_mm256_unpacklo_epi16(xmmp0, xmmp0); - xmmp1 = simde_mm256_mulhrs_epi16(xmmp1, QAM_amp256); - - if (aarx == 0) - { - *rxF_comp256 = xmmp4; - *ul_ch_mag256 = xmmp1; - } - else - { - *rxF_comp256 = simde_mm256_adds_epi16(*rxF_comp256, xmmp4); - *ul_ch_mag256 = simde_mm256_adds_epi16(*ul_ch_mag256, xmmp1); - } - rxF_comp256++; - ul_ch_mag256++; - } -#endif - } - } - c16_t *rho0 = (c16_t *)rho[1]; - c16_t *rho1 = (c16_t *)rho[2]; - c16_t *llr_0 = (c16_t *)&llr[0][pusch_vars->llr_offset[symbol]]; - c16_t *llr_1 = (c16_t *)&llr[1][pusch_vars->llr_offset[symbol]]; - c16_t *ul_ch_mag0 = (c16_t *)ul_ch_mag[0]; - c16_t *ul_ch_mag1 = (c16_t *)ul_ch_mag[1]; - nr_ulsch_qam64_qam64((c16_t *)rxFext_comp[0], (c16_t *)rxFext_comp[1], ul_ch_mag0, ul_ch_mag1, llr_0, rho0, length); - nr_ulsch_qam64_qam64((c16_t *)rxFext_comp[1], (c16_t *)rxFext_comp[0], ul_ch_mag1, ul_ch_mag0, llr_1, rho1, length); -} - -void nr_pusch_symbol_processing_noprecoding(void *arg) -{ - - puschSymbolProc_t *rdata=(puschSymbolProc_t*)arg; - - PHY_VARS_gNB *gNB = rdata->gNB; - NR_DL_FRAME_PARMS *frame_parms = rdata->frame_parms; - nfapi_nr_pusch_pdu_t *rel15_ul = rdata->rel15_ul; - int ulsch_id = rdata->ulsch_id; - int slot = rdata->slot; - NR_gNB_PUSCH *pusch_vars = &gNB->pusch_vars[ulsch_id]; - int16_t *s = rdata->s; - for (int symbol = rdata->startSymbol; symbol < rdata->startSymbol+rdata->numSymbols; symbol++) - { - int dmrs_symbol_flag = (rel15_ul->ul_dmrs_symb_pos >> symbol) & 0x01; - int nb_re_pusch = gNB->pusch_vars[ulsch_id].ul_valid_re_per_slot[symbol]; - // this needs to be reworded for parrellization, we need a table which give dmrs symbol location - // used for chennel estimate, they are being run in parallel! - if (dmrs_symbol_flag == 1) - { - if ((rel15_ul->ul_dmrs_symb_pos >> ((symbol + 1) % frame_parms->symbols_per_slot)) & 0x01) - AssertFatal(1==0,"Double DMRS configuration is not yet supported\n"); - - gNB->pusch_vars[ulsch_id].dmrs_symbol = symbol; - } - - LOG_I(PHY,"symbol %d: nb_re_pusch %d, DMRS symbl used for Chest :%d \n", symbol, nb_re_pusch, gNB->pusch_vars[ulsch_id].dmrs_symbol); - - if (nb_re_pusch == 0) continue; - if (rel15_ul->nrOfLayers == 1) - { - int16_t *llr = &rdata->llr[pusch_vars->llr_offset[symbol]]; - void (*inner_rx)(int *,int *,int16_t *,int,int,int); - if (rel15_ul->qam_mod_order == 2) inner_rx = inner_rx_qpsk; - else if (rel15_ul->qam_mod_order == 4) inner_rx = inner_rx_16qam; - else if (rel15_ul->qam_mod_order == 6) inner_rx = inner_rx_64qam; - else if (rel15_ul->qam_mod_order == 8) inner_rx = inner_rx_256qam; - else AssertFatal(1==0,"rel15_ul->qam_mod_order %d, pusch_pdu->dmrs_config_type %d\n", - rel15_ul->qam_mod_order,rel15_ul->dmrs_config_type); - - int soffset = (slot&3)*frame_parms->symbols_per_slot*frame_parms->ofdm_symbol_size; - int32_t rxFext[nb_re_pusch+8] __attribute__((aligned(32))); - int32_t chFext[nb_re_pusch+8] __attribute__((aligned(32))); - int16_t llr_temp[(nb_re_pusch*rel15_ul->qam_mod_order)+16] __attribute__((aligned(32))); - for (int aa = 0; aa < frame_parms->nb_antennas_rx; aa++) - { - nr_ulsch_extract_rbs0(gNB->common_vars.rxdataF[aa], - gNB->pusch_vars[ulsch_id].ul_ch_estimates[aa], - rxFext, - chFext, - soffset+(symbol * frame_parms->ofdm_symbol_size), - gNB->pusch_vars[ulsch_id].dmrs_symbol*frame_parms->ofdm_symbol_size, - aa, - dmrs_symbol_flag, - rel15_ul, - frame_parms); - // demodulation - inner_rx(rxFext, chFext, llr_temp, aa, nb_re_pusch, gNB->pusch_vars[ulsch_id].log2_maxh); - } - - // unscrambling - simde__m64 *llr64 = (simde__m64 *) llr; - for (int i=0;i<(nb_re_pusch*rel15_ul->qam_mod_order)>>2;i++) - llr64[i] = simde_mm_mullo_pi16(((simde__m64 *)llr_temp)[i],((simde__m64 *)s)[i]); - - s += nb_re_pusch * rel15_ul->qam_mod_order; - llr += nb_re_pusch * rel15_ul->qam_mod_order; - } - else // MIMO for 2x2 - { - int soffset = (slot&3)*frame_parms->symbols_per_slot*frame_parms->ofdm_symbol_size; - void (*inner_rx)(NR_DL_FRAME_PARMS *, - NR_gNB_PUSCH *, - nfapi_nr_pusch_pdu_t *, - int32_t **, - int32_t **, - int16_t **, - int32_t, - int32_t, - int32_t, - int32_t, - int32_t, - int16_t, - int32_t, - int32_t); - if (rel15_ul->qam_mod_order == 2) inner_rx = inner_rx_qpsk_2layer; - else if (rel15_ul->qam_mod_order == 4) inner_rx = inner_rx_16qam_2layer; - else if (rel15_ul->qam_mod_order == 6) inner_rx = inner_rx_64qam_2layer; - else AssertFatal(1==0,"rel15_ul->qam_mod_order %d, pusch_pdu->dmrs_config_type %d\n", - rel15_ul->qam_mod_order,rel15_ul->dmrs_config_type); - - inner_rx(frame_parms, - pusch_vars, - rel15_ul, - (int32_t**)gNB->common_vars.rxdataF, - gNB->pusch_vars[ulsch_id].ul_ch_estimates, - rdata->llr_layers, - rel15_ul->nrOfLayers, - frame_parms->nb_antennas_rx, - soffset, - nb_re_pusch, // length - symbol, // symbol index - rel15_ul->rb_size, // ofdm size - dmrs_symbol_flag, - gNB->pusch_vars[ulsch_id].log2_maxh); - - // layer de-mapping - int16_t* llr_cw = &rdata->llr[pusch_vars->llr_offset[symbol] * rel15_ul->nrOfLayers]; - for (int i = 0; i < (nb_re_pusch); i++) - for (int l = 0; l < rel15_ul->nrOfLayers; l++) - for (int m = 0; m < rel15_ul->qam_mod_order; m++) - llr_cw[i*rel15_ul->nrOfLayers*rel15_ul->qam_mod_order+l*rel15_ul->qam_mod_order+m] = rdata->llr_layers[l][pusch_vars->llr_offset[symbol] + i*rel15_ul->qam_mod_order+m]; - - // unscrambling - simde__m64 *llr64 = (simde__m64 *) &rdata->llr[pusch_vars->llr_offset[symbol] * rel15_ul->nrOfLayers]; - for (int i = 0; i < (nb_re_pusch*rel15_ul->qam_mod_order*rel15_ul->nrOfLayers)>>2; i++) - llr64[i] = simde_mm_mullo_pi16(((simde__m64 *)llr64)[i], ((simde__m64 *)s)[i]); - - s += (nb_re_pusch*rel15_ul->qam_mod_order*rel15_ul->nrOfLayers); - } - } -} - //compute average channel_level on each (TX,RX) antenna pair void nr_ulsch_channel_level(int **ul_ch_estimates_ext, NR_DL_FRAME_PARMS *frame_parms, @@ -2088,8 +548,6 @@ void nr_ulsch_channel_level(int **ul_ch_estimates_ext, uint8_t nrOfLayers, unsigned short nb_rb) { - - short rb; unsigned char aatx, aarx; simde__m128i *ul_ch128, avg128U; @@ -2124,10 +582,9 @@ void nr_ulsch_channel_level(int **ul_ch_estimates_ext, simde_mm_empty(); simde_m_empty(); - } -static simde__m128i a_mult_conjb(simde__m128i a, simde__m128i b, unsigned char output_shift) +static simde__m128i a_mult_conjb(__m128i a, __m128i b, unsigned char output_shift) { simde__m128i mmtmpD0 = simde_mm_madd_epi16(b, a); simde__m128i mmtmpD1 = simde_mm_shufflelo_epi16(b, SIMDE_MM_SHUFFLE(2, 3, 0, 1)); @@ -2138,7 +595,7 @@ static simde__m128i a_mult_conjb(simde__m128i a, simde__m128i b, unsigned char o mmtmpD1 = simde_mm_srai_epi32(mmtmpD1, output_shift); simde__m128i mmtmpD2 = simde_mm_unpacklo_epi32(mmtmpD0, mmtmpD1); simde__m128i mmtmpD3 = simde_mm_unpackhi_epi32(mmtmpD0, mmtmpD1); - return simde_mm_packs_epi32(mmtmpD2, mmtmpD3); + return _mm_packs_epi32(mmtmpD2, mmtmpD3); } //============================================================================================== @@ -2202,29 +659,30 @@ void nr_ulsch_channel_compensation(int **rxdataF_ext, #endif +#if defined(__i386) || defined(__x86_64__) unsigned short rb; unsigned char aatx,aarx; - simde__m128i *ul_ch128,*ul_ch128_2,*ul_ch_mag128,*ul_ch_mag128b,*ul_ch_mag128c,*rxdataF128,*rxdataF_comp128,*rho128; - simde__m128i mmtmpD0,mmtmpD1,mmtmpD2,mmtmpD3,QAM_amp128={0},QAM_amp128b={0},QAM_amp128c={0}; - QAM_amp128b = simde_mm_setzero_si128(); + __m128i *ul_ch128,*ul_ch128_2,*ul_ch_mag128,*ul_ch_mag128b,*ul_ch_mag128c,*rxdataF128,*rxdataF_comp128,*rho128; + __m128i mmtmpD0,mmtmpD1,mmtmpD2,mmtmpD3,QAM_amp128={0},QAM_amp128b={0},QAM_amp128c={0}; + QAM_amp128b = _mm_setzero_si128(); uint32_t nb_rb_0 = length/12 + ((length%12)?1:0); for (aatx=0; aatxnb_antennas_rx][symbol * (off + nb_rb * 12)]; - ul_ch_mag128b_1 = (simde__m128i *)&ul_ch_magb[frame_parms->nb_antennas_rx][symbol * (off + nb_rb * 12)]; - ul_ch_mag128c_1 = (simde__m128i *)&ul_ch_magc[frame_parms->nb_antennas_rx][symbol * (off + nb_rb * 12)]; + ul_ch_mag128_1 = (simde__m128i *) &((int *)ul_ch_mag)[1 * (length + length_aligned)]; + ul_ch_mag128b_1 = (simde__m128i *)&((int *)ul_ch_magb)[1 * (length + length_aligned)]; + ul_ch_mag128c_1 = (simde__m128i *)&((int *)ul_ch_magc)[1 * (length + length_aligned)]; } for (int rb = 0; rb < 3 * nb_rb_0; rb++) { @@ -3245,340 +1696,981 @@ uint8_t nr_ulsch_mmse_2layers(NR_DL_FRAME_PARMS *frame_parms, return(0); } -//============================================================================================== +void inner_rx_qpsk (NR_DL_FRAME_PARMS *frame_parms, + NR_gNB_PUSCH *pusch_vars, + nfapi_nr_pusch_pdu_t *rel15_ul, + c16_t **rxF, + c16_t **ul_ch, + int16_t *llr, + int nb_layer, + int nb_rx_ant, + int soffset, + int length, + int symbol, + int short nb_rb, + int dmrs_symbol_flag, + int output_shift) +{ + int length_aligned = (length % 8) ? (8 - length % 8) : 0; -/* Main Function */ -void nr_rx_pusch(PHY_VARS_gNB *gNB, - uint8_t ulsch_id, - uint32_t frame, - uint8_t slot, - unsigned char harq_pid) + c16_t rxFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + c16_t chFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + for (int aarx = 0; aarx < nb_rx_ant; aarx++) + { + nr_ulsch_extract_rbs0(rxF[aarx], + (c16_t*)pusch_vars->ul_ch_estimates[aarx], + rxFext[aarx], + chFext[aarx], + soffset+(symbol * frame_parms->ofdm_symbol_size), + pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, + aarx, + dmrs_symbol_flag, + rel15_ul, + frame_parms); + simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; + simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); + + simde__m256i *rxF256 = (simde__m256i*)rxFext[aarx]; + simde__m256i *ulch256 = (simde__m256i*)chFext[aarx]; + simde__m256i *llr256 = (simde__m256i *)llr; + for (int i = 0; i < ((length>>3) + ((length&7) > 0 ? 1:0)); i++) + { + xmmp0 = simde_mm256_madd_epi16(ulch256[i], rxF256[i]); + // xmmp0 contains real part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp1 = simde_mm256_shuffle_epi8(ulch256[i], complex_shuffle256); + xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); + xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF256[i]); + // xmmp1 contains imag part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); + xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); + xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); + xmmp4 = simde_mm256_packs_epi32(xmmp2,xmmp3); + + // MRC & LLR + llr256[i] = simde_mm256_add_epi16(llr256[i], xmmp4); + } + } + // Perform IDFT if transform precoding is enabled + if(rel15_ul->transform_precoding == transformPrecoder_enabled) + nr_idft((int32_t *)llr, length); +} + +void inner_rx_16qam (NR_DL_FRAME_PARMS *frame_parms, + NR_gNB_PUSCH *pusch_vars, + nfapi_nr_pusch_pdu_t *rel15_ul, + c16_t **rxF, + c16_t **ul_ch, + int16_t *llr, + int nb_layer, + int nb_rx_ant, + int soffset, + int length, + int symbol, + int short nb_rb, + int dmrs_symbol_flag, + int output_shift) +{ + int length_aligned = (length % 8) ? (8 - length % 8) : 0; + + c16_t rxFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + c16_t chFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + c16_t rxComp[length + length_aligned] __attribute__((aligned(32))); + c16_t rxF_ch_maga[length + length_aligned] __attribute__((aligned(32))); + + memset(rxComp, 0, sizeof(c16_t) * (length + length_aligned)); + memset(rxF_ch_maga, 0, sizeof(c16_t) * (length + length_aligned)); + + simde__m256i *rxComp_256 = (simde__m256i *)rxComp; + simde__m256i *rxF_ch_maga_256 = (simde__m256i *)rxF_ch_maga; + + for (int aarx = 0; aarx < nb_rx_ant; aarx++) + { + nr_ulsch_extract_rbs0(rxF[aarx], + (c16_t*)pusch_vars->ul_ch_estimates[aarx], + rxFext[aarx], + chFext[aarx], + soffset+(symbol * frame_parms->ofdm_symbol_size), + pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, + aarx, + dmrs_symbol_flag, + rel15_ul, + frame_parms); + simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; + simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); + simde__m256i QAM_amp256 = simde_mm256_set1_epi16(QAM16_n1); + + simde__m256i *rxF_256 = (simde__m256i*)rxFext[aarx]; + simde__m256i *chF_256 = (simde__m256i*)chFext[aarx]; + for (int i = 0; i < ((length>>3) + ((length&7) > 0 ? 1:0)); i++) + { + xmmp0 = simde_mm256_madd_epi16(chF_256[i], rxF_256[i]); + // xmmp0 contains real part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp1 = simde_mm256_shuffle_epi8(chF_256[i], complex_shuffle256); + xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); + xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF_256[i]); + // xmmp1 contains imag part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); + xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); + xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); + xmmp4 = simde_mm256_packs_epi32(xmmp2,xmmp3); + + xmmp0 = simde_mm256_madd_epi16(chF_256[i], chF_256[i]); // |h|^2 + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp0 = simde_mm256_packs_epi32(xmmp0, xmmp0); + xmmp0 = simde_mm256_unpacklo_epi16(xmmp0, xmmp0); + + xmmp1 = simde_mm256_mulhrs_epi16(xmmp0, QAM_amp256); + + // MRC + rxF_ch_maga_256[i] = simde_mm256_add_epi16(rxF_ch_maga_256[i], xmmp1); + rxComp_256[i] = simde_mm256_add_epi16(rxComp_256[i], xmmp4); + } + } + + if (rel15_ul->transform_precoding == transformPrecoder_enabled) + { + nr_freq_equalization(frame_parms, + (int *)rxComp, + (int *)rxF_ch_maga, + (int *)rxF_ch_maga, + symbol, + length, + rel15_ul->qam_mod_order); + nr_idft((int32_t*)rxComp, length); + } + nr_ulsch_compute_llr((int32_t *)rxComp, + (int32_t *)rxF_ch_maga, + NULL, + NULL, + llr, + rel15_ul->rb_size, + length, + symbol, + rel15_ul->qam_mod_order); +} + +void inner_rx_64qam (NR_DL_FRAME_PARMS *frame_parms, + NR_gNB_PUSCH *pusch_vars, + nfapi_nr_pusch_pdu_t *rel15_ul, + c16_t **rxF, + c16_t **ul_ch, + int16_t *llr, + int nb_layer, + int nb_rx_ant, + int soffset, + int length, + int symbol, + int short nb_rb, + int dmrs_symbol_flag, + int output_shift) +{ + int length_aligned = (length % 8) ? (8 - length % 8) : 0; + + c16_t rxFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + c16_t chFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + + c16_t rxComp[length + length_aligned] __attribute__((aligned(32))); + c16_t rxF_ch_maga[length + length_aligned] __attribute__((aligned(32))); + c16_t rxF_ch_magb[length + length_aligned] __attribute__((aligned(32))); + + memset(rxComp, 0, sizeof(c16_t) * (length + length_aligned)); + memset(rxF_ch_maga, 0, sizeof(c16_t) * (length + length_aligned)); + memset(rxF_ch_magb, 0, sizeof(c16_t) * (length + length_aligned)); + + simde__m256i *rxComp_256 = (simde__m256i *)rxComp; + simde__m256i *rxF_ch_maga_256 = (simde__m256i *)rxF_ch_maga; + simde__m256i *rxF_ch_magb_256 = (simde__m256i *)rxF_ch_magb; + + for (int aarx = 0; aarx < nb_rx_ant; aarx++) + { + nr_ulsch_extract_rbs0(rxF[aarx], + (c16_t*)pusch_vars->ul_ch_estimates[aarx], + rxFext[aarx], + chFext[aarx], + soffset+(symbol * frame_parms->ofdm_symbol_size), + pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, + aarx, + dmrs_symbol_flag, + rel15_ul, + frame_parms); + simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; + simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); + simde__m256i QAM_amp256 = simde_mm256_set1_epi16(QAM64_n1); // 2/sqrt(10) + simde__m256i QAM_amp256b = simde_mm256_set1_epi16(QAM64_n2); + + simde__m256i *rxF_256 = (simde__m256i*)rxFext[aarx]; + simde__m256i *chF_256 = (simde__m256i*)chFext[aarx]; + for (int i = 0; i < ((length>>3) + ((length&7) > 0 ? 1:0)); i++) + { + xmmp0 = simde_mm256_madd_epi16(chF_256[i], rxF_256[i]); + // xmmp0 contains real part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp1 = simde_mm256_shuffle_epi8(chF_256[i], complex_shuffle256); + xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); + xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF_256[i]); + // xmmp1 contains imag part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); + xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); + xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); + xmmp4 = simde_mm256_packs_epi32(xmmp2,xmmp3); + + xmmp0 = simde_mm256_madd_epi16(chF_256[i], chF_256[i]); // |h|^2 + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp0 = simde_mm256_packs_epi32(xmmp0, xmmp0); + xmmp0 = simde_mm256_unpacklo_epi16(xmmp0, xmmp0); + + xmmp1 = simde_mm256_mulhrs_epi16(xmmp0, QAM_amp256); + xmmp2 = simde_mm256_mulhrs_epi16(xmmp0, QAM_amp256b); + + // MRC + rxComp_256[i] = simde_mm256_add_epi16(rxComp_256[i], xmmp4); + rxF_ch_maga_256[i] = simde_mm256_add_epi16(rxF_ch_maga_256[i], xmmp1); + rxF_ch_magb_256[i] = simde_mm256_add_epi16(rxF_ch_magb_256[i], xmmp2); + } + } + + if (rel15_ul->transform_precoding == transformPrecoder_enabled) { + nr_freq_equalization(frame_parms, + (int *)rxComp, + (int *)rxF_ch_maga, + (int *)rxF_ch_magb, + symbol, + length, + rel15_ul->qam_mod_order); + nr_idft((int32_t*)rxComp, length); + } + nr_ulsch_compute_llr((int32_t*)rxComp, + (int32_t*)rxF_ch_maga, + (int32_t*)rxF_ch_magb, + NULL, + llr, + rel15_ul->rb_size, + length, + symbol, + rel15_ul->qam_mod_order); +} + +void inner_rx_256qam (NR_DL_FRAME_PARMS *frame_parms, + NR_gNB_PUSCH *pusch_vars, + nfapi_nr_pusch_pdu_t *rel15_ul, + c16_t **rxF, + c16_t **ul_ch, + int16_t *llr, + int nb_layer, + int nb_rx_ant, + int soffset, + int length, + int symbol, + int short nb_rb, + int dmrs_symbol_flag, + int output_shift) +{ + int length_aligned = (length % 8) ? (8 - length % 8) : 0; + + c16_t rxFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + c16_t chFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + c16_t rxComp[length + length_aligned] __attribute__((aligned(32))); + c16_t rxF_ch_maga[length + length_aligned] __attribute__((aligned(32))); + c16_t rxF_ch_magb[length + length_aligned] __attribute__((aligned(32))); + c16_t rxF_ch_magc[length + length_aligned] __attribute__((aligned(32))); + + memset(rxComp, 0, sizeof(c16_t) * (length + length_aligned)); + memset(rxF_ch_maga, 0, sizeof(c16_t) * (length + length_aligned)); + memset(rxF_ch_magb, 0, sizeof(c16_t) * (length + length_aligned)); + memset(rxF_ch_magc, 0, sizeof(c16_t) * (length + length_aligned)); + + simde__m256i *rxComp_256 = (simde__m256i *)rxComp; + simde__m256i *rxF_ch_maga_256 = (simde__m256i *)rxF_ch_maga; + simde__m256i *rxF_ch_magb_256 = (simde__m256i *)rxF_ch_magb; + simde__m256i *rxF_ch_magc_256 = (simde__m256i *)rxF_ch_magc; + + for (int aarx = 0; aarx < nb_rx_ant; aarx++) + { + nr_ulsch_extract_rbs0(rxF[aarx], + (c16_t*)pusch_vars->ul_ch_estimates[aarx], + rxFext[aarx], + chFext[aarx], + soffset+(symbol * frame_parms->ofdm_symbol_size), + pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, + aarx, + dmrs_symbol_flag, + rel15_ul, + frame_parms); + simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; + simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); + + simde__m256i QAM_amp256 = simde_mm256_set1_epi16(QAM256_n1); + simde__m256i QAM_amp256b = simde_mm256_set1_epi16(QAM256_n2); + simde__m256i QAM_amp256c = simde_mm256_set1_epi16(QAM256_n3); + + simde__m256i *rxF256 = (simde__m256i*)rxFext[aarx]; + simde__m256i *ulch256 = (simde__m256i*)chFext[aarx]; + for (int i = 0; i < ((length >> 3) + (( length & 7) > 0 ? 1 : 0)); i++) + { + xmmp0 = simde_mm256_madd_epi16(ulch256[i], rxF256[i]); + // xmmp0 contains real part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp1 = simde_mm256_shuffle_epi8(ulch256[i], complex_shuffle256); + xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); + xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF256[i]); + // xmmp1 contains imag part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); + xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); + xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); + xmmp4 = simde_mm256_packs_epi32(xmmp2, xmmp3); + + // compute channel amplitude for LLR + xmmp0 = simde_mm256_madd_epi16(ulch256[i], ulch256[i]); + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp0 = simde_mm256_packs_epi32(xmmp0, xmmp0); // contains 16 LLRs + xmmp2 = simde_mm256_unpacklo_epi16(xmmp0, xmmp0); + xmmp0 = simde_mm256_mulhrs_epi16(xmmp2, QAM_amp256); + xmmp1 = simde_mm256_mulhrs_epi16(xmmp2, QAM_amp256b); + xmmp2 = simde_mm256_mulhrs_epi16(xmmp2, QAM_amp256c); + + // MRC + rxComp_256[i] = simde_mm256_add_epi16(rxComp_256[i], xmmp4); + rxF_ch_maga_256[i] = simde_mm256_add_epi16(rxF_ch_maga_256[i], xmmp0); + rxF_ch_magb_256[i] = simde_mm256_add_epi16(rxF_ch_magb_256[i], xmmp1); + rxF_ch_magc_256[i] = simde_mm256_add_epi16(rxF_ch_magc_256[i], xmmp2); + } + } + nr_ulsch_compute_llr((int32_t*)rxComp, + (int32_t*)rxF_ch_maga, + (int32_t*)rxF_ch_magb, + (int32_t*)rxF_ch_magc, + llr, + rel15_ul->rb_size, + length, + symbol, + rel15_ul->qam_mod_order); +} + +void inner_rx_qpsk_2layer (NR_DL_FRAME_PARMS *frame_parms, + NR_gNB_PUSCH *pusch_vars, + nfapi_nr_pusch_pdu_t *rel15_ul, + c16_t **rxF, + c16_t **ul_ch, + int16_t **llr, + int nb_layer, + int nb_rx_ant, + int soffset, + int length, + int symbol, + int short nb_rb, + int dmrs_symbol_flag, + int output_shift, + uint32_t nvar) +{ + int length_aligned = (length % 8) ? (8 - length % 8) : 0; + + c16_t rxFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + c16_t chFext[nb_layer][nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + for (int aarx = 0; aarx < nb_rx_ant; aarx++) + { + for (int aatx = 0; aatx < nb_layer; aatx++) + { + nr_ulsch_extract_rbs0(rxF[aarx], + (c16_t*)pusch_vars->ul_ch_estimates[aatx * nb_rx_ant + aarx], + rxFext[aarx], + chFext[aatx][aarx], + soffset+(symbol * frame_parms->ofdm_symbol_size), + pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, + aarx, + dmrs_symbol_flag, + rel15_ul, + frame_parms); + } + } + c16_t rxComp[nb_layer][length + length_aligned] __attribute__((aligned(32))); + c16_t rho[nb_layer][nb_layer][length + length_aligned] __attribute__((aligned(32))); + + memset(rxComp, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); + memset(rho, 0, sizeof(c16_t) * nb_layer * nb_layer * (length + length_aligned)); + + for (int aarx = 0; aarx < nb_rx_ant; aarx++) { + for (int aatx = 0; aatx < nb_layer; aatx++) { + for (int atx = 0; atx < nb_layer; atx++) { + simde__m256i mmtmpD0, mmtmpD1, mmtmpD2, mmtmpD3, mmtmpD4; + simde__m256i *rho_256 = (simde__m256i *)rho[aatx][atx]; + simde__m256i *chF_256 = (simde__m256i *)chFext[aatx][aarx]; + simde__m256i *chF2_256 = (simde__m256i *)chFext[atx][aarx]; + simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); + for (int i = 0; i < ((length >> 3)+((length&7)?1:0)); i++) + { + // multiply by conjugated channel + mmtmpD0 = simde_mm256_madd_epi16(chF_256[i], chF2_256[i]); + // mmtmpD0 contains real part of 4 consecutive outputs (32-bit) + mmtmpD1 = simde_mm256_shuffle_epi8(chF_256[i], complex_shuffle256); + mmtmpD1 = simde_mm256_sign_epi16(mmtmpD1, conj256); + mmtmpD1 = simde_mm256_madd_epi16(mmtmpD1, chF2_256[i]); + // mmtmpD1 contains imag part of 4 consecutive outputs (32-bit) + mmtmpD0 = simde_mm256_srai_epi32(mmtmpD0, output_shift); + mmtmpD1 = simde_mm256_srai_epi32(mmtmpD1, output_shift); + mmtmpD2 = simde_mm256_unpacklo_epi32(mmtmpD0, mmtmpD1); + mmtmpD3 = simde_mm256_unpackhi_epi32(mmtmpD0, mmtmpD1); + mmtmpD4 = simde_mm256_packs_epi32(mmtmpD2, mmtmpD3); + + rho_256[i] = simde_mm256_adds_epi16(rho_256[i], mmtmpD4); + } + } + // compensation + simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; + simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); + simde__m256i *rxF_256 = (simde__m256i*)rxFext[aarx]; + simde__m256i *chF_256 = (simde__m256i*)chFext[aatx][aarx]; + simde__m256i *rxComp_256 = (simde__m256i*)rxComp[aatx]; + for (int i = 0; i < (length>>3) + ((length&7)?1:0); i++) + { + xmmp0 = simde_mm256_madd_epi16(chF_256[i], rxF_256[i]); + // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp1 = simde_mm256_shuffle_epi8(chF_256[i], complex_shuffle256); + xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); + xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF_256[i]); + // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); + xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); + xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); + xmmp4 = simde_mm256_packs_epi32(xmmp2, xmmp3); + + rxComp_256[i] = simde_mm256_adds_epi16(rxComp_256[i], xmmp4); + } + } + } + c16_t *rho0 = rho[0][1]; + c16_t *rho1 = rho[1][0]; + c16_t *llr_0 = (c16_t *)&llr[0][pusch_vars->llr_offset[symbol]]; + c16_t *llr_1 = (c16_t *)&llr[1][pusch_vars->llr_offset[symbol]]; + + nr_ulsch_qpsk_qpsk(rxComp[0], rxComp[1], llr_0, rho0, length); + nr_ulsch_qpsk_qpsk(rxComp[1], rxComp[0], llr_1, rho1, length); + + nr_ulsch_shift_llr(pusch_vars->llr_layers, length, pusch_vars->llr_offset[symbol] >> 1, 2, 4); +} + + +void inner_rx_16qam_2layer (NR_DL_FRAME_PARMS *frame_parms, + NR_gNB_PUSCH *pusch_vars, + nfapi_nr_pusch_pdu_t *rel15_ul, + c16_t **rxF, + c16_t **ul_ch, + int16_t **llr, + int nb_layer, + int nb_rx_ant, + int soffset, + int length, + int symbol, + int short nb_rb, + int dmrs_symbol_flag, + int output_shift, + uint32_t nvar) +{ + int length_aligned = (length % 8) ? (8 - length % 8) : 0; + + c16_t rxFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + c16_t chFext[nb_layer][nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + for (int aarx = 0; aarx < nb_rx_ant; aarx++) { + for (int aatx = 0; aatx < nb_layer; aatx++) { + nr_ulsch_extract_rbs0(rxF[aarx], + (c16_t*)pusch_vars->ul_ch_estimates[aatx * nb_rx_ant + aarx], + rxFext[aarx], + chFext[aatx][aarx], + soffset+(symbol * frame_parms->ofdm_symbol_size), + pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, + aarx, + dmrs_symbol_flag, + rel15_ul, + frame_parms); + } + } + + c16_t rxComp[nb_layer][length + length_aligned] __attribute__((aligned(32))); + c16_t rho[nb_layer][nb_layer][length + length_aligned] __attribute__((aligned(32))); + c16_t ul_ch_maga[nb_layer][length + length_aligned] __attribute__((aligned(32))); + + memset(rxComp, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); + memset(rho, 0, sizeof(c16_t) * nb_layer * nb_layer * (length + length_aligned)); + memset(ul_ch_maga, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); + + for (int aatx = 0; aatx < nb_layer; aatx++) { + for (int aarx = 0; aarx < nb_rx_ant; aarx++) { + for (int atx = 0; atx < nb_layer; atx++) { + simde__m256i mmtmpD0, mmtmpD1, mmtmpD2, mmtmpD3; + simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); + simde__m256i *chF_256 = (simde__m256i *)chFext[aatx][aarx]; + simde__m256i *chF2_256 = (simde__m256i *)chFext[atx][aarx]; + simde__m256i *rho_256 = (simde__m256i *)rho[aatx][atx]; + for (int i = 0; i < (length >> 3)+((length&7)?1:0); i++) + { + // multiply by conjugated channel + mmtmpD0 = simde_mm256_madd_epi16(chF_256[i], chF2_256[i]); + // mmtmpD0 contains real part of 4 consecutive outputs (32-bit) + mmtmpD1 = simde_mm256_shuffle_epi8(chF_256[i], complex_shuffle256); + mmtmpD1 = simde_mm256_sign_epi16(mmtmpD1, conj256); + mmtmpD1 = simde_mm256_madd_epi16(mmtmpD1, chF2_256[i]); + // mmtmpD1 contains imag part of 4 consecutive outputs (32-bit) + mmtmpD0 = simde_mm256_srai_epi32(mmtmpD0, output_shift); + mmtmpD1 = simde_mm256_srai_epi32(mmtmpD1, output_shift); + mmtmpD2 = simde_mm256_unpacklo_epi32(mmtmpD0, mmtmpD1); + mmtmpD3 = simde_mm256_unpackhi_epi32(mmtmpD0, mmtmpD1); + + rho_256[i] = simde_mm256_adds_epi16(rho_256[i], simde_mm256_packs_epi32(mmtmpD2, mmtmpD3)); + } + } + // compensation + simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; + simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); + simde__m256i QAM_amp256 = simde_mm256_set1_epi16(QAM16_n1); // 2/sqrt(10) + simde__m256i *rxF_256 = (simde__m256i*)rxFext[aarx]; + simde__m256i *chF_256 = (simde__m256i*)chFext[aatx][aarx]; + simde__m256i *rxComp_256 = (simde__m256i*)rxComp[aatx]; + simde__m256i *ul_ch_maga_256 = (simde__m256i*)ul_ch_maga[aatx]; + for (int i = 0; i < (length>>3) + ((length&7)?1:0); i++) + { + xmmp0 = simde_mm256_madd_epi16(chF_256[i], rxF_256[i]); + // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp1 = simde_mm256_shuffle_epi8(chF_256[i], complex_shuffle256); + xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); + xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF_256[i]); + // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); + xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); + xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); + xmmp4 = simde_mm256_packs_epi32(xmmp2, xmmp3); + + xmmp0 = simde_mm256_madd_epi16(chF_256[i], chF_256[i]); // |h|^2 + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp0 = simde_mm256_packs_epi32(xmmp0, xmmp0); + xmmp1 = simde_mm256_unpacklo_epi16(xmmp0, xmmp0); + xmmp1 = simde_mm256_mulhrs_epi16(xmmp1, QAM_amp256); + + rxComp_256[i] = simde_mm256_adds_epi16(rxComp_256[i], xmmp4); + ul_ch_maga_256[i] = simde_mm256_adds_epi16(ul_ch_maga_256[i], xmmp1); + } + } + } + c16_t *rho0 = rho[0][1]; + c16_t *rho1 = rho[1][0]; + c16_t *llr_0 = (c16_t *)&llr[0][pusch_vars->llr_offset[symbol]]; + c16_t *llr_1 = (c16_t *)&llr[1][pusch_vars->llr_offset[symbol]]; + c16_t *ul_ch_mag0 = ul_ch_maga[0]; + c16_t *ul_ch_mag1 = ul_ch_maga[1]; + nr_ulsch_qam16_qam16(rxComp[0], rxComp[1], ul_ch_mag0, ul_ch_mag1, llr_0, rho0, length); + nr_ulsch_qam16_qam16(rxComp[1], rxComp[0], ul_ch_mag1, ul_ch_mag0, llr_1, rho1, length); +} + +void inner_rx_64qam_2layer (NR_DL_FRAME_PARMS *frame_parms, + NR_gNB_PUSCH *pusch_vars, + nfapi_nr_pusch_pdu_t *rel15_ul, + c16_t **rxF, + c16_t **ul_ch, + int16_t **llr, + int nb_layer, + int nb_rx_ant, + int soffset, + int length, + int symbol, + int short nb_rb, + int dmrs_symbol_flag, + int output_shift, + uint32_t nvar) +{ + int length_aligned = (length % 8) ? (8 - length % 8) : 0; + + c16_t rxFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + c16_t chFext[nb_layer][nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + for (int aarx = 0; aarx < nb_rx_ant; aarx++) { + for (int aatx = 0; aatx < nb_layer; aatx++) { + nr_ulsch_extract_rbs0(rxF[aarx], + (c16_t *)pusch_vars->ul_ch_estimates[aatx * nb_rx_ant + aarx], + rxFext[aarx], + chFext[aatx][aarx], + soffset+(symbol * frame_parms->ofdm_symbol_size), + pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, + aarx, + dmrs_symbol_flag, + rel15_ul, + frame_parms); + } + } + c16_t rho[nb_layer][nb_layer][length + length_aligned] __attribute__((aligned(32))); + c16_t rxComp[nb_layer][length + length_aligned] __attribute__((aligned(32))); + c16_t ul_ch_maga[nb_layer][length + length_aligned] __attribute__((aligned(32))); + c16_t ul_ch_magb[nb_layer][length + length_aligned] __attribute__((aligned(32))); + c16_t ul_ch_magc[nb_layer][length + length_aligned] __attribute__((aligned(32))); + + memset(rho, 0, sizeof(c16_t) * nb_layer * nb_layer* (length + length_aligned)); + memset(rxComp, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); + memset(ul_ch_maga, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); + memset(ul_ch_magb, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); + memset(ul_ch_magc, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); + + for (int aatx = 0; aatx < nb_layer; aatx++) { + for (int aarx = 0; aarx < nb_rx_ant; aarx++) { +#ifdef USE_ML + for (int atx = 0; atx < nb_layer; atx++) { + simde__m256i mmtmpD0, mmtmpD1, mmtmpD2, mmtmpD3; + simde__m256i *rho_256 = (simde__m256i *)rho[aatx][atx]; + simde__m256i *chF_256 = (simde__m256i *)chFext[aatx][aarx]; + simde__m256i *chF2_256 = (simde__m256i *)chFext[atx][aarx]; + simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); + for (int i = 0; i < (length >> 3)+((length&7)?1:0); i++) + { + // multiply by conjugated channel + mmtmpD0 = simde_mm256_madd_epi16(chF_256[i], chF2_256[i]); + // mmtmpD0 contains real part of 4 consecutive outputs (32-bit) + mmtmpD1 = simde_mm256_shuffle_epi8(chF_256[i], complex_shuffle256); + mmtmpD1 = simde_mm256_sign_epi16(mmtmpD1, conj256); + mmtmpD1 = simde_mm256_madd_epi16(mmtmpD1, chF2_256[i]); + // mmtmpD1 contains imag part of 4 consecutive outputs (32-bit) + mmtmpD0 = simde_mm256_srai_epi32(mmtmpD0, output_shift); + mmtmpD1 = simde_mm256_srai_epi32(mmtmpD1, output_shift); + mmtmpD2 = simde_mm256_unpacklo_epi32(mmtmpD0, mmtmpD1); + mmtmpD3 = simde_mm256_unpackhi_epi32(mmtmpD0, mmtmpD1); + + rho_256[i] = simde_mm256_adds_epi16(rho_256[i], simde_mm256_packs_epi32(mmtmpD2, mmtmpD3)); + } + } +#endif + // compensation + simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; + simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); + simde__m256i QAM_ampa_256 = simde_mm256_set1_epi16(QAM64_n1); // 2/sqrt(10) + simde__m256i QAM_ampb_256 = simde_mm256_set1_epi16(QAM64_n2); // 2/sqrt(10) + simde__m256i *rxF_256 = (simde__m256i*)rxFext[aarx]; + simde__m256i *chF_256 = (simde__m256i*)chFext[aatx][aarx]; + simde__m256i *rxComp_256 = (simde__m256i*)rxComp[aatx]; + simde__m256i *ul_ch_maga_256 = (simde__m256i*)ul_ch_maga[aatx]; + simde__m256i *ul_ch_magb_256 = (simde__m256i*)ul_ch_magb[aatx]; + for (int i = 0; i < (length>>3) + ((length&7)?1:0); i++) + { + xmmp0 = simde_mm256_madd_epi16(chF_256[i], rxF_256[i]); + // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp1 = simde_mm256_shuffle_epi8(chF_256[i], complex_shuffle256); + xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); + xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF_256[i]); + // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); + xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); + xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); + xmmp4 = simde_mm256_packs_epi32(xmmp2, xmmp3); + + xmmp0 = simde_mm256_madd_epi16(chF_256[i], chF_256[i]); // |h|^2 + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp0 = simde_mm256_packs_epi32(xmmp0, xmmp0); + xmmp1 = simde_mm256_unpacklo_epi16(xmmp0, xmmp0); + + xmmp2 = simde_mm256_mulhrs_epi16(xmmp1, QAM_ampa_256); + xmmp3 = simde_mm256_mulhrs_epi16(xmmp1, QAM_ampb_256); + + rxComp_256[i] = simde_mm256_adds_epi16(rxComp_256[i], xmmp4); + ul_ch_maga_256[i] = simde_mm256_adds_epi16(ul_ch_maga_256[i], xmmp2); + ul_ch_magb_256[i] = simde_mm256_adds_epi16(ul_ch_magb_256[i], xmmp3); + } + } + } +#ifdef USE_ML + c16_t *rho0 = rho[0][1]; + c16_t *rho1 = rho[1][0]; + c16_t *llr_0 = (c16_t *)&llr[0][pusch_vars->llr_offset[symbol]]; + c16_t *llr_1 = (c16_t *)&llr[1][pusch_vars->llr_offset[symbol]]; + c16_t *ul_ch_mag0 = ul_ch_maga[0]; + c16_t *ul_ch_mag1 = ul_ch_maga[1]; + nr_ulsch_qam64_qam64(rxComp[0], rxComp[1], ul_ch_mag0, ul_ch_mag1, llr_0, rho0, length); + nr_ulsch_qam64_qam64(rxComp[1], rxComp[0], ul_ch_mag1, ul_ch_mag0, llr_1, rho1, length); +#else + nr_ulsch_mmse_2layers(frame_parms, + (int **)rxComp, + (int **)ul_ch_maga, + (int **)ul_ch_magb, + (int **)ul_ch_magc, + (int **)chFext, + rel15_ul->rb_size, + frame_parms->nb_antennas_rx, + rel15_ul->qam_mod_order, + pusch_vars->log2_maxh, + symbol, + length, + nvar); + for (int aatx = 0; aatx < nb_layer; aatx++) + nr_ulsch_compute_llr((int32_t*)rxComp[aatx], + (int32_t*)ul_ch_maga[aatx], + (int32_t*)ul_ch_magb[aatx], + (int32_t*)ul_ch_magc[aatx], + &llr[aatx][pusch_vars->llr_offset[symbol]], + rel15_ul->rb_size, + length, + symbol, + rel15_ul->qam_mod_order); +#endif +} + +void inner_rx_256qam_2layer (NR_DL_FRAME_PARMS *frame_parms, + NR_gNB_PUSCH *pusch_vars, + nfapi_nr_pusch_pdu_t *rel15_ul, + c16_t **rxF, + c16_t **ul_ch, + int16_t **llr, + int nb_layer, + int nb_rx_ant, + int soffset, + int length, + int symbol, + int short nb_rb, + int dmrs_symbol_flag, + int output_shift, + uint32_t nvar) +{ + int length_aligned = (length % 8) ? (8 - length % 8) : 0; + + c16_t rxFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + c16_t chFext[nb_layer][nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + for (int aarx = 0; aarx < nb_rx_ant; aarx++) { + for (int aatx = 0; aatx < nb_layer; aatx++) { + nr_ulsch_extract_rbs0(rxF[aarx], + (c16_t *)pusch_vars->ul_ch_estimates[aatx * nb_rx_ant + aarx], + rxFext[aarx], + chFext[aatx][aarx], + soffset+(symbol * frame_parms->ofdm_symbol_size), + pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, + aarx, + dmrs_symbol_flag, + rel15_ul, + frame_parms); + } + } + c16_t rho[nb_layer][nb_layer][length + length_aligned] __attribute__((aligned(32))); + c16_t rxComp[nb_layer][length + length_aligned] __attribute__((aligned(32))); + c16_t ul_ch_maga[nb_layer][length + length_aligned] __attribute__((aligned(32))); + c16_t ul_ch_magb[nb_layer][length + length_aligned] __attribute__((aligned(32))); + c16_t ul_ch_magc[nb_layer][length + length_aligned] __attribute__((aligned(32))); + + memset(rho, 0, sizeof(c16_t) * nb_layer * nb_layer* (length + length_aligned)); + memset(rxComp, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); + memset(ul_ch_maga, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); + memset(ul_ch_magb, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); + memset(ul_ch_magc, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); + + // compensation + for (int aatx = 0; aatx < nb_layer; aatx++) { + for (int aarx = 0; aarx < nb_rx_ant; aarx++) { + simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; + simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); + simde__m256i QAM_ampa_256 = simde_mm256_set1_epi16(QAM256_n1); // 2/sqrt(10) + simde__m256i QAM_ampb_256 = simde_mm256_set1_epi16(QAM256_n2); // 2/sqrt(10) + simde__m256i QAM_ampc_256 = simde_mm256_set1_epi16(QAM256_n3); // 2/sqrt(10) + simde__m256i *rxF_256 = (simde__m256i*)rxFext[aarx]; + simde__m256i *chF_256 = (simde__m256i*)chFext[aatx][aarx]; + simde__m256i *rxComp_256 = (simde__m256i*)rxComp[aatx]; + simde__m256i *ul_ch_maga_256 = (simde__m256i*)ul_ch_maga[aatx]; + simde__m256i *ul_ch_magb_256 = (simde__m256i*)ul_ch_magb[aatx]; + simde__m256i *ul_ch_magc_256 = (simde__m256i*)ul_ch_magc[aatx]; + for (int i = 0; i < (length>>3) + ((length&7)?1:0); i++) + { + xmmp0 = simde_mm256_madd_epi16(chF_256[i], rxF_256[i]); + // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp1 = simde_mm256_shuffle_epi8(chF_256[i], complex_shuffle256); + xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); + xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF_256[i]); + // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); + xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); + xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); + xmmp4 = simde_mm256_packs_epi32(xmmp2, xmmp3); + + xmmp0 = simde_mm256_madd_epi16(chF_256[i], chF_256[i]); // |h|^2 + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp0 = simde_mm256_packs_epi32(xmmp0, xmmp0); + xmmp1 = simde_mm256_unpacklo_epi16(xmmp0, xmmp0); + + xmmp2 = simde_mm256_mulhrs_epi16(xmmp1, QAM_ampa_256); + xmmp3 = simde_mm256_mulhrs_epi16(xmmp1, QAM_ampb_256); + xmmp1 = simde_mm256_mulhrs_epi16(xmmp1, QAM_ampc_256); + + rxComp_256[i] = simde_mm256_adds_epi16(rxComp_256[i], xmmp4); + ul_ch_maga_256[i] = simde_mm256_adds_epi16(ul_ch_maga_256[i], xmmp2); + ul_ch_magb_256[i] = simde_mm256_adds_epi16(ul_ch_magb_256[i], xmmp3); + ul_ch_magc_256[i] = simde_mm256_adds_epi16(ul_ch_magc_256[i], xmmp1); + } + } + } + + nr_ulsch_mmse_2layers(frame_parms, + (int **)rxComp, + (int **)ul_ch_maga, + (int **)ul_ch_magb, + (int **)ul_ch_magc, + (int **)chFext, + rel15_ul->rb_size, + frame_parms->nb_antennas_rx, + rel15_ul->qam_mod_order, + pusch_vars->log2_maxh, + symbol, + length, + nvar); + for (int aatx = 0; aatx < nb_layer; aatx++) + nr_ulsch_compute_llr((int32_t*)rxComp[aatx], + (int32_t*)ul_ch_maga[aatx], + (int32_t*)ul_ch_magb[aatx], + (int32_t*)ul_ch_magc[aatx], + &llr[aatx][pusch_vars->llr_offset[symbol]], + rel15_ul->rb_size, + length, + symbol, + rel15_ul->qam_mod_order); +} + +void nr_pusch_symbol_processing_noprecoding(void *arg) { - uint8_t aarx, aatx; - uint32_t nb_re_pusch, bwp_start_subcarrier; - int avgs = 0; - - NR_DL_FRAME_PARMS *frame_parms = &gNB->frame_parms; - NR_gNB_ULSCH_t *ulsch = &gNB->ulsch[ulsch_id]; - nfapi_nr_pusch_pdu_t *rel15_ul = &ulsch->harq_process->ulsch_pdu; - int avg[frame_parms->nb_antennas_rx*rel15_ul->nrOfLayers]; + puschSymbolProc_t *rdata=(puschSymbolProc_t*)arg; + PHY_VARS_gNB *gNB = rdata->gNB; + NR_DL_FRAME_PARMS *frame_parms = rdata->frame_parms; + nfapi_nr_pusch_pdu_t *rel15_ul = rdata->rel15_ul; + int ulsch_id = rdata->ulsch_id; + int slot = rdata->slot; NR_gNB_PUSCH *pusch_vars = &gNB->pusch_vars[ulsch_id]; - pusch_vars->dmrs_symbol = INVALID_VALUE; - pusch_vars->cl_done = 0; - - bwp_start_subcarrier = ((rel15_ul->rb_start + rel15_ul->bwp_start)*NR_NB_SC_PER_RB + frame_parms->first_carrier_offset) % frame_parms->ofdm_symbol_size; - LOG_D(PHY,"pusch %d.%d : bwp_start_subcarrier %d, rb_start %d, first_carrier_offset %d\n", frame,slot,bwp_start_subcarrier, rel15_ul->rb_start, frame_parms->first_carrier_offset); - LOG_D(PHY,"pusch %d.%d : ul_dmrs_symb_pos %x\n",frame,slot,rel15_ul->ul_dmrs_symb_pos); - LOG_D(PHY,"ulsch RX %x : start_rb %d nb_rb %d mcs %d Nl %d Tpmi %d bwp_start %d start_sc %d start_symbol %d num_symbols %d cdmgrpsnodata %d num_dmrs %d dmrs_ports %d\n", - rel15_ul->rnti,rel15_ul->rb_start,rel15_ul->rb_size,rel15_ul->mcs_index, - rel15_ul->nrOfLayers,0,rel15_ul->bwp_start,0,rel15_ul->start_symbol_index,rel15_ul->nr_of_symbols, - rel15_ul->num_dmrs_cdm_grps_no_data,rel15_ul->ul_dmrs_symb_pos,rel15_ul->dmrs_ports); - //---------------------------------------------------------- - //--------------------- Channel estimation --------------------- - //---------------------------------------------------------- - start_meas(&gNB->ulsch_channel_estimation_stats); - int max_ch = 0; - uint32_t nvar = 0; - for(uint8_t symbol = rel15_ul->start_symbol_index; symbol < (rel15_ul->start_symbol_index + rel15_ul->nr_of_symbols); symbol++) { - uint8_t dmrs_symbol_flag = (rel15_ul->ul_dmrs_symb_pos >> symbol) & 0x01; - LOG_D(PHY, "symbol %d, dmrs_symbol_flag :%d\n", symbol, dmrs_symbol_flag); - - if (dmrs_symbol_flag == 1) { - if (pusch_vars->dmrs_symbol == INVALID_VALUE) - pusch_vars->dmrs_symbol = symbol; - - for (int nl=0; nlnrOfLayers; nl++) { - uint32_t nvar_tmp = 0; - nr_pusch_channel_estimation(gNB, - slot, - get_dmrs_port(nl,rel15_ul->dmrs_ports), - symbol, - ulsch_id, - bwp_start_subcarrier, - rel15_ul, - &max_ch, - &nvar_tmp); - nvar += nvar_tmp; - } - - nr_gnb_measurements(gNB, ulsch, pusch_vars, symbol, rel15_ul->nrOfLayers); - allocCast2D(n0_subband_power, - unsigned int, - gNB->measurements.n0_subband_power, - frame_parms->nb_antennas_rx, - frame_parms->N_RB_UL, - false); - for (aarx = 0; aarx < frame_parms->nb_antennas_rx; aarx++) { - if (symbol == rel15_ul->start_symbol_index) { - pusch_vars->ulsch_power[aarx] = 0; - pusch_vars->ulsch_noise_power[aarx] = 0; - } - for (aatx = 0; aatx < rel15_ul->nrOfLayers; aatx++) { - pusch_vars->ulsch_power[aarx] += signal_energy_nodc( - &pusch_vars->ul_ch_estimates[aatx * gNB->frame_parms.nb_antennas_rx + aarx][symbol * frame_parms->ofdm_symbol_size], - rel15_ul->rb_size * 12); - } - for (int rb = 0; rb < rel15_ul->rb_size; rb++) { - pusch_vars->ulsch_noise_power[aarx] += - n0_subband_power[aarx][rel15_ul->bwp_start + rel15_ul->rb_start + rb] / rel15_ul->rb_size; - } - LOG_D(PHY, - "aa %d, bwp_start%d, rb_start %d, rb_size %d: ulsch_power %d, ulsch_noise_power %d\n", - aarx, - rel15_ul->bwp_start, - rel15_ul->rb_start, - rel15_ul->rb_size, - pusch_vars->ulsch_power[aarx], - pusch_vars->ulsch_noise_power[aarx]); - } - } - } - - nvar /= (rel15_ul->nr_of_symbols * rel15_ul->nrOfLayers * frame_parms->nb_antennas_rx); - - if (gNB->chest_time == 1) { // averaging time domain channel estimates - nr_chest_time_domain_avg(frame_parms, - pusch_vars->ul_ch_estimates, - rel15_ul->nr_of_symbols, - rel15_ul->start_symbol_index, - rel15_ul->ul_dmrs_symb_pos, - rel15_ul->rb_size); - - pusch_vars->dmrs_symbol = - get_next_dmrs_symbol_in_slot(rel15_ul->ul_dmrs_symb_pos, rel15_ul->start_symbol_index, rel15_ul->nr_of_symbols); - } - stop_meas(&gNB->ulsch_channel_estimation_stats); - - int off = ((rel15_ul->rb_size&1) == 1)? 4:0; - uint32_t rxdataF_ext_offset = 0; - uint8_t shift_ch_ext = rel15_ul->nrOfLayers > 1 ? log2_approx(max_ch >> 11) : 0; - - // Flag to select the receiver: (true) Nonlinear ML receiver, (false) Linear MMSE receiver - // By default, we are using the Nonlinear ML receiver, except - // - for 256QAM as Nonlinear ML receiver is not implemented for 256QAM - // - for 64QAM as Nonlinear ML receiver requires more processing time than MMSE, and many machines are not powerful enough - bool ml_rx = true; - if (rel15_ul->nrOfLayers != 2 || rel15_ul->qam_mod_order >= 6) { - ml_rx = false; - } - - int ad_shift = 0; - if (rel15_ul->nrOfLayers == 1) { - ad_shift = 1 + log2_approx(frame_parms->nb_antennas_rx >> 2); - } else if (ml_rx == false) { - ad_shift = -3; // For 2-layers, we are already doing a bit shift in the nr_ulsch_mmse_2layers() function, so we can use more bits - } - - int num_re_total = 0; - for(uint8_t symbol = rel15_ul->start_symbol_index; symbol < (rel15_ul->start_symbol_index + rel15_ul->nr_of_symbols); symbol++) { - uint8_t dmrs_symbol_flag = (rel15_ul->ul_dmrs_symb_pos >> symbol) & 0x01; - if (dmrs_symbol_flag == 1) { + simde__m64 *s = (simde__m64 *)rdata->s; + for (int symbol = rdata->startSymbol; symbol < rdata->startSymbol+rdata->numSymbols; symbol++) + { + int dmrs_symbol_flag = (rel15_ul->ul_dmrs_symb_pos >> symbol) & 0x01; + int nb_re_pusch = gNB->pusch_vars[ulsch_id].ul_valid_re_per_slot[symbol]; + // this needs to be reworded for parrellization, we need a table which give dmrs symbol location + // used for chennel estimate, they are being run in parallel! + if (dmrs_symbol_flag == 1) + { if ((rel15_ul->ul_dmrs_symb_pos >> ((symbol + 1) % frame_parms->symbols_per_slot)) & 0x01) AssertFatal(1==0,"Double DMRS configuration is not yet supported\n"); - - if (gNB->chest_time == 0) // Non averaging time domain channel estimates - pusch_vars->dmrs_symbol = symbol; - - if (rel15_ul->dmrs_config_type == 0) { - // if no data in dmrs cdm group is 1 only even REs have no data - // if no data in dmrs cdm group is 2 both odd and even REs have no data - nb_re_pusch = rel15_ul->rb_size *(12 - (rel15_ul->num_dmrs_cdm_grps_no_data*6)); - } - else { - nb_re_pusch = rel15_ul->rb_size *(12 - (rel15_ul->num_dmrs_cdm_grps_no_data*4)); - } - } - else { - nb_re_pusch = rel15_ul->rb_size * NR_NB_SC_PER_RB; + gNB->pusch_vars[ulsch_id].dmrs_symbol = symbol; } - num_re_total += nb_re_pusch; - pusch_vars->ul_valid_re_per_slot[symbol] = nb_re_pusch; - LOG_D(PHY, "symbol %d: nb_re_pusch %d, DMRS symbl used for Chest :%d \n", symbol, nb_re_pusch, pusch_vars->dmrs_symbol); + LOG_I(PHY,"symbol %d: nb_re_pusch %d, DMRS symbl used for Chest :%d \n", symbol, nb_re_pusch, gNB->pusch_vars[ulsch_id].dmrs_symbol); - //---------------------------------------------------------- - //--------------------- RBs extraction --------------------- - //---------------------------------------------------------- - if (nb_re_pusch > 0) { - start_meas(&gNB->ulsch_rbs_extraction_stats); - nr_ulsch_extract_rbs(gNB->common_vars.rxdataF, pusch_vars, slot, symbol, dmrs_symbol_flag, rel15_ul, frame_parms); - stop_meas(&gNB->ulsch_rbs_extraction_stats); + if (nb_re_pusch == 0) continue; + if (rel15_ul->nrOfLayers == 1) + { + void (*inner_rx)(NR_DL_FRAME_PARMS *, + NR_gNB_PUSCH *, + nfapi_nr_pusch_pdu_t *, + c16_t **, + c16_t **, + int16_t *, + int32_t, + int32_t, + int32_t, + int32_t, + int32_t, + int16_t, + int32_t, + int32_t); + + if (rel15_ul->qam_mod_order == 2) inner_rx = inner_rx_qpsk; + else if (rel15_ul->qam_mod_order == 4) inner_rx = inner_rx_16qam; + else if (rel15_ul->qam_mod_order == 6) inner_rx = inner_rx_64qam; + else if (rel15_ul->qam_mod_order == 8) inner_rx = inner_rx_256qam; + else AssertFatal(1==0,"rel15_ul->qam_mod_order %d, pusch_pdu->dmrs_config_type %d\n", + rel15_ul->qam_mod_order,rel15_ul->dmrs_config_type); - //---------------------------------------------------------- - //--------------------- Channel Scaling -------------------- - //---------------------------------------------------------- - nr_ulsch_scale_channel(pusch_vars->ul_ch_estimates_ext, - frame_parms, - ulsch, - symbol, - dmrs_symbol_flag, - nb_re_pusch, - rel15_ul->nrOfLayers, - rel15_ul->rb_size, - shift_ch_ext); + int soffset = (slot&3)*frame_parms->symbols_per_slot*frame_parms->ofdm_symbol_size; + int16_t llr_tmp[(nb_re_pusch + 8) * rel15_ul->qam_mod_order] __attribute__((aligned(32))); + memset(llr_tmp, 0, sizeof(int16_t) * (nb_re_pusch + 8) * rel15_ul->qam_mod_order); + inner_rx(frame_parms, + pusch_vars, + rel15_ul, + gNB->common_vars.rxdataF, + (c16_t**)gNB->pusch_vars[ulsch_id].ul_ch_estimates, + llr_tmp, + rel15_ul->nrOfLayers, + frame_parms->nb_antennas_rx, + soffset, + nb_re_pusch, // length + symbol, // symbol index + rel15_ul->rb_size, // ofdm size + dmrs_symbol_flag, + gNB->pusch_vars[ulsch_id].log2_maxh); + + // unscrambling + simde__m64 *llr64 = (simde__m64 *) &rdata->llr[pusch_vars->llr_offset[symbol]]; + for (int i = 0; i < nb_re_pusch*rel15_ul->qam_mod_order >> 2; i++) + llr64[i] = simde_mm_mullo_pi16(((simde__m64 *)llr_tmp)[i], s[i]); + + } + else // MIMO for 2x2 + { + int soffset = (slot&3)*frame_parms->symbols_per_slot*frame_parms->ofdm_symbol_size; + void (*inner_rx)(NR_DL_FRAME_PARMS *, + NR_gNB_PUSCH *, + nfapi_nr_pusch_pdu_t *, + c16_t **, + c16_t **, + int16_t **, + int32_t, + int32_t, + int32_t, + int32_t, + int32_t, + int16_t, + int32_t, + int32_t, + uint32_t); + if (rel15_ul->qam_mod_order == 2) inner_rx = inner_rx_qpsk_2layer; + else if (rel15_ul->qam_mod_order == 4) inner_rx = inner_rx_16qam_2layer; + else if (rel15_ul->qam_mod_order == 6) inner_rx = inner_rx_64qam_2layer; + else if (rel15_ul->qam_mod_order == 8) inner_rx = inner_rx_256qam_2layer; + else AssertFatal(1==0,"rel15_ul->qam_mod_order %d, pusch_pdu->dmrs_config_type %d\n", + rel15_ul->qam_mod_order,rel15_ul->dmrs_config_type); + + inner_rx(frame_parms, + pusch_vars, + rel15_ul, + gNB->common_vars.rxdataF, + (c16_t**)gNB->pusch_vars[ulsch_id].ul_ch_estimates, + rdata->llr_layers, + rel15_ul->nrOfLayers, + frame_parms->nb_antennas_rx, + soffset, + nb_re_pusch, // length + symbol, // symbol index + rel15_ul->rb_size, // ofdm size + dmrs_symbol_flag, + gNB->pusch_vars[ulsch_id].log2_maxh, + rdata->nvar); + + // layer de-mapping + int16_t* llr_cw = &rdata->llr[pusch_vars->llr_offset[symbol] * rel15_ul->nrOfLayers]; + for (int i = 0; i < (nb_re_pusch); i++) + for (int l = 0; l < rel15_ul->nrOfLayers; l++) + for (int m = 0; m < rel15_ul->qam_mod_order; m++) + llr_cw[i*rel15_ul->nrOfLayers*rel15_ul->qam_mod_order+l*rel15_ul->qam_mod_order+m] = rdata->llr_layers[l][pusch_vars->llr_offset[symbol] + i*rel15_ul->qam_mod_order+m]; + + // unscrambling + simde__m64 *llr64 = (simde__m64 *) &rdata->llr[pusch_vars->llr_offset[symbol] * rel15_ul->nrOfLayers]; + for (int i = 0; i < (nb_re_pusch*rel15_ul->qam_mod_order * rel15_ul->nrOfLayers) >> 2; i++) + llr64[i] = simde_mm_mullo_pi16(llr64[i], s[i]); - if (pusch_vars->cl_done == 0) { - nr_ulsch_channel_level(pusch_vars->ul_ch_estimates_ext, - frame_parms, - avg, - symbol, - nb_re_pusch, - rel15_ul->nrOfLayers, - rel15_ul->rb_size); - - avgs = 0; - - for (aatx=0;aatxnrOfLayers;aatx++) - for (aarx=0;aarxnb_antennas_rx;aarx++) - avgs = cmax(avgs,avg[aatx*frame_parms->nb_antennas_rx+aarx]); - - pusch_vars->log2_maxh = (log2_approx(avgs) >> 1) + ad_shift; - if (pusch_vars->log2_maxh < 0) { - pusch_vars->log2_maxh = 0; - } - pusch_vars->cl_done = 1; - } - - //---------------------------------------------------------- - //--------------------- Channel Compensation --------------- - //---------------------------------------------------------- - start_meas(&gNB->ulsch_channel_compensation_stats); - LOG_D(PHY, "Doing channel compensations log2_maxh %d, avgs %d (%d,%d)\n" ,pusch_vars->log2_maxh, avgs,avg[0], avg[1]); - nr_ulsch_channel_compensation(pusch_vars->rxdataF_ext, - pusch_vars->ul_ch_estimates_ext, - pusch_vars->ul_ch_mag0, - pusch_vars->ul_ch_magb0, - pusch_vars->ul_ch_magc0, - pusch_vars->rxdataF_comp, - (rel15_ul->nrOfLayers > 1) ? pusch_vars->rho : NULL, - frame_parms, - symbol, - nb_re_pusch, - dmrs_symbol_flag, - rel15_ul->qam_mod_order, - rel15_ul->nrOfLayers, - rel15_ul->rb_size, - pusch_vars->log2_maxh); - stop_meas(&gNB->ulsch_channel_compensation_stats); - - start_meas(&gNB->ulsch_mrc_stats); - nr_ulsch_detection_mrc(frame_parms, - pusch_vars->rxdataF_comp, - pusch_vars->ul_ch_mag0, - pusch_vars->ul_ch_magb0, - pusch_vars->ul_ch_magc0, - (rel15_ul->nrOfLayers > 1) ? pusch_vars->rho : NULL, - rel15_ul->nrOfLayers, - symbol, - rel15_ul->rb_size, - nb_re_pusch); - - // Apply MMSE for 2 Tx layers - if (ml_rx == false && rel15_ul->nrOfLayers == 2) { - nr_ulsch_mmse_2layers(frame_parms, - pusch_vars->rxdataF_comp, - pusch_vars->ul_ch_mag0, - pusch_vars->ul_ch_magb0, - pusch_vars->ul_ch_magc0, - pusch_vars->ul_ch_estimates_ext, - rel15_ul->rb_size, - frame_parms->nb_antennas_rx, - rel15_ul->qam_mod_order, - pusch_vars->log2_maxh, - symbol, - nb_re_pusch, - nvar); - } - - stop_meas(&gNB->ulsch_mrc_stats); - - if (rel15_ul->transform_precoding == transformPrecoder_enabled) { - // For odd number of resource blocks need byte alignment to multiple of 8 - int nb_re_pusch2 = nb_re_pusch + (nb_re_pusch&7); - - // perform IDFT operation on the compensated rxdata if transform precoding is enabled - nr_idft(&pusch_vars->rxdataF_comp[0][symbol * nb_re_pusch2], nb_re_pusch); - LOG_D(PHY,"Transform precoding being done on data- symbol: %d, nb_re_pusch: %d\n", symbol, nb_re_pusch); - } - - //---------------------------------------------------------- - //--------------------- PTRS Processing -------------------- - //---------------------------------------------------------- - /* In case PTRS is enabled then LLR will be calculated after PTRS symbols are processed * - * otherwise LLR are calculated for each symbol based upon DMRS channel estimates only. */ - if (rel15_ul->pdu_bit_map & PUSCH_PDU_BITMAP_PUSCH_PTRS) { - start_meas(&gNB->ulsch_ptrs_processing_stats); - nr_pusch_ptrs_processing(gNB, - frame_parms, - rel15_ul, - ulsch_id, - slot, - symbol, - nb_re_pusch); - stop_meas(&gNB->ulsch_ptrs_processing_stats); - - /* Subtract total PTRS RE's in the symbol from PUSCH RE's */ - pusch_vars->ul_valid_re_per_slot[symbol] -= pusch_vars->ptrs_re_per_slot; - } - - /*---------------------------------------------------------------------------------------------------- */ - /*-------------------- LLRs computation -------------------------------------------------------------*/ - /*-----------------------------------------------------------------------------------------------------*/ - start_meas(&gNB->ulsch_llr_stats); - if (ml_rx == false || rel15_ul->nrOfLayers == 1) { - for (aatx=0; aatx < rel15_ul->nrOfLayers; aatx++) { - nr_ulsch_compute_llr(&pusch_vars->rxdataF_comp[aatx * frame_parms->nb_antennas_rx][symbol * (off + rel15_ul->rb_size * NR_NB_SC_PER_RB)], - pusch_vars->ul_ch_mag0[aatx * frame_parms->nb_antennas_rx], - pusch_vars->ul_ch_magb0[aatx * frame_parms->nb_antennas_rx], - pusch_vars->ul_ch_magc0[aatx * frame_parms->nb_antennas_rx], - &pusch_vars->llr_layers[aatx][rxdataF_ext_offset * rel15_ul->qam_mod_order], - rel15_ul->rb_size, - pusch_vars->ul_valid_re_per_slot[symbol], - symbol, - rel15_ul->qam_mod_order); - } - } else { - nr_ulsch_compute_ML_llr(pusch_vars->rxdataF_comp, - pusch_vars->ul_ch_mag0, - pusch_vars->rho, - pusch_vars->llr_layers, - frame_parms->nb_antennas_rx, - rel15_ul->rb_size, - nb_re_pusch, - symbol, - rxdataF_ext_offset, - rel15_ul->qam_mod_order); - - if (rel15_ul->qam_mod_order == 2) { - nr_ulsch_shift_llr(pusch_vars->llr_layers, nb_re_pusch, rxdataF_ext_offset, rel15_ul->qam_mod_order, 4); - } - -#ifdef ML_DEBUG - c16_t *llr_layers0 = (c16_t *)&pusch_vars->llr_layers[0][rxdataF_ext_offset * rel15_ul->qam_mod_order]; - c16_t *llr_layers1 = (c16_t *)&pusch_vars->llr_layers[1][rxdataF_ext_offset * rel15_ul->qam_mod_order]; - printf("===============================\n"); - printf("AFTER nr_ulsch_compute_ML_llr()\n"); - printf("===============================\n"); - for (int k = 0; k < nb_re_pusch; k++) { - printf("[%3i] llr_layers0 = (%6i, %6i), llr_layers1 = (%6i, %6i)\n", - k, llr_layers0[k].r, llr_layers0[k].i, llr_layers1[k].r, llr_layers1[k].i); - } - printf("\n"); -#endif - } - stop_meas(&gNB->ulsch_llr_stats); - rxdataF_ext_offset += pusch_vars->ul_valid_re_per_slot[symbol]; } - } // symbol loop - if (!(frame % 128)) { - int num_llr = num_re_total*rel15_ul->qam_mod_order; - GnbScopeUpdate(gNB, puschLLRe, num_llr); - GnbScopeUpdate(gNB, puschIQe, num_re_total); } } + int nr_rx_pusch_tp(PHY_VARS_gNB *gNB, uint8_t ulsch_id, uint32_t frame, @@ -3605,16 +2697,17 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB, //---------------------------------------------------------- start_meas(&gNB->ulsch_channel_estimation_stats); int max_ch = 0; - for (uint8_t symbol = rel15_ul->start_symbol_index; symbol < (rel15_ul->start_symbol_index + rel15_ul->nr_of_symbols); symbol++) - { + uint32_t nvar = 0; + for(uint8_t symbol = rel15_ul->start_symbol_index; symbol < (rel15_ul->start_symbol_index + rel15_ul->nr_of_symbols); symbol++) { uint8_t dmrs_symbol_flag = (rel15_ul->ul_dmrs_symb_pos >> symbol) & 0x01; LOG_D(PHY, "symbol %d, dmrs_symbol_flag :%d\n", symbol, dmrs_symbol_flag); - if (dmrs_symbol_flag == 1) - { + + if (dmrs_symbol_flag == 1) { if (pusch_vars->dmrs_symbol == INVALID_VALUE) pusch_vars->dmrs_symbol = symbol; - for (int nl=0; nlnrOfLayers; nl++) - { + + for (int nl=0; nlnrOfLayers; nl++) { + uint32_t nvar_tmp = 0; nr_pusch_channel_estimation(gNB, slot, get_dmrs_port(nl,rel15_ul->dmrs_ports), @@ -3623,7 +2716,8 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB, bwp_start_subcarrier, rel15_ul, &max_ch, - 0 /* nvar*/); + &nvar_tmp); + nvar += nvar_tmp; } // measure the SNR from the channel estimation nr_gnb_measurements(gNB, @@ -3656,6 +2750,8 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB, } } + nvar /= (rel15_ul->nr_of_symbols * rel15_ul->nrOfLayers * frame_parms->nb_antennas_rx); + // averaging time domain channel estimates if (gNB->chest_time == 1) { @@ -3778,7 +2874,7 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB, { union puschSymbolReqUnion id = {.s={ulsch_id,frame,slot,0}}; id.p=1+symbol; - notifiedFIFO_elt_t *req = newNotifiedFIFO_elt(sizeof(puschSymbolProc_t), id.p, gNB->respPuschSymb, &nr_pusch_symbol_processing_noprecoding); // create a job for Tpool + notifiedFIFO_elt_t *req = newNotifiedFIFO_elt(sizeof(puschSymbolProc_t), id.p, &gNB->respPuschSymb, &nr_pusch_symbol_processing_noprecoding); // create a job for Tpool puschSymbolProc_t *rdata = (puschSymbolProc_t*)NotifiedFifoData(req); // data for the job rdata->gNB = gNB; @@ -3791,6 +2887,7 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB, rdata->llr = pusch_vars->llr; rdata->llr_layers = pusch_vars->llr_layers; rdata->s = &s[pusch_vars->llr_offset[symbol]*rel15_ul->nrOfLayers]; + rdata->nvar = nvar; pushTpool(&gNB->threadPool, req); gNB->nbSymb++; @@ -3801,7 +2898,7 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB, while (gNB->nbSymb > 0) { - notifiedFIFO_elt_t *req=pullTpool(gNB->respPuschSymb, &gNB->threadPool); + notifiedFIFO_elt_t *req=pullTpool(&gNB->respPuschSymb, &gNB->threadPool); gNB->nbSymb--; delNotifiedFIFO_elt(req); } diff --git a/openair1/PHY/NR_TRANSPORT/nr_ulsch_llr_computation.c b/openair1/PHY/NR_TRANSPORT/nr_ulsch_llr_computation.c index f4d526960e..ddbec269b3 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_ulsch_llr_computation.c +++ b/openair1/PHY/NR_TRANSPORT/nr_ulsch_llr_computation.c @@ -70,68 +70,28 @@ void nr_ulsch_16qam_llr(int32_t *rxdataF_comp, uint32_t nb_re, uint8_t symbol) { + simde__m256i *rxF_256 = (simde__m256i*)rxdataF_comp; + simde__m256i *ch_mag = (simde__m256i*)ul_ch_mag; + simde__m64 *llr_64 = (simde__m64*)ulsch_llr; - simde__m256i *rxF = (simde__m256i*)rxdataF_comp; - simde__m256i *ch_mag; - simde__m256i llr256[2]; - register simde__m256i xmm0; - uint32_t *llr32; - int i; + simde__m256i xmm0, xmm1, xmm2; - int off = ((nb_rb&1) == 1)? 4:0; - - llr32 = (uint32_t*)ulsch_llr; - - ch_mag = (simde__m256i*)&ul_ch_mag[(symbol*(off+(nb_rb*12)))]; - unsigned char len_mod8 = nb_re&7; - nb_re >>= 3; // length in quad words (4 REs) - nb_re += (len_mod8 == 0 ? 0 : 1); - - for (i=0; i |y_R|, registers of odd index in xmm0-> |y_I| - xmm0 = simde_mm256_subs_epi16(ch_mag[i],xmm0); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 + for (int i = 0; i < ((nb_re>>3) + ((nb_re&7) ? 1 : 0)); i++) { + xmm0 = simde_mm256_abs_epi16(rxF_256[i]); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| + xmm0 = simde_mm256_subs_epi16(ch_mag[i], xmm0); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 - llr256[0] = simde_mm256_unpacklo_epi32(rxF[i],xmm0); // llr128[0] contains the llrs of the 1st,2nd,5th and 6th REs - llr256[1] = simde_mm256_unpackhi_epi32(rxF[i],xmm0); // llr128[1] contains the llrs of the 3rd, 4th, 7th and 8th REs + xmm1 = simde_mm256_unpacklo_epi32(rxF_256[i], xmm0); // llr128[0] contains the llrs of the 1st,2nd,5th and 6th REs + xmm2 = simde_mm256_unpackhi_epi32(rxF_256[i], xmm0); // llr128[1] contains the llrs of the 3rd, 4th, 7th and 8th REs - // 1st RE - llr32[0] = simde_mm256_extract_epi32(llr256[0],0); // llr32[0] low 16 bits-> y_R , high 16 bits-> y_I - llr32[1] = simde_mm256_extract_epi32(llr256[0],1); // llr32[1] low 16 bits-> |h|-|y_R|^2, high 16 bits-> |h|-|y_I|^2 - - // 2nd RE - llr32[2] = simde_mm256_extract_epi32(llr256[0],2); // llr32[2] low 16 bits-> y_R , high 16 bits-> y_I - llr32[3] = simde_mm256_extract_epi32(llr256[0],3); // llr32[3] low 16 bits-> |h|-|y_R|^2, high 16 bits-> |h|-|y_I|^2 - - // 3rd RE - llr32[4] = simde_mm256_extract_epi32(llr256[1],0); // llr32[4] low 16 bits-> y_R , high 16 bits-> y_I - llr32[5] = simde_mm256_extract_epi32(llr256[1],1); // llr32[5] low 16 bits-> |h|-|y_R|^2, high 16 bits-> |h|-|y_I|^2 - - // 4th RE - llr32[6] = simde_mm256_extract_epi32(llr256[1],2); // llr32[6] low 16 bits-> y_R , high 16 bits-> y_I - llr32[7] = simde_mm256_extract_epi32(llr256[1],3); // llr32[7] low 16 bits-> |h|-|y_R|^2, high 16 bits-> |h|-|y_I|^2 - - // 5th RE - llr32[8] = simde_mm256_extract_epi32(llr256[0],4); // llr32[8] low 16 bits-> y_R , high 16 bits-> y_I - llr32[9] = simde_mm256_extract_epi32(llr256[0],5); // llr32[9] low 16 bits-> |h|-|y_R|^2, high 16 bits-> |h|-|y_I|^2 - - // 6th RE - llr32[10] = simde_mm256_extract_epi32(llr256[0],6); // llr32[10] low 16 bits-> y_R , high 16 bits-> y_I - llr32[11] = simde_mm256_extract_epi32(llr256[0],7); // llr32[11] low 16 bits-> |h|-|y_R|^2, high 16 bits-> |h|-|y_I|^2 - - // 7th RE - llr32[12] = simde_mm256_extract_epi32(llr256[1],4); // llr32[12] low 16 bits-> y_R , high 16 bits-> y_I - llr32[13] = simde_mm256_extract_epi32(llr256[1],5); // llr32[13] low 16 bits-> |h|-|y_R|^2, high 16 bits-> |h|-|y_I|^2 - - // 8th RE - llr32[14] = simde_mm256_extract_epi32(llr256[1],6); // llr32[14] low 16 bits-> y_R , high 16 bits-> y_I - llr32[15] = simde_mm256_extract_epi32(llr256[1],7); // llr32[15] low 16 bits-> |h|-|y_R|^2, high 16 bits-> |h|-|y_I|^2 - - llr32+=16; - + *llr_64++ = (simde__m64)simde_mm256_extract_epi64(xmm1, 0); + *llr_64++ = (simde__m64)simde_mm256_extract_epi64(xmm1, 1); + *llr_64++ = (simde__m64)simde_mm256_extract_epi64(xmm2, 0); + *llr_64++ = (simde__m64)simde_mm256_extract_epi64(xmm2, 1); + *llr_64++ = (simde__m64)simde_mm256_extract_epi64(xmm1, 2); + *llr_64++ = (simde__m64)simde_mm256_extract_epi64(xmm1, 3); + *llr_64++ = (simde__m64)simde_mm256_extract_epi64(xmm2, 2); + *llr_64++ = (simde__m64)simde_mm256_extract_epi64(xmm2, 3); } - - simde_mm_empty(); - simde_m_empty(); } //---------------------------------------------------------------------------------------------- @@ -146,112 +106,53 @@ void nr_ulsch_64qam_llr(int32_t *rxdataF_comp, uint32_t nb_re, uint8_t symbol) { - int off = ((nb_rb&1) == 1)? 4:0; - simde__m256i *rxF = (simde__m256i*)rxdataF_comp; - simde__m256i *ch_mag,*ch_magb; - register simde__m256i xmm0,xmm1,xmm2; + simde__m256i xmm0, xmm1, xmm2; - int i; + simde__m256i *ch_maga = (simde__m256i*)ul_ch_mag; + simde__m256i *ch_magb = (simde__m256i*)ul_ch_magb; - ch_mag = (simde__m256i*)&ul_ch_mag[(symbol*(off+(nb_rb*12)))]; - ch_magb = (simde__m256i*)&ul_ch_magb[(symbol*(off+(nb_rb*12)))]; + int32_t *llr_32 = (int32_t *)ulsch_llr; - int len_mod8 = nb_re&7; - nb_re = nb_re>>3; // length in quad words (4 REs) - nb_re += ((len_mod8 == 0) ? 0 : 1); - - for (i=0; i>3) + ((nb_re&7) ? 1 : 0)); i++) { xmm0 = rxF[i]; xmm1 = simde_mm256_abs_epi16(xmm0); - xmm1 = simde_mm256_subs_epi16(ch_mag[i],xmm1); + xmm1 = simde_mm256_subs_epi16(ch_maga[i], xmm1); xmm2 = simde_mm256_abs_epi16(xmm1); - xmm2 = simde_mm256_subs_epi16(ch_magb[i],xmm2); + xmm2 = simde_mm256_subs_epi16(ch_magb[i], xmm2); - // --------------------------------------- - // 1st RE - // --------------------------------------- - ulsch_llr[0] = simde_mm256_extract_epi16(xmm0,0); - ulsch_llr[1] = simde_mm256_extract_epi16(xmm0,1); - ulsch_llr[2] = simde_mm256_extract_epi16(xmm1,0); - ulsch_llr[3] = simde_mm256_extract_epi16(xmm1,1); - ulsch_llr[4] = simde_mm256_extract_epi16(xmm2,0); - ulsch_llr[5] = simde_mm256_extract_epi16(xmm2,1); - // --------------------------------------- + *llr_32++ = simde_mm256_extract_epi32(xmm0,0); + *llr_32++ = simde_mm256_extract_epi32(xmm1,0); + *llr_32++ = simde_mm256_extract_epi32(xmm2,0); - ulsch_llr+=6; - - // --------------------------------------- - // 2nd RE - // --------------------------------------- - ulsch_llr[0] = simde_mm256_extract_epi16(xmm0,2); - ulsch_llr[1] = simde_mm256_extract_epi16(xmm0,3); - ulsch_llr[2] = simde_mm256_extract_epi16(xmm1,2); - ulsch_llr[3] = simde_mm256_extract_epi16(xmm1,3); - ulsch_llr[4] = simde_mm256_extract_epi16(xmm2,2); - ulsch_llr[5] = simde_mm256_extract_epi16(xmm2,3); - // --------------------------------------- + *llr_32++ = simde_mm256_extract_epi32(xmm0,1); + *llr_32++ = simde_mm256_extract_epi32(xmm1,1); + *llr_32++ = simde_mm256_extract_epi32(xmm2,1); - ulsch_llr+=6; - - // --------------------------------------- - // 3rd RE - // --------------------------------------- - ulsch_llr[0] = simde_mm256_extract_epi16(xmm0,4); - ulsch_llr[1] = simde_mm256_extract_epi16(xmm0,5); - ulsch_llr[2] = simde_mm256_extract_epi16(xmm1,4); - ulsch_llr[3] = simde_mm256_extract_epi16(xmm1,5); - ulsch_llr[4] = simde_mm256_extract_epi16(xmm2,4); - ulsch_llr[5] = simde_mm256_extract_epi16(xmm2,5); - // --------------------------------------- + *llr_32++ = simde_mm256_extract_epi32(xmm0,2); + *llr_32++ = simde_mm256_extract_epi32(xmm1,2); + *llr_32++ = simde_mm256_extract_epi32(xmm2,2); - ulsch_llr+=6; - - // --------------------------------------- - // 4th RE - // --------------------------------------- - ulsch_llr[0] = simde_mm256_extract_epi16(xmm0,6); - ulsch_llr[1] = simde_mm256_extract_epi16(xmm0,7); - ulsch_llr[2] = simde_mm256_extract_epi16(xmm1,6); - ulsch_llr[3] = simde_mm256_extract_epi16(xmm1,7); - ulsch_llr[4] = simde_mm256_extract_epi16(xmm2,6); - ulsch_llr[5] = simde_mm256_extract_epi16(xmm2,7); - // --------------------------------------- + *llr_32++ = simde_mm256_extract_epi32(xmm0,3); + *llr_32++ = simde_mm256_extract_epi32(xmm1,3); + *llr_32++ = simde_mm256_extract_epi32(xmm2,3); - ulsch_llr+=6; - ulsch_llr[0] = simde_mm256_extract_epi16(xmm0,8); - ulsch_llr[1] = simde_mm256_extract_epi16(xmm0,9); - ulsch_llr[2] = simde_mm256_extract_epi16(xmm1,8); - ulsch_llr[3] = simde_mm256_extract_epi16(xmm1,9); - ulsch_llr[4] = simde_mm256_extract_epi16(xmm2,8); - ulsch_llr[5] = simde_mm256_extract_epi16(xmm2,9); + *llr_32++ = simde_mm256_extract_epi32(xmm0,4); + *llr_32++ = simde_mm256_extract_epi32(xmm1,4); + *llr_32++ = simde_mm256_extract_epi32(xmm2,4); - ulsch_llr[6] = simde_mm256_extract_epi16(xmm0,10); - ulsch_llr[7] = simde_mm256_extract_epi16(xmm0,11); - ulsch_llr[8] = simde_mm256_extract_epi16(xmm1,10); - ulsch_llr[9] = simde_mm256_extract_epi16(xmm1,11); - ulsch_llr[10] = simde_mm256_extract_epi16(xmm2,10); - ulsch_llr[11] = simde_mm256_extract_epi16(xmm2,11); + *llr_32++ = simde_mm256_extract_epi32(xmm0,5); + *llr_32++ = simde_mm256_extract_epi32(xmm1,5); + *llr_32++ = simde_mm256_extract_epi32(xmm2,5); - ulsch_llr[12] = simde_mm256_extract_epi16(xmm0,12); - ulsch_llr[13] = simde_mm256_extract_epi16(xmm0,13); - ulsch_llr[14] = simde_mm256_extract_epi16(xmm1,12); - ulsch_llr[15] = simde_mm256_extract_epi16(xmm1,13); - ulsch_llr[16] = simde_mm256_extract_epi16(xmm2,12); - ulsch_llr[17] = simde_mm256_extract_epi16(xmm2,13); + *llr_32++ = simde_mm256_extract_epi32(xmm0,6); + *llr_32++ = simde_mm256_extract_epi32(xmm1,6); + *llr_32++ = simde_mm256_extract_epi32(xmm2,6); - ulsch_llr[18] = simde_mm256_extract_epi16(xmm0,14); - ulsch_llr[19] = simde_mm256_extract_epi16(xmm0,15); - ulsch_llr[20] = simde_mm256_extract_epi16(xmm1,14); - ulsch_llr[21] = simde_mm256_extract_epi16(xmm1,15); - ulsch_llr[22] = simde_mm256_extract_epi16(xmm2,14); - ulsch_llr[23] = simde_mm256_extract_epi16(xmm2,15); - - ulsch_llr+=24; + *llr_32++ = simde_mm256_extract_epi32(xmm0,7); + *llr_32++ = simde_mm256_extract_epi32(xmm1,7); + *llr_32++ = simde_mm256_extract_epi32(xmm2,7); } - - simde_mm_empty(); - simde_m_empty(); } void nr_ulsch_256qam_llr(int32_t *rxdataF_comp, @@ -263,105 +164,42 @@ void nr_ulsch_256qam_llr(int32_t *rxdataF_comp, uint32_t nb_re, uint8_t symbol) { - int off = ((nb_rb&1) == 1)? 4:0; - simde__m256i *rxF = (simde__m256i*)rxdataF_comp; - simde__m256i *ch_mag,*ch_magb,*ch_magc; - register simde__m256i xmm0,xmm1,xmm2,xmm3,xmm4,xmm5,xmm6; + simde__m256i xmm0,xmm1,xmm2,xmm3,xmm4,xmm5,xmm6; simde__m256i *llr256=(simde__m256i*)ulsch_llr; - ch_mag = (simde__m256i*)&ul_ch_mag[(symbol*(off+(nb_rb*12)))]; - ch_magb = (simde__m256i*)&ul_ch_magb[(symbol*(off+(nb_rb*12)))]; - ch_magc = (simde__m256i*)&ul_ch_magc[(symbol*(off+(nb_rb*12)))]; - int len_mod8 = nb_re&7; - int nb_re256 = nb_re>>3; // length in 256-bit words (8 REs) + simde__m256i* ch_maga = (simde__m256i*)ul_ch_mag; + simde__m256i* ch_magb = (simde__m256i*)ul_ch_magb; + simde__m256i* ch_magc = (simde__m256i*)ul_ch_magc; - for (int i=0; i |y_R|, registers of odd index in xmm0-> |y_I| - xmm0 = simde_mm256_subs_epi16(ch_mag[i],xmm0); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 - // xmmtmpD2 contains 16 LLRs - xmm1 = simde_mm256_abs_epi16(xmm0); - xmm1 = simde_mm256_subs_epi16(ch_magb[i],xmm1); // contains 16 LLRs - xmm2 = simde_mm256_abs_epi16(xmm1); - xmm2 = simde_mm256_subs_epi16(ch_magc[i],xmm2); // contains 16 LLRs - // rxF[i] A0 A1 A2 A3 A4 A5 A6 A7 bits 7,6 - // xmm0 B0 B1 B2 B3 B4 B5 B6 B7 bits 5,4 - // xmm1 C0 C1 C2 C3 C4 C5 C6 C7 bits 3,2 - // xmm2 D0 D1 D2 D3 D4 D5 D6 D7 bits 1,0 - xmm3 = simde_mm256_unpacklo_epi32(rxF[i],xmm0); // A0 B0 A1 B1 A4 B4 A5 B5 - xmm4 = simde_mm256_unpackhi_epi32(rxF[i],xmm0); // A2 B2 A3 B3 A6 B6 A7 B7 - xmm5 = simde_mm256_unpacklo_epi32(xmm1,xmm2); // C0 D0 C1 D1 C4 D4 C5 D5 - xmm6 = simde_mm256_unpackhi_epi32(xmm1,xmm2); // C2 D2 C3 D3 C6 D6 C7 D7 - - xmm0 = simde_mm256_unpacklo_epi64(xmm3,xmm5); // A0 B0 C0 D0 A4 B4 C4 D4 - xmm1 = simde_mm256_unpackhi_epi64(xmm3,xmm5); // A1 B1 C1 D1 A5 B5 C5 D5 - xmm2 = simde_mm256_unpacklo_epi64(xmm4,xmm6); // A2 B2 C2 D2 A6 B6 C6 D6 - xmm3 = simde_mm256_unpackhi_epi64(xmm4,xmm6); // A3 B3 C3 D3 A7 B7 C7 D7 - llr256[0] = simde_mm256_permute2x128_si256(xmm0, xmm1, 0x20); // A0 B0 C0 D0 A1 B1 C1 D1 - llr256[1] = simde_mm256_permute2x128_si256(xmm2, xmm3, 0x20); // A2 B2 C2 D2 A3 B3 C3 D3 - llr256[2] = simde_mm256_permute2x128_si256(xmm0, xmm1, 0x31); // A4 B4 C4 D4 A5 B5 C5 D5 - llr256[3] = simde_mm256_permute2x128_si256(xmm2, xmm3, 0x31); // A6 B6 C6 D6 A7 B7 C7 D7 - llr256+=4; + for (int i = 0; i < ((nb_re>>3) + ((nb_re&7) ? 1 : 0)); i++) { + xmm0 = simde_mm256_abs_epi16(rxF[i]); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| + xmm0 = simde_mm256_subs_epi16(ch_maga[i], xmm0); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 + // xmmtmpD2 contains 16 LLRs + xmm1 = simde_mm256_abs_epi16(xmm0); + xmm1 = simde_mm256_subs_epi16(ch_magb[i], xmm1); // contains 16 LLRs + xmm2 = simde_mm256_abs_epi16(xmm1); + xmm2 = simde_mm256_subs_epi16(ch_magc[i], xmm2); // contains 16 LLRs + // rxF[i] A0 A1 A2 A3 A4 A5 A6 A7 bits 7,6 + // xmm0 B0 B1 B2 B3 B4 B5 B6 B7 bits 5,4 + // xmm1 C0 C1 C2 C3 C4 C5 C6 C7 bits 3,2 + // xmm2 D0 D1 D2 D3 D4 D5 D6 D7 bits 1,0 + xmm3 = simde_mm256_unpacklo_epi32(rxF[i], xmm0); // A0 B0 A1 B1 A4 B4 A5 B5 + xmm4 = simde_mm256_unpackhi_epi32(rxF[i], xmm0); // A2 B2 A3 B3 A6 B6 A7 B7 + xmm5 = simde_mm256_unpacklo_epi32(xmm1, xmm2); // C0 D0 C1 D1 C4 D4 C5 D5 + xmm6 = simde_mm256_unpackhi_epi32(xmm1, xmm2); // C2 D2 C3 D3 C6 D6 C7 D7 + xmm0 = simde_mm256_unpacklo_epi64(xmm3, xmm5); // A0 B0 C0 D0 A4 B4 C4 D4 + xmm1 = simde_mm256_unpackhi_epi64(xmm3, xmm5); // A1 B1 C1 D1 A5 B5 C5 D5 + xmm2 = simde_mm256_unpacklo_epi64(xmm4, xmm6); // A2 B2 C2 D2 A6 B6 C6 D6 + xmm3 = simde_mm256_unpackhi_epi64(xmm4, xmm6); // A3 B3 C3 D3 A7 B7 C7 D7 + *llr256++ = simde_mm256_permute2x128_si256(xmm0, xmm1, 0x20); // A0 B0 C0 D0 A1 B1 C1 D1 + *llr256++ = simde_mm256_permute2x128_si256(xmm2, xmm3, 0x20); // A2 B2 C2 D2 A3 B3 C3 D3 + *llr256++ = simde_mm256_permute2x128_si256(xmm0, xmm1, 0x31); // A4 B4 C4 D4 A5 B5 C5 D5 + *llr256++ = simde_mm256_permute2x128_si256(xmm2, xmm3, 0x31); // A6 B6 C6 D6 A7 B7 C7 D7 } - simde__m128i *llr128 = (simde__m128i*)llr256; - if (len_mod8 >= 4) { - int nb_re128 = nb_re>>2; - simde__m128i xmm0,xmm1,xmm2,xmm3,xmm4,xmm5,xmm6; - simde__m128i *rxF = (simde__m128i*)rxdataF_comp; - simde__m128i *ch_mag = (simde__m128i*)&ul_ch_mag[(symbol*(off+(nb_rb*12)))]; - simde__m128i *ch_magb = (simde__m128i*)&ul_ch_magb[(symbol*(off+(nb_rb*12)))]; - simde__m128i *ch_magc = (simde__m128i*)&ul_ch_magc[(symbol*(off+(nb_rb*12)))]; - - xmm0 = simde_mm_abs_epi16(rxF[nb_re128-1]); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| - xmm0 = simde_mm_subs_epi16(ch_mag[nb_re128-1],xmm0); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 - // xmmtmpD2 contains 8 LLRs - xmm1 = simde_mm_abs_epi16(xmm0); - xmm1 = simde_mm_subs_epi16(ch_magb[nb_re128-1],xmm1); // contains 8 LLRs - xmm2 = simde_mm_abs_epi16(xmm1); - xmm2 = simde_mm_subs_epi16(ch_magc[nb_re128-1],xmm2); // contains 8 LLRs - // rxF[i] A0 A1 A2 A3 - // xmm0 B0 B1 B2 B3 - // xmm1 C0 C1 C2 C3 - // xmm2 D0 D1 D2 D3 - xmm3 = simde_mm_unpacklo_epi32(rxF[nb_re128-1],xmm0); // A0 B0 A1 B1 - xmm4 = simde_mm_unpackhi_epi32(rxF[nb_re128-1],xmm0); // A2 B2 A3 B3 - xmm5 = simde_mm_unpacklo_epi32(xmm1,xmm2); // C0 D0 C1 D1 - xmm6 = simde_mm_unpackhi_epi32(xmm1,xmm2); // C2 D2 C3 D3 - - llr128[0] = simde_mm_unpacklo_epi64(xmm3,xmm5); // A0 B0 C0 D0 - llr128[1] = simde_mm_unpackhi_epi64(xmm3,xmm5); // A1 B1 C1 D1 - llr128[2] = simde_mm_unpacklo_epi64(xmm4,xmm6); // A2 B2 C2 D2 - llr128[3] = simde_mm_unpackhi_epi64(xmm4,xmm6); // A3 B3 C3 D3 - llr128+=4; - } - if (len_mod8 == 6) { - int nb_re64 = nb_re>>1; - simde__m64 *llr64 = (simde__m64 *)llr128; - simde__m64 xmm0,xmm1,xmm2; - simde__m64 *rxF = (simde__m64*)rxdataF_comp; - simde__m64 *ch_mag = (simde__m64*)&ul_ch_mag[(symbol*(off+(nb_rb*12)))]; - simde__m64 *ch_magb = (simde__m64*)&ul_ch_magb[(symbol*(off+(nb_rb*12)))]; - simde__m64 *ch_magc = (simde__m64*)&ul_ch_magc[(symbol*(off+(nb_rb*12)))]; - - xmm0 = simde_mm_abs_pi16(rxF[nb_re64-1]); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| - xmm0 = simde_mm_subs_pi16(ch_mag[nb_re-1],xmm0); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 - // xmmtmpD2 contains 4 LLRs - xmm1 = simde_mm_abs_pi16(xmm0); - xmm1 = simde_mm_subs_pi16(ch_magb[nb_re64-1],xmm1); // contains 4 LLRs - xmm2 = simde_mm_abs_pi16(xmm1); - xmm2 = simde_mm_subs_pi16(ch_magc[nb_re64-1],xmm2); // contains 4 LLRs - // rxF[i] A0 A1 - // xmm0 B0 B1 - // xmm1 C0 C1 - // xmm2 D0 D1 - llr64[0] = simde_m_punpckldq(rxF[nb_re64-1],xmm0); // A0 B0 - llr64[2] = simde_m_punpckhdq(rxF[nb_re64-1],xmm0); // A1 B1 - llr64[1] = simde_m_punpckldq(xmm1,xmm2); // C0 D0 - llr64[3] = simde_m_punpckhdq(xmm1,xmm2); // C1 D1 - } - } + void nr_ulsch_compute_llr(int32_t *rxdataF_comp, int32_t *ul_ch_mag, int32_t *ul_ch_magb, diff --git a/openair1/PHY/defs_gNB.h b/openair1/PHY/defs_gNB.h index f1f931e1f4..88367741d4 100644 --- a/openair1/PHY/defs_gNB.h +++ b/openair1/PHY/defs_gNB.h @@ -760,7 +760,7 @@ typedef struct PHY_VARS_gNB_s { time_stats_t rx_dft_stats; time_stats_t ulsch_freq_offset_estimation_stats; */ - notifiedFIFO_t *respPuschSymb; + notifiedFIFO_t respPuschSymb; notifiedFIFO_t respDecode; notifiedFIFO_t resp_L1; notifiedFIFO_t L1_tx_free; @@ -769,7 +769,6 @@ typedef struct PHY_VARS_gNB_s { notifiedFIFO_t resp_RU_tx; tpool_t threadPool; int nbSymb; - int use_pusch_tp; int num_pusch_symbols_per_thread; pthread_t L1_rx_thread; int L1_rx_thread_core; @@ -792,6 +791,7 @@ typedef struct puschSymbolProc_s { int16_t *llr; int16_t **llr_layers; int16_t *s; + uint32_t nvar; } puschSymbolProc_t; struct puschSymbolReqId { diff --git a/openair1/SCHED_NR/phy_procedures_nr_gNB.c b/openair1/SCHED_NR/phy_procedures_nr_gNB.c index 35afb44fe1..64eae8422c 100644 --- a/openair1/SCHED_NR/phy_procedures_nr_gNB.c +++ b/openair1/SCHED_NR/phy_procedures_nr_gNB.c @@ -398,20 +398,20 @@ static int nr_ulsch_procedures(PHY_VARS_gNB *gNB, int frame_rx, int slot_rx, int pusch_pdu->qam_mod_order, pusch_pdu->nrOfLayers); - if (gNB->use_pusch_tp == 0 ) - { - nr_ulsch_layer_demapping(gNB->pusch_vars[ULSCH_id].llr, - pusch_pdu->nrOfLayers, - pusch_pdu->qam_mod_order, - G, - gNB->pusch_vars[ULSCH_id].llr_layers); - //---------------------------------------------------------- - //------------------- ULSCH unscrambling ------------------- - //---------------------------------------------------------- - start_meas(&gNB->ulsch_unscrambling_stats); - nr_ulsch_unscrambling(gNB->pusch_vars[ULSCH_id].llr, G, pusch_pdu->data_scrambling_id, pusch_pdu->rnti); - stop_meas(&gNB->ulsch_unscrambling_stats); - } + + + // nr_ulsch_layer_demapping(gNB->pusch_vars[ULSCH_id].llr, + // pusch_pdu->nrOfLayers, + // pusch_pdu->qam_mod_order, + // G, + // gNB->pusch_vars[ULSCH_id].llr_layers); + // //---------------------------------------------------------- + // //------------------- ULSCH unscrambling ------------------- + // //---------------------------------------------------------- + // start_meas(&gNB->ulsch_unscrambling_stats); + // nr_ulsch_unscrambling(gNB->pusch_vars[ULSCH_id].llr, G, pusch_pdu->data_scrambling_id, pusch_pdu->rnti); + // stop_meas(&gNB->ulsch_unscrambling_stats); + //---------------------------------------------------------- //--------------------- ULSCH decoding --------------------- @@ -902,8 +902,9 @@ int phy_procedures_gNB_uespec_RX(PHY_VARS_gNB *gNB, int frame_rx, int slot_rx) VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_NR_RX_PUSCH, 1); start_meas(&gNB->rx_pusch_stats); - if (gNB->use_pusch_tp) nr_rx_pusch_tp(gNB, ULSCH_id, frame_rx, slot_rx, ulsch->harq_pid); - else nr_rx_pusch(gNB, ULSCH_id, frame_rx, slot_rx, ulsch->harq_pid); + // if (gNB->use_pusch_tp) nr_rx_pusch_tp(gNB, ULSCH_id, frame_rx, slot_rx, ulsch->harq_pid); + // else nr_rx_pusch(gNB, ULSCH_id, frame_rx, slot_rx, ulsch->harq_pid); + nr_rx_pusch_tp(gNB, ULSCH_id, frame_rx, slot_rx, ulsch->harq_pid); NR_gNB_PUSCH *pusch_vars = &gNB->pusch_vars[ULSCH_id]; pusch_vars->ulsch_power_tot = 0; pusch_vars->ulsch_noise_power_tot = 0; diff --git a/openair1/SIMULATION/NR_PHY/ulsim.c b/openair1/SIMULATION/NR_PHY/ulsim.c index 879d6deda1..5c3abe09be 100644 --- a/openair1/SIMULATION/NR_PHY/ulsim.c +++ b/openair1/SIMULATION/NR_PHY/ulsim.c @@ -222,7 +222,6 @@ int main(int argc, char *argv[]) int ibwp_rboffset=41; int params_from_file = 0; int threadCnt=0; - int use_tpool = 0; int max_ldpc_iterations = 5; if ( load_configmodule(argc,argv,CONFIG_ENABLECMDLINEONLY) == 0 ) { exit_fun("[NR_ULSIM] Error, configuration module init failed\n"); @@ -364,7 +363,6 @@ int main(int argc, char *argv[]) case 'C': threadCnt = atoi(optarg); - use_tpool = 1; break; case 'u': @@ -563,14 +561,12 @@ int main(int argc, char *argv[]) RC.gNB[0] = calloc(1,sizeof(PHY_VARS_gNB)); gNB = RC.gNB[0]; gNB->ofdm_offset_divisor = UINT_MAX; - initNotifiedFIFO(&gNB->respDecode); - gNB->use_pusch_tp = use_tpool; gNB->num_pusch_symbols_per_thread = 1; initFloatingCoresTpool(threadCnt, &gNB->threadPool, false, "gNB-tpool"); initNotifiedFIFO(&gNB->respDecode); - gNB->respPuschSymb = (notifiedFIFO_t*) malloc(sizeof(notifiedFIFO_t)); - initNotifiedFIFO(gNB->respPuschSymb); + + initNotifiedFIFO(&gNB->respPuschSymb); initNotifiedFIFO(&gNB->L1_tx_free); initNotifiedFIFO(&gNB->L1_tx_filled); initNotifiedFIFO(&gNB->L1_tx_out); @@ -1602,20 +1598,16 @@ int main(int argc, char *argv[]) printDistribution(&gNB->phy_proc_rx,table_rx,"Total PHY proc rx"); printStatIndent(&gNB->rx_pusch_stats,"RX PUSCH time"); printStatIndent2(&gNB->ulsch_channel_estimation_stats,"ULSCH channel estimation time"); - if (use_tpool == 1) - { - printStatIndent2(&gNB->rx_pusch_init_stats,"RX PUSCH Initialization time"); - printStatIndent2(&gNB->rx_pusch_symbol_processing_stats,"RX PUSCH Symbol Processing time"); - } - else - { - printStatIndent2(&gNB->ulsch_ptrs_processing_stats,"ULSCH PTRS Processing time"); - printStatIndent2(&gNB->ulsch_rbs_extraction_stats,"ULSCH rbs extraction time"); - printStatIndent2(&gNB->ulsch_channel_compensation_stats,"ULSCH channel compensation time"); - printStatIndent2(&gNB->ulsch_mrc_stats,"ULSCH mrc computation"); - printStatIndent2(&gNB->ulsch_llr_stats,"ULSCH llr computation"); - printStatIndent(&gNB->ulsch_unscrambling_stats,"ULSCH unscrambling"); - } + printStatIndent2(&gNB->rx_pusch_init_stats,"RX PUSCH Initialization time"); + printStatIndent2(&gNB->rx_pusch_symbol_processing_stats,"RX PUSCH Symbol Processing time"); + + // printStatIndent2(&gNB->ulsch_ptrs_processing_stats,"ULSCH PTRS Processing time"); + // printStatIndent2(&gNB->ulsch_rbs_extraction_stats,"ULSCH rbs extraction time"); + // printStatIndent2(&gNB->ulsch_channel_compensation_stats,"ULSCH channel compensation time"); + // printStatIndent2(&gNB->ulsch_mrc_stats,"ULSCH mrc computation"); + // printStatIndent2(&gNB->ulsch_llr_stats,"ULSCH llr computation"); + // printStatIndent(&gNB->ulsch_unscrambling_stats,"ULSCH unscrambling"); + printStatIndent(&gNB->ulsch_decoding_stats,"ULSCH total decoding time"); // printStatIndent2(&gNB->ulsch_deinterleaving_stats,"ULSCH deinterleaving"); // printStatIndent2(&gNB->ulsch_rate_unmatching_stats,"ULSCH rate matching rx"); From e51d8a9bc9e4b35d6e1a7d52f900a96d7c78120f Mon Sep 17 00:00:00 2001 From: Tsung-Yu Chan Date: Tue, 5 Sep 2023 21:09:34 +0200 Subject: [PATCH 03/13] feat / compact the code - merge the inner_rx - rewrite the ulsch channel compensation --- .../PHY/NR_TRANSPORT/nr_transport_proto.h | 47 +- .../PHY/NR_TRANSPORT/nr_ulsch_demodulation.c | 1442 +++-------------- .../NR_TRANSPORT/nr_ulsch_llr_computation.c | 85 +- 3 files changed, 283 insertions(+), 1291 deletions(-) diff --git a/openair1/PHY/NR_TRANSPORT/nr_transport_proto.h b/openair1/PHY/NR_TRANSPORT/nr_transport_proto.h index 97d32232cf..5d25f623df 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_transport_proto.h +++ b/openair1/PHY/NR_TRANSPORT/nr_transport_proto.h @@ -179,35 +179,6 @@ void nr_ulsch_channel_level(int **ul_ch_estimates_ext, uint8_t nrOfLayers, unsigned short nb_rb); -/** \brief This function performs channel compensation (matched filtering) on the received RBs for this allocation. In addition, it computes the squared-magnitude of the channel with weightings for 16QAM/64QAM detection as well as dual-stream detection (cross-correlation) - @param rxdataF_ext Frequency-domain received signal in RBs to be demodulated - @param ul_ch_estimates_ext Frequency-domain channel estimates in RBs to be demodulated - @param ul_ch_mag First Channel magnitudes (16QAM/64QAM/256QAM) - @param ul_ch_magb Second weighted Channel magnitudes (64QAM/256QAM) - @param ul_ch_magc Third weighted Channel magnitudes (256QAM) - @param rxdataF_comp Compensated received waveform - @param frame_parms Pointer to frame descriptor - @param symbol Symbol on which to operate - @param Qm Modulation order of allocation - @param nb_rb Number of RBs in allocation - @param output_shift Rescaling for compensated output (should be energy-normalizing) -*/ -void nr_ulsch_channel_compensation(int **rxdataF_ext, - int **ul_ch_estimates_ext, - int **ul_ch_mag, - int **ul_ch_magb, - int **ul_ch_magc, - int **rxdataF_comp, - int ***rho, - NR_DL_FRAME_PARMS *frame_parms, - unsigned char symbol, - int length, - uint8_t is_dmrs_symbol, - unsigned char mod_order, - uint8_t nrOfLayers, - unsigned short nb_rb, - unsigned char output_shift); - /*! \brief This function implements the idft transform precoding in PUSCH \param z Pointer to input in frequnecy domain, and it is also the output in time domain @@ -316,15 +287,17 @@ void nr_ulsch_compute_llr(int32_t *rxdataF_comp, void reset_active_stats(PHY_VARS_gNB *gNB, int frame); void reset_active_ulsch(PHY_VARS_gNB *gNB, int frame); -void nr_ulsch_compute_ML_llr(int32_t **rxdataF_comp, - int32_t **ul_ch_mag, - int32_t ***rho, - int16_t **llr_layers, - uint8_t nb_antennas_rx, - uint32_t rb_size, +void nr_ulsch_compute_ML_llr(NR_gNB_PUSCH *pusch_vars, + uint32_t symbol, + c16_t* rxdataF_comp0, + c16_t* rxdataF_comp1, + c16_t* ul_ch_mag0, + c16_t* ul_ch_mag1, + c16_t* llr_layers0, + c16_t* llr_layers1, + c16_t* rho0, + c16_t* rho1, uint32_t nb_re, - uint8_t symbol, - uint32_t rxdataF_ext_offset, uint8_t mod_order); void nr_ulsch_shift_llr(int16_t **llr_layers, uint32_t nb_re, uint32_t rxdataF_ext_offset, uint8_t mod_order, int shift); diff --git a/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c b/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c index 3e0b3506b1..e3dec33a0d 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c +++ b/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c @@ -598,279 +598,110 @@ static simde__m128i a_mult_conjb(__m128i a, __m128i b, unsigned char output_shif return _mm_packs_epi32(mmtmpD2, mmtmpD3); } -//============================================================================================== -// Pre-processing for LLR computation -//============================================================================================== -void nr_ulsch_channel_compensation(int **rxdataF_ext, - int **ul_ch_estimates_ext, - int **ul_ch_mag, - int **ul_ch_magb, - int **ul_ch_magc, - int **rxdataF_comp, - int ***rho, + +void nr_ulsch_channel_compensation(c16_t *rxFext, + c16_t *chFext, + c16_t *ul_ch_maga, + c16_t *ul_ch_magb, + c16_t *ul_ch_magc, + c16_t *rxComp, + c16_t *rho, NR_DL_FRAME_PARMS *frame_parms, - unsigned char symbol, - int length, - uint8_t is_dmrs_symbol, - unsigned char mod_order, + uint32_t symbol, + uint32_t length, + uint32_t mod_order, uint8_t nrOfLayers, - unsigned short nb_rb, - unsigned char output_shift) { + uint8_t nb_rx_ant, + uint32_t nb_rb, + uint32_t output_shift) +{ + simde__m256i QAM_ampa_256, QAM_ampb_256, QAM_ampc_256; - int off = ((nb_rb&1) == 1)? 4:0; + if (mod_order == 4) { + QAM_ampa_256 = simde_mm256_set1_epi16(QAM16_n1); + QAM_ampb_256 = simde_mm256_setzero_si256(); + QAM_ampc_256 = simde_mm256_setzero_si256(); + } + else if (mod_order == 6) { + QAM_ampa_256 = simde_mm256_set1_epi16(QAM64_n1); + QAM_ampb_256 = simde_mm256_set1_epi16(QAM64_n2); + QAM_ampc_256 = simde_mm256_setzero_si256(); + } + else if (mod_order == 8) { + QAM_ampa_256 = simde_mm256_set1_epi16(QAM256_n1); + QAM_ampb_256 = simde_mm256_set1_epi16(QAM256_n2); + QAM_ampc_256 = simde_mm256_set1_epi16(QAM256_n3); + } -#ifdef DEBUG_CH_COMP - int16_t *rxF, *ul_ch; - int prnt_idx; - for (int nl=0; nlnb_antennas_rx; aarx++) { - rxF = (int16_t *) &rxdataF_ext[aarx][symbol * (off + (nb_rb * 12))]; - ul_ch = (int16_t *) &ul_ch_estimates_ext[nl * frame_parms->nb_antennas_rx + aarx][symbol * (off + (nb_rb * 12))]; + simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; + simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); + simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); - printf("--------symbol = %d, mod_order = %d, output_shift = %d, layer %i, antenna rx = %d -----------\n", - symbol, mod_order, output_shift, nl, aarx); - printf("----------------Before compensation------------------\n"); + int length_aligned = (length % 8) ? (8 - length % 8) : 0; + + for (int aatx = 0; aatx < nrOfLayers; aatx++) { + simde__m256i *rxComp_256 = (simde__m256i*) &rxComp[aatx * (length + length_aligned)]; + simde__m256i *rxF_ch_maga_256 = (simde__m256i*)&ul_ch_maga[aatx * (length + length_aligned)]; + simde__m256i *rxF_ch_magb_256 = (simde__m256i*)&ul_ch_magb[aatx * (length + length_aligned)]; + simde__m256i *rxF_ch_magc_256 = (simde__m256i*)&ul_ch_magc[aatx * (length + length_aligned)]; + for (int aarx = 0; aarx < nb_rx_ant; aarx++) { + simde__m256i *rxF_256 = (simde__m256i*) &rxFext[aarx * (length + length_aligned)]; + simde__m256i *chF_256 = (simde__m256i*) &chFext[(aatx * nb_rx_ant + aarx) * (length + length_aligned)]; - for (prnt_idx = 0; prnt_idx < 12 * 5 * 2; prnt_idx += 2) { - printf("rxF[%d] = (%d,%d)\n", prnt_idx >> 1, rxF[prnt_idx], rxF[prnt_idx + 1]); - printf("ul_ch[%d] = (%d,%d)\n", prnt_idx >> 1, ul_ch[prnt_idx], ul_ch[prnt_idx + 1]); + for (int i = 0; i < (length>>3) + ((length&7)?1:0); i++) + { + xmmp0 = simde_mm256_madd_epi16(chF_256[i], rxF_256[i]); + // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp1 = simde_mm256_shuffle_epi8(chF_256[i], complex_shuffle256); + xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); + xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF_256[i]); + // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); + xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); + xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); + xmmp4 = simde_mm256_packs_epi32(xmmp2, xmmp3); + + xmmp0 = simde_mm256_madd_epi16(chF_256[i], chF_256[i]); // |h|^2 + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp0 = simde_mm256_packs_epi32(xmmp0, xmmp0); + xmmp1 = simde_mm256_unpacklo_epi16(xmmp0, xmmp0); + + xmmp2 = simde_mm256_mulhrs_epi16(xmmp1, QAM_ampa_256); + xmmp3 = simde_mm256_mulhrs_epi16(xmmp1, QAM_ampb_256); + xmmp1 = simde_mm256_mulhrs_epi16(xmmp1, QAM_ampc_256); + + // MRC + rxComp_256[i] = simde_mm256_add_epi16(rxComp_256[i], xmmp4); + if (mod_order > 2) + rxF_ch_maga_256[i] = simde_mm256_add_epi16(rxF_ch_maga_256[i], xmmp2); + if (mod_order > 4) + rxF_ch_magb_256[i] = simde_mm256_add_epi16(rxF_ch_magb_256[i], xmmp3); + if (mod_order > 6) + rxF_ch_magc_256[i] = simde_mm256_add_epi16(rxF_ch_magc_256[i], xmmp1); } - } - } -#endif - -#ifdef DEBUG_CH_MAG - int16_t *ch_mag; - int print_idx; - - - for (int ant=0; antnb_antennas_rx; ant++) { - ch_mag = (int16_t *)&ul_ch_mag[ant][symbol*(off+(nb_rb*12))]; - - printf("--------------------symbol = %d, mod_order = %d-----------------------\n", symbol, mod_order); - printf("----------------Before computation------------------\n"); - - for (print_idx=0;print_idx<5;print_idx++){ - - printf("ch_mag[%d] = %d\n", print_idx, ch_mag[print_idx]); - - } - } - -#endif - -#if defined(__i386) || defined(__x86_64__) - - unsigned short rb; - unsigned char aatx,aarx; - __m128i *ul_ch128,*ul_ch128_2,*ul_ch_mag128,*ul_ch_mag128b,*ul_ch_mag128c,*rxdataF128,*rxdataF_comp128,*rho128; - __m128i mmtmpD0,mmtmpD1,mmtmpD2,mmtmpD3,QAM_amp128={0},QAM_amp128b={0},QAM_amp128c={0}; - QAM_amp128b = _mm_setzero_si128(); - - uint32_t nb_rb_0 = length/12 + ((length%12)?1:0); - for (aatx=0; aatxnb_antennas_rx; aarx++) { - ul_ch128 = (simde__m128i *)&ul_ch_estimates_ext[aatx*frame_parms->nb_antennas_rx+aarx][symbol*(off+(nb_rb*12))]; - ul_ch_mag128 = (simde__m128i *)&ul_ch_mag[aatx*frame_parms->nb_antennas_rx+aarx][symbol*(off+(nb_rb*12))]; - ul_ch_mag128b = (simde__m128i *)&ul_ch_magb[aatx*frame_parms->nb_antennas_rx+aarx][symbol*(off+(nb_rb*12))]; - ul_ch_mag128c = (simde__m128i *)&ul_ch_magc[aatx*frame_parms->nb_antennas_rx+aarx][symbol*(off+(nb_rb*12))]; - rxdataF128 = (simde__m128i *)&rxdataF_ext[aarx][symbol*(off+(nb_rb*12))]; - rxdataF_comp128 = (simde__m128i *)&rxdataF_comp[aatx*frame_parms->nb_antennas_rx+aarx][symbol*(off+(nb_rb*12))]; - - - for (rb=0; rb2) { - // get channel amplitude if not QPSK - - //print_shorts("ch:",(int16_t*)&ul_ch128[0]); - - mmtmpD0 = simde_mm_madd_epi16(ul_ch128[0],ul_ch128[0]); - mmtmpD0 = simde_mm_srai_epi32(mmtmpD0,output_shift); - - mmtmpD1 = simde_mm_madd_epi16(ul_ch128[1],ul_ch128[1]); - mmtmpD1 = simde_mm_srai_epi32(mmtmpD1,output_shift); - - mmtmpD0 = simde_mm_packs_epi32(mmtmpD0,mmtmpD1); - - // store channel magnitude here in a new field of ulsch - - ul_ch_mag128[0] = simde_mm_unpacklo_epi16(mmtmpD0,mmtmpD0); - ul_ch_mag128b[0] = ul_ch_mag128[0]; - ul_ch_mag128c[0] = ul_ch_mag128[0]; - ul_ch_mag128[0] = simde_mm_mulhrs_epi16(ul_ch_mag128[0],QAM_amp128); - ul_ch_mag128b[0] = simde_mm_mulhrs_epi16(ul_ch_mag128b[0],QAM_amp128b); - ul_ch_mag128c[0] = simde_mm_mulhrs_epi16(ul_ch_mag128c[0],QAM_amp128c); - - ul_ch_mag128[1] = simde_mm_unpackhi_epi16(mmtmpD0,mmtmpD0); - ul_ch_mag128b[1] = ul_ch_mag128[1]; - ul_ch_mag128c[1] = ul_ch_mag128[1]; - ul_ch_mag128[1] = simde_mm_mulhrs_epi16(ul_ch_mag128[1],QAM_amp128); - ul_ch_mag128b[1] = simde_mm_mulhrs_epi16(ul_ch_mag128b[1],QAM_amp128b); - ul_ch_mag128c[1] = simde_mm_mulhrs_epi16(ul_ch_mag128c[1],QAM_amp128c); - - mmtmpD0 = simde_mm_madd_epi16(ul_ch128[2],ul_ch128[2]); - mmtmpD0 = simde_mm_srai_epi32(mmtmpD0,output_shift); - mmtmpD1 = simde_mm_packs_epi32(mmtmpD0,mmtmpD0); - - ul_ch_mag128[2] = simde_mm_unpacklo_epi16(mmtmpD1,mmtmpD1); - ul_ch_mag128b[2] = ul_ch_mag128[2]; - ul_ch_mag128c[2] = ul_ch_mag128[2]; - - ul_ch_mag128[2] = simde_mm_mulhrs_epi16(ul_ch_mag128[2],QAM_amp128); - ul_ch_mag128b[2] = simde_mm_mulhrs_epi16(ul_ch_mag128b[2],QAM_amp128b); - ul_ch_mag128c[2] = simde_mm_mulhrs_epi16(ul_ch_mag128c[2],QAM_amp128c); - } - - // Multiply received data by conjugated channel - rxdataF_comp128[0] = a_mult_conjb(rxdataF128[0], ul_ch128[0], output_shift); - rxdataF_comp128[1] = a_mult_conjb(rxdataF128[1], ul_ch128[1], output_shift); - rxdataF_comp128[2] = a_mult_conjb(rxdataF128[2], ul_ch128[2], output_shift); - - ul_ch128 += 3; - ul_ch_mag128 += 3; - ul_ch_mag128b += 3; - ul_ch_mag128c += 3; - rxdataF128 += 3; - rxdataF_comp128 += 3; - } - } - } - - if (rho) { - //we compute the Tx correlation matrix for each Rx antenna - //As an example the 2x2 MIMO case requires - //rho[aarx][nb_aatx*nb_aatx] = [cov(H_aarx_0,H_aarx_0) cov(H_aarx_0,H_aarx_1) - // cov(H_aarx_1,H_aarx_0) cov(H_aarx_1,H_aarx_1)], aarx=0,...,nb_antennas_rx-1 - - int avg_rho_re[frame_parms->nb_antennas_rx][nrOfLayers*nrOfLayers]; - int avg_rho_im[frame_parms->nb_antennas_rx][nrOfLayers*nrOfLayers]; - - for (aarx=0; aarx < frame_parms->nb_antennas_rx; aarx++) { - for (aatx=0; aatx < nrOfLayers; aatx++) { - for (int atx=0; atx< nrOfLayers; atx++) { - - avg_rho_re[aarx][aatx*nrOfLayers+atx] = 0; - avg_rho_im[aarx][aatx*nrOfLayers+atx] = 0; - rho128 = (simde__m128i *)&rho[aarx][aatx*nrOfLayers+atx][symbol*(off+(nb_rb*12))]; - ul_ch128 = (simde__m128i *)&ul_ch_estimates_ext[aatx*frame_parms->nb_antennas_rx+aarx][symbol*(off+(nb_rb*12))]; - ul_ch128_2 = (simde__m128i *)&ul_ch_estimates_ext[atx*frame_parms->nb_antennas_rx+aarx][symbol*(off+(nb_rb*12))]; - - for (rb=0; rb> 3)+((length&7)?1:0); i++) + { // multiply by conjugated channel - mmtmpD0 = simde_mm_madd_epi16(ul_ch128[0],ul_ch128_2[0]); - // print_ints("re",&mmtmpD0); - + mmtmpD0 = simde_mm256_madd_epi16(chF_256[i], chF2_256[i]); // mmtmpD0 contains real part of 4 consecutive outputs (32-bit) - mmtmpD1 = simde_mm_shufflelo_epi16(ul_ch128[0], SIMDE_MM_SHUFFLE(2,3,0,1)); - mmtmpD1 = simde_mm_shufflehi_epi16(mmtmpD1, SIMDE_MM_SHUFFLE(2,3,0,1)); - mmtmpD1 = simde_mm_sign_epi16(mmtmpD1,*(simde__m128i*)&conjugate[0]); - // print_ints("im",&mmtmpD1); - mmtmpD1 = simde_mm_madd_epi16(mmtmpD1,ul_ch128_2[0]); + mmtmpD1 = simde_mm256_shuffle_epi8(chF_256[i], complex_shuffle256); + mmtmpD1 = simde_mm256_sign_epi16(mmtmpD1, conj256); + mmtmpD1 = simde_mm256_madd_epi16(mmtmpD1, chF2_256[i]); // mmtmpD1 contains imag part of 4 consecutive outputs (32-bit) - mmtmpD0 = simde_mm_srai_epi32(mmtmpD0,output_shift); - // print_ints("re(shift)",&mmtmpD0); - mmtmpD1 = simde_mm_srai_epi32(mmtmpD1,output_shift); - // print_ints("im(shift)",&mmtmpD1); - mmtmpD2 = simde_mm_unpacklo_epi32(mmtmpD0,mmtmpD1); - mmtmpD3 = simde_mm_unpackhi_epi32(mmtmpD0,mmtmpD1); - // print_ints("c0",&mmtmpD2); - // print_ints("c1",&mmtmpD3); - rho128[0] = simde_mm_packs_epi32(mmtmpD2,mmtmpD3); + mmtmpD0 = simde_mm256_srai_epi32(mmtmpD0, output_shift); + mmtmpD1 = simde_mm256_srai_epi32(mmtmpD1, output_shift); + mmtmpD2 = simde_mm256_unpacklo_epi32(mmtmpD0, mmtmpD1); + mmtmpD3 = simde_mm256_unpackhi_epi32(mmtmpD0, mmtmpD1); - //print_shorts("rx:",ul_ch128_2); - //print_shorts("ch:",ul_ch128); - //print_shorts("pack:",rho128); - - avg_rho_re[aarx][aatx*nrOfLayers+atx] +=(((int16_t*)&rho128[0])[0]+ - ((int16_t*)&rho128[0])[2] + - ((int16_t*)&rho128[0])[4] + - ((int16_t*)&rho128[0])[6])/16;// - - avg_rho_im[aarx][aatx*nrOfLayers+atx] +=(((int16_t*)&rho128[0])[1]+ - ((int16_t*)&rho128[0])[3] + - ((int16_t*)&rho128[0])[5] + - ((int16_t*)&rho128[0])[7])/16;// - // multiply by conjugated channel - mmtmpD0 = simde_mm_madd_epi16(ul_ch128[1],ul_ch128_2[1]); - // mmtmpD0 contains real part of 4 consecutive outputs (32-bit) - mmtmpD1 = simde_mm_shufflelo_epi16(ul_ch128[1], SIMDE_MM_SHUFFLE(2,3,0,1)); - mmtmpD1 = simde_mm_shufflehi_epi16(mmtmpD1, SIMDE_MM_SHUFFLE(2,3,0,1)); - mmtmpD1 = simde_mm_sign_epi16(mmtmpD1,*(simde__m128i*)conjugate); - mmtmpD1 = simde_mm_madd_epi16(mmtmpD1,ul_ch128_2[1]); - // mmtmpD1 contains imag part of 4 consecutive outputs (32-bit) - mmtmpD0 = simde_mm_srai_epi32(mmtmpD0,output_shift); - mmtmpD1 = simde_mm_srai_epi32(mmtmpD1,output_shift); - mmtmpD2 = simde_mm_unpacklo_epi32(mmtmpD0,mmtmpD1); - mmtmpD3 = simde_mm_unpackhi_epi32(mmtmpD0,mmtmpD1); - rho128[1] =simde_mm_packs_epi32(mmtmpD2,mmtmpD3); - //print_shorts("rx:",ul_ch128_2+1); - //print_shorts("ch:",ul_ch128+1); - //print_shorts("pack:",rho128+1); - - // multiply by conjugated channel - avg_rho_re[aarx][aatx*nrOfLayers+atx] +=(((int16_t*)&rho128[1])[0]+ - ((int16_t*)&rho128[1])[2] + - ((int16_t*)&rho128[1])[4] + - ((int16_t*)&rho128[1])[6])/16; - - avg_rho_im[aarx][aatx*nrOfLayers+atx] +=(((int16_t*)&rho128[1])[1]+ - ((int16_t*)&rho128[1])[3] + - ((int16_t*)&rho128[1])[5] + - ((int16_t*)&rho128[1])[7])/16; - - mmtmpD0 = simde_mm_madd_epi16(ul_ch128[2],ul_ch128_2[2]); - // mmtmpD0 contains real part of 4 consecutive outputs (32-bit) - mmtmpD1 = simde_mm_shufflelo_epi16(ul_ch128[2], SIMDE_MM_SHUFFLE(2,3,0,1)); - mmtmpD1 = simde_mm_shufflehi_epi16(mmtmpD1, SIMDE_MM_SHUFFLE(2,3,0,1)); - mmtmpD1 = simde_mm_sign_epi16(mmtmpD1,*(simde__m128i*)conjugate); - mmtmpD1 = simde_mm_madd_epi16(mmtmpD1,ul_ch128_2[2]); - // mmtmpD1 contains imag part of 4 consecutive outputs (32-bit) - mmtmpD0 = simde_mm_srai_epi32(mmtmpD0,output_shift); - mmtmpD1 = simde_mm_srai_epi32(mmtmpD1,output_shift); - mmtmpD2 = simde_mm_unpacklo_epi32(mmtmpD0,mmtmpD1); - mmtmpD3 = simde_mm_unpackhi_epi32(mmtmpD0,mmtmpD1); - - rho128[2] = simde_mm_packs_epi32(mmtmpD2,mmtmpD3); - //print_shorts("rx:",ul_ch128_2+2); - //print_shorts("ch:",ul_ch128+2); - //print_shorts("pack:",rho128+2); - avg_rho_re[aarx][aatx*nrOfLayers+atx] +=(((int16_t*)&rho128[2])[0]+ - ((int16_t*)&rho128[2])[2] + - ((int16_t*)&rho128[2])[4] + - ((int16_t*)&rho128[2])[6])/16; - - avg_rho_im[aarx][aatx*nrOfLayers+atx] +=(((int16_t*)&rho128[2])[1]+ - ((int16_t*)&rho128[2])[3] + - ((int16_t*)&rho128[2])[5] + - ((int16_t*)&rho128[2])[7])/16; - - ul_ch128+=3; - ul_ch128_2+=3; - rho128+=3; - } - if (is_dmrs_symbol==1) { - //measurements->rx_correlation[0][0][aarx] = signal_energy(&rho[aarx][aatx*nb_aatx+atx][symbol*nb_rb*12],rb*12); - avg_rho_re[aarx][aatx*nrOfLayers+atx] = 16*avg_rho_re[aarx][aatx*nrOfLayers+atx]/(nb_rb*12); - avg_rho_im[aarx][aatx*nrOfLayers+atx] = 16*avg_rho_im[aarx][aatx*nrOfLayers+atx]/(nb_rb*12); - //printf("rho[rx]%d tx%d tx%d = Re: %d Im: %d\n",aarx, aatx,atx, avg_rho_re[aarx][aatx*nb_aatx+atx], avg_rho_im[aarx][aatx*nb_aatx+atx]); + rho_256[i] = simde_mm256_adds_epi16(rho_256[i], simde_mm256_packs_epi32(mmtmpD2, mmtmpD3)); } } } @@ -879,39 +710,6 @@ void nr_ulsch_channel_compensation(int **rxdataF_ext, simde_mm_empty(); simde_m_empty(); - - -#ifdef DEBUG_CH_COMP - for (int nl2=0; nl2nb_antennas_rx; aarx2++) { - rxF = (int16_t *)&rxdataF_comp[nl2*frame_parms->nb_antennas_rx+aarx2][(symbol*(off+(nb_rb*12)))]; - - printf("--------After compansation, layer %i, antenna rx %i----------\n", nl2, aarx2); - - for (prnt_idx=0;prnt_idx<12*5*2;prnt_idx+=2){ - printf("rxF[%d] = (%d,%d)\n", prnt_idx>>1, rxF[prnt_idx],rxF[prnt_idx+1]); - } - } - } -#endif - -#ifdef DEBUG_CH_MAG - - - for (int ant=0; antnb_antennas_rx; ant++) { - ch_mag = (int16_t *)&ul_ch_mag[ant][(symbol*(off+(nb_rb*12)))]; - - printf("----------------After computation------------------\n"); - - for (print_idx=0;print_idx<12*5*2;print_idx+=2){ - - printf("ch_mag[%d] = (%d,%d)\n", print_idx>>1, ch_mag[print_idx],ch_mag[print_idx+1]); - - } - } - -#endif - } void nr_ulsch_detection_mrc(NR_DL_FRAME_PARMS *frame_parms, @@ -1696,245 +1494,72 @@ uint8_t nr_ulsch_mmse_2layers(NR_DL_FRAME_PARMS *frame_parms, return(0); } -void inner_rx_qpsk (NR_DL_FRAME_PARMS *frame_parms, - NR_gNB_PUSCH *pusch_vars, - nfapi_nr_pusch_pdu_t *rel15_ul, - c16_t **rxF, - c16_t **ul_ch, - int16_t *llr, - int nb_layer, - int nb_rx_ant, - int soffset, - int length, - int symbol, - int short nb_rb, - int dmrs_symbol_flag, - int output_shift) +void inner_rx (PHY_VARS_gNB *gNB, + int ulsch_id, + int slot, + NR_DL_FRAME_PARMS *frame_parms, + NR_gNB_PUSCH *pusch_vars, + nfapi_nr_pusch_pdu_t *rel15_ul, + c16_t **rxF, + c16_t **ul_ch, + int16_t **llr, + int nb_layer, + int nb_rx_ant, + int soffset, + int length, + int symbol, + int short nb_rb, + int dmrs_symbol_flag, + int output_shift, + uint32_t nvar) { int length_aligned = (length % 8) ? (8 - length % 8) : 0; - + c16_t rxFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); - c16_t chFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); - for (int aarx = 0; aarx < nb_rx_ant; aarx++) - { - nr_ulsch_extract_rbs0(rxF[aarx], - (c16_t*)pusch_vars->ul_ch_estimates[aarx], - rxFext[aarx], - chFext[aarx], - soffset+(symbol * frame_parms->ofdm_symbol_size), - pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, - aarx, - dmrs_symbol_flag, - rel15_ul, - frame_parms); - simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; - simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); - simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); - - simde__m256i *rxF256 = (simde__m256i*)rxFext[aarx]; - simde__m256i *ulch256 = (simde__m256i*)chFext[aarx]; - simde__m256i *llr256 = (simde__m256i *)llr; - for (int i = 0; i < ((length>>3) + ((length&7) > 0 ? 1:0)); i++) - { - xmmp0 = simde_mm256_madd_epi16(ulch256[i], rxF256[i]); - // xmmp0 contains real part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm256_shuffle_epi8(ulch256[i], complex_shuffle256); - xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); - xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF256[i]); - // xmmp1 contains imag part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); - xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); - xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); - xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); - xmmp4 = simde_mm256_packs_epi32(xmmp2,xmmp3); - - // MRC & LLR - llr256[i] = simde_mm256_add_epi16(llr256[i], xmmp4); + c16_t chFext[nb_layer][nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + for (int aarx = 0; aarx < nb_rx_ant; aarx++) { + for (int aatx = 0; aatx < nb_layer; aatx++) { + nr_ulsch_extract_rbs0(rxF[aarx], + (c16_t *)pusch_vars->ul_ch_estimates[aatx * nb_rx_ant + aarx], + rxFext[aarx], + chFext[aatx][aarx], + soffset+(symbol * frame_parms->ofdm_symbol_size), + pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, + aarx, + dmrs_symbol_flag, + rel15_ul, + frame_parms); } } - // Perform IDFT if transform precoding is enabled - if(rel15_ul->transform_precoding == transformPrecoder_enabled) - nr_idft((int32_t *)llr, length); -} + c16_t rho[nb_layer][nb_layer][length + length_aligned] __attribute__((aligned(32))); + c16_t rxComp [nb_layer][length + length_aligned] __attribute__((aligned(32))); + c16_t rxF_ch_maga[nb_layer][length + length_aligned] __attribute__((aligned(32))); + c16_t rxF_ch_magb[nb_layer][length + length_aligned] __attribute__((aligned(32))); + c16_t rxF_ch_magc[nb_layer][length + length_aligned] __attribute__((aligned(32))); -void inner_rx_16qam (NR_DL_FRAME_PARMS *frame_parms, - NR_gNB_PUSCH *pusch_vars, - nfapi_nr_pusch_pdu_t *rel15_ul, - c16_t **rxF, - c16_t **ul_ch, - int16_t *llr, - int nb_layer, - int nb_rx_ant, - int soffset, - int length, - int symbol, - int short nb_rb, - int dmrs_symbol_flag, - int output_shift) -{ - int length_aligned = (length % 8) ? (8 - length % 8) : 0; + memset(rho, 0, sizeof(c16_t) * nb_layer * nb_layer* (length + length_aligned)); + memset(rxComp, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); + memset(rxF_ch_maga, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); + memset(rxF_ch_magb, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); + memset(rxF_ch_magc, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); - c16_t rxFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); - c16_t chFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); - c16_t rxComp[length + length_aligned] __attribute__((aligned(32))); - c16_t rxF_ch_maga[length + length_aligned] __attribute__((aligned(32))); + nr_ulsch_channel_compensation((c16_t*)rxFext, + (c16_t*)chFext, + (c16_t*)rxF_ch_maga, + (c16_t*)rxF_ch_magb, + (c16_t*)rxF_ch_magc, + (c16_t*)rxComp, + (nb_layer == 1) ? NULL : (c16_t*)rho, + frame_parms, + symbol, + length, + rel15_ul->qam_mod_order, + rel15_ul->nrOfLayers, + nb_rx_ant, + rel15_ul->rb_size, + output_shift); - memset(rxComp, 0, sizeof(c16_t) * (length + length_aligned)); - memset(rxF_ch_maga, 0, sizeof(c16_t) * (length + length_aligned)); - - simde__m256i *rxComp_256 = (simde__m256i *)rxComp; - simde__m256i *rxF_ch_maga_256 = (simde__m256i *)rxF_ch_maga; - - for (int aarx = 0; aarx < nb_rx_ant; aarx++) - { - nr_ulsch_extract_rbs0(rxF[aarx], - (c16_t*)pusch_vars->ul_ch_estimates[aarx], - rxFext[aarx], - chFext[aarx], - soffset+(symbol * frame_parms->ofdm_symbol_size), - pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, - aarx, - dmrs_symbol_flag, - rel15_ul, - frame_parms); - simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; - simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); - simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); - simde__m256i QAM_amp256 = simde_mm256_set1_epi16(QAM16_n1); - - simde__m256i *rxF_256 = (simde__m256i*)rxFext[aarx]; - simde__m256i *chF_256 = (simde__m256i*)chFext[aarx]; - for (int i = 0; i < ((length>>3) + ((length&7) > 0 ? 1:0)); i++) - { - xmmp0 = simde_mm256_madd_epi16(chF_256[i], rxF_256[i]); - // xmmp0 contains real part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm256_shuffle_epi8(chF_256[i], complex_shuffle256); - xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); - xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF_256[i]); - // xmmp1 contains imag part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); - xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); - xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); - xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); - xmmp4 = simde_mm256_packs_epi32(xmmp2,xmmp3); - - xmmp0 = simde_mm256_madd_epi16(chF_256[i], chF_256[i]); // |h|^2 - xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); - xmmp0 = simde_mm256_packs_epi32(xmmp0, xmmp0); - xmmp0 = simde_mm256_unpacklo_epi16(xmmp0, xmmp0); - - xmmp1 = simde_mm256_mulhrs_epi16(xmmp0, QAM_amp256); - - // MRC - rxF_ch_maga_256[i] = simde_mm256_add_epi16(rxF_ch_maga_256[i], xmmp1); - rxComp_256[i] = simde_mm256_add_epi16(rxComp_256[i], xmmp4); - } - } - - if (rel15_ul->transform_precoding == transformPrecoder_enabled) - { - nr_freq_equalization(frame_parms, - (int *)rxComp, - (int *)rxF_ch_maga, - (int *)rxF_ch_maga, - symbol, - length, - rel15_ul->qam_mod_order); - nr_idft((int32_t*)rxComp, length); - } - nr_ulsch_compute_llr((int32_t *)rxComp, - (int32_t *)rxF_ch_maga, - NULL, - NULL, - llr, - rel15_ul->rb_size, - length, - symbol, - rel15_ul->qam_mod_order); -} - -void inner_rx_64qam (NR_DL_FRAME_PARMS *frame_parms, - NR_gNB_PUSCH *pusch_vars, - nfapi_nr_pusch_pdu_t *rel15_ul, - c16_t **rxF, - c16_t **ul_ch, - int16_t *llr, - int nb_layer, - int nb_rx_ant, - int soffset, - int length, - int symbol, - int short nb_rb, - int dmrs_symbol_flag, - int output_shift) -{ - int length_aligned = (length % 8) ? (8 - length % 8) : 0; - - c16_t rxFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); - c16_t chFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); - - c16_t rxComp[length + length_aligned] __attribute__((aligned(32))); - c16_t rxF_ch_maga[length + length_aligned] __attribute__((aligned(32))); - c16_t rxF_ch_magb[length + length_aligned] __attribute__((aligned(32))); - - memset(rxComp, 0, sizeof(c16_t) * (length + length_aligned)); - memset(rxF_ch_maga, 0, sizeof(c16_t) * (length + length_aligned)); - memset(rxF_ch_magb, 0, sizeof(c16_t) * (length + length_aligned)); - - simde__m256i *rxComp_256 = (simde__m256i *)rxComp; - simde__m256i *rxF_ch_maga_256 = (simde__m256i *)rxF_ch_maga; - simde__m256i *rxF_ch_magb_256 = (simde__m256i *)rxF_ch_magb; - - for (int aarx = 0; aarx < nb_rx_ant; aarx++) - { - nr_ulsch_extract_rbs0(rxF[aarx], - (c16_t*)pusch_vars->ul_ch_estimates[aarx], - rxFext[aarx], - chFext[aarx], - soffset+(symbol * frame_parms->ofdm_symbol_size), - pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, - aarx, - dmrs_symbol_flag, - rel15_ul, - frame_parms); - simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; - simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); - simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); - simde__m256i QAM_amp256 = simde_mm256_set1_epi16(QAM64_n1); // 2/sqrt(10) - simde__m256i QAM_amp256b = simde_mm256_set1_epi16(QAM64_n2); - - simde__m256i *rxF_256 = (simde__m256i*)rxFext[aarx]; - simde__m256i *chF_256 = (simde__m256i*)chFext[aarx]; - for (int i = 0; i < ((length>>3) + ((length&7) > 0 ? 1:0)); i++) - { - xmmp0 = simde_mm256_madd_epi16(chF_256[i], rxF_256[i]); - // xmmp0 contains real part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm256_shuffle_epi8(chF_256[i], complex_shuffle256); - xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); - xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF_256[i]); - // xmmp1 contains imag part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); - xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); - xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); - xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); - xmmp4 = simde_mm256_packs_epi32(xmmp2,xmmp3); - - xmmp0 = simde_mm256_madd_epi16(chF_256[i], chF_256[i]); // |h|^2 - xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); - xmmp0 = simde_mm256_packs_epi32(xmmp0, xmmp0); - xmmp0 = simde_mm256_unpacklo_epi16(xmmp0, xmmp0); - - xmmp1 = simde_mm256_mulhrs_epi16(xmmp0, QAM_amp256); - xmmp2 = simde_mm256_mulhrs_epi16(xmmp0, QAM_amp256b); - - // MRC - rxComp_256[i] = simde_mm256_add_epi16(rxComp_256[i], xmmp4); - rxF_ch_maga_256[i] = simde_mm256_add_epi16(rxF_ch_maga_256[i], xmmp1); - rxF_ch_magb_256[i] = simde_mm256_add_epi16(rxF_ch_magb_256[i], xmmp2); - } - } - - if (rel15_ul->transform_precoding == transformPrecoder_enabled) { + if (nb_layer == 1 && rel15_ul->transform_precoding == transformPrecoder_enabled && rel15_ul->qam_mod_order <= 6) { nr_freq_equalization(frame_parms, (int *)rxComp, (int *)rxF_ch_maga, @@ -1944,597 +1569,68 @@ void inner_rx_64qam (NR_DL_FRAME_PARMS *frame_parms, rel15_ul->qam_mod_order); nr_idft((int32_t*)rxComp, length); } - nr_ulsch_compute_llr((int32_t*)rxComp, - (int32_t*)rxF_ch_maga, - (int32_t*)rxF_ch_magb, - NULL, - llr, - rel15_ul->rb_size, - length, - symbol, - rel15_ul->qam_mod_order); -} + // if (rel15_ul->pdu_bit_map & PUSCH_PDU_BITMAP_PUSCH_PTRS) { + // nr_pusch_ptrs_processing(gNB, + // frame_parms, + // rel15_ul, + // ulsch_id, + // slot, + // symbol, + // length); + // // Subtract total PTRS RE's in the symbol from PUSCH RE's + // length -= pusch_vars->ptrs_re_per_slot; + // } -void inner_rx_256qam (NR_DL_FRAME_PARMS *frame_parms, - NR_gNB_PUSCH *pusch_vars, - nfapi_nr_pusch_pdu_t *rel15_ul, - c16_t **rxF, - c16_t **ul_ch, - int16_t *llr, - int nb_layer, - int nb_rx_ant, - int soffset, - int length, - int symbol, - int short nb_rb, - int dmrs_symbol_flag, - int output_shift) -{ - int length_aligned = (length % 8) ? (8 - length % 8) : 0; - - c16_t rxFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); - c16_t chFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); - c16_t rxComp[length + length_aligned] __attribute__((aligned(32))); - c16_t rxF_ch_maga[length + length_aligned] __attribute__((aligned(32))); - c16_t rxF_ch_magb[length + length_aligned] __attribute__((aligned(32))); - c16_t rxF_ch_magc[length + length_aligned] __attribute__((aligned(32))); - - memset(rxComp, 0, sizeof(c16_t) * (length + length_aligned)); - memset(rxF_ch_maga, 0, sizeof(c16_t) * (length + length_aligned)); - memset(rxF_ch_magb, 0, sizeof(c16_t) * (length + length_aligned)); - memset(rxF_ch_magc, 0, sizeof(c16_t) * (length + length_aligned)); - - simde__m256i *rxComp_256 = (simde__m256i *)rxComp; - simde__m256i *rxF_ch_maga_256 = (simde__m256i *)rxF_ch_maga; - simde__m256i *rxF_ch_magb_256 = (simde__m256i *)rxF_ch_magb; - simde__m256i *rxF_ch_magc_256 = (simde__m256i *)rxF_ch_magc; - - for (int aarx = 0; aarx < nb_rx_ant; aarx++) + if (nb_layer == 2) { - nr_ulsch_extract_rbs0(rxF[aarx], - (c16_t*)pusch_vars->ul_ch_estimates[aarx], - rxFext[aarx], - chFext[aarx], - soffset+(symbol * frame_parms->ofdm_symbol_size), - pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, - aarx, - dmrs_symbol_flag, - rel15_ul, - frame_parms); - simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; - simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); - simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); - - simde__m256i QAM_amp256 = simde_mm256_set1_epi16(QAM256_n1); - simde__m256i QAM_amp256b = simde_mm256_set1_epi16(QAM256_n2); - simde__m256i QAM_amp256c = simde_mm256_set1_epi16(QAM256_n3); - - simde__m256i *rxF256 = (simde__m256i*)rxFext[aarx]; - simde__m256i *ulch256 = (simde__m256i*)chFext[aarx]; - for (int i = 0; i < ((length >> 3) + (( length & 7) > 0 ? 1 : 0)); i++) + if (rel15_ul->qam_mod_order < 6) { + c16_t *rho0 = rho[0][1]; + c16_t *rho1 = rho[1][0]; + c16_t *llr_0 = (c16_t *)&llr[0][pusch_vars->llr_offset[symbol]]; + c16_t *llr_1 = (c16_t *)&llr[1][pusch_vars->llr_offset[symbol]]; + c16_t *ul_ch_mag0 = rxF_ch_maga[0]; + c16_t *ul_ch_mag1 = rxF_ch_maga[1]; + nr_ulsch_compute_ML_llr(pusch_vars, + symbol, + rxComp[0], + rxComp[1], + ul_ch_mag0, + ul_ch_mag1, + llr_0, + llr_1, + rho0, + rho1, + length, + rel15_ul->qam_mod_order); + } + else { - xmmp0 = simde_mm256_madd_epi16(ulch256[i], rxF256[i]); - // xmmp0 contains real part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm256_shuffle_epi8(ulch256[i], complex_shuffle256); - xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); - xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF256[i]); - // xmmp1 contains imag part of 8 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); - xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); - xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); - xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); - xmmp4 = simde_mm256_packs_epi32(xmmp2, xmmp3); - - // compute channel amplitude for LLR - xmmp0 = simde_mm256_madd_epi16(ulch256[i], ulch256[i]); - xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); - xmmp0 = simde_mm256_packs_epi32(xmmp0, xmmp0); // contains 16 LLRs - xmmp2 = simde_mm256_unpacklo_epi16(xmmp0, xmmp0); - xmmp0 = simde_mm256_mulhrs_epi16(xmmp2, QAM_amp256); - xmmp1 = simde_mm256_mulhrs_epi16(xmmp2, QAM_amp256b); - xmmp2 = simde_mm256_mulhrs_epi16(xmmp2, QAM_amp256c); - - // MRC - rxComp_256[i] = simde_mm256_add_epi16(rxComp_256[i], xmmp4); - rxF_ch_maga_256[i] = simde_mm256_add_epi16(rxF_ch_maga_256[i], xmmp0); - rxF_ch_magb_256[i] = simde_mm256_add_epi16(rxF_ch_magb_256[i], xmmp1); - rxF_ch_magc_256[i] = simde_mm256_add_epi16(rxF_ch_magc_256[i], xmmp2); + nr_ulsch_mmse_2layers(frame_parms, + (int **)rxComp, + (int **)rxF_ch_maga, + (int **)rxF_ch_magb, + (int **)rxF_ch_magc, + (int **)chFext, + rel15_ul->rb_size, + frame_parms->nb_antennas_rx, + rel15_ul->qam_mod_order, + pusch_vars->log2_maxh, + symbol, + length, + nvar); } } - nr_ulsch_compute_llr((int32_t*)rxComp, - (int32_t*)rxF_ch_maga, - (int32_t*)rxF_ch_magb, - (int32_t*)rxF_ch_magc, - llr, - rel15_ul->rb_size, - length, - symbol, - rel15_ul->qam_mod_order); -} - -void inner_rx_qpsk_2layer (NR_DL_FRAME_PARMS *frame_parms, - NR_gNB_PUSCH *pusch_vars, - nfapi_nr_pusch_pdu_t *rel15_ul, - c16_t **rxF, - c16_t **ul_ch, - int16_t **llr, - int nb_layer, - int nb_rx_ant, - int soffset, - int length, - int symbol, - int short nb_rb, - int dmrs_symbol_flag, - int output_shift, - uint32_t nvar) -{ - int length_aligned = (length % 8) ? (8 - length % 8) : 0; - - c16_t rxFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); - c16_t chFext[nb_layer][nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); - for (int aarx = 0; aarx < nb_rx_ant; aarx++) - { + if (nb_layer != 2 || rel15_ul->qam_mod_order >= 6) for (int aatx = 0; aatx < nb_layer; aatx++) - { - nr_ulsch_extract_rbs0(rxF[aarx], - (c16_t*)pusch_vars->ul_ch_estimates[aatx * nb_rx_ant + aarx], - rxFext[aarx], - chFext[aatx][aarx], - soffset+(symbol * frame_parms->ofdm_symbol_size), - pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, - aarx, - dmrs_symbol_flag, - rel15_ul, - frame_parms); - } - } - c16_t rxComp[nb_layer][length + length_aligned] __attribute__((aligned(32))); - c16_t rho[nb_layer][nb_layer][length + length_aligned] __attribute__((aligned(32))); - - memset(rxComp, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); - memset(rho, 0, sizeof(c16_t) * nb_layer * nb_layer * (length + length_aligned)); - - for (int aarx = 0; aarx < nb_rx_ant; aarx++) { - for (int aatx = 0; aatx < nb_layer; aatx++) { - for (int atx = 0; atx < nb_layer; atx++) { - simde__m256i mmtmpD0, mmtmpD1, mmtmpD2, mmtmpD3, mmtmpD4; - simde__m256i *rho_256 = (simde__m256i *)rho[aatx][atx]; - simde__m256i *chF_256 = (simde__m256i *)chFext[aatx][aarx]; - simde__m256i *chF2_256 = (simde__m256i *)chFext[atx][aarx]; - simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); - simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); - for (int i = 0; i < ((length >> 3)+((length&7)?1:0)); i++) - { - // multiply by conjugated channel - mmtmpD0 = simde_mm256_madd_epi16(chF_256[i], chF2_256[i]); - // mmtmpD0 contains real part of 4 consecutive outputs (32-bit) - mmtmpD1 = simde_mm256_shuffle_epi8(chF_256[i], complex_shuffle256); - mmtmpD1 = simde_mm256_sign_epi16(mmtmpD1, conj256); - mmtmpD1 = simde_mm256_madd_epi16(mmtmpD1, chF2_256[i]); - // mmtmpD1 contains imag part of 4 consecutive outputs (32-bit) - mmtmpD0 = simde_mm256_srai_epi32(mmtmpD0, output_shift); - mmtmpD1 = simde_mm256_srai_epi32(mmtmpD1, output_shift); - mmtmpD2 = simde_mm256_unpacklo_epi32(mmtmpD0, mmtmpD1); - mmtmpD3 = simde_mm256_unpackhi_epi32(mmtmpD0, mmtmpD1); - mmtmpD4 = simde_mm256_packs_epi32(mmtmpD2, mmtmpD3); - - rho_256[i] = simde_mm256_adds_epi16(rho_256[i], mmtmpD4); - } - } - // compensation - simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; - simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); - simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); - simde__m256i *rxF_256 = (simde__m256i*)rxFext[aarx]; - simde__m256i *chF_256 = (simde__m256i*)chFext[aatx][aarx]; - simde__m256i *rxComp_256 = (simde__m256i*)rxComp[aatx]; - for (int i = 0; i < (length>>3) + ((length&7)?1:0); i++) - { - xmmp0 = simde_mm256_madd_epi16(chF_256[i], rxF_256[i]); - // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm256_shuffle_epi8(chF_256[i], complex_shuffle256); - xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); - xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF_256[i]); - // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); - xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); - xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); - xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); - xmmp4 = simde_mm256_packs_epi32(xmmp2, xmmp3); - - rxComp_256[i] = simde_mm256_adds_epi16(rxComp_256[i], xmmp4); - } - } - } - c16_t *rho0 = rho[0][1]; - c16_t *rho1 = rho[1][0]; - c16_t *llr_0 = (c16_t *)&llr[0][pusch_vars->llr_offset[symbol]]; - c16_t *llr_1 = (c16_t *)&llr[1][pusch_vars->llr_offset[symbol]]; - - nr_ulsch_qpsk_qpsk(rxComp[0], rxComp[1], llr_0, rho0, length); - nr_ulsch_qpsk_qpsk(rxComp[1], rxComp[0], llr_1, rho1, length); - - nr_ulsch_shift_llr(pusch_vars->llr_layers, length, pusch_vars->llr_offset[symbol] >> 1, 2, 4); -} - - -void inner_rx_16qam_2layer (NR_DL_FRAME_PARMS *frame_parms, - NR_gNB_PUSCH *pusch_vars, - nfapi_nr_pusch_pdu_t *rel15_ul, - c16_t **rxF, - c16_t **ul_ch, - int16_t **llr, - int nb_layer, - int nb_rx_ant, - int soffset, - int length, - int symbol, - int short nb_rb, - int dmrs_symbol_flag, - int output_shift, - uint32_t nvar) -{ - int length_aligned = (length % 8) ? (8 - length % 8) : 0; - - c16_t rxFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); - c16_t chFext[nb_layer][nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); - for (int aarx = 0; aarx < nb_rx_ant; aarx++) { - for (int aatx = 0; aatx < nb_layer; aatx++) { - nr_ulsch_extract_rbs0(rxF[aarx], - (c16_t*)pusch_vars->ul_ch_estimates[aatx * nb_rx_ant + aarx], - rxFext[aarx], - chFext[aatx][aarx], - soffset+(symbol * frame_parms->ofdm_symbol_size), - pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, - aarx, - dmrs_symbol_flag, - rel15_ul, - frame_parms); - } - } - - c16_t rxComp[nb_layer][length + length_aligned] __attribute__((aligned(32))); - c16_t rho[nb_layer][nb_layer][length + length_aligned] __attribute__((aligned(32))); - c16_t ul_ch_maga[nb_layer][length + length_aligned] __attribute__((aligned(32))); - - memset(rxComp, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); - memset(rho, 0, sizeof(c16_t) * nb_layer * nb_layer * (length + length_aligned)); - memset(ul_ch_maga, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); - - for (int aatx = 0; aatx < nb_layer; aatx++) { - for (int aarx = 0; aarx < nb_rx_ant; aarx++) { - for (int atx = 0; atx < nb_layer; atx++) { - simde__m256i mmtmpD0, mmtmpD1, mmtmpD2, mmtmpD3; - simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); - simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); - simde__m256i *chF_256 = (simde__m256i *)chFext[aatx][aarx]; - simde__m256i *chF2_256 = (simde__m256i *)chFext[atx][aarx]; - simde__m256i *rho_256 = (simde__m256i *)rho[aatx][atx]; - for (int i = 0; i < (length >> 3)+((length&7)?1:0); i++) - { - // multiply by conjugated channel - mmtmpD0 = simde_mm256_madd_epi16(chF_256[i], chF2_256[i]); - // mmtmpD0 contains real part of 4 consecutive outputs (32-bit) - mmtmpD1 = simde_mm256_shuffle_epi8(chF_256[i], complex_shuffle256); - mmtmpD1 = simde_mm256_sign_epi16(mmtmpD1, conj256); - mmtmpD1 = simde_mm256_madd_epi16(mmtmpD1, chF2_256[i]); - // mmtmpD1 contains imag part of 4 consecutive outputs (32-bit) - mmtmpD0 = simde_mm256_srai_epi32(mmtmpD0, output_shift); - mmtmpD1 = simde_mm256_srai_epi32(mmtmpD1, output_shift); - mmtmpD2 = simde_mm256_unpacklo_epi32(mmtmpD0, mmtmpD1); - mmtmpD3 = simde_mm256_unpackhi_epi32(mmtmpD0, mmtmpD1); - - rho_256[i] = simde_mm256_adds_epi16(rho_256[i], simde_mm256_packs_epi32(mmtmpD2, mmtmpD3)); - } - } - // compensation - simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; - simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); - simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); - simde__m256i QAM_amp256 = simde_mm256_set1_epi16(QAM16_n1); // 2/sqrt(10) - simde__m256i *rxF_256 = (simde__m256i*)rxFext[aarx]; - simde__m256i *chF_256 = (simde__m256i*)chFext[aatx][aarx]; - simde__m256i *rxComp_256 = (simde__m256i*)rxComp[aatx]; - simde__m256i *ul_ch_maga_256 = (simde__m256i*)ul_ch_maga[aatx]; - for (int i = 0; i < (length>>3) + ((length&7)?1:0); i++) - { - xmmp0 = simde_mm256_madd_epi16(chF_256[i], rxF_256[i]); - // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm256_shuffle_epi8(chF_256[i], complex_shuffle256); - xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); - xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF_256[i]); - // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); - xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); - xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); - xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); - xmmp4 = simde_mm256_packs_epi32(xmmp2, xmmp3); - - xmmp0 = simde_mm256_madd_epi16(chF_256[i], chF_256[i]); // |h|^2 - xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); - xmmp0 = simde_mm256_packs_epi32(xmmp0, xmmp0); - xmmp1 = simde_mm256_unpacklo_epi16(xmmp0, xmmp0); - xmmp1 = simde_mm256_mulhrs_epi16(xmmp1, QAM_amp256); - - rxComp_256[i] = simde_mm256_adds_epi16(rxComp_256[i], xmmp4); - ul_ch_maga_256[i] = simde_mm256_adds_epi16(ul_ch_maga_256[i], xmmp1); - } - } - } - c16_t *rho0 = rho[0][1]; - c16_t *rho1 = rho[1][0]; - c16_t *llr_0 = (c16_t *)&llr[0][pusch_vars->llr_offset[symbol]]; - c16_t *llr_1 = (c16_t *)&llr[1][pusch_vars->llr_offset[symbol]]; - c16_t *ul_ch_mag0 = ul_ch_maga[0]; - c16_t *ul_ch_mag1 = ul_ch_maga[1]; - nr_ulsch_qam16_qam16(rxComp[0], rxComp[1], ul_ch_mag0, ul_ch_mag1, llr_0, rho0, length); - nr_ulsch_qam16_qam16(rxComp[1], rxComp[0], ul_ch_mag1, ul_ch_mag0, llr_1, rho1, length); -} - -void inner_rx_64qam_2layer (NR_DL_FRAME_PARMS *frame_parms, - NR_gNB_PUSCH *pusch_vars, - nfapi_nr_pusch_pdu_t *rel15_ul, - c16_t **rxF, - c16_t **ul_ch, - int16_t **llr, - int nb_layer, - int nb_rx_ant, - int soffset, - int length, - int symbol, - int short nb_rb, - int dmrs_symbol_flag, - int output_shift, - uint32_t nvar) -{ - int length_aligned = (length % 8) ? (8 - length % 8) : 0; - - c16_t rxFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); - c16_t chFext[nb_layer][nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); - for (int aarx = 0; aarx < nb_rx_ant; aarx++) { - for (int aatx = 0; aatx < nb_layer; aatx++) { - nr_ulsch_extract_rbs0(rxF[aarx], - (c16_t *)pusch_vars->ul_ch_estimates[aatx * nb_rx_ant + aarx], - rxFext[aarx], - chFext[aatx][aarx], - soffset+(symbol * frame_parms->ofdm_symbol_size), - pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, - aarx, - dmrs_symbol_flag, - rel15_ul, - frame_parms); - } - } - c16_t rho[nb_layer][nb_layer][length + length_aligned] __attribute__((aligned(32))); - c16_t rxComp[nb_layer][length + length_aligned] __attribute__((aligned(32))); - c16_t ul_ch_maga[nb_layer][length + length_aligned] __attribute__((aligned(32))); - c16_t ul_ch_magb[nb_layer][length + length_aligned] __attribute__((aligned(32))); - c16_t ul_ch_magc[nb_layer][length + length_aligned] __attribute__((aligned(32))); - - memset(rho, 0, sizeof(c16_t) * nb_layer * nb_layer* (length + length_aligned)); - memset(rxComp, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); - memset(ul_ch_maga, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); - memset(ul_ch_magb, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); - memset(ul_ch_magc, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); - - for (int aatx = 0; aatx < nb_layer; aatx++) { - for (int aarx = 0; aarx < nb_rx_ant; aarx++) { -#ifdef USE_ML - for (int atx = 0; atx < nb_layer; atx++) { - simde__m256i mmtmpD0, mmtmpD1, mmtmpD2, mmtmpD3; - simde__m256i *rho_256 = (simde__m256i *)rho[aatx][atx]; - simde__m256i *chF_256 = (simde__m256i *)chFext[aatx][aarx]; - simde__m256i *chF2_256 = (simde__m256i *)chFext[atx][aarx]; - simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); - simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); - for (int i = 0; i < (length >> 3)+((length&7)?1:0); i++) - { - // multiply by conjugated channel - mmtmpD0 = simde_mm256_madd_epi16(chF_256[i], chF2_256[i]); - // mmtmpD0 contains real part of 4 consecutive outputs (32-bit) - mmtmpD1 = simde_mm256_shuffle_epi8(chF_256[i], complex_shuffle256); - mmtmpD1 = simde_mm256_sign_epi16(mmtmpD1, conj256); - mmtmpD1 = simde_mm256_madd_epi16(mmtmpD1, chF2_256[i]); - // mmtmpD1 contains imag part of 4 consecutive outputs (32-bit) - mmtmpD0 = simde_mm256_srai_epi32(mmtmpD0, output_shift); - mmtmpD1 = simde_mm256_srai_epi32(mmtmpD1, output_shift); - mmtmpD2 = simde_mm256_unpacklo_epi32(mmtmpD0, mmtmpD1); - mmtmpD3 = simde_mm256_unpackhi_epi32(mmtmpD0, mmtmpD1); - - rho_256[i] = simde_mm256_adds_epi16(rho_256[i], simde_mm256_packs_epi32(mmtmpD2, mmtmpD3)); - } - } -#endif - // compensation - simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; - simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); - simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); - simde__m256i QAM_ampa_256 = simde_mm256_set1_epi16(QAM64_n1); // 2/sqrt(10) - simde__m256i QAM_ampb_256 = simde_mm256_set1_epi16(QAM64_n2); // 2/sqrt(10) - simde__m256i *rxF_256 = (simde__m256i*)rxFext[aarx]; - simde__m256i *chF_256 = (simde__m256i*)chFext[aatx][aarx]; - simde__m256i *rxComp_256 = (simde__m256i*)rxComp[aatx]; - simde__m256i *ul_ch_maga_256 = (simde__m256i*)ul_ch_maga[aatx]; - simde__m256i *ul_ch_magb_256 = (simde__m256i*)ul_ch_magb[aatx]; - for (int i = 0; i < (length>>3) + ((length&7)?1:0); i++) - { - xmmp0 = simde_mm256_madd_epi16(chF_256[i], rxF_256[i]); - // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm256_shuffle_epi8(chF_256[i], complex_shuffle256); - xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); - xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF_256[i]); - // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); - xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); - xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); - xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); - xmmp4 = simde_mm256_packs_epi32(xmmp2, xmmp3); - - xmmp0 = simde_mm256_madd_epi16(chF_256[i], chF_256[i]); // |h|^2 - xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); - xmmp0 = simde_mm256_packs_epi32(xmmp0, xmmp0); - xmmp1 = simde_mm256_unpacklo_epi16(xmmp0, xmmp0); - - xmmp2 = simde_mm256_mulhrs_epi16(xmmp1, QAM_ampa_256); - xmmp3 = simde_mm256_mulhrs_epi16(xmmp1, QAM_ampb_256); - - rxComp_256[i] = simde_mm256_adds_epi16(rxComp_256[i], xmmp4); - ul_ch_maga_256[i] = simde_mm256_adds_epi16(ul_ch_maga_256[i], xmmp2); - ul_ch_magb_256[i] = simde_mm256_adds_epi16(ul_ch_magb_256[i], xmmp3); - } - } - } -#ifdef USE_ML - c16_t *rho0 = rho[0][1]; - c16_t *rho1 = rho[1][0]; - c16_t *llr_0 = (c16_t *)&llr[0][pusch_vars->llr_offset[symbol]]; - c16_t *llr_1 = (c16_t *)&llr[1][pusch_vars->llr_offset[symbol]]; - c16_t *ul_ch_mag0 = ul_ch_maga[0]; - c16_t *ul_ch_mag1 = ul_ch_maga[1]; - nr_ulsch_qam64_qam64(rxComp[0], rxComp[1], ul_ch_mag0, ul_ch_mag1, llr_0, rho0, length); - nr_ulsch_qam64_qam64(rxComp[1], rxComp[0], ul_ch_mag1, ul_ch_mag0, llr_1, rho1, length); -#else - nr_ulsch_mmse_2layers(frame_parms, - (int **)rxComp, - (int **)ul_ch_maga, - (int **)ul_ch_magb, - (int **)ul_ch_magc, - (int **)chFext, - rel15_ul->rb_size, - frame_parms->nb_antennas_rx, - rel15_ul->qam_mod_order, - pusch_vars->log2_maxh, - symbol, - length, - nvar); - for (int aatx = 0; aatx < nb_layer; aatx++) - nr_ulsch_compute_llr((int32_t*)rxComp[aatx], - (int32_t*)ul_ch_maga[aatx], - (int32_t*)ul_ch_magb[aatx], - (int32_t*)ul_ch_magc[aatx], - &llr[aatx][pusch_vars->llr_offset[symbol]], - rel15_ul->rb_size, - length, - symbol, - rel15_ul->qam_mod_order); -#endif -} - -void inner_rx_256qam_2layer (NR_DL_FRAME_PARMS *frame_parms, - NR_gNB_PUSCH *pusch_vars, - nfapi_nr_pusch_pdu_t *rel15_ul, - c16_t **rxF, - c16_t **ul_ch, - int16_t **llr, - int nb_layer, - int nb_rx_ant, - int soffset, - int length, - int symbol, - int short nb_rb, - int dmrs_symbol_flag, - int output_shift, - uint32_t nvar) -{ - int length_aligned = (length % 8) ? (8 - length % 8) : 0; - - c16_t rxFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); - c16_t chFext[nb_layer][nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); - for (int aarx = 0; aarx < nb_rx_ant; aarx++) { - for (int aatx = 0; aatx < nb_layer; aatx++) { - nr_ulsch_extract_rbs0(rxF[aarx], - (c16_t *)pusch_vars->ul_ch_estimates[aatx * nb_rx_ant + aarx], - rxFext[aarx], - chFext[aatx][aarx], - soffset+(symbol * frame_parms->ofdm_symbol_size), - pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, - aarx, - dmrs_symbol_flag, - rel15_ul, - frame_parms); - } - } - c16_t rho[nb_layer][nb_layer][length + length_aligned] __attribute__((aligned(32))); - c16_t rxComp[nb_layer][length + length_aligned] __attribute__((aligned(32))); - c16_t ul_ch_maga[nb_layer][length + length_aligned] __attribute__((aligned(32))); - c16_t ul_ch_magb[nb_layer][length + length_aligned] __attribute__((aligned(32))); - c16_t ul_ch_magc[nb_layer][length + length_aligned] __attribute__((aligned(32))); - - memset(rho, 0, sizeof(c16_t) * nb_layer * nb_layer* (length + length_aligned)); - memset(rxComp, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); - memset(ul_ch_maga, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); - memset(ul_ch_magb, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); - memset(ul_ch_magc, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); - - // compensation - for (int aatx = 0; aatx < nb_layer; aatx++) { - for (int aarx = 0; aarx < nb_rx_ant; aarx++) { - simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4; - simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); - simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); - simde__m256i QAM_ampa_256 = simde_mm256_set1_epi16(QAM256_n1); // 2/sqrt(10) - simde__m256i QAM_ampb_256 = simde_mm256_set1_epi16(QAM256_n2); // 2/sqrt(10) - simde__m256i QAM_ampc_256 = simde_mm256_set1_epi16(QAM256_n3); // 2/sqrt(10) - simde__m256i *rxF_256 = (simde__m256i*)rxFext[aarx]; - simde__m256i *chF_256 = (simde__m256i*)chFext[aatx][aarx]; - simde__m256i *rxComp_256 = (simde__m256i*)rxComp[aatx]; - simde__m256i *ul_ch_maga_256 = (simde__m256i*)ul_ch_maga[aatx]; - simde__m256i *ul_ch_magb_256 = (simde__m256i*)ul_ch_magb[aatx]; - simde__m256i *ul_ch_magc_256 = (simde__m256i*)ul_ch_magc[aatx]; - for (int i = 0; i < (length>>3) + ((length&7)?1:0); i++) - { - xmmp0 = simde_mm256_madd_epi16(chF_256[i], rxF_256[i]); - // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp1 = simde_mm256_shuffle_epi8(chF_256[i], complex_shuffle256); - xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); - xmmp1 = simde_mm256_madd_epi16(xmmp1, rxF_256[i]); - // xmmp1 contains imag part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] - xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); - xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); - xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); - xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); - xmmp4 = simde_mm256_packs_epi32(xmmp2, xmmp3); - - xmmp0 = simde_mm256_madd_epi16(chF_256[i], chF_256[i]); // |h|^2 - xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); - xmmp0 = simde_mm256_packs_epi32(xmmp0, xmmp0); - xmmp1 = simde_mm256_unpacklo_epi16(xmmp0, xmmp0); - - xmmp2 = simde_mm256_mulhrs_epi16(xmmp1, QAM_ampa_256); - xmmp3 = simde_mm256_mulhrs_epi16(xmmp1, QAM_ampb_256); - xmmp1 = simde_mm256_mulhrs_epi16(xmmp1, QAM_ampc_256); - - rxComp_256[i] = simde_mm256_adds_epi16(rxComp_256[i], xmmp4); - ul_ch_maga_256[i] = simde_mm256_adds_epi16(ul_ch_maga_256[i], xmmp2); - ul_ch_magb_256[i] = simde_mm256_adds_epi16(ul_ch_magb_256[i], xmmp3); - ul_ch_magc_256[i] = simde_mm256_adds_epi16(ul_ch_magc_256[i], xmmp1); - } - } - } - - nr_ulsch_mmse_2layers(frame_parms, - (int **)rxComp, - (int **)ul_ch_maga, - (int **)ul_ch_magb, - (int **)ul_ch_magc, - (int **)chFext, - rel15_ul->rb_size, - frame_parms->nb_antennas_rx, - rel15_ul->qam_mod_order, - pusch_vars->log2_maxh, - symbol, - length, - nvar); - for (int aatx = 0; aatx < nb_layer; aatx++) - nr_ulsch_compute_llr((int32_t*)rxComp[aatx], - (int32_t*)ul_ch_maga[aatx], - (int32_t*)ul_ch_magb[aatx], - (int32_t*)ul_ch_magc[aatx], - &llr[aatx][pusch_vars->llr_offset[symbol]], - rel15_ul->rb_size, - length, - symbol, - rel15_ul->qam_mod_order); + nr_ulsch_compute_llr((int32_t*)rxComp[aatx], + (int32_t*)rxF_ch_maga[aatx], + (int32_t*)rxF_ch_magb[aatx], + (int32_t*)rxF_ch_magc[aatx], + &llr[aatx][pusch_vars->llr_offset[symbol]], + rel15_ul->rb_size, + length, + symbol, + rel15_ul->qam_mod_order); } void nr_pusch_symbol_processing_noprecoding(void *arg) @@ -2564,109 +1660,45 @@ void nr_pusch_symbol_processing_noprecoding(void *arg) LOG_I(PHY,"symbol %d: nb_re_pusch %d, DMRS symbl used for Chest :%d \n", symbol, nb_re_pusch, gNB->pusch_vars[ulsch_id].dmrs_symbol); - if (nb_re_pusch == 0) continue; + if (nb_re_pusch == 0) + continue; + int soffset = (slot&3)*frame_parms->symbols_per_slot*frame_parms->ofdm_symbol_size; + inner_rx(gNB, + ulsch_id, + slot, + frame_parms, + pusch_vars, + rel15_ul, + gNB->common_vars.rxdataF, + (c16_t**)gNB->pusch_vars[ulsch_id].ul_ch_estimates, + rdata->llr_layers, + rel15_ul->nrOfLayers, + frame_parms->nb_antennas_rx, + soffset, + nb_re_pusch, // length + symbol, // symbol index + rel15_ul->rb_size, // ofdm size + dmrs_symbol_flag, + gNB->pusch_vars[ulsch_id].log2_maxh, + rdata->nvar); + + simde__m64 *llr_ptr_64; if (rel15_ul->nrOfLayers == 1) + llr_ptr_64 = (simde__m64 *)&rdata->llr_layers[0][pusch_vars->llr_offset[symbol]]; + else { - void (*inner_rx)(NR_DL_FRAME_PARMS *, - NR_gNB_PUSCH *, - nfapi_nr_pusch_pdu_t *, - c16_t **, - c16_t **, - int16_t *, - int32_t, - int32_t, - int32_t, - int32_t, - int32_t, - int16_t, - int32_t, - int32_t); - - if (rel15_ul->qam_mod_order == 2) inner_rx = inner_rx_qpsk; - else if (rel15_ul->qam_mod_order == 4) inner_rx = inner_rx_16qam; - else if (rel15_ul->qam_mod_order == 6) inner_rx = inner_rx_64qam; - else if (rel15_ul->qam_mod_order == 8) inner_rx = inner_rx_256qam; - else AssertFatal(1==0,"rel15_ul->qam_mod_order %d, pusch_pdu->dmrs_config_type %d\n", - rel15_ul->qam_mod_order,rel15_ul->dmrs_config_type); - - int soffset = (slot&3)*frame_parms->symbols_per_slot*frame_parms->ofdm_symbol_size; - int16_t llr_tmp[(nb_re_pusch + 8) * rel15_ul->qam_mod_order] __attribute__((aligned(32))); - memset(llr_tmp, 0, sizeof(int16_t) * (nb_re_pusch + 8) * rel15_ul->qam_mod_order); - inner_rx(frame_parms, - pusch_vars, - rel15_ul, - gNB->common_vars.rxdataF, - (c16_t**)gNB->pusch_vars[ulsch_id].ul_ch_estimates, - llr_tmp, - rel15_ul->nrOfLayers, - frame_parms->nb_antennas_rx, - soffset, - nb_re_pusch, // length - symbol, // symbol index - rel15_ul->rb_size, // ofdm size - dmrs_symbol_flag, - gNB->pusch_vars[ulsch_id].log2_maxh); - - // unscrambling - simde__m64 *llr64 = (simde__m64 *) &rdata->llr[pusch_vars->llr_offset[symbol]]; - for (int i = 0; i < nb_re_pusch*rel15_ul->qam_mod_order >> 2; i++) - llr64[i] = simde_mm_mullo_pi16(((simde__m64 *)llr_tmp)[i], s[i]); - - } - else // MIMO for 2x2 - { - int soffset = (slot&3)*frame_parms->symbols_per_slot*frame_parms->ofdm_symbol_size; - void (*inner_rx)(NR_DL_FRAME_PARMS *, - NR_gNB_PUSCH *, - nfapi_nr_pusch_pdu_t *, - c16_t **, - c16_t **, - int16_t **, - int32_t, - int32_t, - int32_t, - int32_t, - int32_t, - int16_t, - int32_t, - int32_t, - uint32_t); - if (rel15_ul->qam_mod_order == 2) inner_rx = inner_rx_qpsk_2layer; - else if (rel15_ul->qam_mod_order == 4) inner_rx = inner_rx_16qam_2layer; - else if (rel15_ul->qam_mod_order == 6) inner_rx = inner_rx_64qam_2layer; - else if (rel15_ul->qam_mod_order == 8) inner_rx = inner_rx_256qam_2layer; - else AssertFatal(1==0,"rel15_ul->qam_mod_order %d, pusch_pdu->dmrs_config_type %d\n", - rel15_ul->qam_mod_order,rel15_ul->dmrs_config_type); - - inner_rx(frame_parms, - pusch_vars, - rel15_ul, - gNB->common_vars.rxdataF, - (c16_t**)gNB->pusch_vars[ulsch_id].ul_ch_estimates, - rdata->llr_layers, - rel15_ul->nrOfLayers, - frame_parms->nb_antennas_rx, - soffset, - nb_re_pusch, // length - symbol, // symbol index - rel15_ul->rb_size, // ofdm size - dmrs_symbol_flag, - gNB->pusch_vars[ulsch_id].log2_maxh, - rdata->nvar); - // layer de-mapping int16_t* llr_cw = &rdata->llr[pusch_vars->llr_offset[symbol] * rel15_ul->nrOfLayers]; for (int i = 0; i < (nb_re_pusch); i++) for (int l = 0; l < rel15_ul->nrOfLayers; l++) for (int m = 0; m < rel15_ul->qam_mod_order; m++) llr_cw[i*rel15_ul->nrOfLayers*rel15_ul->qam_mod_order+l*rel15_ul->qam_mod_order+m] = rdata->llr_layers[l][pusch_vars->llr_offset[symbol] + i*rel15_ul->qam_mod_order+m]; - - // unscrambling - simde__m64 *llr64 = (simde__m64 *) &rdata->llr[pusch_vars->llr_offset[symbol] * rel15_ul->nrOfLayers]; - for (int i = 0; i < (nb_re_pusch*rel15_ul->qam_mod_order * rel15_ul->nrOfLayers) >> 2; i++) - llr64[i] = simde_mm_mullo_pi16(llr64[i], s[i]); - + llr_ptr_64 = (simde__m64 *)llr_cw; } + // unscrambling + simde__m64 *llr64 = (simde__m64 *) &rdata->llr[pusch_vars->llr_offset[symbol] * rel15_ul->nrOfLayers]; + for (int i = 0; i < (nb_re_pusch * rel15_ul->qam_mod_order * rel15_ul->nrOfLayers) >> 2; i++) + llr64[i] = simde_mm_mullo_pi16(llr_ptr_64[i], s[i]); } } diff --git a/openair1/PHY/NR_TRANSPORT/nr_ulsch_llr_computation.c b/openair1/PHY/NR_TRANSPORT/nr_ulsch_llr_computation.c index ddbec269b3..476c9d07bf 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_ulsch_llr_computation.c +++ b/openair1/PHY/NR_TRANSPORT/nr_ulsch_llr_computation.c @@ -30,6 +30,7 @@ * \warning */ +#include "PHY/defs_gNB.h" #include "PHY/defs_nr_common.h" #include "PHY/sse_intrin.h" #include "PHY/impl_defs_top.h" @@ -46,16 +47,9 @@ void nr_ulsch_qpsk_llr(int32_t *rxdataF_comp, { c16_t *rxF = (c16_t *)rxdataF_comp; c16_t *llr32 = (c16_t *)ulsch_llr; - - if (!llr32) { - LOG_E(PHY,"nr_ulsch_qpsk_llr: llr is null, symbol %d, llr32 = %p\n",symbol, llr32); - } for (int i = 0; i < nb_re; i++) { - //*llr32 = *rxF; - llr32->r = rxF->r >> 3; - llr32->i = rxF->i >> 3; - rxF++; - llr32++; + llr32[i].r = rxF[i].r >> 3; + llr32[i].i = rxF[i].i >> 3; } } @@ -3624,46 +3618,7 @@ void nr_ulsch_qam64_qam64(c16_t *stream0_in, simde_m_empty(); } -void nr_ulsch_compute_ML_llr(int32_t **rxdataF_comp, - int32_t **ul_ch_mag, - int32_t ***rho, - int16_t **llr_layers, - uint8_t nb_antennas_rx, - uint32_t rb_size, - uint32_t nb_re, - uint8_t symbol, - uint32_t rxdataF_ext_offset, - uint8_t mod_order) -{ - int off = ((rb_size & 1) == 1) ? 4 : 0; - c16_t *rxdataF_comp0 = (c16_t *)&rxdataF_comp[0][symbol * (off + (rb_size * NR_NB_SC_PER_RB))]; - c16_t *rxdataF_comp1 = (c16_t *)&rxdataF_comp[nb_antennas_rx][symbol * (off + (rb_size * NR_NB_SC_PER_RB))]; - c16_t *ul_ch_mag0 = (c16_t *)&ul_ch_mag[0][symbol * (off + (rb_size * NR_NB_SC_PER_RB))]; - c16_t *ul_ch_mag1 = (c16_t *)&ul_ch_mag[nb_antennas_rx][symbol * (off + (rb_size * NR_NB_SC_PER_RB))]; - c16_t *llr_layers0 = (c16_t *)&llr_layers[0][rxdataF_ext_offset * mod_order]; - c16_t *llr_layers1 = (c16_t *)&llr_layers[1][rxdataF_ext_offset * mod_order]; - c16_t *rho0 = (c16_t *)&rho[0][1][symbol * (off + (rb_size * NR_NB_SC_PER_RB))]; - c16_t *rho1 = (c16_t *)&rho[0][2][symbol * (off + (rb_size * NR_NB_SC_PER_RB))]; - - switch (mod_order) { - case 2: - nr_ulsch_qpsk_qpsk(rxdataF_comp0, rxdataF_comp1, llr_layers0, rho0, nb_re); - nr_ulsch_qpsk_qpsk(rxdataF_comp1, rxdataF_comp0, llr_layers1, rho1, nb_re); - break; - case 4: - nr_ulsch_qam16_qam16(rxdataF_comp0, rxdataF_comp1, ul_ch_mag0, ul_ch_mag1, llr_layers0, rho0, nb_re); - nr_ulsch_qam16_qam16(rxdataF_comp1, rxdataF_comp0, ul_ch_mag1, ul_ch_mag0, llr_layers1, rho1, nb_re); - break; - case 6: - nr_ulsch_qam64_qam64(rxdataF_comp0, rxdataF_comp1, ul_ch_mag0, ul_ch_mag1, llr_layers0, rho0, nb_re); - nr_ulsch_qam64_qam64(rxdataF_comp1, rxdataF_comp0, ul_ch_mag1, ul_ch_mag0, llr_layers1, rho1, nb_re); - break; - default: - AssertFatal(1 == 0, "nr_ulsch_compute_llr: invalid Qm value, symbol = %d, Qm = %d\n", symbol, mod_order); - } -} - -void nr_ulsch_shift_llr(int16_t **llr_layers, uint32_t nb_re, uint32_t rxdataF_ext_offset, uint8_t mod_order, int shift) +static void nr_ulsch_shift_llr(int16_t **llr_layers, uint32_t nb_re, uint32_t rxdataF_ext_offset, uint8_t mod_order, int shift) { simde__m128i *llr_layers0 = (simde__m128i *)&llr_layers[0][rxdataF_ext_offset * mod_order]; simde__m128i *llr_layers1 = (simde__m128i *)&llr_layers[1][rxdataF_ext_offset * mod_order]; @@ -3686,3 +3641,35 @@ void nr_ulsch_shift_llr(int16_t **llr_layers, uint32_t nb_re, uint32_t rxdataF_e llr_layers1[i] = simde_mm_srai_epi16(llr_layers1[i], shift); } } + +void nr_ulsch_compute_ML_llr(NR_gNB_PUSCH *pusch_vars, + uint32_t symbol, + c16_t* rxdataF_comp0, + c16_t* rxdataF_comp1, + c16_t* ul_ch_mag0, + c16_t* ul_ch_mag1, + c16_t* llr_layers0, + c16_t* llr_layers1, + c16_t* rho0, + c16_t* rho1, + uint32_t nb_re, + uint8_t mod_order) +{ + switch (mod_order) { + case 2: + nr_ulsch_qpsk_qpsk(rxdataF_comp0, rxdataF_comp1, llr_layers0, rho0, nb_re); + nr_ulsch_qpsk_qpsk(rxdataF_comp1, rxdataF_comp0, llr_layers1, rho1, nb_re); + nr_ulsch_shift_llr(pusch_vars->llr_layers, nb_re, pusch_vars->llr_offset[symbol] >> 1, 2, 4); + break; + case 4: + nr_ulsch_qam16_qam16(rxdataF_comp0, rxdataF_comp1, ul_ch_mag0, ul_ch_mag1, llr_layers0, rho0, nb_re); + nr_ulsch_qam16_qam16(rxdataF_comp1, rxdataF_comp0, ul_ch_mag1, ul_ch_mag0, llr_layers1, rho1, nb_re); + break; + case 6: + nr_ulsch_qam64_qam64(rxdataF_comp0, rxdataF_comp1, ul_ch_mag0, ul_ch_mag1, llr_layers0, rho0, nb_re); + nr_ulsch_qam64_qam64(rxdataF_comp1, rxdataF_comp0, ul_ch_mag1, ul_ch_mag0, llr_layers1, rho1, nb_re); + break; + default: + AssertFatal(1 == 0, "nr_ulsch_compute_llr: invalid Qm value, Qm = %d\n", mod_order); + } +} From 428af852a0b7947d2d301c842cbf4f37d98cb272 Mon Sep 17 00:00:00 2001 From: Tsung-Yu Chan Date: Tue, 5 Sep 2023 21:35:03 +0200 Subject: [PATCH 04/13] fix / clean the unused variable --- openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c | 5 ----- openair1/PHY/defs_gNB.h | 4 ---- openair1/SCHED_NR/phy_procedures_nr_gNB.c | 15 --------------- 3 files changed, 24 deletions(-) diff --git a/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c b/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c index e3dec33a0d..5a079a476a 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c +++ b/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c @@ -1717,8 +1717,6 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB, NR_gNB_PUSCH *pusch_vars = &gNB->pusch_vars[ulsch_id]; pusch_vars->dmrs_symbol = INVALID_VALUE; - pusch_vars->cl_done = 0; - memset(pusch_vars->extraction_done,0,14*sizeof(int)); gNB->nbSymb=0; bwp_start_subcarrier = ((rel15_ul->rb_start + rel15_ul->bwp_start)*NR_NB_SC_PER_RB + frame_parms->first_carrier_offset) % frame_parms->ofdm_symbol_size; LOG_D(PHY,"pusch %d.%d : bwp_start_subcarrier %d, rb_start %d, first_carrier_offset %d\n", frame,slot,bwp_start_subcarrier, rel15_ul->rb_start, frame_parms->first_carrier_offset); @@ -1881,9 +1879,6 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB, else pusch_vars->log2_maxh = (log2_approx(avgs) >> 1); - pusch_vars->cl_done = 1; - pusch_vars->extraction_done[meas_symbol] = 1; - stop_meas(&gNB->rx_pusch_init_stats); start_meas(&gNB->rx_pusch_symbol_processing_stats); diff --git a/openair1/PHY/defs_gNB.h b/openair1/PHY/defs_gNB.h index 88367741d4..a28e4a30f4 100644 --- a/openair1/PHY/defs_gNB.h +++ b/openair1/PHY/defs_gNB.h @@ -411,10 +411,6 @@ typedef struct { int16_t *ul_valid_re_per_slot; /// \brief offset for llr corresponding to each symbol int llr_offset[14]; - /// flag to verify if channel level computation is done - uint8_t cl_done; - /// flag to indicate if channel extraction is done - int extraction_done[14]; /// flag to indicate DTX on reception int DTX; } NR_gNB_PUSCH; diff --git a/openair1/SCHED_NR/phy_procedures_nr_gNB.c b/openair1/SCHED_NR/phy_procedures_nr_gNB.c index 64eae8422c..6c4fdd4c05 100644 --- a/openair1/SCHED_NR/phy_procedures_nr_gNB.c +++ b/openair1/SCHED_NR/phy_procedures_nr_gNB.c @@ -397,21 +397,6 @@ static int nr_ulsch_procedures(PHY_VARS_gNB *gNB, int frame_rx, int slot_rx, int number_dmrs_symbols, // number of dmrs symbols irrespective of single or double symbol dmrs pusch_pdu->qam_mod_order, pusch_pdu->nrOfLayers); - - - - // nr_ulsch_layer_demapping(gNB->pusch_vars[ULSCH_id].llr, - // pusch_pdu->nrOfLayers, - // pusch_pdu->qam_mod_order, - // G, - // gNB->pusch_vars[ULSCH_id].llr_layers); - // //---------------------------------------------------------- - // //------------------- ULSCH unscrambling ------------------- - // //---------------------------------------------------------- - // start_meas(&gNB->ulsch_unscrambling_stats); - // nr_ulsch_unscrambling(gNB->pusch_vars[ULSCH_id].llr, G, pusch_pdu->data_scrambling_id, pusch_pdu->rnti); - // stop_meas(&gNB->ulsch_unscrambling_stats); - //---------------------------------------------------------- //--------------------- ULSCH decoding --------------------- From e54d4a98b6bc4cc4a811158d19a00a1e14917afc Mon Sep 17 00:00:00 2001 From: Tsung-Yu Chan Date: Wed, 6 Sep 2023 17:17:27 +0200 Subject: [PATCH 05/13] fix / fix the log2_maxh for llr - init the variable for simde_m256i avoid error in ubuntu --- .../PHY/NR_TRANSPORT/nr_ulsch_demodulation.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c b/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c index 5a079a476a..309aab6256 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c +++ b/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c @@ -615,7 +615,9 @@ void nr_ulsch_channel_compensation(c16_t *rxFext, uint32_t nb_rb, uint32_t output_shift) { - simde__m256i QAM_ampa_256, QAM_ampb_256, QAM_ampc_256; + simde__m256i QAM_ampa_256 = simde_mm256_setzero_si256(); + simde__m256i QAM_ampb_256 = simde_mm256_setzero_si256(); + simde__m256i QAM_ampc_256 = simde_mm256_setzero_si256(); if (mod_order == 4) { QAM_ampa_256 = simde_mm256_set1_epi16(QAM16_n1); @@ -1874,10 +1876,14 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB, for (int aarx = 0; aarx < frame_parms->nb_antennas_rx; aarx++) avgs = cmax(avgs, avg[aatx*frame_parms->nb_antennas_rx+aarx]); - if (rel15_ul->nrOfLayers == 1) - pusch_vars->log2_maxh = (log2_approx(avgs) >> 1) + 2; - else - pusch_vars->log2_maxh = (log2_approx(avgs) >> 1); + + if (rel15_ul->nrOfLayers == 2 && rel15_ul->qam_mod_order < 6) + pusch_vars->log2_maxh = (log2_approx(avgs) >> 1) - 3; // for MMSE + else + pusch_vars->log2_maxh = (log2_approx(avgs) >> 1) + 1 + log2_approx(frame_parms->nb_antennas_rx >> 2); + + if (pusch_vars->log2_maxh < 0) + pusch_vars->log2_maxh = 0; stop_meas(&gNB->rx_pusch_init_stats); From 090ac4870df34d420261173c9fe9ba25c6fed465 Mon Sep 17 00:00:00 2001 From: Tsung-Yu Chan Date: Wed, 13 Sep 2023 14:15:29 +0200 Subject: [PATCH 06/13] fix / init the variable init the PUSCH symbols per thread to avoid for stucking --- executables/nr-gnb.c | 3 ++- openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/executables/nr-gnb.c b/executables/nr-gnb.c index d436828294..e4a9e4e5e6 100644 --- a/executables/nr-gnb.c +++ b/executables/nr-gnb.c @@ -475,7 +475,8 @@ void init_gNB_Tpool(int inst) { PHY_VARS_gNB *gNB; gNB = RC.gNB[inst]; gNB_L1_proc_t *proc = &gNB->proc; - + // PUSCH symbols per thread need to be calculated by how many threads we have + gNB->num_pusch_symbols_per_thread = 1; // ULSCH decoding threadpool initTpool(get_softmodem_params()->threadPoolConfig, &gNB->threadPool, cpumeas(CPUMEAS_GETSTATE)); // ULSCH decoder result FIFO diff --git a/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c b/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c index 309aab6256..3fa192f7cb 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c +++ b/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c @@ -1931,7 +1931,7 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB, while (gNB->nbSymb > 0) { - notifiedFIFO_elt_t *req=pullTpool(&gNB->respPuschSymb, &gNB->threadPool); + notifiedFIFO_elt_t *req = pullTpool(&gNB->respPuschSymb, &gNB->threadPool); gNB->nbSymb--; delNotifiedFIFO_elt(req); } From 39e7e091520afe4fdadb50e69941741f98da62bb Mon Sep 17 00:00:00 2001 From: Tsung-Yu Chan Date: Wed, 13 Sep 2023 16:12:44 +0200 Subject: [PATCH 07/13] fix / set the output shift to correct value --- openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c b/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c index 3fa192f7cb..e03d9cc198 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c +++ b/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c @@ -1876,10 +1876,11 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB, for (int aarx = 0; aarx < frame_parms->nb_antennas_rx; aarx++) avgs = cmax(avgs, avg[aatx*frame_parms->nb_antennas_rx+aarx]); + pusch_vars->log2_maxh = (log2_approx(avgs) >> 1); - if (rel15_ul->nrOfLayers == 2 && rel15_ul->qam_mod_order < 6) + if (rel15_ul->nrOfLayers == 2 && rel15_ul->qam_mod_order >= 6) pusch_vars->log2_maxh = (log2_approx(avgs) >> 1) - 3; // for MMSE - else + else if (rel15_ul->nrOfLayers == 1) pusch_vars->log2_maxh = (log2_approx(avgs) >> 1) + 1 + log2_approx(frame_parms->nb_antennas_rx >> 2); if (pusch_vars->log2_maxh < 0) From 9db99b2fd2d06afd513564b30327e9b83577e99b Mon Sep 17 00:00:00 2001 From: Tsung-Yu Chan Date: Thu, 14 Sep 2023 01:02:37 +0200 Subject: [PATCH 08/13] feat / add ptrs module --- .../NR_ESTIMATION/nr_ul_channel_estimation.c | 21 ++-- .../PHY/NR_TRANSPORT/nr_ulsch_demodulation.c | 103 +++++++++--------- 2 files changed, 61 insertions(+), 63 deletions(-) diff --git a/openair1/PHY/NR_ESTIMATION/nr_ul_channel_estimation.c b/openair1/PHY/NR_ESTIMATION/nr_ul_channel_estimation.c index 98606e7f3c..11a316d77e 100644 --- a/openair1/PHY/NR_ESTIMATION/nr_ul_channel_estimation.c +++ b/openair1/PHY/NR_ESTIMATION/nr_ul_channel_estimation.c @@ -486,6 +486,7 @@ int nr_pusch_channel_estimation(PHY_VARS_gNB *gNB, * 2) Interpolate PTRS estimated value in TD after all PTRS symbols * 3) Compensated DMRS based estimated signal with PTRS estimation for slot *********************************************************************/ +// #define DEBUG_UL_PTRS void nr_pusch_ptrs_processing(PHY_VARS_gNB *gNB, NR_DL_FRAME_PARMS *frame_parms, nfapi_nr_pusch_pdu_t *rel15_ul, @@ -495,7 +496,6 @@ void nr_pusch_ptrs_processing(PHY_VARS_gNB *gNB, uint32_t nb_re_pusch) { NR_gNB_PUSCH *pusch_vars = &gNB->pusch_vars[ulsch_id]; - //#define DEBUG_UL_PTRS 1 int32_t *ptrs_re_symbol = NULL; int8_t ret = 0; uint8_t symbInSlot = rel15_ul->start_symbol_index + rel15_ul->nr_of_symbols; @@ -573,13 +573,6 @@ void nr_pusch_ptrs_processing(PHY_VARS_gNB *gNB, } } -#ifdef DEBUG_UL_PTRS - LOG_M("ptrsEstUl.m", "est", pusch_vars->ptrs_phase_per_slot[aarx], frame_parms->symbols_per_slot, 1, 1); - LOG_M("rxdataF_bf_ptrs_comp_ul.m","bf_ptrs_cmp", - &gNB->pusch_vars[0]->rxdataF_comp[aarx][rel15_ul->start_symbol_index * NR_NB_SC_PER_RB * rel15_ul->rb_size], - rel15_ul->nr_of_symbols * NR_NB_SC_PER_RB * rel15_ul->rb_size,1,1); -#endif - /*------------------------------------------------------------------------------------------------------- */ /* 3) Compensated DMRS based estimated signal with PTRS estimation */ /*--------------------------------------------------------------------------------------------------------*/ @@ -590,15 +583,15 @@ void nr_pusch_ptrs_processing(PHY_VARS_gNB *gNB, #ifdef DEBUG_UL_PTRS printf("[PHY][UL][PTRS]: Rotate Symbol %2d with %d + j* %d\n", i, phase_per_symbol[i].r,phase_per_symbol[i].i); #endif - rotate_cpx_vector((c16_t *)&pusch_vars->rxdataF_comp[aarx][(i * rel15_ul->rb_size * NR_NB_SC_PER_RB)], + rotate_cpx_vector((c16_t *)&pusch_vars->rxdataF_comp[aarx][i * nb_re_pusch], &phase_per_symbol[i], - (c16_t *)&pusch_vars->rxdataF_comp[aarx][(i * rel15_ul->rb_size * NR_NB_SC_PER_RB)], + (c16_t *)&pusch_vars->rxdataF_comp[aarx][i * nb_re_pusch], ((*nb_rb) * NR_NB_SC_PER_RB), 15); - }// if not DMRS Symbol - }// symbol loop - }// last symbol check - }//Antenna loop + } // if not DMRS Symbol + } // symbol loop + } // last symbol check + } // Antenna loop } uint32_t calc_power(const int16_t *x, const uint32_t size) { diff --git a/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c b/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c index e03d9cc198..ad4d696d4d 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c +++ b/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c @@ -604,7 +604,7 @@ void nr_ulsch_channel_compensation(c16_t *rxFext, c16_t *ul_ch_maga, c16_t *ul_ch_magb, c16_t *ul_ch_magc, - c16_t *rxComp, + int32_t **rxComp, c16_t *rho, NR_DL_FRAME_PARMS *frame_parms, uint32_t symbol, @@ -640,9 +640,9 @@ void nr_ulsch_channel_compensation(c16_t *rxFext, simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); int length_aligned = (length % 8) ? (8 - length % 8) : 0; - + int rxComp_aligned = ((nb_rb * 12) % 8) ? (8 - (nb_rb * 12) % 8) : 0; for (int aatx = 0; aatx < nrOfLayers; aatx++) { - simde__m256i *rxComp_256 = (simde__m256i*) &rxComp[aatx * (length + length_aligned)]; + simde__m256i *rxComp_256 = (simde__m256i*) &rxComp[aatx * nb_rx_ant][symbol * (nb_rb * 12 + rxComp_aligned)]; simde__m256i *rxF_ch_maga_256 = (simde__m256i*)&ul_ch_maga[aatx * (length + length_aligned)]; simde__m256i *rxF_ch_magb_256 = (simde__m256i*)&ul_ch_magb[aatx * (length + length_aligned)]; simde__m256i *rxF_ch_magc_256 = (simde__m256i*)&ul_ch_magc[aatx * (length + length_aligned)]; @@ -1382,13 +1382,14 @@ uint8_t nr_ulsch_mmse_2layers(NR_DL_FRAME_PARMS *frame_parms, simde__m128i *determ_fin_128 = (simde__m128i *)&determ_fin[0]; - simde__m128i *rxdataF_comp128_0 = (simde__m128i *)&((int *)rxdataF_comp)[0 * (length + length_aligned)]; - simde__m128i *rxdataF_comp128_1 = (simde__m128i *)&((int *)rxdataF_comp)[1 * (length + length_aligned)]; simde__m128i *after_mf_a_128 = (simde__m128i *)af_mf_00; simde__m128i *after_mf_b_128 = (simde__m128i *)af_mf_01; simde__m128i *after_mf_c_128 = (simde__m128i *)af_mf_10; simde__m128i *after_mf_d_128 = (simde__m128i *)af_mf_11; - + + int rxComp_aligned = ((nb_rb * 12) % 8) ? (8 - (nb_rb * 12) % 8) : 0; + simde__m128i *rxdataF_comp128_0 = (simde__m128i *)&rxdataF_comp[0][symbol * (nb_rb * 12 + rxComp_aligned)]; + simde__m128i *rxdataF_comp128_1 = (simde__m128i *)&rxdataF_comp[n_rx][symbol * (nb_rb * 12 + rxComp_aligned)]; if (mod_order > 2) { if (mod_order == 4) { @@ -1496,24 +1497,24 @@ uint8_t nr_ulsch_mmse_2layers(NR_DL_FRAME_PARMS *frame_parms, return(0); } -void inner_rx (PHY_VARS_gNB *gNB, - int ulsch_id, - int slot, - NR_DL_FRAME_PARMS *frame_parms, - NR_gNB_PUSCH *pusch_vars, - nfapi_nr_pusch_pdu_t *rel15_ul, - c16_t **rxF, - c16_t **ul_ch, - int16_t **llr, - int nb_layer, - int nb_rx_ant, - int soffset, - int length, - int symbol, - int short nb_rb, - int dmrs_symbol_flag, - int output_shift, - uint32_t nvar) +static void inner_rx (PHY_VARS_gNB *gNB, + int ulsch_id, + int slot, + NR_DL_FRAME_PARMS *frame_parms, + NR_gNB_PUSCH *pusch_vars, + nfapi_nr_pusch_pdu_t *rel15_ul, + c16_t **rxF, + c16_t **ul_ch, + int16_t **llr, + int nb_layer, + int nb_rx_ant, + int soffset, + int length, + int symbol, + int short nb_rb, + int dmrs_symbol_flag, + int output_shift, + uint32_t nvar) { int length_aligned = (length % 8) ? (8 - length % 8) : 0; @@ -1539,18 +1540,21 @@ void inner_rx (PHY_VARS_gNB *gNB, c16_t rxF_ch_magb[nb_layer][length + length_aligned] __attribute__((aligned(32))); c16_t rxF_ch_magc[nb_layer][length + length_aligned] __attribute__((aligned(32))); + int rxComp_aligned = ((nb_rb * 12) % 8) ? (8 - (nb_rb * 12) % 8) : 0; memset(rho, 0, sizeof(c16_t) * nb_layer * nb_layer* (length + length_aligned)); memset(rxComp, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); memset(rxF_ch_maga, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); memset(rxF_ch_magb, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); memset(rxF_ch_magc, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); + for (int i = 0; i < nb_layer; i++) + memset(&pusch_vars->rxdataF_comp[i*nb_rx_ant][symbol * (nb_rb * 12 + rxComp_aligned)], 0, sizeof(int32_t) * (nb_rb * 12 + rxComp_aligned)); nr_ulsch_channel_compensation((c16_t*)rxFext, (c16_t*)chFext, (c16_t*)rxF_ch_maga, (c16_t*)rxF_ch_magb, (c16_t*)rxF_ch_magc, - (c16_t*)rxComp, + pusch_vars->rxdataF_comp, // (c16_t*)rxComp, (nb_layer == 1) ? NULL : (c16_t*)rho, frame_parms, symbol, @@ -1563,25 +1567,26 @@ void inner_rx (PHY_VARS_gNB *gNB, if (nb_layer == 1 && rel15_ul->transform_precoding == transformPrecoder_enabled && rel15_ul->qam_mod_order <= 6) { nr_freq_equalization(frame_parms, - (int *)rxComp, + (int *)&pusch_vars->rxdataF_comp[0][symbol * (nb_rb * 12 + rxComp_aligned)], (int *)rxF_ch_maga, (int *)rxF_ch_magb, symbol, length, rel15_ul->qam_mod_order); - nr_idft((int32_t*)rxComp, length); + nr_idft((int32_t*)&pusch_vars->rxdataF_comp[0][symbol * (nb_rb * 12 + rxComp_aligned)], length); + } + if (rel15_ul->pdu_bit_map & PUSCH_PDU_BITMAP_PUSCH_PTRS) { + nr_pusch_ptrs_processing(gNB, + frame_parms, + rel15_ul, + ulsch_id, + slot, + symbol, + nb_rb * 12 + rxComp_aligned); + // Subtract total PTRS RE's in the symbol from PUSCH RE's + length -= pusch_vars->ptrs_re_per_slot; + pusch_vars->ul_valid_re_per_slot[symbol] -= pusch_vars->ptrs_re_per_slot; } - // if (rel15_ul->pdu_bit_map & PUSCH_PDU_BITMAP_PUSCH_PTRS) { - // nr_pusch_ptrs_processing(gNB, - // frame_parms, - // rel15_ul, - // ulsch_id, - // slot, - // symbol, - // length); - // // Subtract total PTRS RE's in the symbol from PUSCH RE's - // length -= pusch_vars->ptrs_re_per_slot; - // } if (nb_layer == 2) { @@ -1594,8 +1599,8 @@ void inner_rx (PHY_VARS_gNB *gNB, c16_t *ul_ch_mag1 = rxF_ch_maga[1]; nr_ulsch_compute_ML_llr(pusch_vars, symbol, - rxComp[0], - rxComp[1], + (c16_t*)&pusch_vars->rxdataF_comp[0][symbol * (nb_rb * 12 + rxComp_aligned)], + (c16_t*)&pusch_vars->rxdataF_comp[nb_rx_ant][symbol * (nb_rb * 12 + rxComp_aligned)], ul_ch_mag0, ul_ch_mag1, llr_0, @@ -1608,7 +1613,7 @@ void inner_rx (PHY_VARS_gNB *gNB, else { nr_ulsch_mmse_2layers(frame_parms, - (int **)rxComp, + (int32_t **)pusch_vars->rxdataF_comp, (int **)rxF_ch_maga, (int **)rxF_ch_magb, (int **)rxF_ch_magc, @@ -1624,7 +1629,7 @@ void inner_rx (PHY_VARS_gNB *gNB, } if (nb_layer != 2 || rel15_ul->qam_mod_order >= 6) for (int aatx = 0; aatx < nb_layer; aatx++) - nr_ulsch_compute_llr((int32_t*)rxComp[aatx], + nr_ulsch_compute_llr((int32_t*)&pusch_vars->rxdataF_comp[aatx * nb_rx_ant][symbol * (nb_rb * 12 + rxComp_aligned)], (int32_t*)rxF_ch_maga[aatx], (int32_t*)rxF_ch_magb[aatx], (int32_t*)rxF_ch_magc[aatx], @@ -1646,7 +1651,6 @@ void nr_pusch_symbol_processing_noprecoding(void *arg) int ulsch_id = rdata->ulsch_id; int slot = rdata->slot; NR_gNB_PUSCH *pusch_vars = &gNB->pusch_vars[ulsch_id]; - simde__m64 *s = (simde__m64 *)rdata->s; for (int symbol = rdata->startSymbol; symbol < rdata->startSymbol+rdata->numSymbols; symbol++) { int dmrs_symbol_flag = (rel15_ul->ul_dmrs_symb_pos >> symbol) & 0x01; @@ -1684,21 +1688,22 @@ void nr_pusch_symbol_processing_noprecoding(void *arg) gNB->pusch_vars[ulsch_id].log2_maxh, rdata->nvar); - simde__m64 *llr_ptr_64; + simde__m64 *s = (simde__m64*)rdata->s; + int16_t *llr_ptr; if (rel15_ul->nrOfLayers == 1) - llr_ptr_64 = (simde__m64 *)&rdata->llr_layers[0][pusch_vars->llr_offset[symbol]]; + llr_ptr = &rdata->llr_layers[0][pusch_vars->llr_offset[symbol]]; else { // layer de-mapping - int16_t* llr_cw = &rdata->llr[pusch_vars->llr_offset[symbol] * rel15_ul->nrOfLayers]; + llr_ptr = &rdata->llr[pusch_vars->llr_offset[symbol] * rel15_ul->nrOfLayers]; for (int i = 0; i < (nb_re_pusch); i++) for (int l = 0; l < rel15_ul->nrOfLayers; l++) for (int m = 0; m < rel15_ul->qam_mod_order; m++) - llr_cw[i*rel15_ul->nrOfLayers*rel15_ul->qam_mod_order+l*rel15_ul->qam_mod_order+m] = rdata->llr_layers[l][pusch_vars->llr_offset[symbol] + i*rel15_ul->qam_mod_order+m]; - llr_ptr_64 = (simde__m64 *)llr_cw; + llr_ptr[i*rel15_ul->nrOfLayers*rel15_ul->qam_mod_order+l*rel15_ul->qam_mod_order+m] = rdata->llr_layers[l][pusch_vars->llr_offset[symbol] + i*rel15_ul->qam_mod_order+m]; } // unscrambling - simde__m64 *llr64 = (simde__m64 *) &rdata->llr[pusch_vars->llr_offset[symbol] * rel15_ul->nrOfLayers]; + simde__m64* llr_ptr_64 = (simde__m64*)llr_ptr; + simde__m64 *llr64 = (simde__m64*)&rdata->llr[pusch_vars->llr_offset[symbol] * rel15_ul->nrOfLayers]; for (int i = 0; i < (nb_re_pusch * rel15_ul->qam_mod_order * rel15_ul->nrOfLayers) >> 2; i++) llr64[i] = simde_mm_mullo_pi16(llr_ptr_64[i], s[i]); } From f4beb879cc75f77dae8cf71e9cc0e5d7883fab31 Mon Sep 17 00:00:00 2001 From: Tsung-Yu Chan Date: Fri, 15 Sep 2023 16:33:55 +0200 Subject: [PATCH 09/13] fix / unscrambling runtime error - fix the unscrambling runtime error - avoid to many variables pass by function --- .../PHY/NR_TRANSPORT/nr_transport_proto.h | 38 +- .../PHY/NR_TRANSPORT/nr_ulsch_demodulation.c | 474 ++++++------------ .../NR_TRANSPORT/nr_ulsch_llr_computation.c | 15 +- openair1/SCHED_NR/phy_procedures_nr_gNB.c | 2 - 4 files changed, 158 insertions(+), 371 deletions(-) diff --git a/openair1/PHY/NR_TRANSPORT/nr_transport_proto.h b/openair1/PHY/NR_TRANSPORT/nr_transport_proto.h index 5d25f623df..ab25bd7744 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_transport_proto.h +++ b/openair1/PHY/NR_TRANSPORT/nr_transport_proto.h @@ -125,40 +125,17 @@ void free_gNB_dlsch(NR_gNB_DLSCH_t *dlsch, uint16_t N_RB, const NR_DL_FRAME_PARM @param slot Slot number @param harq_pid HARQ process ID */ -void nr_rx_pusch(PHY_VARS_gNB *gNB, - uint8_t UE_id, - uint32_t frame, - uint8_t slot, - unsigned char harq_pid); - int nr_rx_pusch_tp(PHY_VARS_gNB *gNB, uint8_t ulsch_id, uint32_t frame, uint8_t slot, unsigned char harq_pid); -/** \brief This function performs RB extraction (signal and channel estimates) (currently signal only until channel estimation and compensation are implemented) - @param rxdataF pointer to the received frequency domain signal - @param rxdataF_ext pointer to the extracted frequency domain signal - @param rb_alloc RB allocation map (used for Resource Allocation Type 0 in NR) - @param symbol Symbol on which to act (within-in nr_TTI_rx) - @param start_rb The starting RB in the RB allocation (used for Resource Allocation Type 1 in NR) - @param nb_rb_pusch The number of RBs allocated (used for Resource Allocation Type 1 in NR) - @param frame_parms, Pointer to frame descriptor structure -*/ -void nr_ulsch_extract_rbs(c16_t **rxdataF, - NR_gNB_PUSCH *pusch_vars, - int slot, - unsigned char symbol, - uint8_t is_dmrs_symbol, - nfapi_nr_pusch_pdu_t *pusch_pdu, - NR_DL_FRAME_PARMS *frame_parms); - void nr_ulsch_scale_channel(int32_t **ul_ch_estimates_ext, NR_DL_FRAME_PARMS *frame_parms, NR_gNB_ULSCH_t *ulsch_gNB, uint8_t symbol, - uint8_t is_dmrs_symbol, + uint8_t is_dmrs_symbol, uint32_t len, uint8_t nrOfLayers, uint16_t nb_rb, @@ -186,7 +163,11 @@ void nr_ulsch_channel_level(int **ul_ch_estimates_ext, */ void nr_idft(int32_t *z, uint32_t Msc_PUSCH); -void nr_ulsch_qpsk_qpsk(c16_t *stream0_in, c16_t *stream1_in, c16_t *stream0_out, c16_t *rho01, uint32_t length); +void nr_ulsch_qpsk_qpsk(c16_t *stream0_in, + c16_t *stream1_in, + c16_t *stream0_out, + c16_t *rho01, + uint32_t length); void nr_ulsch_qam16_qam16(c16_t *stream0_in, c16_t *stream1_in, @@ -211,7 +192,7 @@ void nr_ulsch_qam64_qam64(c16_t *stream0_in, @param symbol OFDM symbol index in sub-frame */ void nr_ulsch_qpsk_llr(int32_t *rxdataF_comp, - int16_t *ulsch_llr, + int16_t *ulsch_llr, uint32_t nb_re, uint8_t symbol); @@ -226,11 +207,9 @@ void nr_ulsch_qpsk_llr(int32_t *rxdataF_comp, void nr_ulsch_16qam_llr(int32_t *rxdataF_comp, int32_t **ul_ch_mag, int16_t *ulsch_llr, - uint32_t nb_rb, uint32_t nb_re, uint8_t symbol); - /** \brief This function generates log-likelihood ratios (decoder input) for single-stream 64 QAM received waveforms. @param rxdataF_comp Compensated channel output @param ul_ch_mag uplink channel magnitude multiplied by the 1st amplitude threshold in QAM 64 @@ -243,7 +222,6 @@ void nr_ulsch_64qam_llr(int32_t *rxdataF_comp, int32_t **ul_ch_mag, int32_t **ul_ch_magb, int16_t *ulsch_llr, - uint32_t nb_rb, uint32_t nb_re, uint8_t symbol); @@ -261,7 +239,6 @@ void nr_ulsch_256qam_llr(int32_t *rxdataF_comp, int32_t **ul_ch_magb, int32_t **ul_ch_magc, int16_t *ulsch_llr, - uint32_t nb_rb, uint32_t nb_re, uint8_t symbol); @@ -279,7 +256,6 @@ void nr_ulsch_compute_llr(int32_t *rxdataF_comp, int32_t *ul_ch_magb, int32_t *ul_ch_magc, int16_t *ulsch_llr, - uint32_t nb_rb, uint32_t nb_re, uint8_t symbol, uint8_t mod_order); diff --git a/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c b/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c index ad4d696d4d..d22ad70ee5 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c +++ b/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c @@ -11,11 +11,6 @@ #include #include "PHY/sse_intrin.h" -//#define DEBUG_CH_COMP -//#define DEBUG_RB_EXT -//#define DEBUG_CH_MAG -//#define ML_DEBUG - #define INVALID_VALUE 255 void nr_idft(int32_t *z, uint32_t Msc_PUSCH) @@ -281,7 +276,7 @@ void nr_idft(int32_t *z, uint32_t Msc_PUSCH) } -static void nr_ulsch_extract_rbs0 (c16_t* const rxdataF, +static void nr_ulsch_extract_rbs (c16_t* const rxdataF, c16_t* const chF, c16_t *rxFext, c16_t *chFext, @@ -385,99 +380,6 @@ static void nr_ulsch_extract_rbs0 (c16_t* const rxdataF, } } -void nr_ulsch_extract_rbs(c16_t **rxdataF, - NR_gNB_PUSCH *pusch_vars, - int slot, - unsigned char symbol, - uint8_t is_dmrs_symbol, - nfapi_nr_pusch_pdu_t *pusch_pdu, - NR_DL_FRAME_PARMS *frame_parms) { - - unsigned short start_re, re, nb_re_pusch; - unsigned char aarx, aatx; - uint32_t rxF_ext_index = 0; - uint32_t ul_ch0_ext_index = 0; - uint32_t ul_ch0_index = 0; - int16_t *rxF,*rxF_ext; - int *ul_ch0,*ul_ch0_ext; - int soffset = (slot&3)*frame_parms->symbols_per_slot*frame_parms->ofdm_symbol_size; - -#ifdef DEBUG_RB_EXT - printf("--------------------symbol = %d-----------------------\n", symbol); - printf("--------------------ch_ext_index = %d-----------------------\n", symbol*NR_NB_SC_PER_RB * pusch_pdu->rb_size); -#endif - - uint8_t is_data_re; - start_re = (frame_parms->first_carrier_offset + (pusch_pdu->rb_start + pusch_pdu->bwp_start) * NR_NB_SC_PER_RB)%frame_parms->ofdm_symbol_size; - nb_re_pusch = NR_NB_SC_PER_RB * pusch_pdu->rb_size; - - int nb_re_pusch2 = nb_re_pusch + (nb_re_pusch&7); - - for (aarx = 0; aarx < frame_parms->nb_antennas_rx; aarx++) { - - rxF = (int16_t *)&rxdataF[aarx][soffset+(symbol * frame_parms->ofdm_symbol_size)]; - rxF_ext = (int16_t *)&pusch_vars->rxdataF_ext[aarx][symbol * nb_re_pusch2]; // [hna] rxdataF_ext isn't contiguous in order to solve an alignment problem ib llr computation in case of mod_order = 4, 6 - - if (is_dmrs_symbol == 0) { - if (start_re + nb_re_pusch <= frame_parms->ofdm_symbol_size) { - memcpy((void*)rxF_ext, (void*)&rxF[start_re*2], nb_re_pusch*sizeof(int32_t)); - } else { - int neg_length = frame_parms->ofdm_symbol_size-start_re; - int pos_length = nb_re_pusch-neg_length; - memcpy((void*)rxF_ext,(void*)&rxF[start_re*2],neg_length*sizeof(int32_t)); - memcpy((void*)&rxF_ext[2*neg_length],(void*)rxF,pos_length*sizeof(int32_t)); - } - - for (aatx = 0; aatx < pusch_pdu->nrOfLayers; aatx++) { - ul_ch0 = &pusch_vars->ul_ch_estimates[aatx*frame_parms->nb_antennas_rx+aarx][pusch_vars->dmrs_symbol*frame_parms->ofdm_symbol_size]; // update channel estimates if new dmrs symbol are available - ul_ch0_ext = &pusch_vars->ul_ch_estimates_ext[aatx*frame_parms->nb_antennas_rx+aarx][symbol*nb_re_pusch2]; - memcpy((void*)ul_ch0_ext,(void*)ul_ch0,nb_re_pusch*sizeof(int32_t)); - } - - } else { - - for (aatx = 0; aatx < pusch_pdu->nrOfLayers; aatx++) { - ul_ch0 = &pusch_vars->ul_ch_estimates[aatx*frame_parms->nb_antennas_rx+aarx][pusch_vars->dmrs_symbol*frame_parms->ofdm_symbol_size]; // update channel estimates if new dmrs symbol are available - ul_ch0_ext = &pusch_vars->ul_ch_estimates_ext[aatx*frame_parms->nb_antennas_rx+aarx][symbol*nb_re_pusch2]; - - rxF_ext_index = 0; - ul_ch0_ext_index = 0; - ul_ch0_index = 0; - for (re = 0; re < nb_re_pusch; re++) { - uint16_t k = start_re + re; - is_data_re = allowed_xlsch_re_in_dmrs_symbol(k, start_re, frame_parms->ofdm_symbol_size, pusch_pdu->num_dmrs_cdm_grps_no_data, pusch_pdu->dmrs_config_type); - if (++k >= frame_parms->ofdm_symbol_size) { - k -= frame_parms->ofdm_symbol_size; - } - - #ifdef DEBUG_RB_EXT - printf("re = %d, is_dmrs_symbol = %d, symbol = %d\n", re, is_dmrs_symbol, symbol); - #endif - - // save only data and respective channel estimates - if (is_data_re == 1) { - if (aatx == 0) { - rxF_ext[rxF_ext_index] = (rxF[ ((start_re + re)*2) % (frame_parms->ofdm_symbol_size*2)]); - rxF_ext[rxF_ext_index + 1] = (rxF[(((start_re + re)*2) + 1) % (frame_parms->ofdm_symbol_size*2)]); - rxF_ext_index +=2; - } - - ul_ch0_ext[ul_ch0_ext_index] = ul_ch0[ul_ch0_index]; - ul_ch0_ext_index++; - - #ifdef DEBUG_RB_EXT - printf("dmrs symb %d: rxF_ext[%u] = (%d,%d), ul_ch0_ext[%u] = (%d,%d)\n", - is_dmrs_symbol,rxF_ext_index>>1, rxF_ext[rxF_ext_index],rxF_ext[rxF_ext_index+1], - ul_ch0_ext_index, ((int16_t*)&ul_ch0_ext[ul_ch0_ext_index])[0], ((int16_t*)&ul_ch0_ext[ul_ch0_ext_index])[1]); - #endif - } - ul_ch0_index++; - } - } - } - } -} - void nr_ulsch_scale_channel(int **ul_ch_estimates_ext, NR_DL_FRAME_PARMS *frame_parms, NR_gNB_ULSCH_t *ulsch_gNB, @@ -607,14 +509,15 @@ void nr_ulsch_channel_compensation(c16_t *rxFext, int32_t **rxComp, c16_t *rho, NR_DL_FRAME_PARMS *frame_parms, + nfapi_nr_pusch_pdu_t* rel15_ul, uint32_t symbol, - uint32_t length, - uint32_t mod_order, - uint8_t nrOfLayers, - uint8_t nb_rx_ant, - uint32_t nb_rb, - uint32_t output_shift) + uint32_t buffer_length, + uint32_t output_shift) { + int mod_order = rel15_ul->qam_mod_order; + int nrOfLayers = rel15_ul->nrOfLayers; + int nb_rx_ant = frame_parms->nb_antennas_rx; + simde__m256i QAM_ampa_256 = simde_mm256_setzero_si256(); simde__m256i QAM_ampb_256 = simde_mm256_setzero_si256(); simde__m256i QAM_ampc_256 = simde_mm256_setzero_si256(); @@ -639,18 +542,16 @@ void nr_ulsch_channel_compensation(c16_t *rxFext, simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2); simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1); - int length_aligned = (length % 8) ? (8 - length % 8) : 0; - int rxComp_aligned = ((nb_rb * 12) % 8) ? (8 - (nb_rb * 12) % 8) : 0; for (int aatx = 0; aatx < nrOfLayers; aatx++) { - simde__m256i *rxComp_256 = (simde__m256i*) &rxComp[aatx * nb_rx_ant][symbol * (nb_rb * 12 + rxComp_aligned)]; - simde__m256i *rxF_ch_maga_256 = (simde__m256i*)&ul_ch_maga[aatx * (length + length_aligned)]; - simde__m256i *rxF_ch_magb_256 = (simde__m256i*)&ul_ch_magb[aatx * (length + length_aligned)]; - simde__m256i *rxF_ch_magc_256 = (simde__m256i*)&ul_ch_magc[aatx * (length + length_aligned)]; + simde__m256i *rxComp_256 = (simde__m256i*) &rxComp[aatx * nb_rx_ant][symbol * buffer_length]; + simde__m256i *rxF_ch_maga_256 = (simde__m256i*)&ul_ch_maga[aatx * buffer_length]; + simde__m256i *rxF_ch_magb_256 = (simde__m256i*)&ul_ch_magb[aatx * buffer_length]; + simde__m256i *rxF_ch_magc_256 = (simde__m256i*)&ul_ch_magc[aatx * buffer_length]; for (int aarx = 0; aarx < nb_rx_ant; aarx++) { - simde__m256i *rxF_256 = (simde__m256i*) &rxFext[aarx * (length + length_aligned)]; - simde__m256i *chF_256 = (simde__m256i*) &chFext[(aatx * nb_rx_ant + aarx) * (length + length_aligned)]; + simde__m256i *rxF_256 = (simde__m256i*) &rxFext[aarx * buffer_length]; + simde__m256i *chF_256 = (simde__m256i*) &chFext[(aatx * nb_rx_ant + aarx) * buffer_length]; - for (int i = 0; i < (length>>3) + ((length&7)?1:0); i++) + for (int i = 0; i < buffer_length >> 3; i++) { xmmp0 = simde_mm256_madd_epi16(chF_256[i], rxF_256[i]); // xmmp0 contains real part of 4 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i] @@ -682,28 +583,25 @@ void nr_ulsch_channel_compensation(c16_t *rxFext, if (mod_order > 6) rxF_ch_magc_256[i] = simde_mm256_add_epi16(rxF_ch_magc_256[i], xmmp1); } - if (rho != NULL) - { + if (rho != NULL) { for (int atx = 0; atx < nrOfLayers; atx++) { - simde__m256i mmtmpD0, mmtmpD1, mmtmpD2, mmtmpD3; - simde__m256i *rho_256 = (simde__m256i * )&rho[(aatx * nrOfLayers + atx) * (length + length_aligned)]; - simde__m256i *chF_256 = (simde__m256i *)&chFext[(aatx * nb_rx_ant + aarx) * (length + length_aligned)]; - simde__m256i *chF2_256 = (simde__m256i *)&chFext[ (atx * nb_rx_ant + aarx) * (length + length_aligned)]; - for (int i = 0; i < (length >> 3)+((length&7)?1:0); i++) - { + simde__m256i *rho_256 = (simde__m256i * )&rho[(aatx * nrOfLayers + atx) * buffer_length]; + simde__m256i *chF_256 = (simde__m256i *)&chFext[(aatx * nb_rx_ant + aarx) * buffer_length]; + simde__m256i *chF2_256 = (simde__m256i *)&chFext[ (atx * nb_rx_ant + aarx) * buffer_length]; + for (int i = 0; i < buffer_length >> 3; i++) { // multiply by conjugated channel - mmtmpD0 = simde_mm256_madd_epi16(chF_256[i], chF2_256[i]); - // mmtmpD0 contains real part of 4 consecutive outputs (32-bit) - mmtmpD1 = simde_mm256_shuffle_epi8(chF_256[i], complex_shuffle256); - mmtmpD1 = simde_mm256_sign_epi16(mmtmpD1, conj256); - mmtmpD1 = simde_mm256_madd_epi16(mmtmpD1, chF2_256[i]); - // mmtmpD1 contains imag part of 4 consecutive outputs (32-bit) - mmtmpD0 = simde_mm256_srai_epi32(mmtmpD0, output_shift); - mmtmpD1 = simde_mm256_srai_epi32(mmtmpD1, output_shift); - mmtmpD2 = simde_mm256_unpacklo_epi32(mmtmpD0, mmtmpD1); - mmtmpD3 = simde_mm256_unpackhi_epi32(mmtmpD0, mmtmpD1); + xmmp0 = simde_mm256_madd_epi16(chF_256[i], chF2_256[i]); + // xmmp0 contains real part of 4 consecutive outputs (32-bit) + xmmp1 = simde_mm256_shuffle_epi8(chF_256[i], complex_shuffle256); + xmmp1 = simde_mm256_sign_epi16(xmmp1, conj256); + xmmp1 = simde_mm256_madd_epi16(xmmp1, chF2_256[i]); + // xmmp0 contains imag part of 4 consecutive outputs (32-bit) + xmmp0 = simde_mm256_srai_epi32(xmmp0, output_shift); + xmmp1 = simde_mm256_srai_epi32(xmmp1, output_shift); + xmmp2 = simde_mm256_unpacklo_epi32(xmmp0, xmmp1); + xmmp3 = simde_mm256_unpackhi_epi32(xmmp0, xmmp1); - rho_256[i] = simde_mm256_adds_epi16(rho_256[i], simde_mm256_packs_epi32(mmtmpD2, mmtmpD3)); + rho_256[i] = simde_mm256_adds_epi16(rho_256[i], simde_mm256_packs_epi32(xmmp2, xmmp3)); } } } @@ -714,79 +612,15 @@ void nr_ulsch_channel_compensation(c16_t *rxFext, simde_m_empty(); } -void nr_ulsch_detection_mrc(NR_DL_FRAME_PARMS *frame_parms, - int32_t **rxdataF_comp, - int32_t **ul_ch_mag, - int32_t **ul_ch_magb, - int32_t **ul_ch_magc, - int32_t ***rho, - uint8_t nrOfLayers, - uint8_t symbol, - uint16_t nb_rb, - int length) { - int n_rx = frame_parms->nb_antennas_rx; - simde__m128i *rxdataF_comp128[2],*ul_ch_mag128[2],*ul_ch_mag128b[2],*ul_ch_mag128c[2]; - int32_t i; - uint32_t nb_rb_0 = length/12 + ((length%12)?1:0); - - int off = ((nb_rb&1) == 1)? 4:0; - - if (n_rx > 1) { - - int nb_re = nb_rb * 12; - - for (int aatx = 0; aatx < nrOfLayers; aatx++) { - - rxdataF_comp128[0] = (simde__m128i *)&rxdataF_comp[aatx*frame_parms->nb_antennas_rx][(symbol*(nb_re + off))]; - ul_ch_mag128[0] = (simde__m128i *)&ul_ch_mag[aatx*frame_parms->nb_antennas_rx][(symbol*(nb_re + off))]; - ul_ch_mag128b[0] = (simde__m128i *)&ul_ch_magb[aatx*frame_parms->nb_antennas_rx][(symbol*(nb_re + off))]; - ul_ch_mag128c[0] = (simde__m128i *)&ul_ch_magc[aatx*frame_parms->nb_antennas_rx][(symbol*(nb_re + off))]; - - for (int aa=1;aa < n_rx;aa++) { - rxdataF_comp128[1] = (simde__m128i *)&rxdataF_comp[aatx*frame_parms->nb_antennas_rx+aa][(symbol*(nb_re + off))]; - ul_ch_mag128[1] = (simde__m128i *)&ul_ch_mag[aatx*frame_parms->nb_antennas_rx+aa][(symbol*(nb_re + off))]; - ul_ch_mag128b[1] = (simde__m128i *)&ul_ch_magb[aatx*frame_parms->nb_antennas_rx+aa][(symbol*(nb_re + off))]; - ul_ch_mag128c[1] = (simde__m128i *)&ul_ch_magc[aatx*frame_parms->nb_antennas_rx+aa][(symbol*(nb_re + off))]; - - // MRC on each re of rb, both on MF output and magnitude (for 16QAM/64QAM llr computation) - for (i=0; inrOfLayers; + int nb_rx_ant = frame_parms->nb_antennas_rx; + int dmrs_symbol_flag = (rel15_ul->ul_dmrs_symb_pos >> symbol) & 0x01; + + int buffer_length = rel15_ul->rb_size * NR_NB_SC_PER_RB; + if (buffer_length & 7) + buffer_length += (8 - buffer_length%8); - c16_t rxFext[nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); - c16_t chFext[nb_layer][nb_rx_ant][length + length_aligned] __attribute__((aligned(32))); + c16_t rxFext[nb_rx_ant][buffer_length] __attribute__((aligned(32))); + c16_t chFext[nb_layer][nb_rx_ant][buffer_length] __attribute__((aligned(32))); + + memset(rxFext, 0, sizeof(c16_t) * nb_rx_ant * buffer_length); + memset(chFext, 0, sizeof(c16_t) * nb_layer * nb_rx_ant* buffer_length); + for (int aarx = 0; aarx < nb_rx_ant; aarx++) { for (int aatx = 0; aatx < nb_layer; aatx++) { - nr_ulsch_extract_rbs0(rxF[aarx], + nr_ulsch_extract_rbs(rxF[aarx], (c16_t *)pusch_vars->ul_ch_estimates[aatx * nb_rx_ant + aarx], rxFext[aarx], chFext[aatx][aarx], @@ -1534,46 +1376,40 @@ static void inner_rx (PHY_VARS_gNB *gNB, frame_parms); } } - c16_t rho[nb_layer][nb_layer][length + length_aligned] __attribute__((aligned(32))); - c16_t rxComp [nb_layer][length + length_aligned] __attribute__((aligned(32))); - c16_t rxF_ch_maga[nb_layer][length + length_aligned] __attribute__((aligned(32))); - c16_t rxF_ch_magb[nb_layer][length + length_aligned] __attribute__((aligned(32))); - c16_t rxF_ch_magc[nb_layer][length + length_aligned] __attribute__((aligned(32))); + c16_t rho[nb_layer][nb_layer][buffer_length] __attribute__((aligned(32))); + c16_t rxF_ch_maga [nb_layer][buffer_length] __attribute__((aligned(32))); + c16_t rxF_ch_magb [nb_layer][buffer_length] __attribute__((aligned(32))); + c16_t rxF_ch_magc [nb_layer][buffer_length] __attribute__((aligned(32))); - int rxComp_aligned = ((nb_rb * 12) % 8) ? (8 - (nb_rb * 12) % 8) : 0; - memset(rho, 0, sizeof(c16_t) * nb_layer * nb_layer* (length + length_aligned)); - memset(rxComp, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); - memset(rxF_ch_maga, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); - memset(rxF_ch_magb, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); - memset(rxF_ch_magc, 0, sizeof(c16_t) * nb_layer * (length + length_aligned)); + memset(rho, 0, sizeof(c16_t) * nb_layer * nb_layer* buffer_length); + memset(rxF_ch_maga, 0, sizeof(c16_t) * nb_layer * buffer_length); + memset(rxF_ch_magb, 0, sizeof(c16_t) * nb_layer * buffer_length); + memset(rxF_ch_magc, 0, sizeof(c16_t) * nb_layer * buffer_length); for (int i = 0; i < nb_layer; i++) - memset(&pusch_vars->rxdataF_comp[i*nb_rx_ant][symbol * (nb_rb * 12 + rxComp_aligned)], 0, sizeof(int32_t) * (nb_rb * 12 + rxComp_aligned)); + memset(&pusch_vars->rxdataF_comp[i*nb_rx_ant][symbol * buffer_length], 0, sizeof(int32_t) * buffer_length); nr_ulsch_channel_compensation((c16_t*)rxFext, (c16_t*)chFext, (c16_t*)rxF_ch_maga, (c16_t*)rxF_ch_magb, (c16_t*)rxF_ch_magc, - pusch_vars->rxdataF_comp, // (c16_t*)rxComp, + pusch_vars->rxdataF_comp, (nb_layer == 1) ? NULL : (c16_t*)rho, frame_parms, + rel15_ul, symbol, - length, - rel15_ul->qam_mod_order, - rel15_ul->nrOfLayers, - nb_rx_ant, - rel15_ul->rb_size, + buffer_length, output_shift); if (nb_layer == 1 && rel15_ul->transform_precoding == transformPrecoder_enabled && rel15_ul->qam_mod_order <= 6) { nr_freq_equalization(frame_parms, - (int *)&pusch_vars->rxdataF_comp[0][symbol * (nb_rb * 12 + rxComp_aligned)], - (int *)rxF_ch_maga, - (int *)rxF_ch_magb, - symbol, - length, - rel15_ul->qam_mod_order); - nr_idft((int32_t*)&pusch_vars->rxdataF_comp[0][symbol * (nb_rb * 12 + rxComp_aligned)], length); + &pusch_vars->rxdataF_comp[0][symbol * buffer_length], + (int *)rxF_ch_maga, + (int *)rxF_ch_magb, + symbol, + pusch_vars->ul_valid_re_per_slot[symbol], + rel15_ul->qam_mod_order); + nr_idft(&pusch_vars->rxdataF_comp[0][symbol * buffer_length], pusch_vars->ul_valid_re_per_slot[symbol]); } if (rel15_ul->pdu_bit_map & PUSCH_PDU_BITMAP_PUSCH_PTRS) { nr_pusch_ptrs_processing(gNB, @@ -1582,36 +1418,26 @@ static void inner_rx (PHY_VARS_gNB *gNB, ulsch_id, slot, symbol, - nb_rb * 12 + rxComp_aligned); - // Subtract total PTRS RE's in the symbol from PUSCH RE's - length -= pusch_vars->ptrs_re_per_slot; + buffer_length); pusch_vars->ul_valid_re_per_slot[symbol] -= pusch_vars->ptrs_re_per_slot; } - if (nb_layer == 2) - { + if (nb_layer == 2) { if (rel15_ul->qam_mod_order < 6) { - c16_t *rho0 = rho[0][1]; - c16_t *rho1 = rho[1][0]; - c16_t *llr_0 = (c16_t *)&llr[0][pusch_vars->llr_offset[symbol]]; - c16_t *llr_1 = (c16_t *)&llr[1][pusch_vars->llr_offset[symbol]]; - c16_t *ul_ch_mag0 = rxF_ch_maga[0]; - c16_t *ul_ch_mag1 = rxF_ch_maga[1]; nr_ulsch_compute_ML_llr(pusch_vars, symbol, - (c16_t*)&pusch_vars->rxdataF_comp[0][symbol * (nb_rb * 12 + rxComp_aligned)], - (c16_t*)&pusch_vars->rxdataF_comp[nb_rx_ant][symbol * (nb_rb * 12 + rxComp_aligned)], - ul_ch_mag0, - ul_ch_mag1, - llr_0, - llr_1, - rho0, - rho1, - length, + (c16_t*)&pusch_vars->rxdataF_comp[0][symbol * buffer_length], + (c16_t*)&pusch_vars->rxdataF_comp[nb_rx_ant][symbol * buffer_length], + rxF_ch_maga[0], + rxF_ch_maga[1], + (c16_t*)&llr[0][pusch_vars->llr_offset[symbol]], + (c16_t*)&llr[1][pusch_vars->llr_offset[symbol]], + rho[0][1], + rho[1][0], + pusch_vars->ul_valid_re_per_slot[symbol], rel15_ul->qam_mod_order); } - else - { + else { nr_ulsch_mmse_2layers(frame_parms, (int32_t **)pusch_vars->rxdataF_comp, (int **)rxF_ch_maga, @@ -1623,26 +1449,25 @@ static void inner_rx (PHY_VARS_gNB *gNB, rel15_ul->qam_mod_order, pusch_vars->log2_maxh, symbol, - length, - nvar); + pusch_vars->ul_valid_re_per_slot[symbol], + nvar, + buffer_length); } } if (nb_layer != 2 || rel15_ul->qam_mod_order >= 6) for (int aatx = 0; aatx < nb_layer; aatx++) - nr_ulsch_compute_llr((int32_t*)&pusch_vars->rxdataF_comp[aatx * nb_rx_ant][symbol * (nb_rb * 12 + rxComp_aligned)], + nr_ulsch_compute_llr((int32_t*)&pusch_vars->rxdataF_comp[aatx * nb_rx_ant][symbol * buffer_length], (int32_t*)rxF_ch_maga[aatx], (int32_t*)rxF_ch_magb[aatx], (int32_t*)rxF_ch_magc[aatx], &llr[aatx][pusch_vars->llr_offset[symbol]], - rel15_ul->rb_size, - length, + pusch_vars->ul_valid_re_per_slot[symbol], symbol, rel15_ul->qam_mod_order); } -void nr_pusch_symbol_processing_noprecoding(void *arg) +void nr_pusch_symbol_processing(void *arg) { - puschSymbolProc_t *rdata=(puschSymbolProc_t*)arg; PHY_VARS_gNB *gNB = rdata->gNB; @@ -1651,12 +1476,8 @@ void nr_pusch_symbol_processing_noprecoding(void *arg) int ulsch_id = rdata->ulsch_id; int slot = rdata->slot; NR_gNB_PUSCH *pusch_vars = &gNB->pusch_vars[ulsch_id]; - for (int symbol = rdata->startSymbol; symbol < rdata->startSymbol+rdata->numSymbols; symbol++) - { + for (int symbol = rdata->startSymbol; symbol < rdata->startSymbol+rdata->numSymbols; symbol++) { int dmrs_symbol_flag = (rel15_ul->ul_dmrs_symb_pos >> symbol) & 0x01; - int nb_re_pusch = gNB->pusch_vars[ulsch_id].ul_valid_re_per_slot[symbol]; - // this needs to be reworded for parrellization, we need a table which give dmrs symbol location - // used for chennel estimate, they are being run in parallel! if (dmrs_symbol_flag == 1) { if ((rel15_ul->ul_dmrs_symb_pos >> ((symbol + 1) % frame_parms->symbols_per_slot)) & 0x01) @@ -1664,9 +1485,7 @@ void nr_pusch_symbol_processing_noprecoding(void *arg) gNB->pusch_vars[ulsch_id].dmrs_symbol = symbol; } - LOG_I(PHY,"symbol %d: nb_re_pusch %d, DMRS symbl used for Chest :%d \n", symbol, nb_re_pusch, gNB->pusch_vars[ulsch_id].dmrs_symbol); - - if (nb_re_pusch == 0) + if (gNB->pusch_vars[ulsch_id].ul_valid_re_per_slot[symbol] == 0) continue; int soffset = (slot&3)*frame_parms->symbols_per_slot*frame_parms->ofdm_symbol_size; inner_rx(gNB, @@ -1678,23 +1497,16 @@ void nr_pusch_symbol_processing_noprecoding(void *arg) gNB->common_vars.rxdataF, (c16_t**)gNB->pusch_vars[ulsch_id].ul_ch_estimates, rdata->llr_layers, - rel15_ul->nrOfLayers, - frame_parms->nb_antennas_rx, soffset, - nb_re_pusch, // length - symbol, // symbol index - rel15_ul->rb_size, // ofdm size - dmrs_symbol_flag, + gNB->pusch_vars[ulsch_id].ul_valid_re_per_slot[symbol], + symbol, gNB->pusch_vars[ulsch_id].log2_maxh, rdata->nvar); - simde__m64 *s = (simde__m64*)rdata->s; - int16_t *llr_ptr; - if (rel15_ul->nrOfLayers == 1) - llr_ptr = &rdata->llr_layers[0][pusch_vars->llr_offset[symbol]]; - else - { - // layer de-mapping + int nb_re_pusch = gNB->pusch_vars[ulsch_id].ul_valid_re_per_slot[symbol]; + // layer de-mapping + int16_t* llr_ptr = &rdata->llr_layers[0][pusch_vars->llr_offset[symbol]]; + if (rel15_ul->nrOfLayers != 1) { llr_ptr = &rdata->llr[pusch_vars->llr_offset[symbol] * rel15_ul->nrOfLayers]; for (int i = 0; i < (nb_re_pusch); i++) for (int l = 0; l < rel15_ul->nrOfLayers; l++) @@ -1702,10 +1514,9 @@ void nr_pusch_symbol_processing_noprecoding(void *arg) llr_ptr[i*rel15_ul->nrOfLayers*rel15_ul->qam_mod_order+l*rel15_ul->qam_mod_order+m] = rdata->llr_layers[l][pusch_vars->llr_offset[symbol] + i*rel15_ul->qam_mod_order+m]; } // unscrambling - simde__m64* llr_ptr_64 = (simde__m64*)llr_ptr; - simde__m64 *llr64 = (simde__m64*)&rdata->llr[pusch_vars->llr_offset[symbol] * rel15_ul->nrOfLayers]; - for (int i = 0; i < (nb_re_pusch * rel15_ul->qam_mod_order * rel15_ul->nrOfLayers) >> 2; i++) - llr64[i] = simde_mm_mullo_pi16(llr_ptr_64[i], s[i]); + int16_t *llr16 = (int16_t*)&rdata->llr[pusch_vars->llr_offset[symbol] * rel15_ul->nrOfLayers]; + for (int i = 0; i < (nb_re_pusch * rel15_ul->qam_mod_order * rel15_ul->nrOfLayers); i++) + llr16[i] = llr_ptr[i] * rdata->s[i]; } } @@ -1843,13 +1654,21 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB, start_meas(&gNB->ulsch_rbs_extraction_stats); // extract the first dmrs for the channel level computation // extract the data in the OFDM frame, to the start of the array - nr_ulsch_extract_rbs(gNB->common_vars.rxdataF, - pusch_vars, - slot, - meas_symbol, - (rel15_ul->ul_dmrs_symb_pos >> meas_symbol) & 0x01, - rel15_ul, - frame_parms); + int soffset = (slot&3)*frame_parms->symbols_per_slot*frame_parms->ofdm_symbol_size; + int nb_re_pusch2 = nb_re_pusch + (nb_re_pusch&7); + for (int aarx = 0; aarx < frame_parms->nb_antennas_rx; aarx++) + for (int aatx = 0; aatx < rel15_ul->nrOfLayers; aatx++) + nr_ulsch_extract_rbs(gNB->common_vars.rxdataF[aarx], + (c16_t *)pusch_vars->ul_ch_estimates[aatx * frame_parms->nb_antennas_rx + aarx], + (c16_t*)&pusch_vars->rxdataF_ext[aarx][meas_symbol * nb_re_pusch2], + (c16_t*)&pusch_vars->ul_ch_estimates_ext[aatx*frame_parms->nb_antennas_rx+aarx][meas_symbol*nb_re_pusch2], + soffset + meas_symbol * frame_parms->ofdm_symbol_size, + pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, + aarx, + (rel15_ul->ul_dmrs_symb_pos >> meas_symbol) & 0x01, + rel15_ul, + frame_parms); + stop_meas(&gNB->ulsch_rbs_extraction_stats); int avgs = 0; @@ -1901,19 +1720,17 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB, symbol += numSymbols) { int total_res = 0; - for (int s = 0; s < numSymbols;s++) - { + for (int s = 0; s < numSymbols;s++) { pusch_vars->ul_valid_re_per_slot[symbol+s] = get_nb_re_pusch(frame_parms,rel15_ul,symbol+s); pusch_vars->llr_offset[symbol+s] = ((symbol+s) == rel15_ul->start_symbol_index) ? 0 : pusch_vars->llr_offset[symbol+s-1] + pusch_vars->ul_valid_re_per_slot[symbol+s-1] * rel15_ul->qam_mod_order; total_res+=pusch_vars->ul_valid_re_per_slot[symbol+s]; } - if (total_res > 0) - { + if (total_res > 0) { union puschSymbolReqUnion id = {.s={ulsch_id,frame,slot,0}}; id.p=1+symbol; - notifiedFIFO_elt_t *req = newNotifiedFIFO_elt(sizeof(puschSymbolProc_t), id.p, &gNB->respPuschSymb, &nr_pusch_symbol_processing_noprecoding); // create a job for Tpool + notifiedFIFO_elt_t *req = newNotifiedFIFO_elt(sizeof(puschSymbolProc_t), id.p, &gNB->respPuschSymb, &nr_pusch_symbol_processing); // create a job for Tpool puschSymbolProc_t *rdata = (puschSymbolProc_t*)NotifiedFifoData(req); // data for the job rdata->gNB = gNB; @@ -1928,15 +1745,18 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB, rdata->s = &s[pusch_vars->llr_offset[symbol]*rel15_ul->nrOfLayers]; rdata->nvar = nvar; - pushTpool(&gNB->threadPool, req); - gNB->nbSymb++; + if (rel15_ul->pdu_bit_map & PUSCH_PDU_BITMAP_PUSCH_PTRS) { + nr_pusch_symbol_processing(rdata); + } else { + pushTpool(&gNB->threadPool, req); + gNB->nbSymb++; + } LOG_D(PHY,"%d.%d Added symbol %d (count %d) to process, in pipe\n",frame,slot,symbol,gNB->nbSymb); } } // symbol loop - while (gNB->nbSymb > 0) - { + while (gNB->nbSymb > 0 && !(rel15_ul->pdu_bit_map & PUSCH_PDU_BITMAP_PUSCH_PTRS)) { notifiedFIFO_elt_t *req = pullTpool(&gNB->respPuschSymb, &gNB->threadPool); gNB->nbSymb--; delNotifiedFIFO_elt(req); diff --git a/openair1/PHY/NR_TRANSPORT/nr_ulsch_llr_computation.c b/openair1/PHY/NR_TRANSPORT/nr_ulsch_llr_computation.c index 476c9d07bf..a5f4a84a90 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_ulsch_llr_computation.c +++ b/openair1/PHY/NR_TRANSPORT/nr_ulsch_llr_computation.c @@ -60,7 +60,6 @@ void nr_ulsch_qpsk_llr(int32_t *rxdataF_comp, void nr_ulsch_16qam_llr(int32_t *rxdataF_comp, int32_t *ul_ch_mag, int16_t *ulsch_llr, - uint32_t nb_rb, uint32_t nb_re, uint8_t symbol) { @@ -96,7 +95,6 @@ void nr_ulsch_64qam_llr(int32_t *rxdataF_comp, int32_t *ul_ch_mag, int32_t *ul_ch_magb, int16_t *ulsch_llr, - uint32_t nb_rb, uint32_t nb_re, uint8_t symbol) { @@ -152,11 +150,10 @@ void nr_ulsch_64qam_llr(int32_t *rxdataF_comp, void nr_ulsch_256qam_llr(int32_t *rxdataF_comp, int32_t *ul_ch_mag, int32_t *ul_ch_magb, - int32_t *ul_ch_magc, - int16_t *ulsch_llr, - uint32_t nb_rb, - uint32_t nb_re, - uint8_t symbol) + int32_t *ul_ch_magc, + int16_t *ulsch_llr, + uint32_t nb_re, + uint8_t symbol) { simde__m256i *rxF = (simde__m256i*)rxdataF_comp; simde__m256i xmm0,xmm1,xmm2,xmm3,xmm4,xmm5,xmm6; @@ -199,7 +196,6 @@ void nr_ulsch_compute_llr(int32_t *rxdataF_comp, int32_t *ul_ch_magb, int32_t *ul_ch_magc, int16_t *ulsch_llr, - uint32_t nb_rb, uint32_t nb_re, uint8_t symbol, uint8_t mod_order) @@ -215,7 +211,6 @@ void nr_ulsch_compute_llr(int32_t *rxdataF_comp, nr_ulsch_16qam_llr(rxdataF_comp, ul_ch_mag, ulsch_llr, - nb_rb, nb_re, symbol); break; @@ -224,7 +219,6 @@ void nr_ulsch_compute_llr(int32_t *rxdataF_comp, ul_ch_mag, ul_ch_magb, ulsch_llr, - nb_rb, nb_re, symbol); break; @@ -234,7 +228,6 @@ void nr_ulsch_compute_llr(int32_t *rxdataF_comp, ul_ch_magb, ul_ch_magc, ulsch_llr, - nb_rb, nb_re, symbol); break; diff --git a/openair1/SCHED_NR/phy_procedures_nr_gNB.c b/openair1/SCHED_NR/phy_procedures_nr_gNB.c index 6c4fdd4c05..f1c20ff2df 100644 --- a/openair1/SCHED_NR/phy_procedures_nr_gNB.c +++ b/openair1/SCHED_NR/phy_procedures_nr_gNB.c @@ -887,8 +887,6 @@ int phy_procedures_gNB_uespec_RX(PHY_VARS_gNB *gNB, int frame_rx, int slot_rx) VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_NR_RX_PUSCH, 1); start_meas(&gNB->rx_pusch_stats); - // if (gNB->use_pusch_tp) nr_rx_pusch_tp(gNB, ULSCH_id, frame_rx, slot_rx, ulsch->harq_pid); - // else nr_rx_pusch(gNB, ULSCH_id, frame_rx, slot_rx, ulsch->harq_pid); nr_rx_pusch_tp(gNB, ULSCH_id, frame_rx, slot_rx, ulsch->harq_pid); NR_gNB_PUSCH *pusch_vars = &gNB->pusch_vars[ULSCH_id]; pusch_vars->ulsch_power_tot = 0; From 62ad185df9f771918878a3a25f6265fbc667f163 Mon Sep 17 00:00:00 2001 From: Tsung-Yu Chan Date: Mon, 18 Sep 2023 16:10:28 +0200 Subject: [PATCH 10/13] fix / fix the length of ce estimation buffer --- openair1/PHY/INIT/nr_init.c | 4 +- .../PHY/NR_TRANSPORT/nr_transport_proto.h | 14 +--- .../PHY/NR_TRANSPORT/nr_ulsch_demodulation.c | 70 +++++-------------- 3 files changed, 22 insertions(+), 66 deletions(-) diff --git a/openair1/PHY/INIT/nr_init.c b/openair1/PHY/INIT/nr_init.c index 09898d383b..01e332ecf2 100644 --- a/openair1/PHY/INIT/nr_init.c +++ b/openair1/PHY/INIT/nr_init.c @@ -686,7 +686,9 @@ int phy_init_nr_gNB(PHY_VARS_gNB *gNB) int n_buf = Prx*max_ul_mimo_layers; int nb_re_pusch = N_RB_UL * NR_NB_SC_PER_RB; - int nb_re_pusch2 = nb_re_pusch + (nb_re_pusch&7); + int nb_re_pusch2 = nb_re_pusch; + if (nb_re_pusch & 7) + nb_re_pusch2 += 8 - (nb_re_pusch & 7); gNB->pusch_vars = (NR_gNB_PUSCH *)malloc16_clear(gNB->max_nb_pusch * sizeof(NR_gNB_PUSCH)); for (int ULSCH_id = 0; ULSCH_id < gNB->max_nb_pusch; ULSCH_id++) { diff --git a/openair1/PHY/NR_TRANSPORT/nr_transport_proto.h b/openair1/PHY/NR_TRANSPORT/nr_transport_proto.h index ab25bd7744..2c7d7a7d3e 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_transport_proto.h +++ b/openair1/PHY/NR_TRANSPORT/nr_transport_proto.h @@ -131,30 +131,18 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB, uint8_t slot, unsigned char harq_pid); -void nr_ulsch_scale_channel(int32_t **ul_ch_estimates_ext, - NR_DL_FRAME_PARMS *frame_parms, - NR_gNB_ULSCH_t *ulsch_gNB, - uint8_t symbol, - uint8_t is_dmrs_symbol, - uint32_t len, - uint8_t nrOfLayers, - uint16_t nb_rb, - int shift_ch_ext); - /** \brief This function computes the average channel level over all allocated RBs and antennas (TX/RX) in order to compute output shift for compensated signal @param ul_ch_estimates_ext Channel estimates in allocated RBs @param frame_parms Pointer to frame descriptor @param avg Pointer to average signal strength @param pilots_flag Flag to indicate pilots in symbol - @param nb_rb Number of allocated RBs */ void nr_ulsch_channel_level(int **ul_ch_estimates_ext, NR_DL_FRAME_PARMS *frame_parms, int32_t *avg, uint8_t symbol, uint32_t len, - uint8_t nrOfLayers, - unsigned short nb_rb); + uint8_t nrOfLayers); /*! \brief This function implements the idft transform precoding in PUSCH diff --git a/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c b/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c index d22ad70ee5..2dc8741227 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c +++ b/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c @@ -382,7 +382,6 @@ static void nr_ulsch_extract_rbs (c16_t* const rxdataF, void nr_ulsch_scale_channel(int **ul_ch_estimates_ext, NR_DL_FRAME_PARMS *frame_parms, - NR_gNB_ULSCH_t *ulsch_gNB, uint8_t symbol, uint8_t is_dmrs_symbol, uint32_t len, @@ -405,21 +404,12 @@ void nr_ulsch_scale_channel(int **ul_ch_estimates_ext, simde__m128i ch_amp128 = simde_mm_set1_epi16(ch_amp); // Q3.13 LOG_D(PHY, "Scaling PUSCH Chest in OFDM symbol %d by %d, pilots %d nb_rb %d NCP %d symbol %d\n", symbol, ch_amp, is_dmrs_symbol, nb_rb, frame_parms->Ncp, symbol); - uint32_t nb_rb_0 = len / 12 + ((len % 12) ? 1 : 0); - int off = ((nb_rb & 1) == 1) ? 4 : 0; for (int aatx = 0; aatx < nrOfLayers; aatx++) { for (int aarx = 0; aarx < frame_parms->nb_antennas_rx; aarx++) { - simde__m128i *ul_ch128 = (simde__m128i *)&ul_ch_estimates_ext[aatx * frame_parms->nb_antennas_rx + aarx][symbol * (off + (nb_rb * NR_NB_SC_PER_RB))]; - for (int rb = 0; rb < nb_rb_0; rb++) { - ul_ch128[0] = simde_mm_mulhi_epi16(ul_ch128[0], ch_amp128); - ul_ch128[0] = simde_mm_slli_epi16(ul_ch128[0], b); - - ul_ch128[1] = simde_mm_mulhi_epi16(ul_ch128[1], ch_amp128); - ul_ch128[1] = simde_mm_slli_epi16(ul_ch128[1], b); - - ul_ch128[2] = simde_mm_mulhi_epi16(ul_ch128[2], ch_amp128); - ul_ch128[2] = simde_mm_slli_epi16(ul_ch128[2], b); - ul_ch128 += 3; + simde__m128i *ul_ch128 = (simde__m128i *)&ul_ch_estimates_ext[aatx * frame_parms->nb_antennas_rx + aarx][symbol * len]; + for (int i = 0; i < len >> 2; i++) { + ul_ch128[i] = simde_mm_mulhi_epi16(ul_ch128[i], ch_amp128); + ul_ch128[i] = simde_mm_slli_epi16(ul_ch128[i], b); } } } @@ -447,32 +437,22 @@ void nr_ulsch_channel_level(int **ul_ch_estimates_ext, int32_t *avg, uint8_t symbol, uint32_t len, - uint8_t nrOfLayers, - unsigned short nb_rb) + uint8_t nrOfLayers) { - short rb; - unsigned char aatx, aarx; simde__m128i *ul_ch128, avg128U; int16_t x = factor2(len); int16_t y = (len)>>x; - - uint32_t nb_rb_0 = len/12 + ((len%12)?1:0); - int off = ((nb_rb&1) == 1)? 4:0; - - for (aatx = 0; aatx < nrOfLayers; aatx++) { - for (aarx = 0; aarx < frame_parms->nb_antennas_rx; aarx++) { + for (int aatx = 0; aatx < nrOfLayers; aatx++) { + for (int aarx = 0; aarx < frame_parms->nb_antennas_rx; aarx++) { //clear average level avg128U = simde_mm_setzero_si128(); - ul_ch128=(simde__m128i *)&ul_ch_estimates_ext[aatx*frame_parms->nb_antennas_rx+aarx][symbol*(off+(nb_rb*12))]; + ul_ch128 = (simde__m128i *)&ul_ch_estimates_ext[aatx*frame_parms->nb_antennas_rx+aarx][symbol * len]; - for (rb = 0; rb < nb_rb_0; rb++) { - avg128U = simde_mm_add_epi32(avg128U, simde_mm_srai_epi32(simde_mm_madd_epi16(ul_ch128[0], ul_ch128[0]), x)); - avg128U = simde_mm_add_epi32(avg128U, simde_mm_srai_epi32(simde_mm_madd_epi16(ul_ch128[1], ul_ch128[1]), x)); - avg128U = simde_mm_add_epi32(avg128U, simde_mm_srai_epi32(simde_mm_madd_epi16(ul_ch128[2], ul_ch128[2]), x)); - ul_ch128+=3; + for (int i = 0; i < len >> 2; i++) { + avg128U = simde_mm_add_epi32(avg128U, simde_mm_srai_epi32(simde_mm_madd_epi16(ul_ch128[i], ul_ch128[i]), x)); } avg[aatx*frame_parms->nb_antennas_rx+aarx] = (((int32_t*)&avg128U)[0] + @@ -486,21 +466,6 @@ void nr_ulsch_channel_level(int **ul_ch_estimates_ext, simde_m_empty(); } -static simde__m128i a_mult_conjb(__m128i a, __m128i b, unsigned char output_shift) -{ - simde__m128i mmtmpD0 = simde_mm_madd_epi16(b, a); - simde__m128i mmtmpD1 = simde_mm_shufflelo_epi16(b, SIMDE_MM_SHUFFLE(2, 3, 0, 1)); - mmtmpD1 = simde_mm_shufflehi_epi16(mmtmpD1, SIMDE_MM_SHUFFLE(2, 3, 0, 1)); - mmtmpD1 = simde_mm_sign_epi16(mmtmpD1, *(simde__m128i *)&conjugate[0]); - mmtmpD1 = simde_mm_madd_epi16(mmtmpD1, a); - mmtmpD0 = simde_mm_srai_epi32(mmtmpD0, output_shift); - mmtmpD1 = simde_mm_srai_epi32(mmtmpD1, output_shift); - simde__m128i mmtmpD2 = simde_mm_unpacklo_epi32(mmtmpD0, mmtmpD1); - simde__m128i mmtmpD3 = simde_mm_unpackhi_epi32(mmtmpD0, mmtmpD1); - return _mm_packs_epi32(mmtmpD2, mmtmpD3); -} - - void nr_ulsch_channel_compensation(c16_t *rxFext, c16_t *chFext, c16_t *ul_ch_maga, @@ -1354,7 +1319,7 @@ static void inner_rx (PHY_VARS_gNB *gNB, int buffer_length = rel15_ul->rb_size * NR_NB_SC_PER_RB; if (buffer_length & 7) - buffer_length += (8 - buffer_length%8); + buffer_length += (8 - (buffer_length & 7)); c16_t rxFext[nb_rx_ant][buffer_length] __attribute__((aligned(32))); c16_t chFext[nb_layer][nb_rx_ant][buffer_length] __attribute__((aligned(32))); @@ -1655,13 +1620,16 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB, // extract the first dmrs for the channel level computation // extract the data in the OFDM frame, to the start of the array int soffset = (slot&3)*frame_parms->symbols_per_slot*frame_parms->ofdm_symbol_size; - int nb_re_pusch2 = nb_re_pusch + (nb_re_pusch&7); + + if (nb_re_pusch & 7) + nb_re_pusch += 8 - (nb_re_pusch & 7); + for (int aarx = 0; aarx < frame_parms->nb_antennas_rx; aarx++) for (int aatx = 0; aatx < rel15_ul->nrOfLayers; aatx++) nr_ulsch_extract_rbs(gNB->common_vars.rxdataF[aarx], (c16_t *)pusch_vars->ul_ch_estimates[aatx * frame_parms->nb_antennas_rx + aarx], - (c16_t*)&pusch_vars->rxdataF_ext[aarx][meas_symbol * nb_re_pusch2], - (c16_t*)&pusch_vars->ul_ch_estimates_ext[aatx*frame_parms->nb_antennas_rx+aarx][meas_symbol*nb_re_pusch2], + (c16_t*)&pusch_vars->rxdataF_ext[aarx][meas_symbol * nb_re_pusch], + (c16_t*)&pusch_vars->ul_ch_estimates_ext[aatx*frame_parms->nb_antennas_rx+aarx][meas_symbol * nb_re_pusch], soffset + meas_symbol * frame_parms->ofdm_symbol_size, pusch_vars->dmrs_symbol * frame_parms->ofdm_symbol_size, aarx, @@ -1680,7 +1648,6 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB, //---------------------------------------------------------- nr_ulsch_scale_channel(pusch_vars->ul_ch_estimates_ext, frame_parms, - &gNB->ulsch[ulsch_id], meas_symbol, (rel15_ul->ul_dmrs_symb_pos >> meas_symbol) & 0x01, nb_re_pusch, @@ -1693,8 +1660,7 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB, avg, meas_symbol, // index of the start symbol nb_re_pusch, // number of the re in pusch - rel15_ul->nrOfLayers, - rel15_ul->rb_size); + rel15_ul->nrOfLayers); for (int aatx = 0; aatx < rel15_ul->nrOfLayers; aatx++) for (int aarx = 0; aarx < frame_parms->nb_antennas_rx; aarx++) From 5231a3ab74dd5296249053367a72c7d40333e30a Mon Sep 17 00:00:00 2001 From: Tsung Yu Chan Date: Tue, 19 Sep 2023 12:33:09 +0200 Subject: [PATCH 11/13] fix / fix the transfrom precoding error when qpsk - avoid the nr_freq_equalization when qpsk - do the stop_meas when use the lepc offload --- openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c | 15 ++++++++------- openair1/SCHED_NR/phy_procedures_nr_gNB.c | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c b/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c index 2dc8741227..45070b47dd 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c +++ b/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c @@ -1367,13 +1367,14 @@ static void inner_rx (PHY_VARS_gNB *gNB, output_shift); if (nb_layer == 1 && rel15_ul->transform_precoding == transformPrecoder_enabled && rel15_ul->qam_mod_order <= 6) { - nr_freq_equalization(frame_parms, - &pusch_vars->rxdataF_comp[0][symbol * buffer_length], - (int *)rxF_ch_maga, - (int *)rxF_ch_magb, - symbol, - pusch_vars->ul_valid_re_per_slot[symbol], - rel15_ul->qam_mod_order); + if (rel15_ul->qam_mod_order > 2) + nr_freq_equalization(frame_parms, + &pusch_vars->rxdataF_comp[0][symbol * buffer_length], + (int *)rxF_ch_maga, + (int *)rxF_ch_magb, + symbol, + pusch_vars->ul_valid_re_per_slot[symbol], + rel15_ul->qam_mod_order); nr_idft(&pusch_vars->rxdataF_comp[0][symbol * buffer_length], pusch_vars->ul_valid_re_per_slot[symbol]); } if (rel15_ul->pdu_bit_map & PUSCH_PDU_BITMAP_PUSCH_PTRS) { diff --git a/openair1/SCHED_NR/phy_procedures_nr_gNB.c b/openair1/SCHED_NR/phy_procedures_nr_gNB.c index f1c20ff2df..f00ac5c2d4 100644 --- a/openair1/SCHED_NR/phy_procedures_nr_gNB.c +++ b/openair1/SCHED_NR/phy_procedures_nr_gNB.c @@ -950,8 +950,8 @@ int phy_procedures_gNB_uespec_RX(PHY_VARS_gNB *gNB, int frame_rx, int slot_rx) delNotifiedFIFO_elt(req); totalDecode--; } - stop_meas(&gNB->ulsch_decoding_stats); } + stop_meas(&gNB->ulsch_decoding_stats); for (int i = 0; i < gNB->max_nb_srs; i++) { NR_gNB_SRS_t *srs = &gNB->srs[i]; if (srs) { From 0263b4a807f24ce67ab4420534fd19c1621456dc Mon Sep 17 00:00:00 2001 From: Tsung Yu Chan Date: Tue, 19 Sep 2023 17:35:38 +0200 Subject: [PATCH 12/13] chore / remove the unused variable --- .../PHY/NR_TRANSPORT/nr_transport_proto.h | 13 -- openair1/PHY/NR_TRANSPORT/nr_ulsch_decoding.c | 10 - .../PHY/NR_TRANSPORT/nr_ulsch_demodulation.c | 216 +++++++----------- openair1/PHY/defs_gNB.h | 5 - openair1/SIMULATION/NR_PHY/ulsim.c | 30 +-- 5 files changed, 93 insertions(+), 181 deletions(-) diff --git a/openair1/PHY/NR_TRANSPORT/nr_transport_proto.h b/openair1/PHY/NR_TRANSPORT/nr_transport_proto.h index 2c7d7a7d3e..2583996b95 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_transport_proto.h +++ b/openair1/PHY/NR_TRANSPORT/nr_transport_proto.h @@ -131,19 +131,6 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB, uint8_t slot, unsigned char harq_pid); -/** \brief This function computes the average channel level over all allocated RBs and antennas (TX/RX) in order to compute output shift for compensated signal - @param ul_ch_estimates_ext Channel estimates in allocated RBs - @param frame_parms Pointer to frame descriptor - @param avg Pointer to average signal strength - @param pilots_flag Flag to indicate pilots in symbol -*/ -void nr_ulsch_channel_level(int **ul_ch_estimates_ext, - NR_DL_FRAME_PARMS *frame_parms, - int32_t *avg, - uint8_t symbol, - uint32_t len, - uint8_t nrOfLayers); - /*! \brief This function implements the idft transform precoding in PUSCH \param z Pointer to input in frequnecy domain, and it is also the output in time domain diff --git a/openair1/PHY/NR_TRANSPORT/nr_ulsch_decoding.c b/openair1/PHY/NR_TRANSPORT/nr_ulsch_decoding.c index 69015660d8..99218749a4 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_ulsch_decoding.c +++ b/openair1/PHY/NR_TRANSPORT/nr_ulsch_decoding.c @@ -122,7 +122,6 @@ NR_gNB_ULSCH_t new_gNB_ulsch(uint8_t max_ldpc_iterations, uint16_t N_RB_UL) static void nr_processULSegment(void *arg) { ldpcDecode_t *rdata = (ldpcDecode_t *)arg; - PHY_VARS_gNB *phy_vars_gNB = rdata->gNB; NR_UL_gNB_HARQ_t *ulsch_harq = rdata->ulsch_harq; t_nrLDPC_dec_params *p_decoderParms = &rdata->decoderParms; int length_dec; @@ -156,8 +155,6 @@ static void nr_processULSegment(void *arg) t_nrLDPC_time_stats procTime = {0}; t_nrLDPC_time_stats *p_procTime = &procTime; - // start_meas(&phy_vars_gNB->ulsch_deinterleaving_stats); - //////////////////////////////////////////////////////////////////////////////////////////// ///////////////////////////////// nr_deinterleaving_ldpc /////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////////////// @@ -172,7 +169,6 @@ static void nr_processULSegment(void *arg) // for (int i =0; i<16; i++) // printf("rx output deinterleaving w[%d]= %d r_offset %d\n", i,ulsch_harq->w[r][i], r_offset); - stop_meas(&phy_vars_gNB->ulsch_deinterleaving_stats); ////////////////////////////////////////////////////////////////////////////////////////// @@ -182,7 +178,6 @@ static void nr_processULSegment(void *arg) ///////////////////////// ulsch_harq->e =====> ulsch_harq->d ///////////////////////// - // start_meas(&phy_vars_gNB->ulsch_rate_unmatching_stats); if (nr_rate_matching_ldpc_rx(rdata->tbslbrm, p_decoderParms->BG, @@ -196,13 +191,10 @@ static void nr_processULSegment(void *arg) ulsch_harq->F, Kr - ulsch_harq->F - 2 * (p_decoderParms->Z)) == -1) { - stop_meas(&phy_vars_gNB->ulsch_rate_unmatching_stats); LOG_E(PHY, "ulsch_decoding.c: Problem in rate_matching\n"); rdata->decodeIterations = max_ldpc_iterations + 1; return; - } else { - stop_meas(&phy_vars_gNB->ulsch_rate_unmatching_stats); } ulsch_harq->d_to_be_cleared[r] = false; @@ -221,7 +213,6 @@ static void nr_processULSegment(void *arg) length_dec = (ulsch_harq->B + 24 * ulsch_harq->C) / ulsch_harq->C; } - // start_meas(&phy_vars_gNB->ulsch_ldpc_decoding_stats); // set first 2*Z_c bits to zeros memset(&z[0], 0, 2 * ulsch_harq->Z * sizeof(int16_t)); @@ -248,7 +239,6 @@ static void nr_processULSegment(void *arg) if (rdata->decodeIterations <= p_decoderParms->numMaxIter) memcpy(ulsch_harq->c[r],llrProcBuf, Kr>>3); - //stop_meas(&phy_vars_gNB->ulsch_ldpc_decoding_stats); } int nr_ulsch_decoding(PHY_VARS_gNB *phy_vars_gNB, diff --git a/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c b/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c index 45070b47dd..22f3fd4a51 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c +++ b/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c @@ -287,7 +287,6 @@ static void nr_ulsch_extract_rbs (c16_t* const rxdataF, nfapi_nr_pusch_pdu_t *pusch_pdu, NR_DL_FRAME_PARMS *frame_parms) { - uint8_t delta = 0; int start_re = (frame_parms->first_carrier_offset + (pusch_pdu->rb_start + pusch_pdu->bwp_start) * NR_NB_SC_PER_RB)%frame_parms->ofdm_symbol_size; @@ -380,14 +379,14 @@ static void nr_ulsch_extract_rbs (c16_t* const rxdataF, } } -void nr_ulsch_scale_channel(int **ul_ch_estimates_ext, - NR_DL_FRAME_PARMS *frame_parms, - uint8_t symbol, - uint8_t is_dmrs_symbol, - uint32_t len, - uint8_t nrOfLayers, - unsigned short nb_rb, - int shift_ch_ext) +static void nr_ulsch_scale_channel(int **ul_ch_estimates_ext, + NR_DL_FRAME_PARMS *frame_parms, + uint8_t symbol, + uint8_t is_dmrs_symbol, + uint32_t len, + uint8_t nrOfLayers, + unsigned short nb_rb, + int shift_ch_ext) { // Determine scaling amplitude based the symbol int b = 3; @@ -415,7 +414,7 @@ void nr_ulsch_scale_channel(int **ul_ch_estimates_ext, } } -int get_nb_re_pusch (NR_DL_FRAME_PARMS *frame_parms, nfapi_nr_pusch_pdu_t *rel15_ul,int symbol) +static int get_nb_re_pusch (NR_DL_FRAME_PARMS *frame_parms, nfapi_nr_pusch_pdu_t *rel15_ul,int symbol) { uint8_t dmrs_symbol_flag = (rel15_ul->ul_dmrs_symb_pos >> symbol) & 0x01; if (dmrs_symbol_flag == 1) { @@ -431,13 +430,13 @@ int get_nb_re_pusch (NR_DL_FRAME_PARMS *frame_parms, nfapi_nr_pusch_pdu_t *rel15 } else return(rel15_ul->rb_size * NR_NB_SC_PER_RB); } -//compute average channel_level on each (TX,RX) antenna pair -void nr_ulsch_channel_level(int **ul_ch_estimates_ext, - NR_DL_FRAME_PARMS *frame_parms, - int32_t *avg, - uint8_t symbol, - uint32_t len, - uint8_t nrOfLayers) +// compute average channel_level on each (TX,RX) antenna pair +static void nr_ulsch_channel_level(int **ul_ch_estimates_ext, + NR_DL_FRAME_PARMS *frame_parms, + int32_t *avg, + uint8_t symbol, + uint32_t len, + uint8_t nrOfLayers) { simde__m128i *ul_ch128, avg128U; @@ -466,18 +465,18 @@ void nr_ulsch_channel_level(int **ul_ch_estimates_ext, simde_m_empty(); } -void nr_ulsch_channel_compensation(c16_t *rxFext, - c16_t *chFext, - c16_t *ul_ch_maga, - c16_t *ul_ch_magb, - c16_t *ul_ch_magc, - int32_t **rxComp, - c16_t *rho, - NR_DL_FRAME_PARMS *frame_parms, - nfapi_nr_pusch_pdu_t* rel15_ul, - uint32_t symbol, - uint32_t buffer_length, - uint32_t output_shift) +static void nr_ulsch_channel_compensation(c16_t *rxFext, + c16_t *chFext, + c16_t *ul_ch_maga, + c16_t *ul_ch_magb, + c16_t *ul_ch_magc, + int32_t **rxComp, + c16_t *rho, + NR_DL_FRAME_PARMS *frame_parms, + nfapi_nr_pusch_pdu_t* rel15_ul, + uint32_t symbol, + uint32_t buffer_length, + uint32_t output_shift) { int mod_order = rel15_ul->qam_mod_order; int nrOfLayers = rel15_ul->nrOfLayers; @@ -578,14 +577,14 @@ void nr_ulsch_channel_compensation(c16_t *rxFext, } // Zero Forcing Rx function: nr_det_HhH() -void nr_ulsch_det_HhH (int32_t *after_mf_00,//a - int32_t *after_mf_01,//b - int32_t *after_mf_10,//c - int32_t *after_mf_11,//d - int32_t *det_fin,//1/ad-bc - unsigned short nb_rb, - unsigned char symbol, - int32_t shift) +static void nr_ulsch_det_HhH (int32_t *after_mf_00,//a + int32_t *after_mf_01,//b + int32_t *after_mf_10,//c + int32_t *after_mf_11,//d + int32_t *det_fin,//1/ad-bc + unsigned short nb_rb, + unsigned char symbol, + int32_t shift) { int16_t nr_conjug2[8]__attribute__((aligned(16))) = {1,-1,1,-1,1,-1,1,-1} ; unsigned short rb; @@ -640,53 +639,15 @@ void nr_ulsch_det_HhH (int32_t *after_mf_00,//a simde_m_empty(); } -/* Zero Forcing Rx function: nr_inv_comp_muli - * Complex number multi: z = x*y - * = (x_re*y_re - x_im*y_im) + j(x_im*y_re + x_re*y_im) - * */ -simde__m128i nr_ulsch_inv_comp_muli(simde__m128i input_x, - simde__m128i input_y) -{ - int16_t nr_conjug2[8]__attribute__((aligned(16))) = {1,-1,1,-1,1,-1,1,-1} ; - - simde__m128i xy_re_128, xy_im_128; - simde__m128i output_z, tmp_z0, tmp_z1; - - // complex multiplication (x_re + jx_im)*(y_re + jy_im) = (x_re*y_re - x_im*y_im) + j(x_im*y_re + x_re*y_im) - - // the real part - xy_re_128 = simde_mm_sign_epi16(input_x,*(simde__m128i*)&nr_conjug2[0]); - xy_re_128 = simde_mm_madd_epi16(xy_re_128,input_y); //Re: (x_re*y_re - x_im*y_im) - - // the imag part - xy_im_128 = simde_mm_shufflelo_epi16(input_x, SIMDE_MM_SHUFFLE(2,3,0,1));//permutes IQs for the low 64 bits as [I_a0 Q_a1 I_a2 Q_a3]_64bits to [Q_a1 I_a0 Q_a3 I_a2]_64bits - xy_im_128 = simde_mm_shufflehi_epi16(xy_im_128, SIMDE_MM_SHUFFLE(2,3,0,1));//permutes IQs for the high 64 bits as [I_a0 Q_a1 I_a2 Q_a3]_64bits to [Q_a1 I_a0 Q_a3 I_a2]_64bits - xy_im_128 = simde_mm_madd_epi16(xy_im_128,input_y);//Im: (x_im*y_re + x_re*y_im) - - //convert back to Q15 before packing - xy_re_128 = simde_mm_srai_epi32(xy_re_128,4);//(2^15/64*2*16) - xy_im_128 = simde_mm_srai_epi32(xy_im_128,4); - - tmp_z0 = simde_mm_unpacklo_epi32(xy_re_128,xy_im_128); - //print_ints("unpack lo:",&tmp_z0[0]); - tmp_z1 = simde_mm_unpackhi_epi32(xy_re_128,xy_im_128); - //print_ints("unpack hi:",&tmp_z1[0]); - output_z = simde_mm_packs_epi32(tmp_z0,tmp_z1); - - simde_mm_empty(); - simde_m_empty(); - return(output_z); -} - /* Zero Forcing Rx function: nr_conjch0_mult_ch1() * * * */ -void nr_ulsch_conjch0_mult_ch1(int *ch0, - int *ch1, - int32_t *ch0conj_ch1, - unsigned short nb_rb, - unsigned char output_shift0) +static void nr_ulsch_conjch0_mult_ch1(int *ch0, + int *ch1, + int32_t *ch0conj_ch1, + unsigned short nb_rb, + unsigned char output_shift0) { //This function is used to compute multiplications in H_hermitian * H matrix short nr_conjugate[8]__attribute__((aligned(16))) = {-1,1,-1,1,-1,1,-1,1}; @@ -724,11 +685,12 @@ void nr_ulsch_conjch0_mult_ch1(int *ch0, simde_mm_empty(); simde_m_empty(); } -simde__m128i nr_ulsch_comp_muli_sum(simde__m128i input_x, - simde__m128i input_y, - simde__m128i input_w, - simde__m128i input_z, - simde__m128i det) + +static simde__m128i nr_ulsch_comp_muli_sum(simde__m128i input_x, + simde__m128i input_y, + simde__m128i input_w, + simde__m128i input_z, + simde__m128i det) { int16_t nr_conjug2[8]__attribute__((aligned(16))) = {1,-1,1,-1,1,-1,1,-1} ; @@ -786,32 +748,33 @@ simde__m128i nr_ulsch_comp_muli_sum(simde__m128i input_x, simde_m_empty(); return(output); } + /* Zero Forcing Rx function: nr_construct_HhH_elements() * * * */ -void nr_ulsch_construct_HhH_elements(int *conjch00_ch00, - int *conjch01_ch01, - int *conjch11_ch11, - int *conjch10_ch10,// - int *conjch20_ch20, - int *conjch21_ch21, - int *conjch30_ch30, - int *conjch31_ch31, - int *conjch00_ch01,//00_01 - int *conjch01_ch00,//01_00 - int *conjch10_ch11,//10_11 - int *conjch11_ch10,//11_10 - int *conjch20_ch21, - int *conjch21_ch20, - int *conjch30_ch31, - int *conjch31_ch30, - int32_t *after_mf_00, - int32_t *after_mf_01, - int32_t *after_mf_10, - int32_t *after_mf_11, - unsigned short nb_rb, - unsigned char symbol) +static void nr_ulsch_construct_HhH_elements(int *conjch00_ch00, + int *conjch01_ch01, + int *conjch11_ch11, + int *conjch10_ch10,// + int *conjch20_ch20, + int *conjch21_ch21, + int *conjch30_ch30, + int *conjch31_ch31, + int *conjch00_ch01,//00_01 + int *conjch01_ch00,//01_00 + int *conjch10_ch11,//10_11 + int *conjch11_ch10,//11_10 + int *conjch20_ch21, + int *conjch21_ch20, + int *conjch30_ch31, + int *conjch31_ch30, + int32_t *after_mf_00, + int32_t *after_mf_01, + int32_t *after_mf_10, + int32_t *after_mf_11, + unsigned short nb_rb, + unsigned char symbol) { //This function is used to construct the (H_hermitian * H matrix) matrix elements unsigned short rb; @@ -903,23 +866,21 @@ void nr_ulsch_construct_HhH_elements(int *conjch00_ch00, simde_m_empty(); } -/* - * MMSE Rx function: nr_ulsch_mmse_2layers() - */ -uint8_t nr_ulsch_mmse_2layers(NR_DL_FRAME_PARMS *frame_parms, - int **rxdataF_comp, - int **ul_ch_mag, - int **ul_ch_magb, - int **ul_ch_magc, - int **ul_ch_estimates_ext, - unsigned short nb_rb, - unsigned char n_rx, - unsigned char mod_order, - int shift, - unsigned char symbol, - int length, - uint32_t noise_var, - uint32_t buffer_length) +// MMSE Rx function: nr_ulsch_mmse_2layers() +static uint8_t nr_ulsch_mmse_2layers(NR_DL_FRAME_PARMS *frame_parms, + int **rxdataF_comp, + int **ul_ch_mag, + int **ul_ch_magb, + int **ul_ch_magc, + int **ul_ch_estimates_ext, + unsigned short nb_rb, + unsigned char n_rx, + unsigned char mod_order, + int shift, + unsigned char symbol, + int length, + uint32_t noise_var, + uint32_t buffer_length) { int *ch00, *ch01, *ch10, *ch11; int *ch20, *ch30, *ch21, *ch31; @@ -1432,7 +1393,7 @@ static void inner_rx (PHY_VARS_gNB *gNB, rel15_ul->qam_mod_order); } -void nr_pusch_symbol_processing(void *arg) +static void nr_pusch_symbol_processing(void *arg) { puschSymbolProc_t *rdata=(puschSymbolProc_t*)arg; @@ -1617,7 +1578,6 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB, AssertFatal(nb_re_pusch>0 && meas_symbol>=0,"nb_re_pusch %d cannot be 0 or meas_symbol %d cannot be negative here\n",nb_re_pusch,meas_symbol); - start_meas(&gNB->ulsch_rbs_extraction_stats); // extract the first dmrs for the channel level computation // extract the data in the OFDM frame, to the start of the array int soffset = (slot&3)*frame_parms->symbols_per_slot*frame_parms->ofdm_symbol_size; @@ -1638,8 +1598,6 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB, rel15_ul, frame_parms); - stop_meas(&gNB->ulsch_rbs_extraction_stats); - int avgs = 0; int avg[frame_parms->nb_antennas_rx*rel15_ul->nrOfLayers]; uint8_t shift_ch_ext = rel15_ul->nrOfLayers > 1 ? log2_approx(max_ch >> 11) : 0; @@ -1723,7 +1681,7 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB, } } // symbol loop - while (gNB->nbSymb > 0 && !(rel15_ul->pdu_bit_map & PUSCH_PDU_BITMAP_PUSCH_PTRS)) { + while (gNB->nbSymb > 0) { notifiedFIFO_elt_t *req = pullTpool(&gNB->respPuschSymb, &gNB->threadPool); gNB->nbSymb--; delNotifiedFIFO_elt(req); diff --git a/openair1/PHY/defs_gNB.h b/openair1/PHY/defs_gNB.h index a28e4a30f4..83040cb0e0 100644 --- a/openair1/PHY/defs_gNB.h +++ b/openair1/PHY/defs_gNB.h @@ -736,12 +736,7 @@ typedef struct PHY_VARS_gNB_s { time_stats_t ulsch_rate_unmatching_stats; time_stats_t ulsch_ldpc_decoding_stats; time_stats_t ulsch_deinterleaving_stats; - time_stats_t ulsch_unscrambling_stats; time_stats_t ulsch_channel_estimation_stats; - time_stats_t ulsch_ptrs_processing_stats; - time_stats_t ulsch_channel_compensation_stats; - time_stats_t ulsch_rbs_extraction_stats; - time_stats_t ulsch_mrc_stats; time_stats_t ulsch_llr_stats; time_stats_t rx_srs_stats; time_stats_t generate_srs_stats; diff --git a/openair1/SIMULATION/NR_PHY/ulsim.c b/openair1/SIMULATION/NR_PHY/ulsim.c index 5c3abe09be..762487c759 100644 --- a/openair1/SIMULATION/NR_PHY/ulsim.c +++ b/openair1/SIMULATION/NR_PHY/ulsim.c @@ -721,7 +721,6 @@ int main(int argc, char *argv[]) NR_gNB_ULSCH_t *ulsch_gNB = &gNB->ulsch[UE_id]; - // nfapi_nr_ul_config_ulsch_pdu *rel15_ul = &ulsch_gNB->harq_process->ulsch_pdu; NR_Sched_Rsp_t *Sched_INFO = malloc(sizeof(*Sched_INFO)); memset((void*)Sched_INFO,0,sizeof(*Sched_INFO)); nfapi_nr_ul_tti_request_t *UL_tti_req = &Sched_INFO->UL_tti_req; @@ -932,14 +931,7 @@ int main(int argc, char *argv[]) reset_meas(&gNB->rx_pusch_init_stats); reset_meas(&gNB->rx_pusch_symbol_processing_stats); reset_meas(&gNB->ulsch_decoding_stats); - reset_meas(&gNB->ulsch_deinterleaving_stats); - reset_meas(&gNB->ulsch_rate_unmatching_stats); - reset_meas(&gNB->ulsch_ldpc_decoding_stats); - reset_meas(&gNB->ulsch_unscrambling_stats); reset_meas(&gNB->ulsch_channel_estimation_stats); - reset_meas(&gNB->ulsch_llr_stats); - reset_meas(&gNB->ulsch_channel_compensation_stats); - reset_meas(&gNB->ulsch_rbs_extraction_stats); reset_meas(&UE->ulsch_ldpc_encoding_stats); reset_meas(&UE->ulsch_rate_matching_stats); reset_meas(&UE->ulsch_interleaving_stats); @@ -1595,23 +1587,13 @@ int main(int argc, char *argv[]) if (print_perf==1) { printf("gNB RX\n"); - printDistribution(&gNB->phy_proc_rx,table_rx,"Total PHY proc rx"); - printStatIndent(&gNB->rx_pusch_stats,"RX PUSCH time"); - printStatIndent2(&gNB->ulsch_channel_estimation_stats,"ULSCH channel estimation time"); - printStatIndent2(&gNB->rx_pusch_init_stats,"RX PUSCH Initialization time"); - printStatIndent2(&gNB->rx_pusch_symbol_processing_stats,"RX PUSCH Symbol Processing time"); - - // printStatIndent2(&gNB->ulsch_ptrs_processing_stats,"ULSCH PTRS Processing time"); - // printStatIndent2(&gNB->ulsch_rbs_extraction_stats,"ULSCH rbs extraction time"); - // printStatIndent2(&gNB->ulsch_channel_compensation_stats,"ULSCH channel compensation time"); - // printStatIndent2(&gNB->ulsch_mrc_stats,"ULSCH mrc computation"); - // printStatIndent2(&gNB->ulsch_llr_stats,"ULSCH llr computation"); - // printStatIndent(&gNB->ulsch_unscrambling_stats,"ULSCH unscrambling"); - + printDistribution(&gNB->phy_proc_rx,table_rx, "Total PHY proc rx"); + printStatIndent(&gNB->rx_pusch_stats, "RX PUSCH time"); + printStatIndent2(&gNB->ulsch_channel_estimation_stats, "ULSCH channel estimation time"); + printStatIndent2(&gNB->rx_pusch_init_stats, "RX PUSCH Initialization time"); + printStatIndent2(&gNB->rx_pusch_symbol_processing_stats, "RX PUSCH Symbol Processing time"); printStatIndent(&gNB->ulsch_decoding_stats,"ULSCH total decoding time"); - // printStatIndent2(&gNB->ulsch_deinterleaving_stats,"ULSCH deinterleaving"); - // printStatIndent2(&gNB->ulsch_rate_unmatching_stats,"ULSCH rate matching rx"); - // printStatIndent2(&gNB->ulsch_ldpc_decoding_stats,"ULSCH ldpc decoding"); + printf("\nUE TX\n"); printStatIndent(&UE->ulsch_encoding_stats,"ULSCH total encoding time"); printStatIndent2(&UE->ulsch_segmentation_stats,"ULSCH segmentation time"); From fc2432e93bc60addfdda96b9c7d0b2f0e39e45be Mon Sep 17 00:00:00 2001 From: Tsung Yu Chan Date: Mon, 25 Sep 2023 12:45:39 +0200 Subject: [PATCH 13/13] chore / simplify the code and add the comment - simplify the buffer alignment - add the comment for llr - clean unused variable --- openair1/PHY/INIT/nr_init.c | 4 +- .../PHY/NR_ESTIMATION/nr_freq_equalization.c | 11 ++--- openair1/PHY/NR_TRANSPORT/nr_scrambling.c | 17 ++++---- .../PHY/NR_TRANSPORT/nr_ulsch_demodulation.c | 6 +-- .../NR_TRANSPORT/nr_ulsch_llr_computation.c | 42 +++++++++++-------- openair1/PHY/defs_gNB.h | 1 - 6 files changed, 38 insertions(+), 43 deletions(-) diff --git a/openair1/PHY/INIT/nr_init.c b/openair1/PHY/INIT/nr_init.c index 01e332ecf2..9cf5e26cc7 100644 --- a/openair1/PHY/INIT/nr_init.c +++ b/openair1/PHY/INIT/nr_init.c @@ -686,9 +686,7 @@ int phy_init_nr_gNB(PHY_VARS_gNB *gNB) int n_buf = Prx*max_ul_mimo_layers; int nb_re_pusch = N_RB_UL * NR_NB_SC_PER_RB; - int nb_re_pusch2 = nb_re_pusch; - if (nb_re_pusch & 7) - nb_re_pusch2 += 8 - (nb_re_pusch & 7); + int nb_re_pusch2 = (nb_re_pusch + 7) & ~7; gNB->pusch_vars = (NR_gNB_PUSCH *)malloc16_clear(gNB->max_nb_pusch * sizeof(NR_gNB_PUSCH)); for (int ULSCH_id = 0; ULSCH_id < gNB->max_nb_pusch; ULSCH_id++) { diff --git a/openair1/PHY/NR_ESTIMATION/nr_freq_equalization.c b/openair1/PHY/NR_ESTIMATION/nr_freq_equalization.c index e73ac48fab..c593c04cc5 100644 --- a/openair1/PHY/NR_ESTIMATION/nr_freq_equalization.c +++ b/openair1/PHY/NR_ESTIMATION/nr_freq_equalization.c @@ -26,7 +26,7 @@ // Reference of openair1/PHY/LTE_ESTIMATION/freq_equalization.c // This is 4096/(1:4096) in simde__m128i format -simde__m128i nr_inv_ch[4096];/* = {0, 4096/1, 4096/2, 4096/3, 4096/4...}*/ +static simde__m128i nr_inv_ch[4096]; /* = {0, 4096/1, 4096/2, 4096/3, 4096/4...}*/ void nr_init_fde() { @@ -51,8 +51,7 @@ void nr_freq_equalization (NR_DL_FRAME_PARMS *frame_parms, AssertFatal(Msc_RS <= frame_parms->N_RB_UL*12, "Msc_RS %d >= %d\n", Msc_RS, frame_parms->N_RB_UL*12); - for (uint16_t re = 0; re < (Msc_RS >> 2); re++) - { + for (uint16_t re = 0; re < (Msc_RS >> 2); re++) { int16_t amp = (*((int16_t*)&ul_ch_mag128[re])); if (amp > 4095) @@ -62,12 +61,10 @@ void nr_freq_equalization (NR_DL_FRAME_PARMS *frame_parms, if (Qm == 4) ul_ch_mag128[re] = simde_mm_set1_epi16(324); // this is 512*2/sqrt(10) - else if (Qm == 6) - { + else if (Qm == 6) { ul_ch_mag128[re] = simde_mm_set1_epi16(316); // this is 512*4/sqrt(42) ul_ch_magb128[re] = simde_mm_set1_epi16(158); // this is 512*2/sqrt(42) - } - else if(Qm != 2) + } else if(Qm != 2) AssertFatal(1, "nr_freq_equalization(), Qm = %d, should be 2, 4 or 6. symbol=%d, Msc_RS=%d\n", Qm, symbol, Msc_RS); } } diff --git a/openair1/PHY/NR_TRANSPORT/nr_scrambling.c b/openair1/PHY/NR_TRANSPORT/nr_scrambling.c index fd0da127c8..b54a5c2c0c 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_scrambling.c +++ b/openair1/PHY/NR_TRANSPORT/nr_scrambling.c @@ -81,17 +81,16 @@ void nr_codeword_unscrambling_init(int16_t *s2, uint32_t size, uint8_t q, uint32 { uint32_t x1; uint32_t x2 = (n_RNTI << 15) + (q << 14) + Nid; - uint32_t s = 0; - - uint8_t *s8=(uint8_t *)&s; - s = lte_gold_generic(&x1, &x2, 1); simde__m128i *s128=(simde__m128i *)s2; - for (int i = 0, j = 0; i < ((size >> 5) + ((size & 0x1f) > 0 ? 1 : 0)); i++, j += 4) { - s128[j] = byte2m128i[s8[0]]; - s128[j+1] = byte2m128i[s8[1]]; - s128[j+2] = byte2m128i[s8[2]]; - s128[j+3] = byte2m128i[s8[3]]; + uint32_t s = lte_gold_generic(&x1, &x2, 1); + uint8_t *s8=(uint8_t *)&s; + + for (int i = 0; i < (size +31) >> 5; i++) { + *s128++ = byte2m128i[s8[0]]; + *s128++ = byte2m128i[s8[1]]; + *s128++ = byte2m128i[s8[2]]; + *s128++ = byte2m128i[s8[3]]; s = lte_gold_generic(&x1, &x2, 0); } } diff --git a/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c b/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c index 22f3fd4a51..18aea849f7 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c +++ b/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c @@ -1277,11 +1277,7 @@ static void inner_rx (PHY_VARS_gNB *gNB, int nb_layer = rel15_ul->nrOfLayers; int nb_rx_ant = frame_parms->nb_antennas_rx; int dmrs_symbol_flag = (rel15_ul->ul_dmrs_symb_pos >> symbol) & 0x01; - - int buffer_length = rel15_ul->rb_size * NR_NB_SC_PER_RB; - if (buffer_length & 7) - buffer_length += (8 - (buffer_length & 7)); - + int buffer_length = (rel15_ul->rb_size * NR_NB_SC_PER_RB + 7) & ~7; c16_t rxFext[nb_rx_ant][buffer_length] __attribute__((aligned(32))); c16_t chFext[nb_layer][nb_rx_ant][buffer_length] __attribute__((aligned(32))); diff --git a/openair1/PHY/NR_TRANSPORT/nr_ulsch_llr_computation.c b/openair1/PHY/NR_TRANSPORT/nr_ulsch_llr_computation.c index a5f4a84a90..1649f57878 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_ulsch_llr_computation.c +++ b/openair1/PHY/NR_TRANSPORT/nr_ulsch_llr_computation.c @@ -65,25 +65,28 @@ void nr_ulsch_16qam_llr(int32_t *rxdataF_comp, { simde__m256i *rxF_256 = (simde__m256i*)rxdataF_comp; simde__m256i *ch_mag = (simde__m256i*)ul_ch_mag; - simde__m64 *llr_64 = (simde__m64*)ulsch_llr; + int64_t *llr_64 = (int64_t*)ulsch_llr; simde__m256i xmm0, xmm1, xmm2; - for (int i = 0; i < ((nb_re>>3) + ((nb_re&7) ? 1 : 0)); i++) { - xmm0 = simde_mm256_abs_epi16(rxF_256[i]); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| + for (int i = 0; i < ((nb_re + 7) >> 3); i++) { + xmm0 = simde_mm256_abs_epi16(rxF_256[i]); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| xmm0 = simde_mm256_subs_epi16(ch_mag[i], xmm0); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 - xmm1 = simde_mm256_unpacklo_epi32(rxF_256[i], xmm0); // llr128[0] contains the llrs of the 1st,2nd,5th and 6th REs - xmm2 = simde_mm256_unpackhi_epi32(rxF_256[i], xmm0); // llr128[1] contains the llrs of the 3rd, 4th, 7th and 8th REs + xmm1 = simde_mm256_unpacklo_epi32(rxF_256[i], xmm0); + xmm2 = simde_mm256_unpackhi_epi32(rxF_256[i], xmm0); + + // xmm1 |1st 2ed 3rd 4th 9th 10th 13rd 14th| + // xmm2 |5th 6th 7th 8th 11st 12ed 15th 16th| - *llr_64++ = (simde__m64)simde_mm256_extract_epi64(xmm1, 0); - *llr_64++ = (simde__m64)simde_mm256_extract_epi64(xmm1, 1); - *llr_64++ = (simde__m64)simde_mm256_extract_epi64(xmm2, 0); - *llr_64++ = (simde__m64)simde_mm256_extract_epi64(xmm2, 1); - *llr_64++ = (simde__m64)simde_mm256_extract_epi64(xmm1, 2); - *llr_64++ = (simde__m64)simde_mm256_extract_epi64(xmm1, 3); - *llr_64++ = (simde__m64)simde_mm256_extract_epi64(xmm2, 2); - *llr_64++ = (simde__m64)simde_mm256_extract_epi64(xmm2, 3); + *llr_64++ = simde_mm256_extract_epi64(xmm1, 0); + *llr_64++ = simde_mm256_extract_epi64(xmm1, 1); + *llr_64++ = simde_mm256_extract_epi64(xmm2, 0); + *llr_64++ = simde_mm256_extract_epi64(xmm2, 1); + *llr_64++ = simde_mm256_extract_epi64(xmm1, 2); + *llr_64++ = simde_mm256_extract_epi64(xmm1, 3); + *llr_64++ = simde_mm256_extract_epi64(xmm2, 2); + *llr_64++ = simde_mm256_extract_epi64(xmm2, 3); } } @@ -106,13 +109,16 @@ void nr_ulsch_64qam_llr(int32_t *rxdataF_comp, int32_t *llr_32 = (int32_t *)ulsch_llr; - for (int i = 0; i < ((nb_re>>3) + ((nb_re&7) ? 1 : 0)); i++) { + for (int i = 0; i < ((nb_re + 7) >> 3); i++) { xmm0 = rxF[i]; - xmm1 = simde_mm256_abs_epi16(xmm0); - xmm1 = simde_mm256_subs_epi16(ch_maga[i], xmm1); + xmm1 = simde_mm256_abs_epi16(xmm0); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| + xmm1 = simde_mm256_subs_epi16(ch_maga[i], xmm1); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 xmm2 = simde_mm256_abs_epi16(xmm1); xmm2 = simde_mm256_subs_epi16(ch_magb[i], xmm2); - + // xmm0 |1st 4th 7th 10th 13th 16th 19th 22ed| + // xmm1 |2ed 5th 8th 11th 14th 17th 20th 23rd| + // xmm2 |3rd 6th 9th 12th 15th 18th 21st 24th| + *llr_32++ = simde_mm256_extract_epi32(xmm0,0); *llr_32++ = simde_mm256_extract_epi32(xmm1,0); *llr_32++ = simde_mm256_extract_epi32(xmm2,0); @@ -163,7 +169,7 @@ void nr_ulsch_256qam_llr(int32_t *rxdataF_comp, simde__m256i* ch_magb = (simde__m256i*)ul_ch_magb; simde__m256i* ch_magc = (simde__m256i*)ul_ch_magc; - for (int i = 0; i < ((nb_re>>3) + ((nb_re&7) ? 1 : 0)); i++) { + for (int i = 0; i < ((nb_re + 7) >> 3); i++) { xmm0 = simde_mm256_abs_epi16(rxF[i]); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| xmm0 = simde_mm256_subs_epi16(ch_maga[i], xmm0); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 // xmmtmpD2 contains 16 LLRs diff --git a/openair1/PHY/defs_gNB.h b/openair1/PHY/defs_gNB.h index 83040cb0e0..0a070eb67b 100644 --- a/openair1/PHY/defs_gNB.h +++ b/openair1/PHY/defs_gNB.h @@ -733,7 +733,6 @@ typedef struct PHY_VARS_gNB_s { time_stats_t ul_indication_stats; time_stats_t schedule_response_stats; time_stats_t ulsch_decoding_stats; - time_stats_t ulsch_rate_unmatching_stats; time_stats_t ulsch_ldpc_decoding_stats; time_stats_t ulsch_deinterleaving_stats; time_stats_t ulsch_channel_estimation_stats;