From 37c1219e7972320cbff47ccaace51629312df7dd Mon Sep 17 00:00:00 2001 From: Raymond Knopp Date: Sun, 16 Aug 2020 23:07:04 +0200 Subject: [PATCH] AVX2 optimization of 16QAM NR ULSCH LLR --- .../PHY/NR_TRANSPORT/nr_ulsch_demodulation.c | 63 ++++++++++---- .../NR_TRANSPORT/nr_ulsch_llr_computation.c | 83 ++++++++++++++++--- 2 files changed, 116 insertions(+), 30 deletions(-) diff --git a/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c b/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c index 1efdf005b2..03bb1e11dc 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c +++ b/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c @@ -257,16 +257,22 @@ void nr_ulsch_extract_rbs_single(int32_t **rxdataF, uint8_t is_dmrs_re=0,is_ptrs_re=0; start_re = (frame_parms->first_carrier_offset + (pusch_pdu->rb_start * NR_NB_SC_PER_RB))%frame_parms->ofdm_symbol_size; nb_re_pusch = NR_NB_SC_PER_RB * pusch_pdu->rb_size; +#ifdef __AVX2__ + int nb_re_pusch2 = nb_re_pusch + (nb_re_pusch&7); +#else + int nb_re_pusch2 = nb_re_pusch; +#endif + num_ptrs_symbols = 0; for (aarx = 0; aarx < frame_parms->nb_antennas_rx; aarx++) { rxF = (int16_t *)&rxdataF[aarx][symbol * frame_parms->ofdm_symbol_size]; - rxF_ext = (int16_t *)&pusch_vars->rxdataF_ext[aarx][symbol * nb_re_pusch]; // [hna] rxdataF_ext isn't contiguous in order to solve an alignment problem ib llr computation in case of mod_order = 4, 6 + rxF_ext = (int16_t *)&pusch_vars->rxdataF_ext[aarx][symbol * nb_re_pusch2]; // [hna] rxdataF_ext isn't contiguous in order to solve an alignment problem ib llr computation in case of mod_order = 4, 6 ul_ch0 = &pusch_vars->ul_ch_estimates[aarx][pusch_vars->dmrs_symbol*frame_parms->ofdm_symbol_size]; // update channel estimates if new dmrs symbol are available - ul_ch0_ext = &pusch_vars->ul_ch_estimates_ext[aarx][symbol*nb_re_pusch]; + ul_ch0_ext = &pusch_vars->ul_ch_estimates_ext[aarx][symbol*nb_re_pusch2]; ul_ch0_ptrs = &pusch_vars->ul_ch_ptrs_estimates[aarx][pusch_vars->ptrs_symbol_index*frame_parms->ofdm_symbol_size]; // update channel estimates if new dmrs symbol are available @@ -366,10 +372,16 @@ void nr_ulsch_scale_channel(int **ul_ch_estimates_ext, ch_amp128 = _mm_set1_epi16(ch_amp); // Q3.13 +#ifdef __AVX2__ + int off = ((nb_rb&1) == 1)? 4:0; +#else + int off = 0; +#endif + for (aatx=0; aatx < frame_parms->nb_antenna_ports_gNB; aatx++) { for (aarx=0; aarx < frame_parms->nb_antennas_rx; aarx++) { - ul_ch128 = (__m128i *)&ul_ch_estimates_ext[aarx][symbol*nb_rb*NR_NB_SC_PER_RB]; + ul_ch128 = (__m128i *)&ul_ch_estimates_ext[aarx][symbol*(off+(nb_rb*NR_NB_SC_PER_RB))]; if (is_dmrs_symbol==1){ if (pusch_dmrs_type == pusch_dmrs_type1) @@ -418,12 +430,18 @@ void nr_ulsch_channel_level(int **ul_ch_estimates_ext, int16_t x = factor2(len); int16_t y = (len)>>x; +#ifdef __AVX2__ + int off = ((nb_rb&1) == 1)? 4:0; +#else + int off = 0; +#endif + for (aatx = 0; aatx < frame_parms->nb_antennas_tx; aatx++) for (aarx = 0; aarx < frame_parms->nb_antennas_rx; aarx++) { //clear average level avg128U = _mm_setzero_si128(); - ul_ch128=(__m128i *)&ul_ch_estimates_ext[(aatx<<1)+aarx][symbol*nb_rb*12]; + ul_ch128=(__m128i *)&ul_ch_estimates_ext[(aatx<<1)+aarx][symbol*(off+(nb_rb*12))]; for (rb = 0; rb < len/12; rb++) { avg128U = _mm_add_epi32(avg128U, _mm_srai_epi32(_mm_madd_epi16(ul_ch128[0], ul_ch128[0]), x)); @@ -511,13 +529,13 @@ void nr_ulsch_channel_compensation(int **rxdataF_ext, unsigned short nb_rb, unsigned char output_shift) { + #ifdef DEBUG_CH_COMP int16_t *rxF, *ul_ch; int prnt_idx; - - rxF = (int16_t *)&rxdataF_ext[0][(symbol*nb_rb*12)]; - ul_ch = (int16_t *)&ul_ch_estimates_ext[0][symbol*nb_rb*12]; + rxF = (int16_t *)&rxdataF_ext[0][symbol*(off+(nb_rb*12))]; + ul_ch = (int16_t *)&ul_ch_estimates_ext[0][symbol*(off+(nb_rb*1))2]; printf("--------------------symbol = %d, mod_order = %d, output_shift = %d-----------------------\n", symbol, mod_order, output_shift); printf("----------------Before compensation------------------\n"); @@ -536,7 +554,7 @@ void nr_ulsch_channel_compensation(int **rxdataF_ext, int print_idx; - ch_mag = (int16_t *)&ul_ch_mag[0][(symbol*nb_rb*12)]; + ch_mag = (int16_t *)&ul_ch_mag[0][symbol*(off+(nb_rb*12))]; printf("--------------------symbol = %d, mod_order = %d-----------------------\n", symbol, mod_order); printf("----------------Before computation------------------\n"); @@ -549,7 +567,13 @@ void nr_ulsch_channel_compensation(int **rxdataF_ext, #endif -#if defined(__i386) || defined(__x86_64) +#ifdef __AVX2__ + int off = ((nb_rb&1) == 1)? 4:0; +#else + int off = 0; +#endif + +#if defined(__i386) || defined(__x86_64__) unsigned short rb; unsigned char aatx,aarx; @@ -571,11 +595,11 @@ void nr_ulsch_channel_compensation(int **rxdataF_ext, for (aarx=0; aarxnb_antennas_rx; aarx++) { - ul_ch128 = (__m128i *)&ul_ch_estimates_ext[(aatx<<1)+aarx][symbol*nb_rb*12]; - ul_ch_mag128 = (__m128i *)&ul_ch_mag[(aatx<<1)+aarx][symbol*nb_rb*12]; - ul_ch_mag128b = (__m128i *)&ul_ch_magb[(aatx<<1)+aarx][symbol*nb_rb*12]; - rxdataF128 = (__m128i *)&rxdataF_ext[aarx][symbol*nb_rb*12]; - rxdataF_comp128 = (__m128i *)&rxdataF_comp[(aatx<<1)+aarx][symbol*nb_rb*12]; + ul_ch128 = (__m128i *)&ul_ch_estimates_ext[(aatx<<1)+aarx][symbol*(off+(nb_rb*12))]; + ul_ch_mag128 = (__m128i *)&ul_ch_mag[(aatx<<1)+aarx][symbol*(off+(nb_rb*12))]; + ul_ch_mag128b = (__m128i *)&ul_ch_magb[(aatx<<1)+aarx][symbol*(off+(nb_rb*12))]; + rxdataF128 = (__m128i *)&rxdataF_ext[aarx][symbol*(off+(nb_rb*12))]; + rxdataF_comp128 = (__m128i *)&rxdataF_comp[(aatx<<1)+aarx][symbol*(off+(nb_rb*12))]; for (rb=0; rbulsch_llr_stats); AssertFatal(gNB->pusch_vars[ulsch_id]->rxdataF_ext_offset * rel15_ul->qam_mod_order+nb_re_pusch*rel15_ul->qam_mod_order < (8*((3*8*6144)+12)) , "Mysterious llr buffer size check"); - nr_ulsch_compute_llr(&gNB->pusch_vars[ulsch_id]->rxdataF_comp[0][symbol * rel15_ul->rb_size * NR_NB_SC_PER_RB], +#ifdef __AVX2__ + int off = ((rel15_ul->rb_size&1) == 1)? 4:0; +#else + int off = 0; +#endif + nr_ulsch_compute_llr(&gNB->pusch_vars[ulsch_id]->rxdataF_comp[0][symbol * (off+(rel15_ul->rb_size * NR_NB_SC_PER_RB))], gNB->pusch_vars[ulsch_id]->ul_ch_mag0, gNB->pusch_vars[ulsch_id]->ul_ch_magb0, &gNB->pusch_vars[ulsch_id]->llr[gNB->pusch_vars[ulsch_id]->rxdataF_ext_offset * rel15_ul->qam_mod_order], diff --git a/openair1/PHY/NR_TRANSPORT/nr_ulsch_llr_computation.c b/openair1/PHY/NR_TRANSPORT/nr_ulsch_llr_computation.c index 28a1db2c0a..43e7c6f2ce 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_ulsch_llr_computation.c +++ b/openair1/PHY/NR_TRANSPORT/nr_ulsch_llr_computation.c @@ -34,9 +34,6 @@ #include "PHY/sse_intrin.h" #include "PHY/impl_defs_top.h" -__m128i xmm0 __attribute__ ((aligned(32))); -__m128i xmm1 __attribute__ ((aligned(32))); -__m128i xmm2 __attribute__ ((aligned(32))); //---------------------------------------------------------------------------------------------- @@ -47,20 +44,19 @@ void nr_ulsch_qpsk_llr(int32_t *rxdataF_comp, uint32_t nb_re, uint8_t symbol) { - int i; - uint32_t *rxF = (uint32_t*)rxdataF_comp; uint32_t *llr32 = (uint32_t*)ulsch_llr; if (!llr32) { LOG_E(PHY,"nr_ulsch_qpsk_llr: llr is null, symbol %d, llr32 = %p\n",symbol, llr32); } - + /* for (i = 0; i < nb_re; i++) { *llr32 = *rxF; rxF++; llr32++; - } + }*/ + memcpy1((void*)llr32,(void*)rxF,nb_re<<2); } //---------------------------------------------------------------------------------------------- @@ -76,9 +72,17 @@ void nr_ulsch_16qam_llr(int32_t *rxdataF_comp, { #if defined(__x86_64__) || defined(__i386__) +#ifdef __AVX2__ + __m256i *rxF = (__m256i*)rxdataF_comp; + __m256i *ch_mag; + __m256i llr256[2]; + register __m256i xmm0; +#else __m128i *rxF = (__m128i*)rxdataF_comp; __m128i *ch_mag; __m128i llr128[2]; + register __m128i xmm0; +#endif uint32_t *llr32; #elif defined(__arm__) @@ -90,7 +94,12 @@ void nr_ulsch_16qam_llr(int32_t *rxdataF_comp, int i; - unsigned char len_mod4 = 0; + +#ifdef __AVX2__ + int off = ((nb_rb&1) == 1)? 4:0; +#else + int off = 0; +#endif #if defined(__x86_64__) || defined(__i386__) @@ -100,22 +109,69 @@ void nr_ulsch_16qam_llr(int32_t *rxdataF_comp, #endif #if defined(__x86_64__) || defined(__i386__) - ch_mag = (__m128i*)&ul_ch_mag[0][(symbol*nb_rb*12)]; +#ifdef __AVX2__ + ch_mag = (__m256i*)&ul_ch_mag[0][(symbol*(off+(nb_rb*12)))]; +#else + ch_mag = (__m128i*)&ul_ch_mag[0][(symbol*(off+(nb_rb*12)))]; +#endif #elif defined(__arm__) ch_mag = (int16x8_t*)&ul_ch_mag[0][(symbol*nb_rb*12)]; #endif - len_mod4 = nb_re&3; +#ifdef __AVX2__ + unsigned char len_mod8 = nb_re&7; + nb_re >>= 3; // length in quad words (4 REs) + nb_re += (len_mod8 == 0 ? 0 : 1); +#else + unsigned char len_mod4 = nb_re&3; nb_re >>= 2; // length in quad words (4 REs) nb_re += (len_mod4 == 0 ? 0 : 1); +#endif for (i=0; i |y_R|, registers of odd index in xmm0-> |y_I| + xmm0 = _mm256_subs_epi16(ch_mag[i],xmm0); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 + + llr256[0] = _mm256_unpacklo_epi32(rxF[i],xmm0); // llr128[0] contains the llrs of the 1st,2nd,5th and 6th REs + llr256[1] = _mm256_unpackhi_epi32(rxF[i],xmm0); // llr128[1] contains the llrs of the 3rd, 4th, 7th and 8th REs + + // 1st RE + llr32[0] = _mm256_extract_epi32(llr256[0],0); // llr32[0] low 16 bits-> y_R , high 16 bits-> y_I + llr32[1] = _mm256_extract_epi32(llr256[0],1); // llr32[1] low 16 bits-> |h|-|y_R|^2, high 16 bits-> |h|-|y_I|^2 + // 2nd RE + llr32[2] = _mm256_extract_epi32(llr256[0],2); // llr32[2] low 16 bits-> y_R , high 16 bits-> y_I + llr32[3] = _mm256_extract_epi32(llr256[0],3); // llr32[3] low 16 bits-> |h|-|y_R|^2, high 16 bits-> |h|-|y_I|^2 + + // 3rd RE + llr32[4] = _mm256_extract_epi32(llr256[1],0); // llr32[4] low 16 bits-> y_R , high 16 bits-> y_I + llr32[5] = _mm256_extract_epi32(llr256[1],1); // llr32[5] low 16 bits-> |h|-|y_R|^2, high 16 bits-> |h|-|y_I|^2 + + // 4th RE + llr32[6] = _mm256_extract_epi32(llr256[1],2); // llr32[6] low 16 bits-> y_R , high 16 bits-> y_I + llr32[7] = _mm256_extract_epi32(llr256[1],3); // llr32[7] low 16 bits-> |h|-|y_R|^2, high 16 bits-> |h|-|y_I|^2 + + // 5th RE + llr32[8] = _mm256_extract_epi32(llr256[0],4); // llr32[8] low 16 bits-> y_R , high 16 bits-> y_I + llr32[9] = _mm256_extract_epi32(llr256[0],5); // llr32[9] low 16 bits-> |h|-|y_R|^2, high 16 bits-> |h|-|y_I|^2 + + // 6th RE + llr32[10] = _mm256_extract_epi32(llr256[0],6); // llr32[10] low 16 bits-> y_R , high 16 bits-> y_I + llr32[11] = _mm256_extract_epi32(llr256[0],7); // llr32[11] low 16 bits-> |h|-|y_R|^2, high 16 bits-> |h|-|y_I|^2 + + // 7th RE + llr32[12] = _mm256_extract_epi32(llr256[1],4); // llr32[12] low 16 bits-> y_R , high 16 bits-> y_I + llr32[13] = _mm256_extract_epi32(llr256[1],5); // llr32[13] low 16 bits-> |h|-|y_R|^2, high 16 bits-> |h|-|y_I|^2 + + // 8th RE + llr32[14] = _mm256_extract_epi32(llr256[1],6); // llr32[14] low 16 bits-> y_R , high 16 bits-> y_I + llr32[15] = _mm256_extract_epi32(llr256[1],7); // llr32[15] low 16 bits-> |h|-|y_R|^2, high 16 bits-> |h|-|y_I|^2 + + llr32+=16; +#else xmm0 = _mm_abs_epi16(rxF[i]); // registers of even index in xmm0-> |y_R|, registers of odd index in xmm0-> |y_I| - - xmm0 = _mm_subs_epi16(ch_mag[i],xmm0); // registers of even index in xmm0-> |y_R|-|h|^2, registers of odd index in xmm0-> |y_I|-|h|^2 llr128[0] = _mm_unpacklo_epi32(rxF[i],xmm0); // llr128[0] contains the llrs of the 1st and 2nd REs @@ -138,6 +194,7 @@ void nr_ulsch_16qam_llr(int32_t *rxdataF_comp, llr32[7] = _mm_extract_epi32(llr128[1],3); // llr32[7] low 16 bits-> |h|-|y_R|^2, high 16 bits-> |h|-|y_I|^2 llr32+=8; +#endif #elif defined(__arm__) xmm0 = vabsq_s16(rxF[i]); xmm0 = vqsubq_s16((*(__m128i*)&ones[0]),xmm0);