diff --git a/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c b/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c index b7cfa7d1e4..43f6417c10 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c +++ b/openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c @@ -209,25 +209,17 @@ static void nr_ulsch_channel_level(int size_est, uint32_t len, uint8_t nrOfLayers) { - simde__m128i *ul_ch128, avg128U; - int16_t x = factor2(len); int16_t y = (len)>>x; for (int aatx = 0; aatx < nrOfLayers; aatx++) { for (int aarx = 0; aarx < frame_parms->nb_antennas_rx; aarx++) { - //clear average level - avg128U = simde_mm_setzero_si128(); - ul_ch128 = (simde__m128i *)&ul_ch_estimates_ext[aatx * frame_parms->nb_antennas_rx + aarx][symbol * len]; + simde__m128i *ul_ch128 = (simde__m128i *)&ul_ch_estimates_ext[aatx * frame_parms->nb_antennas_rx + aarx][symbol * len]; - for (int i = 0; i < len >> 2; i++) { - avg128U = simde_mm_add_epi32(avg128U, simde_mm_srai_epi32(simde_mm_madd_epi16(ul_ch128[i], ul_ch128[i]), x)); - } - - int32_t *avg32i = (int32_t *)&avg128U; - int64_t avg64 = (int64_t)avg32i[0] + avg32i[1] + avg32i[2] + avg32i[3]; - avg[aatx * frame_parms->nb_antennas_rx + aarx] = avg64 / y; + //compute average level + avg[aatx * frame_parms->nb_antennas_rx + aarx] = simde_mm_average(ul_ch128, len, x, y); + //LOG_D(PHY, "Channel level: %d\n", avg[aatx * frame_parms->nb_antennas_rx + aarx]); } } diff --git a/openair1/PHY/NR_UE_TRANSPORT/dci_nr.c b/openair1/PHY/NR_UE_TRANSPORT/dci_nr.c index 15bbc4e626..36bd5a1ce7 100644 --- a/openair1/PHY/NR_UE_TRANSPORT/dci_nr.c +++ b/openair1/PHY/NR_UE_TRANSPORT/dci_nr.c @@ -226,30 +226,12 @@ void nr_pdcch_channel_level(int32_t rx_size, int nb_rb) { for (int aarx = 0; aarx < frame_parms->nb_antennas_rx; aarx++) { - //clear average level - simde__m128i avg128P = simde_mm_setzero_si128(); + simde__m128i *dl_ch128 = (simde__m128i *)&dl_ch_estimates_ext[aarx][symbol * nb_rb * 12]; - for (int rb = 0; rb < (nb_rb * 3) >> 2; rb++) { - avg128P = simde_mm_add_epi32(avg128P,simde_mm_madd_epi16(dl_ch128[0],dl_ch128[0])); - avg128P = simde_mm_add_epi32(avg128P,simde_mm_madd_epi16(dl_ch128[1],dl_ch128[1])); - avg128P = simde_mm_add_epi32(avg128P,simde_mm_madd_epi16(dl_ch128[2],dl_ch128[2])); - // for (int i=0;i<24;i+=2) printf("pdcch channel re %d (%d,%d)\n",(rb*12)+(i>>1),((int16_t*)dl_ch128)[i],((int16_t*)dl_ch128)[i+1]); - dl_ch128+=3; - /* - if (rb==0) { - print_shorts("dl_ch128",&dl_ch128[0]); - print_shorts("dl_ch128",&dl_ch128[1]); - print_shorts("dl_ch128",&dl_ch128[2]); - } - */ - } - - DevAssert(nb_rb); - avg[aarx] = 0; - for (int i = 0; i < 4; i++) - avg[aarx] += ((int32_t *)&avg128P)[i] / (nb_rb * 9); - LOG_DDD("Channel level : %d\n",avg[aarx]); + //compute average level + avg[aarx] = simde_mm_average(dl_ch128, nb_rb * 12, 0, nb_rb * 12); + //LOG_DDD("Channel level : %d\n", avg[aarx]); } } diff --git a/openair1/PHY/NR_UE_TRANSPORT/nr_dlsch_demodulation.c b/openair1/PHY/NR_UE_TRANSPORT/nr_dlsch_demodulation.c index c0019fb3ec..217a370054 100644 --- a/openair1/PHY/NR_UE_TRANSPORT/nr_dlsch_demodulation.c +++ b/openair1/PHY/NR_UE_TRANSPORT/nr_dlsch_demodulation.c @@ -1098,28 +1098,20 @@ void nr_dlsch_channel_level(uint32_t rx_size_symbol, int32_t avg[MAX_ANT][MAX_ANT], uint32_t len) { - simde__m128i *dl_ch128, avg128D; - //nb_rb*nre = y * 2^x int16_t x = factor2(len); int16_t y = (len) >> x; + uint32_t nb_rb_0 = len / NR_NB_SC_PER_RB + ((len % NR_NB_SC_PER_RB) ? 1 : 0); LOG_D(NR_PHY, "nb_rb_0 %d len %d = %d * 2^(%d)\n", nb_rb_0, len, y, x); + for (int aatx = 0; aatx < n_tx; aatx++) { for (int aarx = 0; aarx < n_rx; aarx++) { - //clear average level - avg128D = simde_mm_setzero_si128(); - dl_ch128 = (simde__m128i *)dl_ch_estimates_ext[(aatx * n_rx) + aarx]; + simde__m128i *dl_ch128 = (simde__m128i *)dl_ch_estimates_ext[(aatx * n_rx) + aarx]; - for (int rb = 0; rb < nb_rb_0; rb++) { - avg128D = simde_mm_add_epi32(avg128D,simde_mm_srai_epi32(simde_mm_madd_epi16(dl_ch128[0],dl_ch128[0]),x)); - avg128D = simde_mm_add_epi32(avg128D,simde_mm_srai_epi32(simde_mm_madd_epi16(dl_ch128[1],dl_ch128[1]),x)); - avg128D = simde_mm_add_epi32(avg128D,simde_mm_srai_epi32(simde_mm_madd_epi16(dl_ch128[2],dl_ch128[2]),x)); - dl_ch128+=3; - } - int32_t *tmp = (int32_t *)&avg128D; - avg[aatx][aarx] = ((int64_t)tmp[0] + tmp[1] + tmp[2] + tmp[3]) / y; - LOG_D(PHY, "Channel level: %d\n", avg[aatx][aarx]); + //compute average level + avg[aatx][aarx] = simde_mm_average(dl_ch128, nb_rb_0 * 12, x, y); + //LOG_D(PHY, "Channel level: %d\n", avg[aatx][aarx]); } } } diff --git a/openair1/PHY/NR_UE_TRANSPORT/nr_pbch.c b/openair1/PHY/NR_UE_TRANSPORT/nr_pbch.c index 5dbab85f12..229191269f 100644 --- a/openair1/PHY/NR_UE_TRANSPORT/nr_pbch.c +++ b/openair1/PHY/NR_UE_TRANSPORT/nr_pbch.c @@ -208,31 +208,14 @@ int nr_pbch_channel_level(struct complex16 dl_ch_estimates_ext[][PBCH_MAX_RE_PER const NR_DL_FRAME_PARMS *frame_parms, int nb_re) { - int16_t nb_rb=nb_re/12; - simde__m128i avg128; - simde__m128i *dl_ch128; - int avg1=0,avg2=0; + int32_t avg2 = 0; for (int aarx=0; aarxnb_antennas_rx; aarx++) { - //clear average level - avg128 = simde_mm_setzero_si128(); - dl_ch128=(simde__m128i *)dl_ch_estimates_ext[aarx]; - for (int rb=0; rbavg2) avg2 = avg1; diff --git a/openair1/PHY/sse_intrin.h b/openair1/PHY/sse_intrin.h index 655c38982e..d6c3157ce8 100644 --- a/openair1/PHY/sse_intrin.h +++ b/openair1/PHY/sse_intrin.h @@ -49,13 +49,6 @@ #define SSE_INTRIN_H -#include -#include -#include -#include -#include -#include -#include #include #include #if defined(__x86_64) || defined(__i386__) @@ -81,31 +74,101 @@ * OAI specific */ -static const short minusConjug128[8]__attribute__((aligned(16))) = {-1,1,-1,1,-1,1,-1,1}; -static inline simde__m128i mulByConjugate128(simde__m128i *a, simde__m128i *b, int8_t output_shift) { - - simde__m128i realPart = simde_mm_madd_epi16(*a,*b); - realPart = simde_mm_srai_epi32(realPart,output_shift); - simde__m128i imagPart = simde_mm_shufflelo_epi16(*b, SIMDE_MM_SHUFFLE(2,3,0,1)); - imagPart = simde_mm_shufflehi_epi16(imagPart, SIMDE_MM_SHUFFLE(2,3,0,1)); - imagPart = simde_mm_sign_epi16(imagPart,*(simde__m128i *)minusConjug128); - imagPart = simde_mm_madd_epi16(imagPart,*a); - imagPart = simde_mm_srai_epi32(imagPart,output_shift); - simde__m128i lowPart = simde_mm_unpacklo_epi32(realPart,imagPart); - simde__m128i highPart = simde_mm_unpackhi_epi32(realPart,imagPart); - return ( simde_mm_packs_epi32(lowPart,highPart)); +static const short minusConjug128[8] __attribute__((aligned(16))) = {-1, 1, -1, 1, -1, 1, -1, 1}; +static inline simde__m128i mulByConjugate128(simde__m128i *a, simde__m128i *b, int8_t output_shift) +{ + simde__m128i realPart = simde_mm_madd_epi16(*a, *b); + realPart = simde_mm_srai_epi32(realPart, output_shift); + simde__m128i imagPart = simde_mm_shufflelo_epi16(*b, SIMDE_MM_SHUFFLE(2, 3, 0, 1)); + imagPart = simde_mm_shufflehi_epi16(imagPart, SIMDE_MM_SHUFFLE(2, 3, 0, 1)); + imagPart = simde_mm_sign_epi16(imagPart, *(simde__m128i *)minusConjug128); + imagPart = simde_mm_madd_epi16(imagPart, *a); + imagPart = simde_mm_srai_epi32(imagPart, output_shift); + simde__m128i lowPart = simde_mm_unpacklo_epi32(realPart, imagPart); + simde__m128i highPart = simde_mm_unpackhi_epi32(realPart, imagPart); + return (simde_mm_packs_epi32(lowPart, highPart)); } -#define displaySamples128(vect) {\ - simde__m128i x=vect; \ - printf("vector: %s = (%hd,%hd) (%hd,%hd) (%hd,%hd) (%hd,%hd)\n", #vect, \ - simde_mm_extract_epi16(x,0), \ - simde_mm_extract_epi16(x,1),\ - simde_mm_extract_epi16(x,2),\ - simde_mm_extract_epi16(x,3),\ - simde_mm_extract_epi16(x,4),\ - simde_mm_extract_epi16(x,5),\ - simde_mm_extract_epi16(x,6),\ - simde_mm_extract_epi16(x,7));\ +#define displaySamples128(vect) \ + { \ + simde__m128i x = vect; \ + printf("vector: %s = (%hd,%hd) (%hd,%hd) (%hd,%hd) (%hd,%hd)\n", \ + #vect, \ + simde_mm_extract_epi16(x, 0), \ + simde_mm_extract_epi16(x, 1), \ + simde_mm_extract_epi16(x, 2), \ + simde_mm_extract_epi16(x, 3), \ + simde_mm_extract_epi16(x, 4), \ + simde_mm_extract_epi16(x, 5), \ + simde_mm_extract_epi16(x, 6), \ + simde_mm_extract_epi16(x, 7)); \ } + +__attribute__((always_inline)) static inline int64_t simde_mm_average_sse(simde__m128i *a, int length, int shift) +{ + // compute average level with shift (64-bit verstion) + simde__m128i avg128 = simde_mm_setzero_si128(); + for (int i = 0; i < length >> 2; i++) { + const simde__m128i in1 = a[i]; + avg128 = simde_mm_add_epi32(avg128, simde_mm_srai_epi32(simde_mm_madd_epi16(in1, in1), shift)); + } + + // Horizontally add pairs + // 1st [A + C, B + D] + simde__m128i sum_pairs = simde_mm_add_epi64(simde_mm_unpacklo_epi32(avg128, simde_mm_setzero_si128()), // [A, B] → [A, 0, B, 0] + simde_mm_unpackhi_epi32(avg128, simde_mm_setzero_si128()) // [C, D] → [C, 0, D, 0] + ); + + // 2nd [A + B + C + D, ...] + simde__m128i total_sum = simde_mm_add_epi64(sum_pairs, simde_mm_shuffle_epi32(sum_pairs, SIMDE_MM_SHUFFLE(1, 0, 3, 2))); + + // Extract horizontal sum as a scalar int64_t result + return simde_mm_cvtsi128_si64(total_sum); +} + +__attribute__((always_inline)) static inline int64_t simde_mm_average_avx2(simde__m256i *a, int length, int shift) +{ + simde__m256i avg256 = simde_mm256_setzero_si256(); + for (int i = 0; i < length >> 3; i++) { + const simde__m256i in1 = simde_mm256_loadu_si256(&a[i]); // unaligned load + avg256 = simde_mm256_add_epi32(avg256, simde_mm256_srai_epi32(simde_mm256_madd_epi16(in1, in1), shift)); + } + + // Split the 256-bit vector into two 128-bit halves and convert to 64-bit + // [A + E, B + F, C + G, D + H] + simde__m256i sum_pairs = simde_mm256_add_epi64( + simde_mm256_cvtepi32_epi64(simde_mm256_castsi256_si128(avg256)), // [A, B, C, D] → [A, 0, B, 0, C, 0, D, 0] + simde_mm256_cvtepi32_epi64(simde_mm256_extracti128_si256(avg256, 1)) // [E, F, G, H] → [E, 0, F, 0, G, 0, H, 0] + ); + + // Horizontal sum within the 256-bit vector + // [A + E + B + F, C + G + D + H] + simde__m128i total_sum = simde_mm_add_epi64(simde_mm256_castsi256_si128(sum_pairs), simde_mm256_extracti128_si256(sum_pairs, 1)); + + // [A + E + B + F + C + G + D + H, ...] + total_sum = simde_mm_add_epi64(total_sum, simde_mm_shuffle_epi32(total_sum, SIMDE_MM_SHUFFLE(1, 0, 3, 2))); + + // Extract horizontal sum as a scalar int64_t result + return simde_mm_cvtsi128_si64(total_sum); +} + +__attribute__((always_inline)) static inline int32_t simde_mm_average(simde__m128i *a, int length, int shift, int16_t scale) +{ + int64_t avg = 0; + +#if defined(__x86_64__) || defined(__i386__) + if (__builtin_cpu_supports("avx2")) { + avg += simde_mm_average_avx2((simde__m256i *)a, length, shift); + + // tail processing by SSE + a += ((length & ~7) >> 2); + length -= (length & ~7); + } +#endif + + avg += simde_mm_average_sse(a, length, shift); + + return (uint32_t)(avg / scale); +} + #endif // SSE_INTRIN_H