From f57f6a8d4e44e0aceece71a15409df6f725aef43 Mon Sep 17 00:00:00 2001 From: Laurent THOMAS Date: Tue, 1 Jul 2025 17:44:11 +0200 Subject: [PATCH] factorize code that initialize all 0xff simd vectors --- .../LTE_UE_TRANSPORT/dlsch_llr_computation.c | 31 +++++++++------ .../dlsch_llr_computation_avx2.c | 38 ++++++------------- .../NR_TRANSPORT/nr_ulsch_llr_computation.c | 20 +++------- openair1/PHY/TOOLS/tools_defs.h | 24 ++++++++++++ 4 files changed, 61 insertions(+), 52 deletions(-) diff --git a/openair1/PHY/LTE_UE_TRANSPORT/dlsch_llr_computation.c b/openair1/PHY/LTE_UE_TRANSPORT/dlsch_llr_computation.c index fd1b2a3e9a..bcab4cd8dd 100644 --- a/openair1/PHY/LTE_UE_TRANSPORT/dlsch_llr_computation.c +++ b/openair1/PHY/LTE_UE_TRANSPORT/dlsch_llr_computation.c @@ -38,12 +38,9 @@ #include "transport_proto_ue.h" #include "PHY/sse_intrin.h" -//#define DEBUG_LLR_SIC - -const int16_t zeros[8] __attribute__((aligned(16))) = {0, 0, 0, 0, 0, 0, 0, 0}; -const int16_t ones[8] __attribute__((aligned(16))) = {0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff}; +// #define DEBUG_LLR_SIC //============================================================================================== -// Auxiliary Makros +// Auxiliary Makros // calculates psi_a = psi_r*a_r + psi_i*a_i #define prodsum_psi_a_epi16(psi_r, a_r, psi_i, a_i, psi_a) \ @@ -54,18 +51,18 @@ const int16_t ones[8] __attribute__((aligned(16))) = {0xffff, 0xffff, 0xffff, 0x simde__m128i psi_a = simde_mm_adds_epi16(tmp_result, tmp_result2); // calculate interference magnitude -#define interference_abs_epi16(psi, int_ch_mag, int_mag, c1, c2) \ - tmp_result = simde_mm_cmplt_epi16(psi, int_ch_mag); \ - tmp_result2 = simde_mm_xor_si128(tmp_result, (*(simde__m128i *)&ones[0])); \ - tmp_result = simde_mm_and_si128(tmp_result, c1); \ - tmp_result2 = simde_mm_and_si128(tmp_result2, c2); \ +#define interference_abs_epi16(psi, int_ch_mag, int_mag, c1, c2) \ + tmp_result = simde_mm_cmplt_epi16(psi, int_ch_mag); \ + tmp_result2 = simde_mm_xor_si128(tmp_result, ones); \ + tmp_result = simde_mm_and_si128(tmp_result, c1); \ + tmp_result2 = simde_mm_and_si128(tmp_result2, c2); \ simde__m128i int_mag = simde_mm_or_si128(tmp_result, tmp_result2); // calculate interference magnitude // tmp_result = ones in shorts corr. to interval 2<=x<=4, tmp_result2 interval < 2, tmp_result3 interval 46 #define interference_abs_64qam_epi16(psi, int_ch_mag, int_two_ch_mag, int_three_ch_mag, a, c1, c3, c5, c7) \ tmp_result = simde_mm_cmplt_epi16(psi, int_two_ch_mag); \ - tmp_result3 = simde_mm_xor_si128(tmp_result, (*(simde__m128i *)&ones[0])); \ + tmp_result3 = simde_mm_xor_si128(tmp_result, ones); \ tmp_result2 = simde_mm_cmplt_epi16(psi, int_ch_mag); \ tmp_result = simde_mm_xor_si128(tmp_result, tmp_result2); \ tmp_result4 = simde_mm_cmpgt_epi16(psi, int_three_ch_mag); \ @@ -1123,6 +1120,8 @@ void qpsk_qam16(int16_t *stream0_in, oai_mm_separate_real_imag_parts(&ch_mag_int, NULL, xmm2, xmm3); simde__m128i tmp_result, tmp_result2; + //generate a const of all ff in case the compiler doesn't optimize it inside the loop + const simde__m128i ones = allones128(); // calculate optimal interference amplitudes interference_abs_epi16(psi_r_p1_p1 , ch_mag_int, a_r_p1_p1 , ONE_OVER_SQRT_10_Q15, THREE_OVER_SQRT_10); interference_abs_epi16(psi_i_p1_p1 , ch_mag_int, a_i_p1_p1 , ONE_OVER_SQRT_10_Q15, THREE_OVER_SQRT_10); @@ -1351,6 +1350,8 @@ void qpsk_qam64(short *stream0_in, two_ch_mag_int_with_sigma2 = ch_mag_int; // *4 three_ch_mag_int_with_sigma2 = simde_mm_adds_epi16(ch_mag_int_with_sigma2, two_ch_mag_int_with_sigma2); // *6 simde__m128i tmp_result, tmp_result2, tmp_result3, tmp_result4; + //generate a const of all ff in case the compiler doesn't optimize it inside the loop + const simde__m128i ones = allones128(); interference_abs_64qam_epi16(psi_r_p1_p1, ch_mag_int_with_sigma2, two_ch_mag_int_with_sigma2, three_ch_mag_int_with_sigma2, a_r_p1_p1, ONE_OVER_SQRT_2_42, THREE_OVER_SQRT_2_42, FIVE_OVER_SQRT_2_42, SEVEN_OVER_SQRT_2_42); interference_abs_64qam_epi16(psi_i_p1_p1, ch_mag_int_with_sigma2, two_ch_mag_int_with_sigma2, three_ch_mag_int_with_sigma2, a_i_p1_p1, ONE_OVER_SQRT_2_42, THREE_OVER_SQRT_2_42, FIVE_OVER_SQRT_2_42, @@ -2081,6 +2082,8 @@ void qam16_qam16(short *stream0_in, simde__m128i y0_p_3_3 = simde_mm_adds_epi16(y0r_three_over_sqrt10, y0i_three_over_sqrt10); simde__m128i y0_m_3_3 = simde_mm_subs_epi16(y0r_three_over_sqrt10, y0i_three_over_sqrt10); simde__m128i tmp_result, tmp_result2; + //generate a const of all ff in case the compiler doesn't optimize it inside the loop + const simde__m128i ones = allones128(); // Compute optimal interfering symbol magnitude interference_abs_epi16(psi_r_p1_p1 ,ch_mag_int,a_r_p1_p1,ONE_OVER_SQRT_10_Q15, THREE_OVER_SQRT_10); interference_abs_epi16(psi_i_p1_p1 ,ch_mag_int,a_i_p1_p1,ONE_OVER_SQRT_10_Q15, THREE_OVER_SQRT_10); @@ -2597,6 +2600,8 @@ void qam16_qam64(int16_t *stream0_in, ch_mag_int_with_sigma2 = simde_mm_srai_epi16(ch_mag_int, 1); // *2 two_ch_mag_int_with_sigma2 = ch_mag_int; // *4 simde__m128i tmp_result, tmp_result2, tmp_result3, tmp_result4; + //generate a const of all ff in case the compiler doesn't optimize it inside the loop + const simde__m128i ones = allones128(); three_ch_mag_int_with_sigma2 = simde_mm_adds_epi16(ch_mag_int_with_sigma2, two_ch_mag_int_with_sigma2); // *6 interference_abs_64qam_epi16(psi_r_p1_p1 ,ch_mag_int_with_sigma2, two_ch_mag_int_with_sigma2, three_ch_mag_int_with_sigma2, a_r_p1_p1,ONE_OVER_SQRT_2_42, THREE_OVER_SQRT_2_42,FIVE_OVER_SQRT_2_42, @@ -4927,6 +4932,8 @@ void qam64_qam16(short *stream0_in, simde__m128i y0_m_7_5 = simde_mm_subs_epi16(y0r_seven_over_sqrt_21, y0i_five_over_sqrt_21); simde__m128i y0_m_7_7 = simde_mm_subs_epi16(y0r_seven_over_sqrt_21, y0i_seven_over_sqrt_21); simde__m128i tmp_result, tmp_result2; + //generate a const of all ff in case the compiler doesn't optimize it inside the loop + const simde__m128i ones = allones128(); interference_abs_epi16(psi_r_p7_p7, ch_mag_int, a_r_p7_p7, ONE_OVER_SQRT_10_Q15, THREE_OVER_SQRT_10); interference_abs_epi16(psi_r_p7_p5, ch_mag_int, a_r_p7_p5, ONE_OVER_SQRT_10_Q15, THREE_OVER_SQRT_10); interference_abs_epi16(psi_r_p7_p3, ch_mag_int, a_r_p7_p3, ONE_OVER_SQRT_10_Q15, THREE_OVER_SQRT_10); @@ -6425,6 +6432,8 @@ void qam64_qam64(short *stream0_in, two_ch_mag_int_with_sigma2 = ch_mag_int; // *4 three_ch_mag_int_with_sigma2 = simde_mm_adds_epi16(ch_mag_int_with_sigma2, two_ch_mag_int_with_sigma2); // *6 simde__m128i tmp_result, tmp_result2, tmp_result3, tmp_result4; + //generate a const of all ff in case the compiler doesn't optimize it inside the loop + const simde__m128i ones = allones128(); interference_abs_64qam_epi16(psi_r_p7_p7, ch_mag_int_with_sigma2, two_ch_mag_int_with_sigma2, three_ch_mag_int_with_sigma2, a_r_p7_p7, ONE_OVER_SQRT_2_42, THREE_OVER_SQRT_2_42, FIVE_OVER_SQRT_2_42, SEVEN_OVER_SQRT_2_42); interference_abs_64qam_epi16(psi_r_p7_p5, ch_mag_int_with_sigma2, two_ch_mag_int_with_sigma2, three_ch_mag_int_with_sigma2, a_r_p7_p5, ONE_OVER_SQRT_2_42, THREE_OVER_SQRT_2_42, FIVE_OVER_SQRT_2_42, diff --git a/openair1/PHY/LTE_UE_TRANSPORT/dlsch_llr_computation_avx2.c b/openair1/PHY/LTE_UE_TRANSPORT/dlsch_llr_computation_avx2.c index 0877b2ba7e..a4f641896b 100644 --- a/openair1/PHY/LTE_UE_TRANSPORT/dlsch_llr_computation_avx2.c +++ b/openair1/PHY/LTE_UE_TRANSPORT/dlsch_llr_computation_avx2.c @@ -37,40 +37,21 @@ #if defined(__x86_64__) - static const int16_t ones256[16] __attribute__((aligned(32))) = {0xffff, - 0xffff, - 0xffff, - 0xffff, - 0xffff, - 0xffff, - 0xffff, - 0xffff, - 0xffff, - 0xffff, - 0xffff, - 0xffff, - 0xffff, - 0xffff, - 0xffff, - 0xffff}; - - - // Auxiliary Makros - + // Auxiliary Macros // calculate interference magnitude #define interference_abs_epi16(psi, int_ch_mag, int_mag, c1, c2) \ - tmp_result = simde_mm256_cmpgt_epi16(int_ch_mag, psi); \ - tmp_result2 = simde_mm256_xor_si256(tmp_result, (*(simde__m256i *)&ones256[0])); \ - tmp_result = simde_mm256_and_si256(tmp_result, c1); \ - tmp_result2 = simde_mm256_and_si256(tmp_result2, c2); \ - const simde__m256i int_mag = simde_mm256_or_si256(tmp_result, tmp_result2); + tmp_result = simde_mm256_cmpgt_epi16(int_ch_mag, psi); \ + tmp_result2 = simde_mm256_xor_si256(tmp_result, ff256); \ + tmp_result = simde_mm256_and_si256(tmp_result, c1); \ + tmp_result2 = simde_mm256_and_si256(tmp_result2, c2); \ + const simde__m256i int_mag = simde_mm256_or_si256(tmp_result, tmp_result2); // calculate interference magnitude // tmp_result = ones in shorts corr. to interval 2<=x<=4, tmp_result2 interval < 2, tmp_result3 interval 46 #define interference_abs_64qam_epi16(psi, int_ch_mag, int_two_ch_mag, int_three_ch_mag, a, c1, c3, c5, c7) \ tmp_result = simde_mm256_cmpgt_epi16(int_two_ch_mag, psi); \ - tmp_result3 = simde_mm256_xor_si256(tmp_result, (*(simde__m256i *)&ones256[0])); \ + tmp_result3 = simde_mm256_xor_si256(tmp_result, ff256); \ tmp_result2 = simde_mm256_cmpgt_epi16(int_ch_mag, psi); \ tmp_result = simde_mm256_xor_si256(tmp_result, tmp_result2); \ tmp_result4 = simde_mm256_cmpgt_epi16(psi, int_three_ch_mag); \ @@ -195,7 +176,8 @@ void qam64_qam16_avx2(short *stream0_in, int i,j; uint32_t len256 = (length)>>3; - + //generate a const of all ff in case the compiler doesn't optimize it inside the loop + const simde__m256i ff256 = allones256(); for (i=0; i>3; + //generate a const of all ff in case the compiler doesn't optimize it inside the loop + const simde__m256i ff256 = allones256(); for (i=0; i6 @@ -409,7 +404,7 @@ static inline simde__m128i interference_abs_64qam_epi16(simde__m128i psi, simde__m128i c7) { simde__m128i tmp_result = simde_mm_cmpgt_epi16(int_two_ch_mag, psi); - simde__m128i tmp_result3 = simde_mm_xor_si128(tmp_result, (*(simde__m128i *)&ones[0])); + simde__m128i tmp_result3 = simde_mm_xor_si128(tmp_result, allones128()); simde__m128i tmp_result2 = simde_mm_cmpgt_epi16(int_ch_mag, psi); tmp_result = simde_mm_xor_si128(tmp_result, tmp_result2); simde__m128i tmp_result4 = simde_mm_cmpgt_epi16(psi, int_three_ch_mag); @@ -437,7 +432,7 @@ static inline simde__m128i prodsum_psi_a_epi16(simde__m128i psi_r, simde__m128i static inline simde__m128i interference_abs_epi16(simde__m128i psi, simde__m128i int_ch_mag, simde__m128i c1, simde__m128i c2) { simde__m128i tmp_result = simde_mm_cmplt_epi16(psi, int_ch_mag); - simde__m128i tmp_result2 = simde_mm_xor_si128(tmp_result, (*(simde__m128i *)&ones[0])); + simde__m128i tmp_result2 = simde_mm_xor_si128(tmp_result,allones128()); tmp_result = simde_mm_and_si128(tmp_result, c1); tmp_result2 = simde_mm_and_si128(tmp_result2, c2); return simde_mm_or_si128(tmp_result, tmp_result2); @@ -492,9 +487,6 @@ static inline simde__m128i max_epi16(simde__m128i m0, simde__m128i m1, simde__m1 #else -static const int16_t ones256[16] __attribute__((aligned(32))) = {0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, - 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff}; - // calculate interference magnitude // tmp_result = ones in shorts corr. to interval 2<=x<=4, tmp_result2 interval < 2, tmp_result3 interval 46 @@ -508,7 +500,7 @@ static inline simde__m256i interference_abs_64qam_epi16_256(simde__m256i psi, simde__m256i c7) { simde__m256i tmp_result = simde_mm256_cmpgt_epi16(int_two_ch_mag, psi); - simde__m256i tmp_result3 = simde_mm256_xor_si256(tmp_result, (*(simde__m256i *)&ones256[0])); + simde__m256i tmp_result3 = simde_mm256_xor_si256(tmp_result, allones256()); simde__m256i tmp_result2 = simde_mm256_cmpgt_epi16(int_ch_mag, psi); tmp_result = simde_mm256_xor_si256(tmp_result, tmp_result2); simde__m256i tmp_result4 = simde_mm256_cmpgt_epi16(psi, int_three_ch_mag); @@ -536,7 +528,7 @@ static inline simde__m256i prodsum_psi_a_epi16_256(simde__m256i psi_r, simde__m2 static inline simde__m256i interference_abs_epi16_256(simde__m256i psi, simde__m256i int_ch_mag, simde__m256i c1, simde__m256i c2) { simde__m256i tmp_result = simde_mm256_cmpgt_epi16(int_ch_mag, psi); - simde__m256i tmp_result2 = simde_mm256_xor_si256(tmp_result, (*(simde__m256i *)&ones256[0])); + simde__m256i tmp_result2 = simde_mm256_xor_si256(tmp_result, allones256()); tmp_result = simde_mm256_and_si256(tmp_result, c1); tmp_result2 = simde_mm256_and_si256(tmp_result2, c2); return simde_mm256_or_si256(tmp_result, tmp_result2); diff --git a/openair1/PHY/TOOLS/tools_defs.h b/openair1/PHY/TOOLS/tools_defs.h index 64e5061476..7d60e19bae 100644 --- a/openair1/PHY/TOOLS/tools_defs.h +++ b/openair1/PHY/TOOLS/tools_defs.h @@ -532,6 +532,20 @@ static inline void multadd_cpx_vector(const c16_t *x1, const c16_t *x2, c16_t *y } } +static const int16_t ones_epi16[16] __attribute__((aligned(32))) = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; +static inline simde__m256i protected_abs256(const simde__m256i in) +{ + const simde__m256i no32768 = + simde_mm256_adds_epi16(simde_mm256_subs_epi16(in, *(simde__m256i *)ones_epi16), *(simde__m256i *)ones_epi16); + return simde_mm256_abs_epi16(no32768); +} + +static inline simde__m128i protected_abs128(const simde__m128i in) +{ + const simde__m128i no32768 = simde_mm_adds_epi16(simde_mm_subs_epi16(in, *(simde__m128i *)ones_epi16), *(simde__m128i *)ones_epi16); + return simde_mm_abs_epi16(no32768); +} + // lte_dfts.c void init_fft(uint16_t size, uint8_t logsize, @@ -919,6 +933,16 @@ c32_t dot_product(const c16_t *x, void oai_mm_separate_real_imag_parts(simde__m128i *out_re, simde__m128i *out_im, simde__m128i in0, simde__m128i in1); void oai_mm256_separate_real_imag_parts(simde__m256i *out_re, simde__m256i *out_im, simde__m256i in0, simde__m256i in1); +// generates vpcmpeqd ymm0, ymm0, ymm0 +static inline simde__m256i allones256(void) +{ + return simde_mm256_set1_epi64x(-1); +} +static inline simde__m128i allones128(void) +{ + return simde_mm_set1_epi32(-1); +} + void InitSinLUT(void); // fast calculation of e^{i*2*pi*phase}