From e056acc896884f791679ba277eb098363d17f1c6 Mon Sep 17 00:00:00 2001 From: Laurent THOMAS Date: Wed, 24 Sep 2025 12:30:55 +0200 Subject: [PATCH] use c16_t type for pm weights, as it is already same type simplify SIMD version of precoder --- nfapi/open-nFAPI/fapi/src/nr_fapi_p5.c | 5 +- nfapi/open-nFAPI/fapi/src/nr_fapi_p5_utils.c | 13 +- .../nfapi/public_inc/nfapi_nr_interface_scf.h | 7 +- nfapi/tests/p5/nr_fapi_config_request_test.c | 4 +- openair1/PHY/MODULATION/nr_modulation.c | 641 +++++++----------- .../MODULATION/tests/test_nr_modulation.cpp | 12 +- openair2/LAYER2/NR_MAC_gNB/config.c | 14 +- 7 files changed, 282 insertions(+), 414 deletions(-) diff --git a/nfapi/open-nFAPI/fapi/src/nr_fapi_p5.c b/nfapi/open-nFAPI/fapi/src/nr_fapi_p5.c index 4bcc9b2a44..c3e36d9999 100644 --- a/nfapi/open-nFAPI/fapi/src/nr_fapi_p5.c +++ b/nfapi/open-nFAPI/fapi/src/nr_fapi_p5.c @@ -760,7 +760,7 @@ static uint8_t pack_pm_table_tlv_value(void *tlv, uint8_t **ppWritePackedMsg, ui for (int k = 0; k < pm_pdu->numLayers; k++) { for (int j = 0; j < pm_pdu->num_ant_ports; j++) { const nfapi_nr_pm_weights_t *pm_weight = &pm_pdu->weights[k][j]; - if (!(push16(pm_weight->precoder_weight_Re, ppWritePackedMsg, end) && push16(pm_weight->precoder_weight_Im, ppWritePackedMsg, end))) { + if (!(push16(pm_weight->r, ppWritePackedMsg, end) && push16(pm_weight->i, ppWritePackedMsg, end))) { return 0; } } @@ -1307,8 +1307,7 @@ static uint8_t unpack_pm_table_tlv_value(void *tlv, uint8_t **ppReadPackedMsg, u for (int k = 0; k < pm_pdu->numLayers; k++) { for (int j = 0; j < pm_pdu->num_ant_ports; j++) { nfapi_nr_pm_weights_t *pm_weight = &pm_pdu->weights[k][j]; - if (!(pulls16(ppReadPackedMsg, &pm_weight->precoder_weight_Re, end) - && pulls16(ppReadPackedMsg, &pm_weight->precoder_weight_Im, end))) { + if (!(pulls16(ppReadPackedMsg, &pm_weight->r, end) && pulls16(ppReadPackedMsg, &pm_weight->i, end))) { return 0; } } diff --git a/nfapi/open-nFAPI/fapi/src/nr_fapi_p5_utils.c b/nfapi/open-nFAPI/fapi/src/nr_fapi_p5_utils.c index a810b199ff..5d1d0ba4e8 100644 --- a/nfapi/open-nFAPI/fapi/src/nr_fapi_p5_utils.c +++ b/nfapi/open-nFAPI/fapi/src/nr_fapi_p5_utils.c @@ -287,8 +287,8 @@ bool eq_config_request(const nfapi_nr_config_request_scf_t *unpacked_req, const for (int k = 0; k < req_pmi_pdu->num_ant_ports; ++k) { const nfapi_nr_pm_weights_t* req_pm_weight = &req_pmi_pdu->weights[j][k]; const nfapi_nr_pm_weights_t* unpacked_pm_weight = &unpacked_pmi_pdu->weights[j][k]; - EQ(unpacked_pm_weight->precoder_weight_Re, req_pm_weight->precoder_weight_Re); - EQ(unpacked_pm_weight->precoder_weight_Im, req_pm_weight->precoder_weight_Im); + EQ(unpacked_pm_weight->r, req_pm_weight->r); + EQ(unpacked_pm_weight->i, req_pm_weight->i); } } } @@ -848,10 +848,7 @@ void copy_config_request(const nfapi_nr_config_request_scf_t *src, nfapi_nr_conf dst_pmi_pdu->num_ant_ports = src_pmi_pdu->num_ant_ports; for (int j = 0; j < dst_pmi_pdu->numLayers; ++j) { for (int k = 0; k < dst_pmi_pdu->num_ant_ports; ++k) { - nfapi_nr_pm_weights_t* dst_pm_weight = &dst_pmi_pdu->weights[j][k]; - const nfapi_nr_pm_weights_t* src_pm_weight = &src_pmi_pdu->weights[j][k]; - dst_pm_weight->precoder_weight_Re = src_pm_weight->precoder_weight_Re; - dst_pm_weight->precoder_weight_Im = src_pm_weight->precoder_weight_Im; + dst_pmi_pdu->weights[j][k] = src_pmi_pdu->weights[j][k]; } } } @@ -1313,8 +1310,8 @@ void dump_config_request(const nfapi_nr_config_request_scf_t *msg) for (int k = 0; k < pm_pdu->numLayers; k++) { for (int l = 0; l < pm_pdu->num_ant_ports; l++) { const nfapi_nr_pm_weights_t *pm_weights = &pm_pdu->weights[k][l]; - INDENTED_GENERIC_PRINT("Dig Beam Weight Real", "0x%02x", pm_weights->precoder_weight_Re); - INDENTED_GENERIC_PRINT("Dig Beam Weight Imaginary", "0x%02x", pm_weights->precoder_weight_Im); + INDENTED_GENERIC_PRINT("Dig Beam Weight Real", "0x%02x", pm_weights->r); + INDENTED_GENERIC_PRINT("Dig Beam Weight Imaginary", "0x%02x", pm_weights->i); } } depth--; diff --git a/nfapi/open-nFAPI/nfapi/public_inc/nfapi_nr_interface_scf.h b/nfapi/open-nFAPI/nfapi/public_inc/nfapi_nr_interface_scf.h index a8ac3f73c1..754ef60e55 100644 --- a/nfapi/open-nFAPI/nfapi/public_inc/nfapi_nr_interface_scf.h +++ b/nfapi/open-nFAPI/nfapi/public_inc/nfapi_nr_interface_scf.h @@ -10,6 +10,7 @@ #define _NFAPI_NR_INTERFACE_SCF_H_ #include "stddef.h" +#include "common/platform_types.h" #include "nfapi_interface.h" #include "nfapi_nr_interface.h" @@ -488,11 +489,7 @@ typedef struct { } nfapi_nr_dbt_tlv_ve_t; // Table 3–62 Precoding matrix (PM) PDU (v.222.10.04) -typedef struct { - int16_t precoder_weight_Re; - int16_t precoder_weight_Im; -} nfapi_nr_pm_weights_t; - +typedef c16_t nfapi_nr_pm_weights_t; typedef struct { uint16_t pm_idx; diff --git a/nfapi/tests/p5/nr_fapi_config_request_test.c b/nfapi/tests/p5/nr_fapi_config_request_test.c index bfb3f7fa21..5a80a89f3d 100644 --- a/nfapi/tests/p5/nr_fapi_config_request_test.c +++ b/nfapi/tests/p5/nr_fapi_config_request_test.c @@ -239,8 +239,8 @@ static void fill_config_request_tlv_tdd_rand(nfapi_nr_config_request_scf_t *nfap for (int j = 0; j < num_layers; j++) { for (int k = 0; k < num_ant_ports; k++) { nfapi_nr_pm_weights_t* pm_weight = &pmi_pdu->weights[j][k]; - pm_weight->precoder_weight_Re = (int16_t)rand16_range(1, 0xffff); - pm_weight->precoder_weight_Im = (int16_t)rand16_range(1, 0xffff); + pm_weight->r = (int16_t)rand16_range(1, 0xffff); + pm_weight->i = (int16_t)rand16_range(1, 0xffff); } } } diff --git a/openair1/PHY/MODULATION/nr_modulation.c b/openair1/PHY/MODULATION/nr_modulation.c index b641ceaad1..5888b17eb0 100644 --- a/openair1/PHY/MODULATION/nr_modulation.c +++ b/openair1/PHY/MODULATION/nr_modulation.c @@ -710,8 +710,7 @@ c16_t nr_layer_precoder_cm(int n_layers, { c16_t precodatatx_F = {0}; for (int al = 0; al < n_layers; al++) { - nfapi_nr_pm_weights_t *w = &pmi_pdu->weights[al][ap]; - c16_t prec_weight = {.r = w->precoder_weight_Re, .i = w->precoder_weight_Im}; + c16_t prec_weight = pmi_pdu->weights[al][ap]; precodatatx_F = c16maddShift(datatx_F_precoding[al][offset], prec_weight, precodatatx_F, 15); } return precodatatx_F; @@ -732,18 +731,9 @@ static inline __attribute__((always_inline)) __m512i cmac0_prec512(__m512i x, __ } static inline __attribute__((always_inline)) __m512i cmac_prec512(__m512i y, __m512i x, __m512i w_c, __m512i w_s) { - - // Multiplication and shift - const __m512i reals = - _mm512_srai_epi32(_mm512_madd_epi16(x, w_c), 15); // (int32_t) .r = (x.r * w.r - x.i * w.i) >> 15 - const __m512i imags = - _mm512_slli_epi32(_mm512_madd_epi16(x, w_s), 1); // (int32_t) .i = (x.r * w.i + x.i * w.r) << 1, since higher 16 bit of each 32 bit is taken by blend_epi16 - - // Re-arrange to match c16_t format - const __m512i produ = _mm512_mask_blend_epi16(0xAAAAAAAA,reals, imags); - - // Accumulate the product - return _mm512_adds_epi16(y, produ); + const __m512i produ = cmac0_prec512(x, w_c, w_s); + // Accumulate the product + return _mm512_adds_epi16(y, produ); } #endif #ifdef __AVX2__ @@ -760,18 +750,9 @@ static inline __attribute__((always_inline)) __m256i cmac0_prec256(__m256i x, __ } static inline __attribute__((always_inline)) __m256i cmac_prec256(__m256i y, __m256i x, __m256i w_c, __m256i w_s) { - - // Multiplication and shift - const __m256i reals = - _mm256_srai_epi32(_mm256_madd_epi16(x, w_c), 15); // (int32_t) .r = (x.r * w.r - x.i * w.i) >> 15 - const __m256i imags = - _mm256_slli_epi32(_mm256_madd_epi16(x, w_s), 1); // (int32_t) .i = (x.r * w.i + x.i * w.r) << 1, since higher 16 bit of each 32 bit is taken by blend_epi16 - - // Re-arrange to match c16_t format - const __m256i produ = _mm256_blend_epi16(reals, imags,0xAA); - - // Accumulate the product - return _mm256_adds_epi16(y, produ); + const __m256i produ = cmac0_prec256(x, w_c, w_s); + // Accumulate the product + return _mm256_adds_epi16(y, produ); } #endif #ifdef __aarch64__ @@ -792,24 +773,41 @@ static inline __attribute__((always_inline)) int16x8_t cmac0_prec128(int16x8_t x return produ.val[0]; } static inline __attribute__((always_inline)) int16x8_t cmac_prec128(int16x8_t y, int16x8_t x, int16x8_t wr, int16x8_t wi) { - // De-interleave real/imag - int16x8_t xr = vuzp1q_s16(x, x); // even lanes - int16x8_t xi = vuzp2q_s16(x, x); // odd lanes - // - // real = ar*br - ai*bi (Q15 scaling via high-half doubling muls) - int16x8_t real = vqdmulhq_s16(xr, wr); // ≈ round((2*ar*br)/2^16) - real = vqrdmlshq_s16(real, xi, wi); // real -= round((2*ai*bi)/2^16) - // - // imag = ar*bi + ai*br - int16x8_t imag = vqdmulhq_s16(xr, wi); - imag = vqrdmlahq_s16(imag, xi, wr); // imag += round((2*ai*br)/2^16) - // - // Re-interleave [real, imag] - int16x8x2_t produ = vzipq_s16(real, imag); - return vaddq_s16(y,produ.val[0]); + int16x8_t produ = cmac0_prec128(x, wr, wi); + return vaddq_s16(y, produ.val[0]); +} +#else +static inline __attribute__((always_inline)) simde__m128i cmac0_prec128(simde__m128i x, simde__m128i w_c, simde__m128i w_s) +{ + // Multiplication and shift + const simde__m128i reals = simde_mm_srai_epi32(simde_mm_madd_epi16(x, w_c), 15); // (int32_t) .r = (x.r * w.r - x.i * w.i) >> 15 + const simde__m128i imags = simde_mm_slli_epi32( + simde_mm_madd_epi16(x, w_s), + 1); // (int32_t) .i = (x.r * w.i + x.i * w.r) << 1, since higher 16 bit of each 32 bit is taken by blend_epi16 + + /* Re-arrange to match c16_t format + bit index: 0 | 16 | 32 | 48 | 64 | 80 | 96 | 112 + reals = {R0.r[15..30] | R0.r[31] (0)*15 | R1.r[15..30] | R1.r[31] (0)*15 | R2.r[15..30] | R2.r[31] (0)*15 | R3.r[15..30] + | R3.r[31] (0)*15} imags = {0 R0.i[0..14]| R0.i[15..30] | 0 R1.i[0..14]| R1.i[15..30] | 0 R2.i[0..14]| R2.i[15..30] + | 0 R3.i[0..14]| R3.i[15..30] } 16b from {reals | imags | reals | imags | reals | imags | reals + | imags } produ = {R0.r[15..30] | R0.i[15..30] | R1.r[15..30] | R1.i[15..30] | R2.r[15..30] | R2.i[15..30] | + R3.r[15..30] | R3.i[15..30] } + */ + return simde_mm_blend_epi16(reals, imags, 0xAA); +} +static inline __attribute__((always_inline)) __m128i cmac_prec128(__m128i y, __m128i x, __m128i w_c, __m128i w_s) +{ + const __m128i produ = cmac0_prec128(x, w_c, w_s); + // Accumulate the product + return simde_mm_adds_epi16(y, produ); } #endif +#define load_consts(Type, Instruct, Rank) \ + const Type w_c##Rank = Instruct(c16toI32(c16conj(pmi_pdu->weights[Rank][ant]))); \ + const Type w_s##Rank = Instruct(c16toI32(c16swap(pmi_pdu->weights[Rank][ant]))); \ + const Type *in##Rank = (Type *)(txdataF_res_mapped[Rank] + sc_offset + (out-beginning)); + void nr_layer_precoder_simd(const int n_layers, const int symSz, const c16_t txdataF_res_mapped[n_layers][symSz], @@ -819,381 +817,264 @@ void nr_layer_precoder_simd(const int n_layers, const int re_cnt, c16_t *txdataF_precoded) { - uint32_t sc = sc_offset; - c16_t prec_weight = {0}; // For x86, use 256 SIMD for every 8 RE and 128 SIMD for last 4 RE // For aarch64, use 128 SIMD for every 4 RE + AssertFatal(n_layers > 0 && n_layers <= 4, "Shouldn't get here, n_layers %d\n", n_layers); // 512/256 SIMD: Do 16/8 RE in one iteration, 3 iterations for 2 RB -#if defined(__AVX512F__) && defined(__AVX512BW__) - const uint32_t re_cnt_align16 = re_cnt & ~15; - if (n_layers==1) { - prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; - const __m512i w_c = _mm512_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const __m512i w_s = _mm512_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - __m512i y = _mm512_set1_epi16(0); - for (; sc < sc_offset + (re_cnt_align16); sc += sizeof(__m512i) / sizeof(prec_weight)) { - - const __m512i x = _mm512_loadu_si512(&txdataF_res_mapped[0][sc]); - // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8) - y=cmac0_prec512(x,w_c,w_s); - _mm512_storeu_si512(&txdataF_precoded[sc],y); + c16_t *beginning = txdataF_precoded + sc_offset; + c16_t *out=beginning; +#if defined(__AVX512F__) && defined(__AVX512BW__) + { + c16_t *end = out + (re_cnt & ~15); + load_consts(__m512i, _mm512_set1_epi32, 0); + if (n_layers == 1) { + for (; out < end; out += sizeof(__m512i) / sizeof(*out)) { + const __m512i x = _mm512_loadu_si512(in0++); + // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8) + __m512i y = cmac0_prec512(x, w_c0, w_s0); + _mm512_storeu_si512(out, y); + } + } else if (n_layers == 2) { + load_consts(__m512i, _mm512_set1_epi32, 1); + for (; out < end; out += sizeof(__m512i) / sizeof(*out)) { + const __m512i x = _mm512_loadu_si512(in0++); + const __m512i x1 = _mm512_loadu_si512(in1++); + // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8) + __m512i y = cmac0_prec512(x, w_c0, w_s0); + y = cmac_prec512(y, x1, w_c1, w_s1); + _mm512_storeu_si512(out, y); + } + } else if (n_layers == 3) { + load_consts(__m512i, _mm512_set1_epi32, 1); + load_consts(__m512i, _mm512_set1_epi32, 2); + for (; out < end; out += sizeof(__m512i) / sizeof(*out)) { + const __m512i x = _mm512_loadu_si512(in0++); + const __m512i x1 = _mm512_loadu_si512(in1++); + const __m512i x2 = _mm512_loadu_si512(in2++); + // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8) + __m512i y = cmac0_prec512(x, w_c0, w_s0); + y = cmac_prec512(y, x1, w_c1, w_s1); + y = cmac_prec512(y, x2, w_c2, w_s2); + _mm512_storeu_si512(out, y); + } + } else if (n_layers == 4) { + load_consts(__m512i, _mm512_set1_epi32, 1); + load_consts(__m512i, _mm512_set1_epi32, 2); + load_consts(__m512i, _mm512_set1_epi32, 3); + for (; out < end; out += sizeof(__m512i) / sizeof(*out)) { + const __m512i x = _mm512_loadu_si512(in0++); + const __m512i x1 = _mm512_loadu_si512(in1++); + const __m512i x2 = _mm512_loadu_si512(in2++); + const __m512i x3 = _mm512_loadu_si512(in3++); + // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8) + __m512i y = cmac0_prec512(x, w_c0, w_s0); + y = cmac_prec512(y, x1, w_c1, w_s1); + y = cmac_prec512(y, x2, w_c2, w_s2); + y = cmac_prec512(y, x3, w_c3, w_s3); + _mm512_storeu_si512(out, y); + } } } - else if (n_layers==2) { - prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; - const __m512i w_c = _mm512_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const __m512i w_s = _mm512_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - prec_weight.r = pmi_pdu->weights[1][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[1][ant].precoder_weight_Im; - const __m512i w_c1 = _mm512_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const __m512i w_s1 = _mm512_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - __m512i y = _mm512_set1_epi16(0); - for (; sc < sc_offset + (re_cnt_align16); sc += sizeof(__m512i) / sizeof(prec_weight)) { - - const __m512i x = _mm512_loadu_si512(&txdataF_res_mapped[0][sc]); - const __m512i x1 = _mm512_loadu_si512(&txdataF_res_mapped[1][sc]); - // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8) - y=cmac0_prec512(x,w_c,w_s); - y=cmac_prec512(y,x1,w_c1,w_s1); - _mm512_storeu_si512(&txdataF_precoded[sc],y); - } - } - else if (n_layers==3) { - prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; - const __m512i w_c = _mm512_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const __m512i w_s = _mm512_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - prec_weight.r = pmi_pdu->weights[1][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[1][ant].precoder_weight_Im; - const __m512i w_c1 = _mm512_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const __m512i w_s1 = _mm512_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - prec_weight.r = pmi_pdu->weights[2][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[2][ant].precoder_weight_Im; - const __m512i w_c2 = _mm512_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const __m512i w_s2 = _mm512_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - __m512i y = _mm512_set1_epi16(0); - for (; sc < sc_offset + (re_cnt_align16); sc += sizeof(__m512i) / sizeof(prec_weight)) { - - const __m512i x = _mm512_loadu_si512(&txdataF_res_mapped[0][sc]); - const __m512i x1 = _mm512_loadu_si512(&txdataF_res_mapped[1][sc]); - const __m512i x2 = _mm512_loadu_si512(&txdataF_res_mapped[2][sc]); - // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8) - y=cmac0_prec512(x,w_c,w_s); - y=cmac_prec512(y,x1,w_c1,w_s1); - y=cmac_prec512(y,x2,w_c2,w_s2); - _mm512_storeu_si512(&txdataF_precoded[sc],y); - } - } - else if (n_layers==4) { - prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; - const __m512i w_c = _mm512_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const __m512i w_s = _mm512_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - prec_weight.r = pmi_pdu->weights[1][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[1][ant].precoder_weight_Im; - const __m512i w_c1 = _mm512_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const __m512i w_s1 = _mm512_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - prec_weight.r = pmi_pdu->weights[2][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[2][ant].precoder_weight_Im; - const __m512i w_c2 = _mm512_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const __m512i w_s2 = _mm512_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - prec_weight.r = pmi_pdu->weights[3][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[3][ant].precoder_weight_Im; - const __m512i w_c3 = _mm512_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const __m512i w_s3 = _mm512_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - __m512i y = _mm512_set1_epi16(0); - for (; sc < sc_offset + (re_cnt_align16); sc += sizeof(__m512i) / sizeof(prec_weight)) { - - const __m512i x = _mm512_loadu_si512(&txdataF_res_mapped[0][sc]); - const __m512i x1 = _mm512_loadu_si512(&txdataF_res_mapped[1][sc]); - const __m512i x2 = _mm512_loadu_si512(&txdataF_res_mapped[2][sc]); - const __m512i x3 = _mm512_loadu_si512(&txdataF_res_mapped[3][sc]); - // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8) - y=cmac0_prec512(x,w_c,w_s); - y=cmac_prec512(y,x1,w_c1,w_s1); - y=cmac_prec512(y,x2,w_c2,w_s2); - y=cmac_prec512(y,x3,w_c3,w_s3); - _mm512_storeu_si512(&txdataF_precoded[sc],y); - } - } - else AssertFatal(1==0,"Shouldn't get here, n_layers %d\n",n_layers); #endif #ifdef __AVX2__ - const uint32_t re_cnt_align8 = re_cnt & ~7; - if (n_layers==1) { - - simde__m256i y = simde_mm256_set1_epi16(0); // Y = W[0]*X[0] + W[1]*X[1] + ... + W[nrOfLayers-1]*X[nrOfLayers-1] - prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; - const simde__m256i w_c0 = simde_mm256_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const simde__m256i w_s0 = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - - for (; sc < sc_offset + (re_cnt_align8); sc += sizeof(simde__m256i) / sizeof(prec_weight)) { - const simde__m256i x0 = simde_mm256_loadu_si256(&txdataF_res_mapped[0][sc]); - // Accumulate the product - y = cmac0_prec256(x0,w_c0,w_s0); - // Store the result to txdataF - simde_mm256_storeu_si256(&txdataF_precoded[sc], y); + { + c16_t *end = beginning + (re_cnt & ~7); + load_consts(simde__m256i, simde_mm256_set1_epi32, 0); + if (n_layers == 1) { + for (; out < end; out += sizeof(simde__m256i) / sizeof(*out)) { + const simde__m256i x0 = simde_mm256_loadu_si256(in0++); + // Accumulate the product + simde__m256i y = cmac0_prec256(x0, w_c0, w_s0); + // Store the result to txdataF + simde_mm256_storeu_si256(out, y); + } + } else if (n_layers == 2) { + load_consts(simde__m256i, simde_mm256_set1_epi32, 1); + for (; out < end; out += sizeof(simde__m256i) / sizeof(*out)) { + const simde__m256i x0 = simde_mm256_loadu_si256(in0++); + const simde__m256i x1 = simde_mm256_loadu_si256(in1++); + // Accumulate the product + simde__m256i y = cmac0_prec256(x0, w_c0, w_s0); + y = cmac_prec256(y, x1, w_c1, w_s1); + // Store the result to txdataF + simde_mm256_storeu_si256(out, y); + } + } else if (n_layers == 3) { + load_consts(simde__m256i, simde_mm256_set1_epi32, 1); + load_consts(simde__m256i, simde_mm256_set1_epi32, 2); + for (; out < end; out += sizeof(simde__m256i) / sizeof(*out)) { + const simde__m256i x0 = simde_mm256_loadu_si256(in0++); + const simde__m256i x1 = simde_mm256_loadu_si256(in1++); + const simde__m256i x2 = simde_mm256_loadu_si256(in2++); + simde__m256i y = cmac0_prec256(x0, w_c0, w_s0); + y = cmac_prec256(y, x1, w_c1, w_s1); + y = cmac_prec256(y, x2, w_c2, w_s2); + // Store the result to txdataF + simde_mm256_storeu_si256(out, y); + } + } else if (n_layers == 4) { + load_consts(simde__m256i, simde_mm256_set1_epi32, 1); + load_consts(simde__m256i, simde_mm256_set1_epi32, 2); + load_consts(simde__m256i, simde_mm256_set1_epi32, 3); + for (; out < end; out += sizeof(simde__m256i) / sizeof(*out)) { + const simde__m256i x0 = simde_mm256_loadu_si256(in0++); + const simde__m256i x1 = simde_mm256_loadu_si256(in1++); + const simde__m256i x2 = simde_mm256_loadu_si256(in2++); + const simde__m256i x3 = simde_mm256_loadu_si256(in3++); + simde__m256i y = cmac0_prec256(x0, w_c0, w_s0); + y = cmac_prec256(y, x1, w_c1, w_s1); + y = cmac_prec256(y, x2, w_c2, w_s2); + y = cmac_prec256(y, x3, w_c3, w_s3); + // Store the result to txdataF + simde_mm256_storeu_si256(out, y); + } } } - else if (n_layers==2) { - - simde__m256i y = simde_mm256_set1_epi16(0); // Y = W[0]*X[0] + W[1]*X[1] + ... + W[nrOfLayers-1]*X[nrOfLayers-1] - prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; - const simde__m256i w_c0 = simde_mm256_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const simde__m256i w_s0 = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - prec_weight.r = pmi_pdu->weights[1][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[1][ant].precoder_weight_Im; - const simde__m256i w_c1 = simde_mm256_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const simde__m256i w_s1 = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - - for (; sc < sc_offset + (re_cnt_align8); sc += sizeof(simde__m256i) / sizeof(prec_weight)) { - const simde__m256i x0 = simde_mm256_loadu_si256(&txdataF_res_mapped[0][sc]); - - const simde__m256i x1 = simde_mm256_loadu_si256(&txdataF_res_mapped[1][sc]); - // Accumulate the product - y = cmac0_prec256(x0,w_c0,w_s0); - y = cmac_prec256(y,x1,w_c1,w_s1); - // Store the result to txdataF - simde_mm256_storeu_si256(&txdataF_precoded[sc], y); - } - } - else if (n_layers==3) { - simde__m256i y = simde_mm256_set1_epi16(0); // Y = W[0]*X[0] + W[1]*X[1] + ... + W[nrOfLayers-1]*X[nrOfLayers-1] - prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; - const simde__m256i w_c0 = simde_mm256_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const simde__m256i w_s0 = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - prec_weight.r = pmi_pdu->weights[1][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[1][ant].precoder_weight_Im; - const simde__m256i w_c1 = simde_mm256_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const simde__m256i w_s1 = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - prec_weight.r = pmi_pdu->weights[2][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[2][ant].precoder_weight_Im; - const simde__m256i w_c2 = simde_mm256_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const simde__m256i w_s2 = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - - for (; sc < sc_offset + (re_cnt_align8); sc += sizeof(simde__m256i) / sizeof(prec_weight)) { - const simde__m256i x0 = simde_mm256_loadu_si256(&txdataF_res_mapped[0][sc]); - const simde__m256i x1 = simde_mm256_loadu_si256(&txdataF_res_mapped[1][sc]); - const simde__m256i x2 = simde_mm256_loadu_si256(&txdataF_res_mapped[2][sc]); - y = cmac0_prec256(x0,w_c0,w_s0); - y = cmac_prec256(y,x1,w_c1,w_s1); - y = cmac_prec256(y,x2,w_c2,w_s2); - // Store the result to txdataF - simde_mm256_storeu_si256(&txdataF_precoded[sc], y); - } - } - else if (n_layers==4) { - simde__m256i y = simde_mm256_set1_epi16(0); // Y = W[0]*X[0] + W[1]*X[1] + ... + W[nrOfLayers-1]*X[nrOfLayers-1] - prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; - const simde__m256i w_c0 = simde_mm256_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const simde__m256i w_s0 = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - prec_weight.r = pmi_pdu->weights[1][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[1][ant].precoder_weight_Im; - const simde__m256i w_c1 = simde_mm256_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const simde__m256i w_s1 = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - prec_weight.r = pmi_pdu->weights[2][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[2][ant].precoder_weight_Im; - const simde__m256i w_c2 = simde_mm256_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const simde__m256i w_s2 = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - - prec_weight.r = pmi_pdu->weights[3][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[3][ant].precoder_weight_Im; - const simde__m256i w_c3 = simde_mm256_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const simde__m256i w_s3 = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - for (; sc < sc_offset + (re_cnt_align8); sc += sizeof(simde__m256i) / sizeof(prec_weight)) { - const simde__m256i x0 = simde_mm256_loadu_si256(&txdataF_res_mapped[0][sc]); - const simde__m256i x1 = simde_mm256_loadu_si256(&txdataF_res_mapped[1][sc]); - const simde__m256i x2 = simde_mm256_loadu_si256(&txdataF_res_mapped[2][sc]); - const simde__m256i x3 = simde_mm256_loadu_si256(&txdataF_res_mapped[3][sc]); - y = cmac0_prec256(x0,w_c0,w_s0); - y = cmac_prec256(y,x1,w_c1,w_s1); - y = cmac_prec256(y,x2,w_c2,w_s2); - y = cmac_prec256(y,x3,w_c3,w_s3); - // Store the result to txdataF - simde_mm256_storeu_si256(&txdataF_precoded[sc], y); - } - } - else AssertFatal(1==0,"Shouldn't get here, n_layers %d\n",n_layers); #endif - // 128 SIMD: Do 4 RE in one iteration, 3 iterations for 1 RB - const uint32_t re_cnt_align4 = re_cnt & ~3; - for (; sc < sc_offset + re_cnt_align4; sc += sizeof(simde__m128i) / sizeof(prec_weight)) { + c16_t *end = beginning + (re_cnt & ~3); #ifdef DEBUG_DLSCH_PRECODING_PRINT_WITH_TRIVIAL // Get result with trivial solution, TODO: To be removed + // 128 SIMD: Do 4 RE in one iteration, 3 iterations for 1 RB + for (; out < end; out += sizeof(simde__m128i) / sizeof(*out)) { c16_t y_triv[4]; for (int i = 0; i < 4; i++) y_triv[i] = nr_layer_precoder_cm(n_layers, symSz, txdataF_res_mapped, ant, pmi_pdu, sc + i); - memcpy(&txdataF_precoded[sc], y_triv, sizeof(y_triv)); + memcpy(out, y_triv, sizeof(y_triv)); + } #endif #ifdef __aarch64__ - - const uint32_t re_cnt_align8 = re_cnt & ~3; - if (n_layers==1) { - - prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; - const int16x8_t wr0 = vdupq_n_s16(prec_weight.r); - const int16x8_t wi0 = vdupq_n_s16(prec_weight.i); - int16x8_t y; - - for (; sc < sc_offset + (re_cnt_align8); sc += sizeof(int16x8_t) / sizeof(prec_weight)) { - const int16x8_t x0 = vld1q_s16((const int16_t*)&txdataF_res_mapped[0][sc]); + load_consts(int16x8_t, vdupq_n_s16, 0); + if (n_layers == 1) { + for (; out < end; out += sizeof(int16x8_t) / sizeof(*out)) { + const int16x8_t x0 = vld1q_s16((const int16_t *)in0++); // Accumulate the product - y = cmac0_prec128(x0,wr0,wi0); + int16x8_t y = cmac0_prec128(x0, w_c0, w_s0); // Store the result to txdataF - *((int16x8_t*)&txdataF_precoded[sc])= y; + *(int16x8_t *)out = y; } } - if (n_layers==2) { - - prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; - const int16x8_t wr0 = vdupq_n_s16(prec_weight.r); - const int16x8_t wi0 = vdupq_n_s16(prec_weight.i); - prec_weight.r = pmi_pdu->weights[1][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[1][ant].precoder_weight_Im; - const int16x8_t wr1 = vdupq_n_s16(prec_weight.r); - const int16x8_t wi1 = vdupq_n_s16(prec_weight.i); - int16x8_t y; - - for (; sc < sc_offset + (re_cnt_align8); sc += sizeof(int16x8_t) / sizeof(prec_weight)) { - const int16x8_t x0 = vld1q_s16((const int16_t*)&txdataF_res_mapped[0][sc]); - const int16x8_t x1 = vld1q_s16((const int16_t*)&txdataF_res_mapped[1][sc]); + if (n_layers == 2) { + load_consts(int16x8_t, vdupq_n_s16, 1); + for (; out < end; out += sizeof(int16x8_t) / sizeof(*out)) { + const int16x8_t x0 = vld1q_s16((const int16_t *)in0++); + const int16x8_t x1 = vld1q_s16((const int16_t *)in1++); // Accumulate the product - y = cmac0_prec128(x0,wr0,wi0); - y = cmac_prec128(y,x1,wr1,wi1); + int16x8_t y = cmac0_prec128(x0, w_c0, w_s0); + y = cmac_prec128(y, x1, w_cr1, w_s1); // Store the result to txdataF - *((int16x8_t*)&txdataF_precoded[sc])= y; + *(int16x8_t *)out = y; } } - if (n_layers==3) { - - prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; - const int16x8_t wr0 = vdupq_n_s16(prec_weight.r); - const int16x8_t wi0 = vdupq_n_s16(prec_weight.i); - prec_weight.r = pmi_pdu->weights[1][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[1][ant].precoder_weight_Im; - const int16x8_t wr1 = vdupq_n_s16(prec_weight.r); - const int16x8_t wi1 = vdupq_n_s16(prec_weight.i); - prec_weight.r = pmi_pdu->weights[2][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[2][ant].precoder_weight_Im; - const int16x8_t wr2 = vdupq_n_s16(prec_weight.r); - const int16x8_t wi2 = vdupq_n_s16(prec_weight.i); - int16x8_t y; - - for (; sc < sc_offset + (re_cnt_align8); sc += sizeof(int16x8_t) / sizeof(prec_weight)) { - const int16x8_t x0 = vld1q_s16((const int16_t*)&txdataF_res_mapped[0][sc]); - const int16x8_t x1 = vld1q_s16((const int16_t*)&txdataF_res_mapped[1][sc]); - const int16x8_t x2 = vld1q_s16((const int16_t*)&txdataF_res_mapped[2][sc]); + if (n_layers == 3) { + load_consts(int16x8_t, vdupq_n_s16, 1); + load_consts(int16x8_t, vdupq_n_s16, 2); + for (; out < end; out += sizeof(int16x8_t) / sizeof(*out)) { + const int16x8_t x0 = vld1q_s16((const int16_t *)in0++); + const int16x8_t x1 = vld1q_s16((const int16_t *)in1++); + const int16x8_t x2 = vld1q_s16((const int16_t *)in2++); // Accumulate the product - y = cmac0_prec128(x0,wr0,wi0); - y = cmac_prec128(y,x1,wr1,wi1); - y = cmac_prec128(y,x2,wr2,wi2); + int16x8_t y = cmac0_prec128(x0, w_c0, w_s0); + ; + y = cmac_prec128(y, x1, w_cr1, w_s1); + y = cmac_prec128(y, x2, w_cr2, w_s2); // Store the result to txdataF - *((int16x8_t*)&txdataF_precoded[sc])= y; + *(int16x8_t *)out = y; } } - if (n_layers==4) { - - prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; - const int16x8_t wr0 = vdupq_n_s16(prec_weight.r); - const int16x8_t wi0 = vdupq_n_s16(prec_weight.i); - prec_weight.r = pmi_pdu->weights[1][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[1][ant].precoder_weight_Im; - const int16x8_t wr1 = vdupq_n_s16(prec_weight.r); - const int16x8_t wi1 = vdupq_n_s16(prec_weight.i); - prec_weight.r = pmi_pdu->weights[2][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[2][ant].precoder_weight_Im; - const int16x8_t wr2 = vdupq_n_s16(prec_weight.r); - const int16x8_t wi2 = vdupq_n_s16(prec_weight.i); - prec_weight.r = pmi_pdu->weights[3][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[3][ant].precoder_weight_Im; - const int16x8_t wr3 = vdupq_n_s16(prec_weight.r); - const int16x8_t wi3 = vdupq_n_s16(prec_weight.i); - int16x8_t y; - - for (; sc < sc_offset + (re_cnt_align8); sc += sizeof(int16x8_t) / sizeof(prec_weight)) { - const int16x8_t x0 = vld1q_s16((const int16_t*)&txdataF_res_mapped[0][sc]); - const int16x8_t x1 = vld1q_s16((const int16_t*)&txdataF_res_mapped[1][sc]); - const int16x8_t x2 = vld1q_s16((const int16_t*)&txdataF_res_mapped[2][sc]); - const int16x8_t x3 = vld1q_s16((const int16_t*)&txdataF_res_mapped[3][sc]); + if (n_layers == 4) { + load_consts(int16x8_t, vdupq_n_s16, 1); + load_consts(int16x8_t, vdupq_n_s16, 2); + load_consts(int16x8_t, vdupq_n_s16, 3); + for (; out < end; out += sizeof(int16x8_t) / sizeof(*out)) { + const int16x8_t x0 = vld1q_s16((const int16_t *)in0++); + const int16x8_t x1 = vld1q_s16((const int16_t *)in1++); + const int16x8_t x2 = vld1q_s16((const int16_t *)in2++); + const int16x8_t x3 = vld1q_s16((const int16_t *)in3++); // Accumulate the product - y = cmac0_prec128(x0,wr0,wi0); - y = cmac_prec128(y,x1,wr1,wi1); - y = cmac_prec128(y,x2,wr2,wi2); - y = cmac_prec128(y,x3,wr3,wi3); + int16x8_t y = cmac0_prec128(x0, w_c0, w_s0); + ; + y = cmac_prec128(y, x1, w_cr1, w_s1); + y = cmac_prec128(y, x2, w_cr2, w_s2); + y = cmac_prec128(y, x3, w_cr3, w_s3); // Store the result to txdataF - *((int16x8_t*)&txdataF_precoded[sc])= y; + *(int16x8_t *)out = y; } } -#else - // Matrix multiplication for 4 elements of the result (sizeof(simde__m128i) / sizeof(c16_t) = 4) - simde__m128i y = simde_mm_set1_epi16(0); // Y = W[0]*X[0] + W[1]*X[1] + ... + W[nrOfLayers-1]*X[nrOfLayers-1] - for (int nl = 0; nl < n_layers; nl++) { - prec_weight.r = pmi_pdu->weights[nl][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[nl][ant].precoder_weight_Im; - - const simde__m128i x = simde_mm_loadu_si128(&txdataF_res_mapped[nl][sc]); - - // Rearrange precoding matrix weight to match complex multiplication and broadcast it to match SIMD size - const simde__m128i w_c = simde_mm_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const simde__m128i w_s = simde_mm_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - - // Multiplication and shift - const simde__m128i reals = - simde_mm_srai_epi32(simde_mm_madd_epi16(x, w_c), 15); // (int32_t) .r = (x.r * w.r - x.i * w.i) >> 15 - const simde__m128i imags = simde_mm_slli_epi32( - simde_mm_madd_epi16(x, w_s), - 1); // (int32_t) .i = (x.r * w.i + x.i * w.r) << 1, since higher 16 bit of each 32 bit is taken by blend_epi16 - - /* Re-arrange to match c16_t format - bit index: 0 | 16 | 32 | 48 | 64 | 80 | 96 | 112 - reals = {R0.r[15..30] | R0.r[31] (0)*15 | R1.r[15..30] | R1.r[31] (0)*15 | R2.r[15..30] | R2.r[31] (0)*15 | R3.r[15..30] - | R3.r[31] (0)*15} imags = {0 R0.i[0..14]| R0.i[15..30] | 0 R1.i[0..14]| R1.i[15..30] | 0 R2.i[0..14]| R2.i[15..30] - | 0 R3.i[0..14]| R3.i[15..30] } 16b from {reals | imags | reals | imags | reals | imags | reals - | imags } produ = {R0.r[15..30] | R0.i[15..30] | R1.r[15..30] | R1.i[15..30] | R2.r[15..30] | R2.i[15..30] | - R3.r[15..30] | R3.i[15..30] } - */ - const simde__m128i produ = simde_mm_blend_epi16(reals, imags, 0xAA); - +#else + load_consts(simde__m128i, simde_mm_set1_epi32, 0); + if (n_layers == 1) { + for (; out < end; out += sizeof(simde__m128i) / sizeof(*out)) { + const simde__m128i x0 = simde_mm_loadu_si128(in0++); // Accumulate the product - y = simde_mm_adds_epi16(y, produ); + simde__m128i y = cmac0_prec128(x0, w_c0, w_s0); + // Store the result to txdataF + simde_mm_storeu_si128(out, y); } - // Store the result to txdataF - simde_mm_storeu_si128(&txdataF_precoded[sc], y); + } else if (n_layers == 2) { + load_consts(simde__m128i, simde_mm_set1_epi32, 1); + for (; out < end; out += sizeof(simde__m128i) / sizeof(*out)) { + const simde__m128i x0 = simde_mm_loadu_si128(in0++); + const simde__m128i x1 = simde_mm_loadu_si128(in1++); + // Accumulate the product + simde__m128i y = cmac0_prec128(x0, w_c0, w_s0); + y = cmac_prec128(y, x1, w_c1, w_s1); + // Store the result to txdataF + simde_mm_storeu_si128(out, y); + } + } else if (n_layers == 3) { + load_consts(simde__m128i, simde_mm_set1_epi32, 1); + load_consts(simde__m128i, simde_mm_set1_epi32, 2); + for (; out < end; out += sizeof(simde__m128i) / sizeof(*out)) { + const simde__m128i x0 = simde_mm_loadu_si128(in0++); + const simde__m128i x1 = simde_mm_loadu_si128(in1++); + const simde__m128i x2 = simde_mm_loadu_si128(in2++); + simde__m128i y = cmac0_prec128(x0, w_c0, w_s0); + y = cmac_prec128(y, x1, w_c1, w_s1); + y = cmac_prec128(y, x2, w_c2, w_s2); + // Store the result to txdataF + simde_mm_storeu_si128(out, y); + } + } else if (n_layers == 4) { + load_consts(simde__m128i, simde_mm_set1_epi32, 1); + load_consts(simde__m128i, simde_mm_set1_epi32, 2); + load_consts(simde__m128i, simde_mm_set1_epi32, 3); + for (; out < end; out += sizeof(simde__m128i) / sizeof(*out)) { + const simde__m128i x0 = simde_mm_loadu_si128(in0++); + const simde__m128i x1 = simde_mm_loadu_si128(in1++); + const simde__m128i x2 = simde_mm_loadu_si128(in2++); + const simde__m128i x3 = simde_mm_loadu_si128(in3++); + simde__m128i y = cmac0_prec128(x0, w_c0, w_s0); + y = cmac_prec128(y, x1, w_c1, w_s1); + y = cmac_prec128(y, x2, w_c2, w_s2); + y = cmac_prec128(y, x3, w_c3, w_s3); + // Store the result to txdataF + simde_mm_storeu_si128(out, y); + } + } #endif #ifdef DEBUG_DLSCH_PRECODING_PRINT_WITH_TRIVIAL // Print simd and trivial result, TODO: To be removed - c16_t *y_simd = (c16_t *)&y; - printf("debug_to_be_removed re_cnt=%d, sc=%u, y_simd=(%+4d,%+4d), (%+4d,%+4d), (%+4d,%+4d), (%+4d,%+4d)\n", - re_cnt, - sc, - y_simd[0].r, - y_simd[0].i, - y_simd[1].r, - y_simd[1].i, - y_simd[2].r, - y_simd[2].i, - y_simd[3].r, - y_simd[3].i); - printf("debug_to_be_removed re_cnt=%d, sc=%u, y_triv=(%+4d,%+4d), (%+4d,%+4d), (%+4d,%+4d), (%+4d,%+4d)\n", - re_cnt, - sc, - y_triv[0].r, - y_triv[0].i, - y_triv[1].r, - y_triv[1].i, - y_triv[2].r, - y_triv[2].i, - y_triv[3].r, - y_triv[3].i); + c16_t *y_simd = (c16_t *)&y; + printf("debug_to_be_removed re_cnt=%d, sc=%u, y_simd=(%+4d,%+4d), (%+4d,%+4d), (%+4d,%+4d), (%+4d,%+4d)\n", + re_cnt, + sc, + y_simd[0].r, + y_simd[0].i, + y_simd[1].r, + y_simd[1].i, + y_simd[2].r, + y_simd[2].i, + y_simd[3].r, + y_simd[3].i); + printf("debug_to_be_removed re_cnt=%d, sc=%u, y_triv=(%+4d,%+4d), (%+4d,%+4d), (%+4d,%+4d), (%+4d,%+4d)\n", + re_cnt, + sc, + y_triv[0].r, + y_triv[0].i, + y_triv[1].r, + y_triv[1].i, + y_triv[2].r, + y_triv[2].i, + y_triv[3].r, + y_triv[3].i); #endif - } } diff --git a/openair1/PHY/MODULATION/tests/test_nr_modulation.cpp b/openair1/PHY/MODULATION/tests/test_nr_modulation.cpp index ca8b61acd7..04d32514a0 100644 --- a/openair1/PHY/MODULATION/tests/test_nr_modulation.cpp +++ b/openair1/PHY/MODULATION/tests/test_nr_modulation.cpp @@ -85,8 +85,7 @@ TEST(NrLayerPrecoderTest, Basic) // Create and populate the pmi_pdu structure nfapi_nr_pm_pdu_t pmi_pdu = {0}; for (int layer = 0; layer < n_layers; ++layer) { - pmi_pdu.weights[layer][ant].precoder_weight_Re = -SHRT_MAX; // Q15 representation of (~-1.0) -0.999969482421875 (-1+2^-15) - pmi_pdu.weights[layer][ant].precoder_weight_Im = 0; + pmi_pdu.weights[layer][ant] = (c16_t){-SHRT_MAX, 0}; } // Call the C function @@ -131,8 +130,7 @@ TEST(NrLayerPrecoderTest, SIMD) // Create and populate the pmi_pdu structure nfapi_nr_pm_pdu_t pmi_pdu = {0}; for (int layer = 0; layer < n_layers; ++layer) { - pmi_pdu.weights[layer][ant].precoder_weight_Re = -SHRT_MAX; // Q15 representation of (~-1.0) -0.999969482421875 (-1+2^-15) - pmi_pdu.weights[layer][ant].precoder_weight_Im = 0; + pmi_pdu.weights[layer][ant] = (c16_t){-SHRT_MAX, 0}; } // Call the C function @@ -182,10 +180,8 @@ TEST(NrLayerPrecoderTest, Compare_CM_SIMD) // Could not use convert_precoder_weight() as complex.h could not be used in googletest // Use the logic in convert_precoder_weight() // precoder [−1,−j] - pmi_pdu.weights[layer][0].precoder_weight_Re = -SHRT_MAX; // Q15 representation of (~-1.0) -0.999969482421875 (-1+2^-15) - pmi_pdu.weights[layer][0].precoder_weight_Im = 0x0000; // Q15 representation of 0 - pmi_pdu.weights[layer][1].precoder_weight_Re = 0x0000; // Q15 representation of 0 - pmi_pdu.weights[layer][1].precoder_weight_Im = -SHRT_MAX; // Q15 representation of (~-1.0) -0.999969482421875 (-1+2^-15) + pmi_pdu.weights[layer][0] = (c16_t){-SHRT_MAX, 0}; + pmi_pdu.weights[layer][1] = (c16_t){0, -SHRT_MAX}; } // Get the results for all the antenna diff --git a/openair2/LAYER2/NR_MAC_gNB/config.c b/openair2/LAYER2/NR_MAC_gNB/config.c index 9a5197181f..a0637ac107 100644 --- a/openair2/LAYER2/NR_MAC_gNB/config.c +++ b/openair2/LAYER2/NR_MAC_gNB/config.c @@ -136,8 +136,7 @@ int precoding_weigths_generation(nfapi_nr_pm_list_t *mat, nfapi_nr_pm_weights_t *weights = &mat->pmi_pdu[pmiq].weights[j_col][i_rows]; res_code = sqrt(1 / (double)L) * v_lm[llc][mmc][i_rows]; c16_t precoder_weight = convert_precoder_weight(res_code); - weights->precoder_weight_Re = precoder_weight.r; - weights->precoder_weight_Im = precoder_weight.i; + *weights = precoder_weight; LOG_D(PHY, "%d Layer Precoding Matrix[pmi %d][antPort %d][layerIdx %d]= %f+j %f -> Fixed Point %d+j %d \n", L, @@ -146,15 +145,14 @@ int precoding_weigths_generation(nfapi_nr_pm_list_t *mat, j_col, creal(res_code), cimag(res_code), - weights->precoder_weight_Re, - weights->precoder_weight_Im); + weights->r, + weights->i); } for (int i_rows = N1 * N2; i_rows < 2 * N1 * N2; i_rows++) { nfapi_nr_pm_weights_t *weights = &mat->pmi_pdu[pmiq].weights[j_col][i_rows]; res_code = sqrt(1 / (double)L) * (phase_sign)*theta_n[nn] * v_lm[llc][mmc][i_rows - N1 * N2]; c16_t precoder_weight = convert_precoder_weight(res_code); - weights->precoder_weight_Re = precoder_weight.r; - weights->precoder_weight_Im = precoder_weight.i; + *weights = precoder_weight; LOG_D(PHY, "%d Layer Precoding Matrix[pmi %d][antPort %d][layerIdx %d]= %f+j %f -> Fixed Point %d+j %d \n", L, @@ -163,8 +161,8 @@ int precoding_weigths_generation(nfapi_nr_pm_list_t *mat, j_col, creal(res_code), cimag(res_code), - weights->precoder_weight_Re, - weights->precoder_weight_Im); + weights->r, + weights->i); } } pmiq++;