From 8b9098b7ca9479a159a9f9d33a5b1597a46662b0 Mon Sep 17 00:00:00 2001 From: Laurent THOMAS Date: Thu, 9 Oct 2025 15:53:51 +0200 Subject: [PATCH 01/10] remove unused declarations in common/platform_types.h --- common/platform_types.h | 32 ----------------------------- openair2/LAYER2/PDCP_v10.1.0/pdcp.c | 3 +-- openair2/LAYER2/RLC/rlc.h | 16 --------------- 3 files changed, 1 insertion(+), 50 deletions(-) diff --git a/common/platform_types.h b/common/platform_types.h index 9827dba974..9404f987ac 100644 --- a/common/platform_types.h +++ b/common/platform_types.h @@ -45,7 +45,6 @@ typedef uint16_t module_id_t; typedef uint8_t slice_id_t; typedef uint8_t eNB_index_t; typedef uint64_t ue_id_t; -typedef int16_t smodule_id_t; typedef long rb_id_t; typedef long srb_id_t; @@ -74,24 +73,7 @@ typedef enum { FDD = 0 } frame_type_t; -typedef enum link_direction_e { UNKNOWN_DIR = 0, DIR_UPLINK = 1, DIR_DOWNLINK = 2 } link_direction_t; - -typedef enum rb_type_e { UNKNOWN_RADIO_BEARER = 0, SIGNALLING_RADIO_BEARER = 1, RADIO_ACCESS_BEARER = 2 } rb_type_t; - -typedef enum { CR_ROUND = 0, CR_SRB12 = 1, CR_HOL = 2, CR_LC = 3, CR_CQI = 4, CR_LCP = 5, CR_NUM = 6 } sorting_criterion_t; - -typedef enum { POL_FAIR = 0, POL_GREEDY = 1, POL_NUM = 2 } accounting_policy_t; -//----------------------------------------------------------------------------- -// PHY TYPES -//----------------------------------------------------------------------------- -typedef uint8_t crc8_t; -typedef uint16_t crc16_t; -typedef uint32_t crc32_t; typedef unsigned int crc_t; - -//----------------------------------------------------------------------------- -// MAC TYPES -//----------------------------------------------------------------------------- typedef sdu_size_t tbs_size_t; typedef sdu_size_t tb_size_t; typedef unsigned int logical_chan_id_t; @@ -103,9 +85,6 @@ typedef uint8_t mac_enb_index_t; //----------------------------------------------------------------------------- typedef unsigned int mui_t; typedef unsigned int confirm_t; -typedef unsigned int rlc_tx_status_t; -typedef int16_t rlc_sn_t; -typedef uint16_t rlc_usn_t; typedef int32_t rlc_buffer_occupancy_t; typedef signed int rlc_op_status_t; @@ -130,7 +109,6 @@ typedef struct { //----------------------------------------------------------------------------- typedef uint16_t pdcp_sn_t; typedef uint32_t pdcp_hfn_t; -typedef int16_t pdcp_hfn_offset_t; typedef enum pdcp_transmission_mode_e { PDCP_TRANSMISSION_MODE_UNKNOWN = 0, @@ -177,7 +155,6 @@ typedef struct net_ip_address_s { typedef uint32_t mbms_session_id_t; typedef uint16_t mbms_service_id_t; typedef uint16_t rnti_t; -typedef uint8_t rrc_enb_index_t; typedef uint8_t mme_code_t; typedef uint32_t m_tmsi_t; @@ -188,7 +165,6 @@ typedef uint32_t m_tmsi_t; #define P_RNTI (rnti_t)0xFFFE #define SI_RNTI (rnti_t)0xFFFF #define CBA_RNTI (rnti_t)0xfff4 -#define OAI_C_RNTI (rnti_t)0x1234 typedef enum config_action_e { CONFIG_ACTION_NULL = 0, CONFIG_ACTION_ADD = 1, @@ -256,8 +232,6 @@ typedef struct protocol_ctxt_s { eNB_index_t eNB_index; /*!< \brief valid for UE indicating the index of connected eNB(s) */ bool brOption; } protocol_ctxt_t; -// warning time hardcoded -#define PROTOCOL_CTXT_TIME_MILLI_SECONDS(CtXt_h) ((CtXt_h)->frame * 10 + (CtXt_h)->subframe) #define UE_MODULE_ID_TO_INSTANCE(mODULE_iD) mODULE_iD + RC.nb_inst #define ENB_MODULE_ID_TO_INSTANCE(mODULE_iD) mODULE_iD @@ -309,12 +283,6 @@ typedef struct protocol_ctxt_s { #define PROTOCOL_CTXT_ARGS(CTXT_Pp) \ (CTXT_Pp)->frame, ((CTXT_Pp)->enb_flag == ENB_FLAG_YES) ? "eNB" : " UE", (CTXT_Pp)->module_id, (CTXT_Pp)->rntiMaybeUEid -#define PROTOCOL_NR_CTXT_ARGS(CTXT_Pp) \ - (CTXT_Pp)->frame, ((CTXT_Pp)->enb_flag == GNB_FLAG_YES) ? "gNB" : " UE", (CTXT_Pp)->module_id, (CTXT_Pp)->rntiMaybeUEid - -#define CHECK_CTXT_ARGS(CTXT_Pp) - - static inline int ceil_mod(const unsigned int v, const unsigned int mod) { return ((v + mod - 1) / mod) * mod; diff --git a/openair2/LAYER2/PDCP_v10.1.0/pdcp.c b/openair2/LAYER2/PDCP_v10.1.0/pdcp.c index f93044f9cd..0b8e9e2d40 100644 --- a/openair2/LAYER2/PDCP_v10.1.0/pdcp.c +++ b/openair2/LAYER2/PDCP_v10.1.0/pdcp.c @@ -231,8 +231,7 @@ bool pdcp_data_req(protocol_ctxt_t *ctxt_pP, hashtable_rc_t h_rc; uint8_t rb_offset= (srb_flagP == 0) ? DTCH -1 : 0; uint16_t pdcp_uid=0; - VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PDCP_DATA_REQ,VCD_FUNCTION_IN); - CHECK_CTXT_ARGS(ctxt_pP); + VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PDCP_DATA_REQ, VCD_FUNCTION_IN); #if T_TRACER if (ctxt_pP->enb_flag != ENB_FLAG_NO) diff --git a/openair2/LAYER2/RLC/rlc.h b/openair2/LAYER2/RLC/rlc.h index 9afad74714..4559eb3273 100644 --- a/openair2/LAYER2/RLC/rlc.h +++ b/openair2/LAYER2/RLC/rlc.h @@ -262,22 +262,6 @@ void rlc_data_ind(const protocol_ctxt_t *const, const sdu_size_t, uint8_t *const); -/*! \fn void rlc_data_conf (const protocol_ctxt_t* const ctxtP, const srb_flag_t srb_flagP, const rb_id_t rb_idP, const mui_t muiP, const rlc_tx_status_t statusP) -* \brief Interface with higher layers, confirm to upper layer the transmission status for a SDU stamped with a MUI, scheduled for transmission. -* \param[in] ctxtP Running context. -* \param[in] srb_flagP Flag to indicate SRB (1) or DRB (0) -* \param[in] rb_idP Radio bearer identifier. -* \param[in] muiP Message Unit identifier. -* \param[in] statusP Status of the transmission (RLC_SDU_CONFIRM_YES, RLC_SDU_CONFIRM_NO). -*/ -void rlc_data_conf( - const protocol_ctxt_t *const, - const srb_flag_t, - const rb_id_t, - const mui_t, - const rlc_tx_status_t ); - - /*! \fn rlc_op_status_t rlc_stat_req ( const protocol_ctxt_t* const ctxtP, const srb_flag_t srb_flagP, From c537e2cabef9befdd4f3f50d3afcbd8fd69c517b Mon Sep 17 00:00:00 2001 From: Laurent THOMAS Date: Thu, 9 Oct 2025 15:59:32 +0200 Subject: [PATCH 02/10] move complex numbers types above phy layer, as they are used in fapi for example --- common/platform_types.h | 33 +++++++++++++++++++++++++++++++ openair1/PHY/TOOLS/tools_defs.h | 35 +-------------------------------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/common/platform_types.h b/common/platform_types.h index 9404f987ac..e2ffb7b304 100644 --- a/common/platform_types.h +++ b/common/platform_types.h @@ -33,6 +33,39 @@ #include #include +#define ALIGNARRAYSIZE(a, b) (((a + b - 1) / b) * b) +#define ALNARS_16_4(a) ALIGNARRAYSIZE(a, 4) + +typedef struct complexd { + double r; + double i; +} cd_t; + +typedef struct complexf { + float r; + float i; +} cf_t; + +typedef struct complex8 { + int8_t r; + int8_t i; +} c8_t; + +typedef struct complex16 { + int16_t r; + int16_t i; +} c16_t; + +typedef struct complex32 { + int32_t r; + int32_t i; +} c32_t; + +typedef struct complex64 { + int64_t r; + int64_t i; +} c64_t; + //----------------------------------------------------------------------------- // GENERIC ACCESS STRATUM TYPES //----------------------------------------------------------------------------- diff --git a/openair1/PHY/TOOLS/tools_defs.h b/openair1/PHY/TOOLS/tools_defs.h index 9ee558c72b..092beebcd7 100644 --- a/openair1/PHY/TOOLS/tools_defs.h +++ b/openair1/PHY/TOOLS/tools_defs.h @@ -32,7 +32,7 @@ #include #include #include - +#include "common/platform_types.h" #include "PHY/sse_intrin.h" #include "common/utils/assertions.h" @@ -51,39 +51,6 @@ extern "C" { #endif -#define ALIGNARRAYSIZE(a, b) (((a + b - 1) / b) * b) -#define ALNARS_16_4(a) ALIGNARRAYSIZE(a, 4) - - typedef struct complexd { - double r; - double i; - } cd_t; - - typedef struct complexf { - float r; - float i; - } cf_t; - - typedef struct complex8 { - int8_t r; - int8_t i; - } c8_t; - - typedef struct complex16 { - int16_t r; - int16_t i; - } c16_t; - - typedef struct complex32 { - int32_t r; - int32_t i; - } c32_t; - - typedef struct complex64 { - int64_t r; - int64_t i; - } c64_t; - typedef struct { int dim1; int dim2; From be236d450e77e6503d4065c823056bb3abd2bea9 Mon Sep 17 00:00:00 2001 From: Raymond Knopp Date: Thu, 18 Sep 2025 14:57:57 +0200 Subject: [PATCH 03/10] Add measurements for performance analysis --- openair1/PHY/NR_TRANSPORT/nr_dlsch.c | 11 +++++++---- openair1/PHY/defs_gNB.h | 2 ++ openair1/SIMULATION/NR_PHY/dlsim.c | 10 +++++++++- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/openair1/PHY/NR_TRANSPORT/nr_dlsch.c b/openair1/PHY/NR_TRANSPORT/nr_dlsch.c index 58d11413f4..7aec1ac5ad 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_dlsch.c +++ b/openair1/PHY/NR_TRANSPORT/nr_dlsch.c @@ -643,6 +643,7 @@ static int do_one_dlsch(unsigned char *input_ptr, PHY_VARS_gNB *gNB, NR_gNB_DLSC #endif } + start_meas(&gNB->dlsch_pdsch_generation_stats); /// Resource mapping // Non interleaved VRB to PRB mapping uint16_t start_sc = frame_parms->first_carrier_offset + (rel15->rbStart + rel15->BWPStart) * NR_NB_SC_PER_RB; @@ -664,6 +665,7 @@ static int do_one_dlsch(unsigned char *input_ptr, PHY_VARS_gNB *gNB, NR_gNB_DLSC c16_t mod_dmrs[(n_dmrs+63)&~63] __attribute__((aligned(64))); unsigned int re_beginning_of_symbol = 0; + start_meas(&gNB->dlsch_layer_mapping_stats); int layerSz2 = (layerSz + 63) & ~63; c16_t tx_layers[rel15->nrOfLayers][layerSz2] __attribute__((aligned(64))); memset(tx_layers, 0, sizeof(tx_layers)); @@ -676,7 +678,6 @@ static int do_one_dlsch(unsigned char *input_ptr, PHY_VARS_gNB *gNB, NR_gNB_DLSC // pmi = prgs_list[rbidx/prg_size].pm_idx, rbidx =0,...,rbSize-1 // The Precoding matrix: // The Codebook Type I - start_meas(&gNB->dlsch_precoding_stats); nfapi_nr_tx_precoding_and_beamforming_t *pb = &rel15->precodingAndBeamforming; // beam number in multi-beam scenario (concurrent beams) int bitmap = SL_to_bitmap(rel15->StartSymbolIndex, rel15->NrOfSymbols); @@ -689,9 +690,10 @@ static int do_one_dlsch(unsigned char *input_ptr, PHY_VARS_gNB *gNB, NR_gNB_DLSC bitmap); c16_t **txdataF = gNB->common_vars.txdataF[beam_nb]; - + stop_meas(&gNB->dlsch_layer_mapping_stats); // Loop Over OFDM symbols: for (int l_symbol = rel15->StartSymbolIndex; l_symbol < rel15->StartSymbolIndex + rel15->NrOfSymbols; l_symbol++) { + start_meas(&gNB->dlsch_resource_mapping_stats); int l_prime = 0; // single symbol layer 0 int l_overline = get_l0(rel15->dlDmrsSymbPos); @@ -758,13 +760,14 @@ static int do_one_dlsch(unsigned char *input_ptr, PHY_VARS_gNB *gNB, NR_gNB_DLSC re_beginning_of_symbol += layer_sz; stop_meas(&gNB->dlsch_resource_mapping_stats); + start_meas(&gNB->dlsch_precoding_stats); for (int ant = 0; ant < frame_parms->nb_antennas_tx; ant++) { const size_t txdataF_offset_per_symbol = l_symbol * symbol_sz + txdataF_offset; do_txdataF(txdataF, symbol_sz, txdataF_precoding, gNB, rel15, ant, start_sc, txdataF_offset_per_symbol); } + stop_meas(&gNB->dlsch_precoding_stats); } - - stop_meas(&gNB->dlsch_precoding_stats); + stop_meas(&gNB->dlsch_pdsch_generation_stats); /* output and its parts for each dlsch should be aligned on 64 bytes (or 8 * 64 bits) * should remain a multiple of 8 * 64 with enough offset to fit each dlsch */ diff --git a/openair1/PHY/defs_gNB.h b/openair1/PHY/defs_gNB.h index 987f77d953..395fb9d822 100644 --- a/openair1/PHY/defs_gNB.h +++ b/openair1/PHY/defs_gNB.h @@ -501,6 +501,8 @@ typedef struct PHY_VARS_gNB_s { time_stats_t dlsch_encoding_stats; time_stats_t dlsch_modulation_stats; time_stats_t dlsch_scrambling_stats; + time_stats_t dlsch_pdsch_generation_stats; + time_stats_t dlsch_layer_mapping_stats; time_stats_t dlsch_resource_mapping_stats; time_stats_t dlsch_precoding_stats; time_stats_t tinput; diff --git a/openair1/SIMULATION/NR_PHY/dlsim.c b/openair1/SIMULATION/NR_PHY/dlsim.c index d1d046505e..82fe1f2f7b 100644 --- a/openair1/SIMULATION/NR_PHY/dlsim.c +++ b/openair1/SIMULATION/NR_PHY/dlsim.c @@ -1029,7 +1029,10 @@ int main(int argc, char **argv) reset_meas(&gNB->dlsch_rate_matching_stats); reset_meas(&gNB->dlsch_segmentation_stats); reset_meas(&gNB->dlsch_modulation_stats); + reset_meas(&gNB->dlsch_pdsch_generation_stats); reset_meas(&gNB->dlsch_precoding_stats); + reset_meas(&gNB->dlsch_layer_mapping_stats); + reset_meas(&gNB->dlsch_resource_mapping_stats); reset_meas(&gNB->dlsch_encoding_stats); reset_meas(&gNB->dci_generation_stats); reset_meas(&gNB->tinput); @@ -1343,7 +1346,12 @@ int main(int argc, char **argv) printStatIndent3(&gNB->dlsch_interleaving_stats, "DLSCH Interleaving time"); printStatIndent2(&gNB->dlsch_modulation_stats,"DLSCH modulation time"); printStatIndent2(&gNB->dlsch_scrambling_stats, "DLSCH scrambling time"); - printStatIndent2(&gNB->dlsch_precoding_stats,"DLSCH Mapping/Precoding time"); + printStatIndent2(&gNB->dlsch_pdsch_generation_stats,"DLSCH PDSCH Generation time"); + printStatIndent3(&gNB->dlsch_layer_mapping_stats,"DLSCH Layer Mapping time"); + gNB->dlsch_resource_mapping_stats.trials = gNB->dlsch_layer_mapping_stats.trials; + printStatIndent3(&gNB->dlsch_resource_mapping_stats,"DLSCH Resource Mapping time"); + gNB->dlsch_precoding_stats.trials = gNB->dlsch_layer_mapping_stats.trials; + printStatIndent3(&gNB->dlsch_precoding_stats,"DLSCH Precoding time"); if (gNB->phase_comp) printStatIndent2(&gNB->phase_comp_stats, "Phase Compensation"); From 8711556d9e9a41aca6f49529deab9bc699de4d1f Mon Sep 17 00:00:00 2001 From: Raymond Knopp Date: Thu, 18 Sep 2025 19:03:48 +0200 Subject: [PATCH 04/10] Handle larger RB step size in precoding --- openair1/PHY/NR_TRANSPORT/nr_dlsch.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/openair1/PHY/NR_TRANSPORT/nr_dlsch.c b/openair1/PHY/NR_TRANSPORT/nr_dlsch.c index 7aec1ac5ad..535c68f419 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_dlsch.c +++ b/openair1/PHY/NR_TRANSPORT/nr_dlsch.c @@ -470,12 +470,14 @@ static inline void do_txdataF(c16_t **txdataF, // get pmi info const int pmi = (pb->prg_size > 0) ? (pb->prgs_list[(int)rb / pb->prg_size].pm_idx) : 0; const int pmi2 = (rb < (rel15->rbSize - 1) && pb->prg_size > 0) ? (pb->prgs_list[(int)(rb + 1) / pb->prg_size].pm_idx) : -1; + const int pmi3 = (rb < (rel15->rbSize - 2) && pb->prg_size > 0) ? (pb->prgs_list[(int)(rb + 2) / pb->prg_size].pm_idx) : -1; + const int pmi4 = (rb < (rel15->rbSize - 3) && pb->prg_size > 0) ? (pb->prgs_list[(int)(rb + 3) / pb->prg_size].pm_idx) : -1; // If pmi of next RB and pmi of current RB are the same, we do 2 RB in a row // if pmi differs, or current rb is the end (rel15->rbSize - 1), than we do 1 RB in a row - const int rb_step = pmi == pmi2 ? 2 : 1; + int rb_step0 = pmi == pmi2 ? 2 : 1; + const int rb_step = rb_step0==2 && pmi3==pmi && pmi4==pmi ? 4 : rb_step0; const int re_cnt = NR_NB_SC_PER_RB * rb_step; - if (pmi == 0) { // unitary Precoding if (subCarrier + re_cnt <= symbol_sz) { // RB does not cross DC if (ant < rel15->nrOfLayers) From 34ba85f3b1a70817832b96264ad765f57ffcd0b3 Mon Sep 17 00:00:00 2001 From: Raymond Knopp Date: Fri, 29 Aug 2025 15:11:19 +0200 Subject: [PATCH 05/10] precoder optimizations for AVX512 --- openair1/PHY/MODULATION/nr_modulation.c | 154 +++++++++++++++++++++++- 1 file changed, 149 insertions(+), 5 deletions(-) diff --git a/openair1/PHY/MODULATION/nr_modulation.c b/openair1/PHY/MODULATION/nr_modulation.c index 84f7e356e7..9b6103edd8 100644 --- a/openair1/PHY/MODULATION/nr_modulation.c +++ b/openair1/PHY/MODULATION/nr_modulation.c @@ -295,7 +295,7 @@ void nr_layer_mapping(int nbCodes, c16_t *tx0 = tx_layers[0]; c16_t *tx1 = tx_layers[1]; c16_t *tx2 = tx_layers[2]; -#if defined(__AVX512BW__) +#if defined(__AVX512F) && defined(__AVX512VBMI__) simde__m512i perm3_0 = simde_mm512_set_epi32(13 + 16, 10 + 16, 7 + 16, @@ -433,7 +433,7 @@ void nr_layer_mapping(int nbCodes, c16_t *tx1 = tx_layers[1]; c16_t *tx2 = tx_layers[2]; c16_t *tx3 = tx_layers[3]; -#if defined(__AVX512BW__) +#if defined(__AVX512VBMI__) simde__m512i perm4 = simde_mm512_set_epi32(15, 11, 7, 3, 14, 10, 6, 2, 13, 9, 5, 1, 12, 8, 4, 0); for (; i < (n_symbs & ~15); i += 16) { simde__m512i e = simde_mm512_permutexvar_epi32(perm4, *(simde__m512i *)(mod + i)); @@ -717,6 +717,35 @@ c16_t nr_layer_precoder_cm(int n_layers, return precodatatx_F; } +#if defined(__AVX512F__) && defined(__AVX512BW__) + +static inline __attribute__((always_inline)) __m512i cmac0_prec512(__m512i x, __m512i w_c, __m512i w_s) { + + // Multiplication and shift + const __m512i reals = + _mm512_srai_epi32(_mm512_madd_epi16(x, w_c), 15); // (int32_t) .r = (x.r * w.r - x.i * w.i) >> 15 + const __m512i imags = + _mm512_slli_epi32(_mm512_madd_epi16(x, w_s), 1); // (int32_t) .i = (x.r * w.i + x.i * w.r) << 1, since higher 16 bit of each 32 bit is taken by blend_epi16 + + // Re-arrange to match c16_t format + return _mm512_mask_blend_epi16(0xAAAAAAAA,reals, imags); + +} +static inline __attribute__((always_inline)) __m512i cmac_prec512(__m512i y, __m512i x, __m512i w_c, __m512i w_s) { + + // Multiplication and shift + const __m512i reals = + _mm512_srai_epi32(_mm512_madd_epi16(x, w_c), 15); // (int32_t) .r = (x.r * w.r - x.i * w.i) >> 15 + const __m512i imags = + _mm512_slli_epi32(_mm512_madd_epi16(x, w_s), 1); // (int32_t) .i = (x.r * w.i + x.i * w.r) << 1, since higher 16 bit of each 32 bit is taken by blend_epi16 + + // Re-arrange to match c16_t format + const __m512i produ = _mm512_mask_blend_epi16(0xAAAAAAAA,reals, imags); + + // Accumulate the product + return _mm512_adds_epi16(y, produ); +} +#endif void nr_layer_precoder_simd(const int n_layers, const int symSz, const c16_t txdataF_res_mapped[n_layers][symSz], @@ -731,11 +760,125 @@ void nr_layer_precoder_simd(const int n_layers, // For x86, use 256 SIMD for every 8 RE and 128 SIMD for last 4 RE // For aarch64, use 128 SIMD for every 4 RE - // 256 SIMD: Do 8 RE in one iteration, 3 iterations for 2 RB + // 512/256 SIMD: Do 16/8 RE in one iteration, 3 iterations for 2 RB +#if defined(__AVX512F__) && defined(__AVX512BW__) + const uint32_t re_cnt_align16 = re_cnt & ~15; + if (n_layers==2) { + prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; + prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; + const __m512i w_c = _mm512_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w + const __m512i w_s = _mm512_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w + prec_weight.r = pmi_pdu->weights[1][ant].precoder_weight_Re; + prec_weight.i = pmi_pdu->weights[1][ant].precoder_weight_Im; + const __m512i w_c1 = _mm512_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w + const __m512i w_s1 = _mm512_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w + __m512i y = _mm512_set1_epi16(0); + for (; sc < sc_offset + (re_cnt_align16); sc += sizeof(__m512i) / sizeof(prec_weight)) { + + const __m512i x = _mm512_loadu_si512(&txdataF_res_mapped[0][sc]); + const __m512i x1 = _mm512_loadu_si512(&txdataF_res_mapped[1][sc]); + // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8) + y=cmac0_prec512(x,w_c,w_s); + y=cmac_prec512(y,x1,w_c1,w_s1); + _mm512_storeu_si512(&txdataF_precoded[sc],y); + } + } + else if (n_layers==3) { + prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; + prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; + const __m512i w_c = _mm512_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w + const __m512i w_s = _mm512_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w + prec_weight.r = pmi_pdu->weights[1][ant].precoder_weight_Re; + prec_weight.i = pmi_pdu->weights[1][ant].precoder_weight_Im; + const __m512i w_c1 = _mm512_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w + const __m512i w_s1 = _mm512_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w + prec_weight.r = pmi_pdu->weights[2][ant].precoder_weight_Re; + prec_weight.i = pmi_pdu->weights[2][ant].precoder_weight_Im; + const __m512i w_c2 = _mm512_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w + const __m512i w_s2 = _mm512_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w + __m512i y = _mm512_set1_epi16(0); + for (; sc < sc_offset + (re_cnt_align16); sc += sizeof(__m512i) / sizeof(prec_weight)) { + + const __m512i x = _mm512_loadu_si512(&txdataF_res_mapped[0][sc]); + const __m512i x1 = _mm512_loadu_si512(&txdataF_res_mapped[1][sc]); + const __m512i x2 = _mm512_loadu_si512(&txdataF_res_mapped[2][sc]); + // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8) + y=cmac0_prec512(x,w_c,w_s); + y=cmac_prec512(y,x1,w_c1,w_s1); + y=cmac_prec512(y,x2,w_c2,w_s2); + _mm512_storeu_si512(&txdataF_precoded[sc],y); + } + } + else if (n_layers==4) { + prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; + prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; + const __m512i w_c = _mm512_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w + const __m512i w_s = _mm512_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w + prec_weight.r = pmi_pdu->weights[1][ant].precoder_weight_Re; + prec_weight.i = pmi_pdu->weights[1][ant].precoder_weight_Im; + const __m512i w_c1 = _mm512_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w + const __m512i w_s1 = _mm512_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w + prec_weight.r = pmi_pdu->weights[2][ant].precoder_weight_Re; + prec_weight.i = pmi_pdu->weights[2][ant].precoder_weight_Im; + const __m512i w_c2 = _mm512_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w + const __m512i w_s2 = _mm512_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w + prec_weight.r = pmi_pdu->weights[3][ant].precoder_weight_Re; + prec_weight.i = pmi_pdu->weights[3][ant].precoder_weight_Im; + const __m512i w_c3 = _mm512_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w + const __m512i w_s3 = _mm512_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w + __m512i y = _mm512_set1_epi16(0); + for (; sc < sc_offset + (re_cnt_align16); sc += sizeof(__m512i) / sizeof(prec_weight)) { + + const __m512i x = _mm512_loadu_si512(&txdataF_res_mapped[0][sc]); + const __m512i x1 = _mm512_loadu_si512(&txdataF_res_mapped[1][sc]); + const __m512i x2 = _mm512_loadu_si512(&txdataF_res_mapped[2][sc]); + const __m512i x3 = _mm512_loadu_si512(&txdataF_res_mapped[3][sc]); + // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8) + y=cmac0_prec512(x,w_c,w_s); + y=cmac_prec512(y,x1,w_c1,w_s1); + y=cmac_prec512(y,x2,w_c2,w_s2); + y=cmac_prec512(y,x3,w_c3,w_s3); + _mm512_storeu_si512(&txdataF_precoded[sc],y); + } + } + /* + const uint32_t re_cnt_align16 = re_cnt & ~15; + for (; sc < sc_offset + (re_cnt_align16); sc += sizeof(simde__m256i) / sizeof(prec_weight)) { + // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8) + + __m512i y = _mm512_set1_epi16(0); // Y = W[0]*X[0] + W[1]*X[1] + ... + W[nrOfLayers-1]*X[nrOfLayers-1] + for (int nl = 0; nl < n_layers; nl++) { + prec_weight.r = pmi_pdu->weights[nl][ant].precoder_weight_Re; + prec_weight.i = pmi_pdu->weights[nl][ant].precoder_weight_Im; + + const __m512i x = _mm512_loadu_si512(&txdataF_res_mapped[nl][sc]); + + // Rearrange precoding matrix weight to match complex multiplication and broadcast it to match SIMD size + const __m512i w_c = _mm512_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w + const __m512i w_s = _mm512_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w + + // Multiplication and shift + const __m512i reals = + _mm512_srai_epi32(_mm512_madd_epi16(x, w_c), 15); // (int32_t) .r = (x.r * w.r - x.i * w.i) >> 15 + const __m512i imags = + _mm512_slli_epi32(_mm512_madd_epi16(x, w_s), 1); // (int32_t) .i = (x.r * w.i + x.i * w.r) << 1, since higher 16 bit of each 32 bit is taken by blend_epi16 + + // Re-arrange to match c16_t format + const __m512i produ = _mm512_mask_blend_epi16(0xAAAAAAAA,reals, imags); + + // Accumulate the product + y = _mm512_adds_epi16(y, produ); + } + // Store the result to txdataF + _mm512_storeu_si512(&txdataF_precoded[sc], y); + } + */ +#endif #ifdef __AVX2__ const uint32_t re_cnt_align8 = re_cnt & ~7; for (; sc < sc_offset + (re_cnt_align8); sc += sizeof(simde__m256i) / sizeof(prec_weight)) { // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8) + simde__m256i y = simde_mm256_set1_epi16(0); // Y = W[0]*X[0] + W[1]*X[1] + ... + W[nrOfLayers-1]*X[nrOfLayers-1] for (int nl = 0; nl < n_layers; nl++) { prec_weight.r = pmi_pdu->weights[nl][ant].precoder_weight_Re; @@ -763,7 +906,6 @@ void nr_layer_precoder_simd(const int n_layers, simde_mm256_storeu_si256(&txdataF_precoded[sc], y); } #endif - // 128 SIMD: Do 4 RE in one iteration, 3 iterations for 1 RB const uint32_t re_cnt_align4 = re_cnt & ~3; for (; sc < sc_offset + re_cnt_align4; sc += sizeof(simde__m128i) / sizeof(prec_weight)) { @@ -773,7 +915,9 @@ void nr_layer_precoder_simd(const int n_layers, y_triv[i] = nr_layer_precoder_cm(n_layers, symSz, txdataF_res_mapped, ant, pmi_pdu, sc + i); memcpy(&txdataF_precoded[sc], y_triv, sizeof(y_triv)); #endif +#ifdef __aarch64__ +#else // Matrix multiplication for 4 elements of the result (sizeof(simde__m128i) / sizeof(c16_t) = 4) simde__m128i y = simde_mm_set1_epi16(0); // Y = W[0]*X[0] + W[1]*X[1] + ... + W[nrOfLayers-1]*X[nrOfLayers-1] for (int nl = 0; nl < n_layers; nl++) { @@ -808,7 +952,7 @@ void nr_layer_precoder_simd(const int n_layers, } // Store the result to txdataF simde_mm_storeu_si128(&txdataF_precoded[sc], y); - +#endif #ifdef DEBUG_DLSCH_PRECODING_PRINT_WITH_TRIVIAL // Print simd and trivial result, TODO: To be removed c16_t *y_simd = (c16_t *)&y; printf("debug_to_be_removed re_cnt=%d, sc=%u, y_simd=(%+4d,%+4d), (%+4d,%+4d), (%+4d,%+4d), (%+4d,%+4d)\n", From e44d498cbad4efd0f5edd70e167be41e13022b4d Mon Sep 17 00:00:00 2001 From: Raymond Knopp Date: Fri, 29 Aug 2025 17:38:56 +0200 Subject: [PATCH 06/10] optimization of DLSCH precoding for AVX2 --- openair1/PHY/MODULATION/nr_modulation.c | 158 ++++++++++++++++-------- 1 file changed, 104 insertions(+), 54 deletions(-) diff --git a/openair1/PHY/MODULATION/nr_modulation.c b/openair1/PHY/MODULATION/nr_modulation.c index 9b6103edd8..7f230cf420 100644 --- a/openair1/PHY/MODULATION/nr_modulation.c +++ b/openair1/PHY/MODULATION/nr_modulation.c @@ -746,6 +746,34 @@ static inline __attribute__((always_inline)) __m512i cmac_prec512(__m512i y, __m return _mm512_adds_epi16(y, produ); } #endif +#ifdef __AVX2__ +static inline __attribute__((always_inline)) __m256i cmac0_prec256(__m256i x, __m256i w_c, __m256i w_s) { + + // Multiplication and shift + const __m256i reals = + _mm256_srai_epi32(_mm256_madd_epi16(x, w_c), 15); // (int32_t) .r = (x.r * w.r - x.i * w.i) >> 15 + const __m256i imags = + _mm256_slli_epi32(_mm256_madd_epi16(x, w_s), 1); // (int32_t) .i = (x.r * w.i + x.i * w.r) << 1, since higher 16 bit of each 32 bit is taken by blend_epi16 + + // Re-arrange to match c16_t format + return _mm256_blend_epi16(reals, imags,0xAA); + +} +static inline __attribute__((always_inline)) __m256i cmac_prec256(__m256i y, __m256i x, __m256i w_c, __m256i w_s) { + + // Multiplication and shift + const __m256i reals = + _mm256_srai_epi32(_mm256_madd_epi16(x, w_c), 15); // (int32_t) .r = (x.r * w.r - x.i * w.i) >> 15 + const __m256i imags = + _mm256_slli_epi32(_mm256_madd_epi16(x, w_s), 1); // (int32_t) .i = (x.r * w.i + x.i * w.r) << 1, since higher 16 bit of each 32 bit is taken by blend_epi16 + + // Re-arrange to match c16_t format + const __m256i produ = _mm256_blend_epi16(reals, imags,0xAA); + + // Accumulate the product + return _mm256_adds_epi16(y, produ); +} +#endif void nr_layer_precoder_simd(const int n_layers, const int symSz, const c16_t txdataF_res_mapped[n_layers][symSz], @@ -841,70 +869,92 @@ void nr_layer_precoder_simd(const int n_layers, _mm512_storeu_si512(&txdataF_precoded[sc],y); } } - /* - const uint32_t re_cnt_align16 = re_cnt & ~15; - for (; sc < sc_offset + (re_cnt_align16); sc += sizeof(simde__m256i) / sizeof(prec_weight)) { - // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8) - - __m512i y = _mm512_set1_epi16(0); // Y = W[0]*X[0] + W[1]*X[1] + ... + W[nrOfLayers-1]*X[nrOfLayers-1] - for (int nl = 0; nl < n_layers; nl++) { - prec_weight.r = pmi_pdu->weights[nl][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[nl][ant].precoder_weight_Im; - - const __m512i x = _mm512_loadu_si512(&txdataF_res_mapped[nl][sc]); - - // Rearrange precoding matrix weight to match complex multiplication and broadcast it to match SIMD size - const __m512i w_c = _mm512_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const __m512i w_s = _mm512_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - - // Multiplication and shift - const __m512i reals = - _mm512_srai_epi32(_mm512_madd_epi16(x, w_c), 15); // (int32_t) .r = (x.r * w.r - x.i * w.i) >> 15 - const __m512i imags = - _mm512_slli_epi32(_mm512_madd_epi16(x, w_s), 1); // (int32_t) .i = (x.r * w.i + x.i * w.r) << 1, since higher 16 bit of each 32 bit is taken by blend_epi16 - - // Re-arrange to match c16_t format - const __m512i produ = _mm512_mask_blend_epi16(0xAAAAAAAA,reals, imags); - - // Accumulate the product - y = _mm512_adds_epi16(y, produ); - } - // Store the result to txdataF - _mm512_storeu_si512(&txdataF_precoded[sc], y); - } - */ + else AssertFatal(1==0,"Shouldn't get here, n_layers %d\n",n_layers); #endif #ifdef __AVX2__ const uint32_t re_cnt_align8 = re_cnt & ~7; - for (; sc < sc_offset + (re_cnt_align8); sc += sizeof(simde__m256i) / sizeof(prec_weight)) { - // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8) + if (n_layers==2) { simde__m256i y = simde_mm256_set1_epi16(0); // Y = W[0]*X[0] + W[1]*X[1] + ... + W[nrOfLayers-1]*X[nrOfLayers-1] - for (int nl = 0; nl < n_layers; nl++) { - prec_weight.r = pmi_pdu->weights[nl][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[nl][ant].precoder_weight_Im; + prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; + prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; + const simde__m256i w_c0 = simde_mm256_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w + const simde__m256i w_s0 = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w + prec_weight.r = pmi_pdu->weights[1][ant].precoder_weight_Re; + prec_weight.i = pmi_pdu->weights[1][ant].precoder_weight_Im; + const simde__m256i w_c1 = simde_mm256_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w + const simde__m256i w_s1 = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - const simde__m256i x = simde_mm256_loadu_si256(&txdataF_res_mapped[nl][sc]); - - // Rearrange precoding matrix weight to match complex multiplication and broadcast it to match SIMD size - const simde__m256i w_c = simde_mm256_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const simde__m256i w_s = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - - // Multiplication and shift - const simde__m256i reals = - simde_mm256_srai_epi32(simde_mm256_madd_epi16(x, w_c), 15); // (int32_t) .r = (x.r * w.r - x.i * w.i) >> 15 - const simde__m256i imags = - simde_mm256_slli_epi32(simde_mm256_madd_epi16(x, w_s), 1); // (int32_t) .i = (x.r * w.i + x.i * w.r) << 1, since higher 16 bit of each 32 bit is taken by blend_epi16 - - // Re-arrange to match c16_t format - const simde__m256i produ = simde_mm256_blend_epi16(reals, imags, 0xAA); + for (; sc < sc_offset + (re_cnt_align8); sc += sizeof(simde__m256i) / sizeof(prec_weight)) { + const simde__m256i x0 = simde_mm256_loadu_si256(&txdataF_res_mapped[0][sc]); + const simde__m256i x1 = simde_mm256_loadu_si256(&txdataF_res_mapped[1][sc]); // Accumulate the product - y = simde_mm256_adds_epi16(y, produ); + y = cmac0_prec256(x0,w_c0,w_s0); + y = cmac_prec256(y,x1,w_c1,w_s1); + // Store the result to txdataF + simde_mm256_storeu_si256(&txdataF_precoded[sc], y); } - // Store the result to txdataF - simde_mm256_storeu_si256(&txdataF_precoded[sc], y); } + else if (n_layers==3) { + simde__m256i y = simde_mm256_set1_epi16(0); // Y = W[0]*X[0] + W[1]*X[1] + ... + W[nrOfLayers-1]*X[nrOfLayers-1] + prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; + prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; + const simde__m256i w_c0 = simde_mm256_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w + const simde__m256i w_s0 = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w + prec_weight.r = pmi_pdu->weights[1][ant].precoder_weight_Re; + prec_weight.i = pmi_pdu->weights[1][ant].precoder_weight_Im; + const simde__m256i w_c1 = simde_mm256_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w + const simde__m256i w_s1 = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w + prec_weight.r = pmi_pdu->weights[2][ant].precoder_weight_Re; + prec_weight.i = pmi_pdu->weights[2][ant].precoder_weight_Im; + const simde__m256i w_c2 = simde_mm256_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w + const simde__m256i w_s2 = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w + + for (; sc < sc_offset + (re_cnt_align8); sc += sizeof(simde__m256i) / sizeof(prec_weight)) { + const simde__m256i x0 = simde_mm256_loadu_si256(&txdataF_res_mapped[0][sc]); + const simde__m256i x1 = simde_mm256_loadu_si256(&txdataF_res_mapped[1][sc]); + const simde__m256i x2 = simde_mm256_loadu_si256(&txdataF_res_mapped[2][sc]); + y = cmac0_prec256(x0,w_c0,w_s0); + y = cmac_prec256(y,x1,w_c1,w_s1); + y = cmac_prec256(y,x2,w_c2,w_s2); + // Store the result to txdataF + simde_mm256_storeu_si256(&txdataF_precoded[sc], y); + } + } + else if (n_layers==4) { + simde__m256i y = simde_mm256_set1_epi16(0); // Y = W[0]*X[0] + W[1]*X[1] + ... + W[nrOfLayers-1]*X[nrOfLayers-1] + prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; + prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; + const simde__m256i w_c0 = simde_mm256_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w + const simde__m256i w_s0 = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w + prec_weight.r = pmi_pdu->weights[1][ant].precoder_weight_Re; + prec_weight.i = pmi_pdu->weights[1][ant].precoder_weight_Im; + const simde__m256i w_c1 = simde_mm256_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w + const simde__m256i w_s1 = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w + prec_weight.r = pmi_pdu->weights[2][ant].precoder_weight_Re; + prec_weight.i = pmi_pdu->weights[2][ant].precoder_weight_Im; + const simde__m256i w_c2 = simde_mm256_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w + const simde__m256i w_s2 = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w + + prec_weight.r = pmi_pdu->weights[3][ant].precoder_weight_Re; + prec_weight.i = pmi_pdu->weights[3][ant].precoder_weight_Im; + const simde__m256i w_c3 = simde_mm256_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w + const simde__m256i w_s3 = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w + for (; sc < sc_offset + (re_cnt_align8); sc += sizeof(simde__m256i) / sizeof(prec_weight)) { + const simde__m256i x0 = simde_mm256_loadu_si256(&txdataF_res_mapped[0][sc]); + const simde__m256i x1 = simde_mm256_loadu_si256(&txdataF_res_mapped[1][sc]); + const simde__m256i x2 = simde_mm256_loadu_si256(&txdataF_res_mapped[2][sc]); + const simde__m256i x3 = simde_mm256_loadu_si256(&txdataF_res_mapped[3][sc]); + y = cmac0_prec256(x0,w_c0,w_s0); + y = cmac_prec256(y,x1,w_c1,w_s1); + y = cmac_prec256(y,x2,w_c2,w_s2); + y = cmac_prec256(y,x3,w_c3,w_s3); + // Store the result to txdataF + simde_mm256_storeu_si256(&txdataF_precoded[sc], y); + } + } + else AssertFatal(1==0,"Shouldn't get here, n_layers %d\n",n_layers); #endif // 128 SIMD: Do 4 RE in one iteration, 3 iterations for 1 RB const uint32_t re_cnt_align4 = re_cnt & ~3; From 264f7cb817e6551b5c8d5ae177a932fed4f46594 Mon Sep 17 00:00:00 2001 From: Raymond Knopp Date: Sat, 30 Aug 2025 11:02:59 +0000 Subject: [PATCH 07/10] aarch64 optimizations for precoding, more complete timing reporting in nr_dlsim --- openair1/PHY/MODULATION/nr_modulation.c | 121 ++++++++++++++++++++++++ 1 file changed, 121 insertions(+) diff --git a/openair1/PHY/MODULATION/nr_modulation.c b/openair1/PHY/MODULATION/nr_modulation.c index 7f230cf420..e1548787ec 100644 --- a/openair1/PHY/MODULATION/nr_modulation.c +++ b/openair1/PHY/MODULATION/nr_modulation.c @@ -774,6 +774,42 @@ static inline __attribute__((always_inline)) __m256i cmac_prec256(__m256i y, __m return _mm256_adds_epi16(y, produ); } #endif +#ifdef __aarch64__ +static inline __attribute__((always_inline)) int16x8_t cmac0_prec128(int16x8_t x, int16x8_t wr, int16x8_t wi) { + // + int16x8_t xr = vuzp1q_s16(x, x); // even lanes + int16x8_t xi = vuzp2q_s16(x, x); // odd lanes + // real = ar*br - ai*bi (Q15 scaling via high-half doubling muls) + int16x8_t real = vqdmulhq_s16(xr, wr); // ≈ round((2*xr*wr)/2^16) + real = vqrdmlshq_s16(real, xi, wi); // real -= round((2*xi*wi)/2^16) + // + // imag = ar*bi + ai*br + int16x8_t imag = vqdmulhq_s16(xr, wi); + imag = vqrdmlahq_s16(imag, xi, wr); // imag += round((2*xi*wr)/2^16) + // + // Re-interleave [real, imag] + int16x8x2_t produ = vzipq_s16(real, imag); + return produ.val[0]; +} +static inline __attribute__((always_inline)) int16x8_t cmac_prec128(int16x8_t y, int16x8_t x, int16x8_t wr, int16x8_t wi) { + // De-interleave real/imag + int16x8_t xr = vuzp1q_s16(x, x); // even lanes + int16x8_t xi = vuzp2q_s16(x, x); // odd lanes + // + // real = ar*br - ai*bi (Q15 scaling via high-half doubling muls) + int16x8_t real = vqdmulhq_s16(xr, wr); // ≈ round((2*ar*br)/2^16) + real = vqrdmlshq_s16(real, xi, wi); // real -= round((2*ai*bi)/2^16) + // + // imag = ar*bi + ai*br + int16x8_t imag = vqdmulhq_s16(xr, wi); + imag = vqrdmlahq_s16(imag, xi, wr); // imag += round((2*ai*br)/2^16) + // + // Re-interleave [real, imag] + int16x8x2_t produ = vzipq_s16(real, imag); + return vaddq_s16(y,produ.val[0]); +} +#endif + void nr_layer_precoder_simd(const int n_layers, const int symSz, const c16_t txdataF_res_mapped[n_layers][symSz], @@ -967,6 +1003,91 @@ void nr_layer_precoder_simd(const int n_layers, #endif #ifdef __aarch64__ + const uint32_t re_cnt_align8 = re_cnt & ~3; + if (n_layers==2) { + + prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; + prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; + const int16x8_t wr0 = vdupq_n_s16(prec_weight.r); + const int16x8_t wi0 = vdupq_n_s16(prec_weight.i); + prec_weight.r = pmi_pdu->weights[1][ant].precoder_weight_Re; + prec_weight.i = pmi_pdu->weights[1][ant].precoder_weight_Im; + const int16x8_t wr1 = vdupq_n_s16(prec_weight.r); + const int16x8_t wi1 = vdupq_n_s16(prec_weight.i); + int16x8_t y; + + for (; sc < sc_offset + (re_cnt_align8); sc += sizeof(int16x8_t) / sizeof(prec_weight)) { + const int16x8_t x0 = vld1q_s16((const int16_t*)&txdataF_res_mapped[0][sc]); + const int16x8_t x1 = vld1q_s16((const int16_t*)&txdataF_res_mapped[1][sc]); + // Accumulate the product + y = cmac0_prec128(x0,wr0,wi0); + y = cmac_prec128(y,x1,wr1,wi1); + // Store the result to txdataF + *((int16x8_t*)&txdataF_precoded[sc])= y; + } + } + if (n_layers==3) { + + prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; + prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; + const int16x8_t wr0 = vdupq_n_s16(prec_weight.r); + const int16x8_t wi0 = vdupq_n_s16(prec_weight.i); + prec_weight.r = pmi_pdu->weights[1][ant].precoder_weight_Re; + prec_weight.i = pmi_pdu->weights[1][ant].precoder_weight_Im; + const int16x8_t wr1 = vdupq_n_s16(prec_weight.r); + const int16x8_t wi1 = vdupq_n_s16(prec_weight.i); + prec_weight.r = pmi_pdu->weights[2][ant].precoder_weight_Re; + prec_weight.i = pmi_pdu->weights[2][ant].precoder_weight_Im; + const int16x8_t wr2 = vdupq_n_s16(prec_weight.r); + const int16x8_t wi2 = vdupq_n_s16(prec_weight.i); + int16x8_t y; + + for (; sc < sc_offset + (re_cnt_align8); sc += sizeof(int16x8_t) / sizeof(prec_weight)) { + const int16x8_t x0 = vld1q_s16((const int16_t*)&txdataF_res_mapped[0][sc]); + const int16x8_t x1 = vld1q_s16((const int16_t*)&txdataF_res_mapped[1][sc]); + const int16x8_t x2 = vld1q_s16((const int16_t*)&txdataF_res_mapped[2][sc]); + // Accumulate the product + y = cmac0_prec128(x0,wr0,wi0); + y = cmac_prec128(y,x1,wr1,wi1); + y = cmac_prec128(y,x2,wr2,wi2); + // Store the result to txdataF + *((int16x8_t*)&txdataF_precoded[sc])= y; + } + } + if (n_layers==4) { + + prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; + prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; + const int16x8_t wr0 = vdupq_n_s16(prec_weight.r); + const int16x8_t wi0 = vdupq_n_s16(prec_weight.i); + prec_weight.r = pmi_pdu->weights[1][ant].precoder_weight_Re; + prec_weight.i = pmi_pdu->weights[1][ant].precoder_weight_Im; + const int16x8_t wr1 = vdupq_n_s16(prec_weight.r); + const int16x8_t wi1 = vdupq_n_s16(prec_weight.i); + prec_weight.r = pmi_pdu->weights[2][ant].precoder_weight_Re; + prec_weight.i = pmi_pdu->weights[2][ant].precoder_weight_Im; + const int16x8_t wr2 = vdupq_n_s16(prec_weight.r); + const int16x8_t wi2 = vdupq_n_s16(prec_weight.i); + prec_weight.r = pmi_pdu->weights[3][ant].precoder_weight_Re; + prec_weight.i = pmi_pdu->weights[3][ant].precoder_weight_Im; + const int16x8_t wr3 = vdupq_n_s16(prec_weight.r); + const int16x8_t wi3 = vdupq_n_s16(prec_weight.i); + int16x8_t y; + + for (; sc < sc_offset + (re_cnt_align8); sc += sizeof(int16x8_t) / sizeof(prec_weight)) { + const int16x8_t x0 = vld1q_s16((const int16_t*)&txdataF_res_mapped[0][sc]); + const int16x8_t x1 = vld1q_s16((const int16_t*)&txdataF_res_mapped[1][sc]); + const int16x8_t x2 = vld1q_s16((const int16_t*)&txdataF_res_mapped[2][sc]); + const int16x8_t x3 = vld1q_s16((const int16_t*)&txdataF_res_mapped[3][sc]); + // Accumulate the product + y = cmac0_prec128(x0,wr0,wi0); + y = cmac_prec128(y,x1,wr1,wi1); + y = cmac_prec128(y,x2,wr2,wi2); + y = cmac_prec128(y,x3,wr3,wi3); + // Store the result to txdataF + *((int16x8_t*)&txdataF_precoded[sc])= y; + } + } #else // Matrix multiplication for 4 elements of the result (sizeof(simde__m128i) / sizeof(c16_t) = 4) simde__m128i y = simde_mm_set1_epi16(0); // Y = W[0]*X[0] + W[1]*X[1] + ... + W[nrOfLayers-1]*X[nrOfLayers-1] From fb47748faeee7faded53be0139ac4548dc07eb4d Mon Sep 17 00:00:00 2001 From: Raymond Knopp Date: Mon, 1 Sep 2025 17:40:07 +0200 Subject: [PATCH 08/10] fixed precoder_simd after CI run (test for n_layers=1) failed --- openair1/PHY/MODULATION/nr_modulation.c | 50 ++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/openair1/PHY/MODULATION/nr_modulation.c b/openair1/PHY/MODULATION/nr_modulation.c index e1548787ec..b641ceaad1 100644 --- a/openair1/PHY/MODULATION/nr_modulation.c +++ b/openair1/PHY/MODULATION/nr_modulation.c @@ -827,7 +827,21 @@ void nr_layer_precoder_simd(const int n_layers, // 512/256 SIMD: Do 16/8 RE in one iteration, 3 iterations for 2 RB #if defined(__AVX512F__) && defined(__AVX512BW__) const uint32_t re_cnt_align16 = re_cnt & ~15; - if (n_layers==2) { + if (n_layers==1) { + prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; + prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; + const __m512i w_c = _mm512_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w + const __m512i w_s = _mm512_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w + __m512i y = _mm512_set1_epi16(0); + for (; sc < sc_offset + (re_cnt_align16); sc += sizeof(__m512i) / sizeof(prec_weight)) { + + const __m512i x = _mm512_loadu_si512(&txdataF_res_mapped[0][sc]); + // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8) + y=cmac0_prec512(x,w_c,w_s); + _mm512_storeu_si512(&txdataF_precoded[sc],y); + } + } + else if (n_layers==2) { prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; const __m512i w_c = _mm512_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w @@ -909,7 +923,23 @@ void nr_layer_precoder_simd(const int n_layers, #endif #ifdef __AVX2__ const uint32_t re_cnt_align8 = re_cnt & ~7; - if (n_layers==2) { + if (n_layers==1) { + + simde__m256i y = simde_mm256_set1_epi16(0); // Y = W[0]*X[0] + W[1]*X[1] + ... + W[nrOfLayers-1]*X[nrOfLayers-1] + prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; + prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; + const simde__m256i w_c0 = simde_mm256_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w + const simde__m256i w_s0 = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w + + for (; sc < sc_offset + (re_cnt_align8); sc += sizeof(simde__m256i) / sizeof(prec_weight)) { + const simde__m256i x0 = simde_mm256_loadu_si256(&txdataF_res_mapped[0][sc]); + // Accumulate the product + y = cmac0_prec256(x0,w_c0,w_s0); + // Store the result to txdataF + simde_mm256_storeu_si256(&txdataF_precoded[sc], y); + } + } + else if (n_layers==2) { simde__m256i y = simde_mm256_set1_epi16(0); // Y = W[0]*X[0] + W[1]*X[1] + ... + W[nrOfLayers-1]*X[nrOfLayers-1] prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; @@ -1004,6 +1034,22 @@ void nr_layer_precoder_simd(const int n_layers, #ifdef __aarch64__ const uint32_t re_cnt_align8 = re_cnt & ~3; + if (n_layers==1) { + + prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; + prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; + const int16x8_t wr0 = vdupq_n_s16(prec_weight.r); + const int16x8_t wi0 = vdupq_n_s16(prec_weight.i); + int16x8_t y; + + for (; sc < sc_offset + (re_cnt_align8); sc += sizeof(int16x8_t) / sizeof(prec_weight)) { + const int16x8_t x0 = vld1q_s16((const int16_t*)&txdataF_res_mapped[0][sc]); + // Accumulate the product + y = cmac0_prec128(x0,wr0,wi0); + // Store the result to txdataF + *((int16x8_t*)&txdataF_precoded[sc])= y; + } + } if (n_layers==2) { prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; From e056acc896884f791679ba277eb098363d17f1c6 Mon Sep 17 00:00:00 2001 From: Laurent THOMAS Date: Wed, 24 Sep 2025 12:30:55 +0200 Subject: [PATCH 09/10] use c16_t type for pm weights, as it is already same type simplify SIMD version of precoder --- nfapi/open-nFAPI/fapi/src/nr_fapi_p5.c | 5 +- nfapi/open-nFAPI/fapi/src/nr_fapi_p5_utils.c | 13 +- .../nfapi/public_inc/nfapi_nr_interface_scf.h | 7 +- nfapi/tests/p5/nr_fapi_config_request_test.c | 4 +- openair1/PHY/MODULATION/nr_modulation.c | 641 +++++++----------- .../MODULATION/tests/test_nr_modulation.cpp | 12 +- openair2/LAYER2/NR_MAC_gNB/config.c | 14 +- 7 files changed, 282 insertions(+), 414 deletions(-) diff --git a/nfapi/open-nFAPI/fapi/src/nr_fapi_p5.c b/nfapi/open-nFAPI/fapi/src/nr_fapi_p5.c index 4bcc9b2a44..c3e36d9999 100644 --- a/nfapi/open-nFAPI/fapi/src/nr_fapi_p5.c +++ b/nfapi/open-nFAPI/fapi/src/nr_fapi_p5.c @@ -760,7 +760,7 @@ static uint8_t pack_pm_table_tlv_value(void *tlv, uint8_t **ppWritePackedMsg, ui for (int k = 0; k < pm_pdu->numLayers; k++) { for (int j = 0; j < pm_pdu->num_ant_ports; j++) { const nfapi_nr_pm_weights_t *pm_weight = &pm_pdu->weights[k][j]; - if (!(push16(pm_weight->precoder_weight_Re, ppWritePackedMsg, end) && push16(pm_weight->precoder_weight_Im, ppWritePackedMsg, end))) { + if (!(push16(pm_weight->r, ppWritePackedMsg, end) && push16(pm_weight->i, ppWritePackedMsg, end))) { return 0; } } @@ -1307,8 +1307,7 @@ static uint8_t unpack_pm_table_tlv_value(void *tlv, uint8_t **ppReadPackedMsg, u for (int k = 0; k < pm_pdu->numLayers; k++) { for (int j = 0; j < pm_pdu->num_ant_ports; j++) { nfapi_nr_pm_weights_t *pm_weight = &pm_pdu->weights[k][j]; - if (!(pulls16(ppReadPackedMsg, &pm_weight->precoder_weight_Re, end) - && pulls16(ppReadPackedMsg, &pm_weight->precoder_weight_Im, end))) { + if (!(pulls16(ppReadPackedMsg, &pm_weight->r, end) && pulls16(ppReadPackedMsg, &pm_weight->i, end))) { return 0; } } diff --git a/nfapi/open-nFAPI/fapi/src/nr_fapi_p5_utils.c b/nfapi/open-nFAPI/fapi/src/nr_fapi_p5_utils.c index a810b199ff..5d1d0ba4e8 100644 --- a/nfapi/open-nFAPI/fapi/src/nr_fapi_p5_utils.c +++ b/nfapi/open-nFAPI/fapi/src/nr_fapi_p5_utils.c @@ -287,8 +287,8 @@ bool eq_config_request(const nfapi_nr_config_request_scf_t *unpacked_req, const for (int k = 0; k < req_pmi_pdu->num_ant_ports; ++k) { const nfapi_nr_pm_weights_t* req_pm_weight = &req_pmi_pdu->weights[j][k]; const nfapi_nr_pm_weights_t* unpacked_pm_weight = &unpacked_pmi_pdu->weights[j][k]; - EQ(unpacked_pm_weight->precoder_weight_Re, req_pm_weight->precoder_weight_Re); - EQ(unpacked_pm_weight->precoder_weight_Im, req_pm_weight->precoder_weight_Im); + EQ(unpacked_pm_weight->r, req_pm_weight->r); + EQ(unpacked_pm_weight->i, req_pm_weight->i); } } } @@ -848,10 +848,7 @@ void copy_config_request(const nfapi_nr_config_request_scf_t *src, nfapi_nr_conf dst_pmi_pdu->num_ant_ports = src_pmi_pdu->num_ant_ports; for (int j = 0; j < dst_pmi_pdu->numLayers; ++j) { for (int k = 0; k < dst_pmi_pdu->num_ant_ports; ++k) { - nfapi_nr_pm_weights_t* dst_pm_weight = &dst_pmi_pdu->weights[j][k]; - const nfapi_nr_pm_weights_t* src_pm_weight = &src_pmi_pdu->weights[j][k]; - dst_pm_weight->precoder_weight_Re = src_pm_weight->precoder_weight_Re; - dst_pm_weight->precoder_weight_Im = src_pm_weight->precoder_weight_Im; + dst_pmi_pdu->weights[j][k] = src_pmi_pdu->weights[j][k]; } } } @@ -1313,8 +1310,8 @@ void dump_config_request(const nfapi_nr_config_request_scf_t *msg) for (int k = 0; k < pm_pdu->numLayers; k++) { for (int l = 0; l < pm_pdu->num_ant_ports; l++) { const nfapi_nr_pm_weights_t *pm_weights = &pm_pdu->weights[k][l]; - INDENTED_GENERIC_PRINT("Dig Beam Weight Real", "0x%02x", pm_weights->precoder_weight_Re); - INDENTED_GENERIC_PRINT("Dig Beam Weight Imaginary", "0x%02x", pm_weights->precoder_weight_Im); + INDENTED_GENERIC_PRINT("Dig Beam Weight Real", "0x%02x", pm_weights->r); + INDENTED_GENERIC_PRINT("Dig Beam Weight Imaginary", "0x%02x", pm_weights->i); } } depth--; diff --git a/nfapi/open-nFAPI/nfapi/public_inc/nfapi_nr_interface_scf.h b/nfapi/open-nFAPI/nfapi/public_inc/nfapi_nr_interface_scf.h index a8ac3f73c1..754ef60e55 100644 --- a/nfapi/open-nFAPI/nfapi/public_inc/nfapi_nr_interface_scf.h +++ b/nfapi/open-nFAPI/nfapi/public_inc/nfapi_nr_interface_scf.h @@ -10,6 +10,7 @@ #define _NFAPI_NR_INTERFACE_SCF_H_ #include "stddef.h" +#include "common/platform_types.h" #include "nfapi_interface.h" #include "nfapi_nr_interface.h" @@ -488,11 +489,7 @@ typedef struct { } nfapi_nr_dbt_tlv_ve_t; // Table 3–62 Precoding matrix (PM) PDU (v.222.10.04) -typedef struct { - int16_t precoder_weight_Re; - int16_t precoder_weight_Im; -} nfapi_nr_pm_weights_t; - +typedef c16_t nfapi_nr_pm_weights_t; typedef struct { uint16_t pm_idx; diff --git a/nfapi/tests/p5/nr_fapi_config_request_test.c b/nfapi/tests/p5/nr_fapi_config_request_test.c index bfb3f7fa21..5a80a89f3d 100644 --- a/nfapi/tests/p5/nr_fapi_config_request_test.c +++ b/nfapi/tests/p5/nr_fapi_config_request_test.c @@ -239,8 +239,8 @@ static void fill_config_request_tlv_tdd_rand(nfapi_nr_config_request_scf_t *nfap for (int j = 0; j < num_layers; j++) { for (int k = 0; k < num_ant_ports; k++) { nfapi_nr_pm_weights_t* pm_weight = &pmi_pdu->weights[j][k]; - pm_weight->precoder_weight_Re = (int16_t)rand16_range(1, 0xffff); - pm_weight->precoder_weight_Im = (int16_t)rand16_range(1, 0xffff); + pm_weight->r = (int16_t)rand16_range(1, 0xffff); + pm_weight->i = (int16_t)rand16_range(1, 0xffff); } } } diff --git a/openair1/PHY/MODULATION/nr_modulation.c b/openair1/PHY/MODULATION/nr_modulation.c index b641ceaad1..5888b17eb0 100644 --- a/openair1/PHY/MODULATION/nr_modulation.c +++ b/openair1/PHY/MODULATION/nr_modulation.c @@ -710,8 +710,7 @@ c16_t nr_layer_precoder_cm(int n_layers, { c16_t precodatatx_F = {0}; for (int al = 0; al < n_layers; al++) { - nfapi_nr_pm_weights_t *w = &pmi_pdu->weights[al][ap]; - c16_t prec_weight = {.r = w->precoder_weight_Re, .i = w->precoder_weight_Im}; + c16_t prec_weight = pmi_pdu->weights[al][ap]; precodatatx_F = c16maddShift(datatx_F_precoding[al][offset], prec_weight, precodatatx_F, 15); } return precodatatx_F; @@ -732,18 +731,9 @@ static inline __attribute__((always_inline)) __m512i cmac0_prec512(__m512i x, __ } static inline __attribute__((always_inline)) __m512i cmac_prec512(__m512i y, __m512i x, __m512i w_c, __m512i w_s) { - - // Multiplication and shift - const __m512i reals = - _mm512_srai_epi32(_mm512_madd_epi16(x, w_c), 15); // (int32_t) .r = (x.r * w.r - x.i * w.i) >> 15 - const __m512i imags = - _mm512_slli_epi32(_mm512_madd_epi16(x, w_s), 1); // (int32_t) .i = (x.r * w.i + x.i * w.r) << 1, since higher 16 bit of each 32 bit is taken by blend_epi16 - - // Re-arrange to match c16_t format - const __m512i produ = _mm512_mask_blend_epi16(0xAAAAAAAA,reals, imags); - - // Accumulate the product - return _mm512_adds_epi16(y, produ); + const __m512i produ = cmac0_prec512(x, w_c, w_s); + // Accumulate the product + return _mm512_adds_epi16(y, produ); } #endif #ifdef __AVX2__ @@ -760,18 +750,9 @@ static inline __attribute__((always_inline)) __m256i cmac0_prec256(__m256i x, __ } static inline __attribute__((always_inline)) __m256i cmac_prec256(__m256i y, __m256i x, __m256i w_c, __m256i w_s) { - - // Multiplication and shift - const __m256i reals = - _mm256_srai_epi32(_mm256_madd_epi16(x, w_c), 15); // (int32_t) .r = (x.r * w.r - x.i * w.i) >> 15 - const __m256i imags = - _mm256_slli_epi32(_mm256_madd_epi16(x, w_s), 1); // (int32_t) .i = (x.r * w.i + x.i * w.r) << 1, since higher 16 bit of each 32 bit is taken by blend_epi16 - - // Re-arrange to match c16_t format - const __m256i produ = _mm256_blend_epi16(reals, imags,0xAA); - - // Accumulate the product - return _mm256_adds_epi16(y, produ); + const __m256i produ = cmac0_prec256(x, w_c, w_s); + // Accumulate the product + return _mm256_adds_epi16(y, produ); } #endif #ifdef __aarch64__ @@ -792,24 +773,41 @@ static inline __attribute__((always_inline)) int16x8_t cmac0_prec128(int16x8_t x return produ.val[0]; } static inline __attribute__((always_inline)) int16x8_t cmac_prec128(int16x8_t y, int16x8_t x, int16x8_t wr, int16x8_t wi) { - // De-interleave real/imag - int16x8_t xr = vuzp1q_s16(x, x); // even lanes - int16x8_t xi = vuzp2q_s16(x, x); // odd lanes - // - // real = ar*br - ai*bi (Q15 scaling via high-half doubling muls) - int16x8_t real = vqdmulhq_s16(xr, wr); // ≈ round((2*ar*br)/2^16) - real = vqrdmlshq_s16(real, xi, wi); // real -= round((2*ai*bi)/2^16) - // - // imag = ar*bi + ai*br - int16x8_t imag = vqdmulhq_s16(xr, wi); - imag = vqrdmlahq_s16(imag, xi, wr); // imag += round((2*ai*br)/2^16) - // - // Re-interleave [real, imag] - int16x8x2_t produ = vzipq_s16(real, imag); - return vaddq_s16(y,produ.val[0]); + int16x8_t produ = cmac0_prec128(x, wr, wi); + return vaddq_s16(y, produ.val[0]); +} +#else +static inline __attribute__((always_inline)) simde__m128i cmac0_prec128(simde__m128i x, simde__m128i w_c, simde__m128i w_s) +{ + // Multiplication and shift + const simde__m128i reals = simde_mm_srai_epi32(simde_mm_madd_epi16(x, w_c), 15); // (int32_t) .r = (x.r * w.r - x.i * w.i) >> 15 + const simde__m128i imags = simde_mm_slli_epi32( + simde_mm_madd_epi16(x, w_s), + 1); // (int32_t) .i = (x.r * w.i + x.i * w.r) << 1, since higher 16 bit of each 32 bit is taken by blend_epi16 + + /* Re-arrange to match c16_t format + bit index: 0 | 16 | 32 | 48 | 64 | 80 | 96 | 112 + reals = {R0.r[15..30] | R0.r[31] (0)*15 | R1.r[15..30] | R1.r[31] (0)*15 | R2.r[15..30] | R2.r[31] (0)*15 | R3.r[15..30] + | R3.r[31] (0)*15} imags = {0 R0.i[0..14]| R0.i[15..30] | 0 R1.i[0..14]| R1.i[15..30] | 0 R2.i[0..14]| R2.i[15..30] + | 0 R3.i[0..14]| R3.i[15..30] } 16b from {reals | imags | reals | imags | reals | imags | reals + | imags } produ = {R0.r[15..30] | R0.i[15..30] | R1.r[15..30] | R1.i[15..30] | R2.r[15..30] | R2.i[15..30] | + R3.r[15..30] | R3.i[15..30] } + */ + return simde_mm_blend_epi16(reals, imags, 0xAA); +} +static inline __attribute__((always_inline)) __m128i cmac_prec128(__m128i y, __m128i x, __m128i w_c, __m128i w_s) +{ + const __m128i produ = cmac0_prec128(x, w_c, w_s); + // Accumulate the product + return simde_mm_adds_epi16(y, produ); } #endif +#define load_consts(Type, Instruct, Rank) \ + const Type w_c##Rank = Instruct(c16toI32(c16conj(pmi_pdu->weights[Rank][ant]))); \ + const Type w_s##Rank = Instruct(c16toI32(c16swap(pmi_pdu->weights[Rank][ant]))); \ + const Type *in##Rank = (Type *)(txdataF_res_mapped[Rank] + sc_offset + (out-beginning)); + void nr_layer_precoder_simd(const int n_layers, const int symSz, const c16_t txdataF_res_mapped[n_layers][symSz], @@ -819,381 +817,264 @@ void nr_layer_precoder_simd(const int n_layers, const int re_cnt, c16_t *txdataF_precoded) { - uint32_t sc = sc_offset; - c16_t prec_weight = {0}; // For x86, use 256 SIMD for every 8 RE and 128 SIMD for last 4 RE // For aarch64, use 128 SIMD for every 4 RE + AssertFatal(n_layers > 0 && n_layers <= 4, "Shouldn't get here, n_layers %d\n", n_layers); // 512/256 SIMD: Do 16/8 RE in one iteration, 3 iterations for 2 RB -#if defined(__AVX512F__) && defined(__AVX512BW__) - const uint32_t re_cnt_align16 = re_cnt & ~15; - if (n_layers==1) { - prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; - const __m512i w_c = _mm512_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const __m512i w_s = _mm512_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - __m512i y = _mm512_set1_epi16(0); - for (; sc < sc_offset + (re_cnt_align16); sc += sizeof(__m512i) / sizeof(prec_weight)) { - - const __m512i x = _mm512_loadu_si512(&txdataF_res_mapped[0][sc]); - // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8) - y=cmac0_prec512(x,w_c,w_s); - _mm512_storeu_si512(&txdataF_precoded[sc],y); + c16_t *beginning = txdataF_precoded + sc_offset; + c16_t *out=beginning; +#if defined(__AVX512F__) && defined(__AVX512BW__) + { + c16_t *end = out + (re_cnt & ~15); + load_consts(__m512i, _mm512_set1_epi32, 0); + if (n_layers == 1) { + for (; out < end; out += sizeof(__m512i) / sizeof(*out)) { + const __m512i x = _mm512_loadu_si512(in0++); + // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8) + __m512i y = cmac0_prec512(x, w_c0, w_s0); + _mm512_storeu_si512(out, y); + } + } else if (n_layers == 2) { + load_consts(__m512i, _mm512_set1_epi32, 1); + for (; out < end; out += sizeof(__m512i) / sizeof(*out)) { + const __m512i x = _mm512_loadu_si512(in0++); + const __m512i x1 = _mm512_loadu_si512(in1++); + // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8) + __m512i y = cmac0_prec512(x, w_c0, w_s0); + y = cmac_prec512(y, x1, w_c1, w_s1); + _mm512_storeu_si512(out, y); + } + } else if (n_layers == 3) { + load_consts(__m512i, _mm512_set1_epi32, 1); + load_consts(__m512i, _mm512_set1_epi32, 2); + for (; out < end; out += sizeof(__m512i) / sizeof(*out)) { + const __m512i x = _mm512_loadu_si512(in0++); + const __m512i x1 = _mm512_loadu_si512(in1++); + const __m512i x2 = _mm512_loadu_si512(in2++); + // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8) + __m512i y = cmac0_prec512(x, w_c0, w_s0); + y = cmac_prec512(y, x1, w_c1, w_s1); + y = cmac_prec512(y, x2, w_c2, w_s2); + _mm512_storeu_si512(out, y); + } + } else if (n_layers == 4) { + load_consts(__m512i, _mm512_set1_epi32, 1); + load_consts(__m512i, _mm512_set1_epi32, 2); + load_consts(__m512i, _mm512_set1_epi32, 3); + for (; out < end; out += sizeof(__m512i) / sizeof(*out)) { + const __m512i x = _mm512_loadu_si512(in0++); + const __m512i x1 = _mm512_loadu_si512(in1++); + const __m512i x2 = _mm512_loadu_si512(in2++); + const __m512i x3 = _mm512_loadu_si512(in3++); + // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8) + __m512i y = cmac0_prec512(x, w_c0, w_s0); + y = cmac_prec512(y, x1, w_c1, w_s1); + y = cmac_prec512(y, x2, w_c2, w_s2); + y = cmac_prec512(y, x3, w_c3, w_s3); + _mm512_storeu_si512(out, y); + } } } - else if (n_layers==2) { - prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; - const __m512i w_c = _mm512_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const __m512i w_s = _mm512_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - prec_weight.r = pmi_pdu->weights[1][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[1][ant].precoder_weight_Im; - const __m512i w_c1 = _mm512_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const __m512i w_s1 = _mm512_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - __m512i y = _mm512_set1_epi16(0); - for (; sc < sc_offset + (re_cnt_align16); sc += sizeof(__m512i) / sizeof(prec_weight)) { - - const __m512i x = _mm512_loadu_si512(&txdataF_res_mapped[0][sc]); - const __m512i x1 = _mm512_loadu_si512(&txdataF_res_mapped[1][sc]); - // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8) - y=cmac0_prec512(x,w_c,w_s); - y=cmac_prec512(y,x1,w_c1,w_s1); - _mm512_storeu_si512(&txdataF_precoded[sc],y); - } - } - else if (n_layers==3) { - prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; - const __m512i w_c = _mm512_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const __m512i w_s = _mm512_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - prec_weight.r = pmi_pdu->weights[1][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[1][ant].precoder_weight_Im; - const __m512i w_c1 = _mm512_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const __m512i w_s1 = _mm512_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - prec_weight.r = pmi_pdu->weights[2][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[2][ant].precoder_weight_Im; - const __m512i w_c2 = _mm512_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const __m512i w_s2 = _mm512_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - __m512i y = _mm512_set1_epi16(0); - for (; sc < sc_offset + (re_cnt_align16); sc += sizeof(__m512i) / sizeof(prec_weight)) { - - const __m512i x = _mm512_loadu_si512(&txdataF_res_mapped[0][sc]); - const __m512i x1 = _mm512_loadu_si512(&txdataF_res_mapped[1][sc]); - const __m512i x2 = _mm512_loadu_si512(&txdataF_res_mapped[2][sc]); - // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8) - y=cmac0_prec512(x,w_c,w_s); - y=cmac_prec512(y,x1,w_c1,w_s1); - y=cmac_prec512(y,x2,w_c2,w_s2); - _mm512_storeu_si512(&txdataF_precoded[sc],y); - } - } - else if (n_layers==4) { - prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; - const __m512i w_c = _mm512_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const __m512i w_s = _mm512_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - prec_weight.r = pmi_pdu->weights[1][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[1][ant].precoder_weight_Im; - const __m512i w_c1 = _mm512_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const __m512i w_s1 = _mm512_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - prec_weight.r = pmi_pdu->weights[2][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[2][ant].precoder_weight_Im; - const __m512i w_c2 = _mm512_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const __m512i w_s2 = _mm512_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - prec_weight.r = pmi_pdu->weights[3][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[3][ant].precoder_weight_Im; - const __m512i w_c3 = _mm512_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const __m512i w_s3 = _mm512_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - __m512i y = _mm512_set1_epi16(0); - for (; sc < sc_offset + (re_cnt_align16); sc += sizeof(__m512i) / sizeof(prec_weight)) { - - const __m512i x = _mm512_loadu_si512(&txdataF_res_mapped[0][sc]); - const __m512i x1 = _mm512_loadu_si512(&txdataF_res_mapped[1][sc]); - const __m512i x2 = _mm512_loadu_si512(&txdataF_res_mapped[2][sc]); - const __m512i x3 = _mm512_loadu_si512(&txdataF_res_mapped[3][sc]); - // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8) - y=cmac0_prec512(x,w_c,w_s); - y=cmac_prec512(y,x1,w_c1,w_s1); - y=cmac_prec512(y,x2,w_c2,w_s2); - y=cmac_prec512(y,x3,w_c3,w_s3); - _mm512_storeu_si512(&txdataF_precoded[sc],y); - } - } - else AssertFatal(1==0,"Shouldn't get here, n_layers %d\n",n_layers); #endif #ifdef __AVX2__ - const uint32_t re_cnt_align8 = re_cnt & ~7; - if (n_layers==1) { - - simde__m256i y = simde_mm256_set1_epi16(0); // Y = W[0]*X[0] + W[1]*X[1] + ... + W[nrOfLayers-1]*X[nrOfLayers-1] - prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; - const simde__m256i w_c0 = simde_mm256_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const simde__m256i w_s0 = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - - for (; sc < sc_offset + (re_cnt_align8); sc += sizeof(simde__m256i) / sizeof(prec_weight)) { - const simde__m256i x0 = simde_mm256_loadu_si256(&txdataF_res_mapped[0][sc]); - // Accumulate the product - y = cmac0_prec256(x0,w_c0,w_s0); - // Store the result to txdataF - simde_mm256_storeu_si256(&txdataF_precoded[sc], y); + { + c16_t *end = beginning + (re_cnt & ~7); + load_consts(simde__m256i, simde_mm256_set1_epi32, 0); + if (n_layers == 1) { + for (; out < end; out += sizeof(simde__m256i) / sizeof(*out)) { + const simde__m256i x0 = simde_mm256_loadu_si256(in0++); + // Accumulate the product + simde__m256i y = cmac0_prec256(x0, w_c0, w_s0); + // Store the result to txdataF + simde_mm256_storeu_si256(out, y); + } + } else if (n_layers == 2) { + load_consts(simde__m256i, simde_mm256_set1_epi32, 1); + for (; out < end; out += sizeof(simde__m256i) / sizeof(*out)) { + const simde__m256i x0 = simde_mm256_loadu_si256(in0++); + const simde__m256i x1 = simde_mm256_loadu_si256(in1++); + // Accumulate the product + simde__m256i y = cmac0_prec256(x0, w_c0, w_s0); + y = cmac_prec256(y, x1, w_c1, w_s1); + // Store the result to txdataF + simde_mm256_storeu_si256(out, y); + } + } else if (n_layers == 3) { + load_consts(simde__m256i, simde_mm256_set1_epi32, 1); + load_consts(simde__m256i, simde_mm256_set1_epi32, 2); + for (; out < end; out += sizeof(simde__m256i) / sizeof(*out)) { + const simde__m256i x0 = simde_mm256_loadu_si256(in0++); + const simde__m256i x1 = simde_mm256_loadu_si256(in1++); + const simde__m256i x2 = simde_mm256_loadu_si256(in2++); + simde__m256i y = cmac0_prec256(x0, w_c0, w_s0); + y = cmac_prec256(y, x1, w_c1, w_s1); + y = cmac_prec256(y, x2, w_c2, w_s2); + // Store the result to txdataF + simde_mm256_storeu_si256(out, y); + } + } else if (n_layers == 4) { + load_consts(simde__m256i, simde_mm256_set1_epi32, 1); + load_consts(simde__m256i, simde_mm256_set1_epi32, 2); + load_consts(simde__m256i, simde_mm256_set1_epi32, 3); + for (; out < end; out += sizeof(simde__m256i) / sizeof(*out)) { + const simde__m256i x0 = simde_mm256_loadu_si256(in0++); + const simde__m256i x1 = simde_mm256_loadu_si256(in1++); + const simde__m256i x2 = simde_mm256_loadu_si256(in2++); + const simde__m256i x3 = simde_mm256_loadu_si256(in3++); + simde__m256i y = cmac0_prec256(x0, w_c0, w_s0); + y = cmac_prec256(y, x1, w_c1, w_s1); + y = cmac_prec256(y, x2, w_c2, w_s2); + y = cmac_prec256(y, x3, w_c3, w_s3); + // Store the result to txdataF + simde_mm256_storeu_si256(out, y); + } } } - else if (n_layers==2) { - - simde__m256i y = simde_mm256_set1_epi16(0); // Y = W[0]*X[0] + W[1]*X[1] + ... + W[nrOfLayers-1]*X[nrOfLayers-1] - prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; - const simde__m256i w_c0 = simde_mm256_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const simde__m256i w_s0 = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - prec_weight.r = pmi_pdu->weights[1][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[1][ant].precoder_weight_Im; - const simde__m256i w_c1 = simde_mm256_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const simde__m256i w_s1 = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - - for (; sc < sc_offset + (re_cnt_align8); sc += sizeof(simde__m256i) / sizeof(prec_weight)) { - const simde__m256i x0 = simde_mm256_loadu_si256(&txdataF_res_mapped[0][sc]); - - const simde__m256i x1 = simde_mm256_loadu_si256(&txdataF_res_mapped[1][sc]); - // Accumulate the product - y = cmac0_prec256(x0,w_c0,w_s0); - y = cmac_prec256(y,x1,w_c1,w_s1); - // Store the result to txdataF - simde_mm256_storeu_si256(&txdataF_precoded[sc], y); - } - } - else if (n_layers==3) { - simde__m256i y = simde_mm256_set1_epi16(0); // Y = W[0]*X[0] + W[1]*X[1] + ... + W[nrOfLayers-1]*X[nrOfLayers-1] - prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; - const simde__m256i w_c0 = simde_mm256_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const simde__m256i w_s0 = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - prec_weight.r = pmi_pdu->weights[1][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[1][ant].precoder_weight_Im; - const simde__m256i w_c1 = simde_mm256_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const simde__m256i w_s1 = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - prec_weight.r = pmi_pdu->weights[2][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[2][ant].precoder_weight_Im; - const simde__m256i w_c2 = simde_mm256_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const simde__m256i w_s2 = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - - for (; sc < sc_offset + (re_cnt_align8); sc += sizeof(simde__m256i) / sizeof(prec_weight)) { - const simde__m256i x0 = simde_mm256_loadu_si256(&txdataF_res_mapped[0][sc]); - const simde__m256i x1 = simde_mm256_loadu_si256(&txdataF_res_mapped[1][sc]); - const simde__m256i x2 = simde_mm256_loadu_si256(&txdataF_res_mapped[2][sc]); - y = cmac0_prec256(x0,w_c0,w_s0); - y = cmac_prec256(y,x1,w_c1,w_s1); - y = cmac_prec256(y,x2,w_c2,w_s2); - // Store the result to txdataF - simde_mm256_storeu_si256(&txdataF_precoded[sc], y); - } - } - else if (n_layers==4) { - simde__m256i y = simde_mm256_set1_epi16(0); // Y = W[0]*X[0] + W[1]*X[1] + ... + W[nrOfLayers-1]*X[nrOfLayers-1] - prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; - const simde__m256i w_c0 = simde_mm256_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const simde__m256i w_s0 = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - prec_weight.r = pmi_pdu->weights[1][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[1][ant].precoder_weight_Im; - const simde__m256i w_c1 = simde_mm256_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const simde__m256i w_s1 = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - prec_weight.r = pmi_pdu->weights[2][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[2][ant].precoder_weight_Im; - const simde__m256i w_c2 = simde_mm256_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const simde__m256i w_s2 = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - - prec_weight.r = pmi_pdu->weights[3][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[3][ant].precoder_weight_Im; - const simde__m256i w_c3 = simde_mm256_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const simde__m256i w_s3 = simde_mm256_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - for (; sc < sc_offset + (re_cnt_align8); sc += sizeof(simde__m256i) / sizeof(prec_weight)) { - const simde__m256i x0 = simde_mm256_loadu_si256(&txdataF_res_mapped[0][sc]); - const simde__m256i x1 = simde_mm256_loadu_si256(&txdataF_res_mapped[1][sc]); - const simde__m256i x2 = simde_mm256_loadu_si256(&txdataF_res_mapped[2][sc]); - const simde__m256i x3 = simde_mm256_loadu_si256(&txdataF_res_mapped[3][sc]); - y = cmac0_prec256(x0,w_c0,w_s0); - y = cmac_prec256(y,x1,w_c1,w_s1); - y = cmac_prec256(y,x2,w_c2,w_s2); - y = cmac_prec256(y,x3,w_c3,w_s3); - // Store the result to txdataF - simde_mm256_storeu_si256(&txdataF_precoded[sc], y); - } - } - else AssertFatal(1==0,"Shouldn't get here, n_layers %d\n",n_layers); #endif - // 128 SIMD: Do 4 RE in one iteration, 3 iterations for 1 RB - const uint32_t re_cnt_align4 = re_cnt & ~3; - for (; sc < sc_offset + re_cnt_align4; sc += sizeof(simde__m128i) / sizeof(prec_weight)) { + c16_t *end = beginning + (re_cnt & ~3); #ifdef DEBUG_DLSCH_PRECODING_PRINT_WITH_TRIVIAL // Get result with trivial solution, TODO: To be removed + // 128 SIMD: Do 4 RE in one iteration, 3 iterations for 1 RB + for (; out < end; out += sizeof(simde__m128i) / sizeof(*out)) { c16_t y_triv[4]; for (int i = 0; i < 4; i++) y_triv[i] = nr_layer_precoder_cm(n_layers, symSz, txdataF_res_mapped, ant, pmi_pdu, sc + i); - memcpy(&txdataF_precoded[sc], y_triv, sizeof(y_triv)); + memcpy(out, y_triv, sizeof(y_triv)); + } #endif #ifdef __aarch64__ - - const uint32_t re_cnt_align8 = re_cnt & ~3; - if (n_layers==1) { - - prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; - const int16x8_t wr0 = vdupq_n_s16(prec_weight.r); - const int16x8_t wi0 = vdupq_n_s16(prec_weight.i); - int16x8_t y; - - for (; sc < sc_offset + (re_cnt_align8); sc += sizeof(int16x8_t) / sizeof(prec_weight)) { - const int16x8_t x0 = vld1q_s16((const int16_t*)&txdataF_res_mapped[0][sc]); + load_consts(int16x8_t, vdupq_n_s16, 0); + if (n_layers == 1) { + for (; out < end; out += sizeof(int16x8_t) / sizeof(*out)) { + const int16x8_t x0 = vld1q_s16((const int16_t *)in0++); // Accumulate the product - y = cmac0_prec128(x0,wr0,wi0); + int16x8_t y = cmac0_prec128(x0, w_c0, w_s0); // Store the result to txdataF - *((int16x8_t*)&txdataF_precoded[sc])= y; + *(int16x8_t *)out = y; } } - if (n_layers==2) { - - prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; - const int16x8_t wr0 = vdupq_n_s16(prec_weight.r); - const int16x8_t wi0 = vdupq_n_s16(prec_weight.i); - prec_weight.r = pmi_pdu->weights[1][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[1][ant].precoder_weight_Im; - const int16x8_t wr1 = vdupq_n_s16(prec_weight.r); - const int16x8_t wi1 = vdupq_n_s16(prec_weight.i); - int16x8_t y; - - for (; sc < sc_offset + (re_cnt_align8); sc += sizeof(int16x8_t) / sizeof(prec_weight)) { - const int16x8_t x0 = vld1q_s16((const int16_t*)&txdataF_res_mapped[0][sc]); - const int16x8_t x1 = vld1q_s16((const int16_t*)&txdataF_res_mapped[1][sc]); + if (n_layers == 2) { + load_consts(int16x8_t, vdupq_n_s16, 1); + for (; out < end; out += sizeof(int16x8_t) / sizeof(*out)) { + const int16x8_t x0 = vld1q_s16((const int16_t *)in0++); + const int16x8_t x1 = vld1q_s16((const int16_t *)in1++); // Accumulate the product - y = cmac0_prec128(x0,wr0,wi0); - y = cmac_prec128(y,x1,wr1,wi1); + int16x8_t y = cmac0_prec128(x0, w_c0, w_s0); + y = cmac_prec128(y, x1, w_cr1, w_s1); // Store the result to txdataF - *((int16x8_t*)&txdataF_precoded[sc])= y; + *(int16x8_t *)out = y; } } - if (n_layers==3) { - - prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; - const int16x8_t wr0 = vdupq_n_s16(prec_weight.r); - const int16x8_t wi0 = vdupq_n_s16(prec_weight.i); - prec_weight.r = pmi_pdu->weights[1][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[1][ant].precoder_weight_Im; - const int16x8_t wr1 = vdupq_n_s16(prec_weight.r); - const int16x8_t wi1 = vdupq_n_s16(prec_weight.i); - prec_weight.r = pmi_pdu->weights[2][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[2][ant].precoder_weight_Im; - const int16x8_t wr2 = vdupq_n_s16(prec_weight.r); - const int16x8_t wi2 = vdupq_n_s16(prec_weight.i); - int16x8_t y; - - for (; sc < sc_offset + (re_cnt_align8); sc += sizeof(int16x8_t) / sizeof(prec_weight)) { - const int16x8_t x0 = vld1q_s16((const int16_t*)&txdataF_res_mapped[0][sc]); - const int16x8_t x1 = vld1q_s16((const int16_t*)&txdataF_res_mapped[1][sc]); - const int16x8_t x2 = vld1q_s16((const int16_t*)&txdataF_res_mapped[2][sc]); + if (n_layers == 3) { + load_consts(int16x8_t, vdupq_n_s16, 1); + load_consts(int16x8_t, vdupq_n_s16, 2); + for (; out < end; out += sizeof(int16x8_t) / sizeof(*out)) { + const int16x8_t x0 = vld1q_s16((const int16_t *)in0++); + const int16x8_t x1 = vld1q_s16((const int16_t *)in1++); + const int16x8_t x2 = vld1q_s16((const int16_t *)in2++); // Accumulate the product - y = cmac0_prec128(x0,wr0,wi0); - y = cmac_prec128(y,x1,wr1,wi1); - y = cmac_prec128(y,x2,wr2,wi2); + int16x8_t y = cmac0_prec128(x0, w_c0, w_s0); + ; + y = cmac_prec128(y, x1, w_cr1, w_s1); + y = cmac_prec128(y, x2, w_cr2, w_s2); // Store the result to txdataF - *((int16x8_t*)&txdataF_precoded[sc])= y; + *(int16x8_t *)out = y; } } - if (n_layers==4) { - - prec_weight.r = pmi_pdu->weights[0][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[0][ant].precoder_weight_Im; - const int16x8_t wr0 = vdupq_n_s16(prec_weight.r); - const int16x8_t wi0 = vdupq_n_s16(prec_weight.i); - prec_weight.r = pmi_pdu->weights[1][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[1][ant].precoder_weight_Im; - const int16x8_t wr1 = vdupq_n_s16(prec_weight.r); - const int16x8_t wi1 = vdupq_n_s16(prec_weight.i); - prec_weight.r = pmi_pdu->weights[2][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[2][ant].precoder_weight_Im; - const int16x8_t wr2 = vdupq_n_s16(prec_weight.r); - const int16x8_t wi2 = vdupq_n_s16(prec_weight.i); - prec_weight.r = pmi_pdu->weights[3][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[3][ant].precoder_weight_Im; - const int16x8_t wr3 = vdupq_n_s16(prec_weight.r); - const int16x8_t wi3 = vdupq_n_s16(prec_weight.i); - int16x8_t y; - - for (; sc < sc_offset + (re_cnt_align8); sc += sizeof(int16x8_t) / sizeof(prec_weight)) { - const int16x8_t x0 = vld1q_s16((const int16_t*)&txdataF_res_mapped[0][sc]); - const int16x8_t x1 = vld1q_s16((const int16_t*)&txdataF_res_mapped[1][sc]); - const int16x8_t x2 = vld1q_s16((const int16_t*)&txdataF_res_mapped[2][sc]); - const int16x8_t x3 = vld1q_s16((const int16_t*)&txdataF_res_mapped[3][sc]); + if (n_layers == 4) { + load_consts(int16x8_t, vdupq_n_s16, 1); + load_consts(int16x8_t, vdupq_n_s16, 2); + load_consts(int16x8_t, vdupq_n_s16, 3); + for (; out < end; out += sizeof(int16x8_t) / sizeof(*out)) { + const int16x8_t x0 = vld1q_s16((const int16_t *)in0++); + const int16x8_t x1 = vld1q_s16((const int16_t *)in1++); + const int16x8_t x2 = vld1q_s16((const int16_t *)in2++); + const int16x8_t x3 = vld1q_s16((const int16_t *)in3++); // Accumulate the product - y = cmac0_prec128(x0,wr0,wi0); - y = cmac_prec128(y,x1,wr1,wi1); - y = cmac_prec128(y,x2,wr2,wi2); - y = cmac_prec128(y,x3,wr3,wi3); + int16x8_t y = cmac0_prec128(x0, w_c0, w_s0); + ; + y = cmac_prec128(y, x1, w_cr1, w_s1); + y = cmac_prec128(y, x2, w_cr2, w_s2); + y = cmac_prec128(y, x3, w_cr3, w_s3); // Store the result to txdataF - *((int16x8_t*)&txdataF_precoded[sc])= y; + *(int16x8_t *)out = y; } } -#else - // Matrix multiplication for 4 elements of the result (sizeof(simde__m128i) / sizeof(c16_t) = 4) - simde__m128i y = simde_mm_set1_epi16(0); // Y = W[0]*X[0] + W[1]*X[1] + ... + W[nrOfLayers-1]*X[nrOfLayers-1] - for (int nl = 0; nl < n_layers; nl++) { - prec_weight.r = pmi_pdu->weights[nl][ant].precoder_weight_Re; - prec_weight.i = pmi_pdu->weights[nl][ant].precoder_weight_Im; - - const simde__m128i x = simde_mm_loadu_si128(&txdataF_res_mapped[nl][sc]); - - // Rearrange precoding matrix weight to match complex multiplication and broadcast it to match SIMD size - const simde__m128i w_c = simde_mm_set1_epi32(c16toI32(c16conj(prec_weight))); // broadcast conjugate of w - const simde__m128i w_s = simde_mm_set1_epi32(c16toI32(c16swap(prec_weight))); // broadcast swapped real and img of w - - // Multiplication and shift - const simde__m128i reals = - simde_mm_srai_epi32(simde_mm_madd_epi16(x, w_c), 15); // (int32_t) .r = (x.r * w.r - x.i * w.i) >> 15 - const simde__m128i imags = simde_mm_slli_epi32( - simde_mm_madd_epi16(x, w_s), - 1); // (int32_t) .i = (x.r * w.i + x.i * w.r) << 1, since higher 16 bit of each 32 bit is taken by blend_epi16 - - /* Re-arrange to match c16_t format - bit index: 0 | 16 | 32 | 48 | 64 | 80 | 96 | 112 - reals = {R0.r[15..30] | R0.r[31] (0)*15 | R1.r[15..30] | R1.r[31] (0)*15 | R2.r[15..30] | R2.r[31] (0)*15 | R3.r[15..30] - | R3.r[31] (0)*15} imags = {0 R0.i[0..14]| R0.i[15..30] | 0 R1.i[0..14]| R1.i[15..30] | 0 R2.i[0..14]| R2.i[15..30] - | 0 R3.i[0..14]| R3.i[15..30] } 16b from {reals | imags | reals | imags | reals | imags | reals - | imags } produ = {R0.r[15..30] | R0.i[15..30] | R1.r[15..30] | R1.i[15..30] | R2.r[15..30] | R2.i[15..30] | - R3.r[15..30] | R3.i[15..30] } - */ - const simde__m128i produ = simde_mm_blend_epi16(reals, imags, 0xAA); - +#else + load_consts(simde__m128i, simde_mm_set1_epi32, 0); + if (n_layers == 1) { + for (; out < end; out += sizeof(simde__m128i) / sizeof(*out)) { + const simde__m128i x0 = simde_mm_loadu_si128(in0++); // Accumulate the product - y = simde_mm_adds_epi16(y, produ); + simde__m128i y = cmac0_prec128(x0, w_c0, w_s0); + // Store the result to txdataF + simde_mm_storeu_si128(out, y); } - // Store the result to txdataF - simde_mm_storeu_si128(&txdataF_precoded[sc], y); + } else if (n_layers == 2) { + load_consts(simde__m128i, simde_mm_set1_epi32, 1); + for (; out < end; out += sizeof(simde__m128i) / sizeof(*out)) { + const simde__m128i x0 = simde_mm_loadu_si128(in0++); + const simde__m128i x1 = simde_mm_loadu_si128(in1++); + // Accumulate the product + simde__m128i y = cmac0_prec128(x0, w_c0, w_s0); + y = cmac_prec128(y, x1, w_c1, w_s1); + // Store the result to txdataF + simde_mm_storeu_si128(out, y); + } + } else if (n_layers == 3) { + load_consts(simde__m128i, simde_mm_set1_epi32, 1); + load_consts(simde__m128i, simde_mm_set1_epi32, 2); + for (; out < end; out += sizeof(simde__m128i) / sizeof(*out)) { + const simde__m128i x0 = simde_mm_loadu_si128(in0++); + const simde__m128i x1 = simde_mm_loadu_si128(in1++); + const simde__m128i x2 = simde_mm_loadu_si128(in2++); + simde__m128i y = cmac0_prec128(x0, w_c0, w_s0); + y = cmac_prec128(y, x1, w_c1, w_s1); + y = cmac_prec128(y, x2, w_c2, w_s2); + // Store the result to txdataF + simde_mm_storeu_si128(out, y); + } + } else if (n_layers == 4) { + load_consts(simde__m128i, simde_mm_set1_epi32, 1); + load_consts(simde__m128i, simde_mm_set1_epi32, 2); + load_consts(simde__m128i, simde_mm_set1_epi32, 3); + for (; out < end; out += sizeof(simde__m128i) / sizeof(*out)) { + const simde__m128i x0 = simde_mm_loadu_si128(in0++); + const simde__m128i x1 = simde_mm_loadu_si128(in1++); + const simde__m128i x2 = simde_mm_loadu_si128(in2++); + const simde__m128i x3 = simde_mm_loadu_si128(in3++); + simde__m128i y = cmac0_prec128(x0, w_c0, w_s0); + y = cmac_prec128(y, x1, w_c1, w_s1); + y = cmac_prec128(y, x2, w_c2, w_s2); + y = cmac_prec128(y, x3, w_c3, w_s3); + // Store the result to txdataF + simde_mm_storeu_si128(out, y); + } + } #endif #ifdef DEBUG_DLSCH_PRECODING_PRINT_WITH_TRIVIAL // Print simd and trivial result, TODO: To be removed - c16_t *y_simd = (c16_t *)&y; - printf("debug_to_be_removed re_cnt=%d, sc=%u, y_simd=(%+4d,%+4d), (%+4d,%+4d), (%+4d,%+4d), (%+4d,%+4d)\n", - re_cnt, - sc, - y_simd[0].r, - y_simd[0].i, - y_simd[1].r, - y_simd[1].i, - y_simd[2].r, - y_simd[2].i, - y_simd[3].r, - y_simd[3].i); - printf("debug_to_be_removed re_cnt=%d, sc=%u, y_triv=(%+4d,%+4d), (%+4d,%+4d), (%+4d,%+4d), (%+4d,%+4d)\n", - re_cnt, - sc, - y_triv[0].r, - y_triv[0].i, - y_triv[1].r, - y_triv[1].i, - y_triv[2].r, - y_triv[2].i, - y_triv[3].r, - y_triv[3].i); + c16_t *y_simd = (c16_t *)&y; + printf("debug_to_be_removed re_cnt=%d, sc=%u, y_simd=(%+4d,%+4d), (%+4d,%+4d), (%+4d,%+4d), (%+4d,%+4d)\n", + re_cnt, + sc, + y_simd[0].r, + y_simd[0].i, + y_simd[1].r, + y_simd[1].i, + y_simd[2].r, + y_simd[2].i, + y_simd[3].r, + y_simd[3].i); + printf("debug_to_be_removed re_cnt=%d, sc=%u, y_triv=(%+4d,%+4d), (%+4d,%+4d), (%+4d,%+4d), (%+4d,%+4d)\n", + re_cnt, + sc, + y_triv[0].r, + y_triv[0].i, + y_triv[1].r, + y_triv[1].i, + y_triv[2].r, + y_triv[2].i, + y_triv[3].r, + y_triv[3].i); #endif - } } diff --git a/openair1/PHY/MODULATION/tests/test_nr_modulation.cpp b/openair1/PHY/MODULATION/tests/test_nr_modulation.cpp index ca8b61acd7..04d32514a0 100644 --- a/openair1/PHY/MODULATION/tests/test_nr_modulation.cpp +++ b/openair1/PHY/MODULATION/tests/test_nr_modulation.cpp @@ -85,8 +85,7 @@ TEST(NrLayerPrecoderTest, Basic) // Create and populate the pmi_pdu structure nfapi_nr_pm_pdu_t pmi_pdu = {0}; for (int layer = 0; layer < n_layers; ++layer) { - pmi_pdu.weights[layer][ant].precoder_weight_Re = -SHRT_MAX; // Q15 representation of (~-1.0) -0.999969482421875 (-1+2^-15) - pmi_pdu.weights[layer][ant].precoder_weight_Im = 0; + pmi_pdu.weights[layer][ant] = (c16_t){-SHRT_MAX, 0}; } // Call the C function @@ -131,8 +130,7 @@ TEST(NrLayerPrecoderTest, SIMD) // Create and populate the pmi_pdu structure nfapi_nr_pm_pdu_t pmi_pdu = {0}; for (int layer = 0; layer < n_layers; ++layer) { - pmi_pdu.weights[layer][ant].precoder_weight_Re = -SHRT_MAX; // Q15 representation of (~-1.0) -0.999969482421875 (-1+2^-15) - pmi_pdu.weights[layer][ant].precoder_weight_Im = 0; + pmi_pdu.weights[layer][ant] = (c16_t){-SHRT_MAX, 0}; } // Call the C function @@ -182,10 +180,8 @@ TEST(NrLayerPrecoderTest, Compare_CM_SIMD) // Could not use convert_precoder_weight() as complex.h could not be used in googletest // Use the logic in convert_precoder_weight() // precoder [−1,−j] - pmi_pdu.weights[layer][0].precoder_weight_Re = -SHRT_MAX; // Q15 representation of (~-1.0) -0.999969482421875 (-1+2^-15) - pmi_pdu.weights[layer][0].precoder_weight_Im = 0x0000; // Q15 representation of 0 - pmi_pdu.weights[layer][1].precoder_weight_Re = 0x0000; // Q15 representation of 0 - pmi_pdu.weights[layer][1].precoder_weight_Im = -SHRT_MAX; // Q15 representation of (~-1.0) -0.999969482421875 (-1+2^-15) + pmi_pdu.weights[layer][0] = (c16_t){-SHRT_MAX, 0}; + pmi_pdu.weights[layer][1] = (c16_t){0, -SHRT_MAX}; } // Get the results for all the antenna diff --git a/openair2/LAYER2/NR_MAC_gNB/config.c b/openair2/LAYER2/NR_MAC_gNB/config.c index 9a5197181f..a0637ac107 100644 --- a/openair2/LAYER2/NR_MAC_gNB/config.c +++ b/openair2/LAYER2/NR_MAC_gNB/config.c @@ -136,8 +136,7 @@ int precoding_weigths_generation(nfapi_nr_pm_list_t *mat, nfapi_nr_pm_weights_t *weights = &mat->pmi_pdu[pmiq].weights[j_col][i_rows]; res_code = sqrt(1 / (double)L) * v_lm[llc][mmc][i_rows]; c16_t precoder_weight = convert_precoder_weight(res_code); - weights->precoder_weight_Re = precoder_weight.r; - weights->precoder_weight_Im = precoder_weight.i; + *weights = precoder_weight; LOG_D(PHY, "%d Layer Precoding Matrix[pmi %d][antPort %d][layerIdx %d]= %f+j %f -> Fixed Point %d+j %d \n", L, @@ -146,15 +145,14 @@ int precoding_weigths_generation(nfapi_nr_pm_list_t *mat, j_col, creal(res_code), cimag(res_code), - weights->precoder_weight_Re, - weights->precoder_weight_Im); + weights->r, + weights->i); } for (int i_rows = N1 * N2; i_rows < 2 * N1 * N2; i_rows++) { nfapi_nr_pm_weights_t *weights = &mat->pmi_pdu[pmiq].weights[j_col][i_rows]; res_code = sqrt(1 / (double)L) * (phase_sign)*theta_n[nn] * v_lm[llc][mmc][i_rows - N1 * N2]; c16_t precoder_weight = convert_precoder_weight(res_code); - weights->precoder_weight_Re = precoder_weight.r; - weights->precoder_weight_Im = precoder_weight.i; + *weights = precoder_weight; LOG_D(PHY, "%d Layer Precoding Matrix[pmi %d][antPort %d][layerIdx %d]= %f+j %f -> Fixed Point %d+j %d \n", L, @@ -163,8 +161,8 @@ int precoding_weigths_generation(nfapi_nr_pm_list_t *mat, j_col, creal(res_code), cimag(res_code), - weights->precoder_weight_Re, - weights->precoder_weight_Im); + weights->r, + weights->i); } } pmiq++; From 0dcea09e8737037c9c389ddd26376062c938b453 Mon Sep 17 00:00:00 2001 From: Raymond Knopp Date: Fri, 3 Oct 2025 13:47:33 +0000 Subject: [PATCH 10/10] bugfix for arm-v8x/v9 --- openair1/PHY/MODULATION/nr_modulation.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/openair1/PHY/MODULATION/nr_modulation.c b/openair1/PHY/MODULATION/nr_modulation.c index 5888b17eb0..eb4dc50bc9 100644 --- a/openair1/PHY/MODULATION/nr_modulation.c +++ b/openair1/PHY/MODULATION/nr_modulation.c @@ -774,7 +774,7 @@ static inline __attribute__((always_inline)) int16x8_t cmac0_prec128(int16x8_t x } static inline __attribute__((always_inline)) int16x8_t cmac_prec128(int16x8_t y, int16x8_t x, int16x8_t wr, int16x8_t wi) { int16x8_t produ = cmac0_prec128(x, wr, wi); - return vaddq_s16(y, produ.val[0]); + return vaddq_s16(y, produ); } #else static inline __attribute__((always_inline)) simde__m128i cmac0_prec128(simde__m128i x, simde__m128i w_c, simde__m128i w_s) @@ -960,7 +960,7 @@ void nr_layer_precoder_simd(const int n_layers, const int16x8_t x1 = vld1q_s16((const int16_t *)in1++); // Accumulate the product int16x8_t y = cmac0_prec128(x0, w_c0, w_s0); - y = cmac_prec128(y, x1, w_cr1, w_s1); + y = cmac_prec128(y, x1, w_c1, w_s1); // Store the result to txdataF *(int16x8_t *)out = y; } @@ -975,8 +975,8 @@ void nr_layer_precoder_simd(const int n_layers, // Accumulate the product int16x8_t y = cmac0_prec128(x0, w_c0, w_s0); ; - y = cmac_prec128(y, x1, w_cr1, w_s1); - y = cmac_prec128(y, x2, w_cr2, w_s2); + y = cmac_prec128(y, x1, w_c1, w_s1); + y = cmac_prec128(y, x2, w_c2, w_s2); // Store the result to txdataF *(int16x8_t *)out = y; } @@ -993,9 +993,9 @@ void nr_layer_precoder_simd(const int n_layers, // Accumulate the product int16x8_t y = cmac0_prec128(x0, w_c0, w_s0); ; - y = cmac_prec128(y, x1, w_cr1, w_s1); - y = cmac_prec128(y, x2, w_cr2, w_s2); - y = cmac_prec128(y, x3, w_cr3, w_s3); + y = cmac_prec128(y, x1, w_c1, w_s1); + y = cmac_prec128(y, x2, w_c2, w_s2); + y = cmac_prec128(y, x3, w_c3, w_s3); // Store the result to txdataF *(int16x8_t *)out = y; }