From 9a91ed0eeb1282a4cc1919c9bb4c390d4e636d5f Mon Sep 17 00:00:00 2001 From: Raymond Knopp Date: Mon, 6 Mar 2023 06:54:55 +0100 Subject: [PATCH 1/6] priorities/policy on usrp-tx-thread. Thread id for ru_thread --- executables/nr-ru.c | 4 ++-- radio/USRP/USERSPACE/LIB/usrp_lib.cpp | 31 ++++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/executables/nr-ru.c b/executables/nr-ru.c index 48cf6ebe31..e6b6eeac26 100644 --- a/executables/nr-ru.c +++ b/executables/nr-ru.c @@ -1083,7 +1083,7 @@ void *ru_thread( void *param ) { ru_thread_status = 0; // set default return value sprintf(threadname,"ru_thread %u",ru->idx); - LOG_I(PHY,"Starting RU %d (%s,%s),\n",ru->idx,NB_functions[ru->function],NB_timing[ru->if_timing]); + LOG_I(PHY,"Starting RU %d (%s,%s) on core %d\n",ru->idx,NB_functions[ru->function],NB_timing[ru->if_timing],sched_getcpu()); memcpy((void *)&ru->config,(void *)&RC.gNB[0]->gNB_config,sizeof(ru->config)); if(emulate_rf) { @@ -1342,7 +1342,7 @@ void init_RU_proc(RU_t *ru) { pthread_mutex_init( &proc->mutex_emulateRF,NULL); pthread_cond_init( &proc->cond_emulateRF, NULL); - threadCreate( &proc->pthread_FH, ru_thread, (void *)ru, "ru_thread", ru->tpcores[0], OAI_PRIORITY_RT_MAX ); + threadCreate( &proc->pthread_FH, ru_thread, (void *)ru, "ru_thread", -1 /*ru->tpcores[0]*/, OAI_PRIORITY_RT_MAX ); if(emulate_rf) threadCreate( &proc->pthread_emulateRF, emulatedRF_thread, (void *)proc, "emulateRF", -1, OAI_PRIORITY_RT ); diff --git a/radio/USRP/USERSPACE/LIB/usrp_lib.cpp b/radio/USRP/USERSPACE/LIB/usrp_lib.cpp index 227c2f461c..cb7c9fe3b3 100644 --- a/radio/USRP/USERSPACE/LIB/usrp_lib.cpp +++ b/radio/USRP/USERSPACE/LIB/usrp_lib.cpp @@ -556,6 +556,7 @@ void *trx_usrp_write_thread(void * arg){ signed char last_packet; int flags_gpio; + printf("trx_usrp_write_thread started on cpu %d\n",sched_getcpu()); while(1){ pthread_mutex_lock(&write_thread->mutex_write); while (write_thread->count_write == 0) { @@ -658,7 +659,35 @@ int trx_usrp_write_init(openair0_device *device){ printf("end of tx write thread\n"); pthread_mutex_init(&write_thread->mutex_write, NULL); pthread_cond_init(&write_thread->cond_write, NULL); - pthread_create(&write_thread->pthread_write,NULL,trx_usrp_write_thread,(void *)device); + struct sched_param sparam={0}; + sparam.sched_priority = sched_get_priority_max(SCHED_RR); + pthread_attr_t attr; + int ret=pthread_attr_init(&attr); + if (ret!=0) { + printf("error initializing USRP tx-thread attribute ret %d, errno: %d\n",ret,errno); + exit(-1); + } + else printf("USRP tx-thread attribute initialized\n"); + ret=pthread_attr_setinheritsched(&attr, PTHREAD_EXPLICIT_SCHED); + if (ret!=0) { + printf("error setting USRP tx-thread inheritance ret %d, errno: %d\n",ret,errno); + exit(-1); + } + else printf("USRP tx-thread inheritance set\n"); + ret=pthread_attr_setschedpolicy(&attr, SCHED_RR); + if (ret!=0) { + printf("error setting USRP tx-thread scheduling policy ret %d, errno: %d\n",ret,errno); + exit(-1); + } + else printf("USRP tx-thread scheduling policy set to SCHED_RR\n"); + ret = pthread_attr_setschedparam(&attr, &sparam); + if (ret!=0) { + printf("error setting USRP tx-thread priority ret %d, errno: %d\n",ret,errno); + exit(-1); + } + else printf("USRP tx-thread priority set to %d\n",sparam.sched_priority); + + pthread_create(&write_thread->pthread_write,&attr,trx_usrp_write_thread,(void *)device); return(0); } From e3425b1f9e4b71211caebe9fe3e3457e759af236 Mon Sep 17 00:00:00 2001 From: Raymond Knopp Date: Tue, 7 Mar 2023 11:24:26 +0100 Subject: [PATCH 2/6] added msg-q threading options (and CPU thread pinning) from O-RAN 7.2 branch to develop-based for testing with USRP. --- executables/nr-gnb.c | 123 ++++++++++++++++++---- executables/nr-ru.c | 56 ++++++---- executables/softmodem-common.c | 1 + executables/softmodem-common.h | 13 ++- openair1/PHY/defs_RU.h | 2 + openair1/PHY/defs_gNB.h | 7 ++ openair1/SCHED_NR/fapi_nr_l1.c | 24 ++++- openair1/SCHED_NR/phy_procedures_nr_gNB.c | 7 +- openair2/ENB_APP/enb_paramdef.h | 4 + openair2/GNB_APP/L1_nr_paramdef.h | 7 ++ openair2/GNB_APP/gnb_config.c | 3 + 11 files changed, 197 insertions(+), 50 deletions(-) diff --git a/executables/nr-gnb.c b/executables/nr-gnb.c index d958e772a9..e797b685df 100644 --- a/executables/nr-gnb.c +++ b/executables/nr-gnb.c @@ -87,6 +87,7 @@ #include #include +#define USE_MSGQ 1 //#define USRP_DEBUG 1 // Fix per CC openair rf/if device update // extern openair0_device openair0; @@ -118,6 +119,7 @@ void tx_func(void *param) { int absslot_tx = info->timestamp_tx/info->gNB->frame_parms.get_samples_per_slot(slot_tx,&info->gNB->frame_parms); int absslot_rx = absslot_tx-info->gNB->RU_list[0]->sl_ahead; int rt_prof_idx = absslot_rx % RT_PROF_DEPTH; + start_meas(&info->gNB->phy_proc_tx); clock_gettime(CLOCK_MONOTONIC,&info->gNB->rt_L1_profiling.start_L1_TX[rt_prof_idx]); phy_procedures_gNB_TX(info, @@ -126,8 +128,43 @@ void tx_func(void *param) { 1); clock_gettime(CLOCK_MONOTONIC,&info->gNB->rt_L1_profiling.return_L1_TX[rt_prof_idx]); + if (get_softmodem_params()->reorder_thread_disable) { + PHY_VARS_gNB *gNB = info->gNB; + processingData_RU_t syncMsgRU; + syncMsgRU.frame_tx = frame_tx; + syncMsgRU.slot_tx = slot_tx; + syncMsgRU.ru = gNB->RU_list[0]; + syncMsgRU.timestamp_tx = info->timestamp_tx; + LOG_D(PHY,"gNB: %d.%d : calling RU TX function\n",syncMsgRU.frame_tx,syncMsgRU.slot_tx); + ru_tx_func((void*)&syncMsgRU); + } + stop_meas(&info->gNB->phy_proc_tx); } + +void *L1_rx_thread(void *arg) { + PHY_VARS_gNB *gNB = (PHY_VARS_gNB*)arg; + + while (oai_exit == 0) { + notifiedFIFO_elt_t *res = pullNotifiedFIFO(&gNB->resp_L1); + processingData_L1_t *info = (processingData_L1_t *)NotifiedFifoData(res); + rx_func(info); + delNotifiedFIFO_elt(res); + } + return NULL; +} + +void *L1_tx_thread(void *arg) { + PHY_VARS_gNB *gNB = (PHY_VARS_gNB*)arg; + + while (oai_exit == 0) { + notifiedFIFO_elt_t *res = pullNotifiedFIFO(&gNB->L1_tx_out); + processingData_L1tx_t *info = (processingData_L1tx_t *)NotifiedFifoData(res); + tx_func(info); + delNotifiedFIFO_elt(res); + } + return NULL; +} void rx_func(void *param) { processingData_L1_t *info = (processingData_L1_t *) param; PHY_VARS_gNB *gNB = info->gNB; @@ -220,13 +257,16 @@ void rx_func(void *param) { // Do PRACH RU processing L1_nr_prach_procedures(gNB,frame_rx,slot_rx); - //apply the rx signal rotation here - for (int aa = 0; aa < gNB->frame_parms.nb_antennas_rx; aa++) { - apply_nr_rotation_ul(&gNB->frame_parms, - gNB->common_vars.rxdataF[aa], - slot_rx, - 0, - gNB->frame_parms.Ncp==EXTENDED?12:14); + //WA: comment rotation in tx/rx + if((gNB->num_RU == 1) && (gNB->RU_list[0]->if_south != REMOTE_IF4p5)) { + //apply the rx signal rotation here + for (int aa = 0; aa < gNB->frame_parms.nb_antennas_rx; aa++) { + apply_nr_rotation_ul(&gNB->frame_parms, + gNB->common_vars.rxdataF[aa], + slot_rx, + 0, + gNB->frame_parms.Ncp==EXTENDED?12:14); + } } phy_procedures_gNB_uespec_RX(gNB, frame_rx, slot_rx); } @@ -237,22 +277,26 @@ void rx_func(void *param) { // Call the scheduler start_meas(&gNB->ul_indication_stats); - pthread_mutex_lock(&gNB->UL_INFO_mutex); +// pthread_mutex_lock(&gNB->UL_INFO_mutex); gNB->UL_INFO.frame = frame_rx; gNB->UL_INFO.slot = slot_rx; gNB->UL_INFO.module_id = gNB->Mod_id; gNB->UL_INFO.CC_id = gNB->CC_id; gNB->if_inst->NR_UL_indication(&gNB->UL_INFO); - pthread_mutex_unlock(&gNB->UL_INFO_mutex); +// pthread_mutex_unlock(&gNB->UL_INFO_mutex); stop_meas(&gNB->ul_indication_stats); - int tx_slot_type = nr_slot_select(cfg,frame_rx,slot_tx); + int tx_slot_type = nr_slot_select(cfg,frame_tx,slot_tx); if ((tx_slot_type == NR_DOWNLINK_SLOT || tx_slot_type == NR_MIXED_SLOT) && NFAPI_MODE != NFAPI_MODE_PNF) { notifiedFIFO_elt_t *res; processingData_L1tx_t *syncMsg; // Its a FIFO so it maitains the order in which the MAC fills the messages // so no need for checking for right slot - res = pullTpool(&gNB->L1_tx_filled, &gNB->threadPool); + #ifndef USE_MSGQ + if (get_softmodem_params()->reorder_thread_disable) + res = pullTpool(&gNB->L1_tx_out, &gNB->threadPool); + else + res = pullTpool(&gNB->L1_tx_filled, &gNB->threadPool); if (res == NULL) return; // Tpool has been stopped syncMsg = (processingData_L1tx_t *)NotifiedFifoData(res); @@ -260,6 +304,12 @@ void rx_func(void *param) { syncMsg->timestamp_tx = info->timestamp_tx; res->key = slot_tx; pushTpool(&gNB->threadPool, res); +#else + syncMsg = gNB->msgDataTx; + syncMsg->gNB = gNB; + syncMsg->timestamp_tx = info->timestamp_tx; + tx_func(syncMsg); +#endif } else if (get_softmodem_params()->continuous_tx) { notifiedFIFO_elt_t *res = pullTpool(&gNB->L1_tx_free, &gNB->threadPool); if (res == NULL) @@ -318,6 +368,11 @@ static size_t dump_L1_meas_stats(PHY_VARS_gNB *gNB, RU_t *ru, char *output, size const char *end = output + outputlen; output += print_meas_log(&gNB->phy_proc_tx, "L1 Tx processing", NULL, NULL, output, end - output); output += print_meas_log(&gNB->dlsch_encoding_stats, "DLSCH encoding", NULL, NULL, output, end - output); + output += print_meas_log(&gNB->dlsch_scrambling_stats, "DLSCH scrambling", NULL, NULL, output, end-output); + output += print_meas_log(&gNB->dlsch_modulation_stats, "DLSCH modulation", NULL, NULL, output, end-output); + output += print_meas_log(&gNB->dlsch_layer_mapping_stats, "DLSCH layer mapping", NULL, NULL, output,end-output); + output += print_meas_log(&gNB->dlsch_resource_mapping_stats, "DLSCH resource mapping", NULL, NULL, output,end-output); + output += print_meas_log(&gNB->dlsch_precoding_stats, "DLSCH precoding", NULL, NULL, output,end-output); output += print_meas_log(&gNB->phy_proc_rx, "L1 Rx processing", NULL, NULL, output, end - output); output += print_meas_log(&gNB->ul_indication_stats, "UL Indication", NULL, NULL, output, end - output); output += print_meas_log(&gNB->rx_pusch_stats, "PUSCH inner-receiver", NULL, NULL, output, end - output); @@ -362,7 +417,11 @@ void *nrL1_stats_thread(void *param) { reset_meas(&gNB->rx_pusch_stats); reset_meas(&gNB->ulsch_decoding_stats); reset_meas(&gNB->schedule_response_stats); - + reset_meas(&gNB->dlsch_scrambling_stats); + reset_meas(&gNB->dlsch_modulation_stats); + reset_meas(&gNB->dlsch_layer_mapping_stats); + reset_meas(&gNB->dlsch_resource_mapping_stats); + reset_meas(&gNB->dlsch_precoding_stats); while (!oai_exit) { sleep(1); dump_nr_I0_stats(fd,gNB); @@ -445,31 +504,51 @@ void init_gNB_Tpool(int inst) { // L1 RX result FIFO initNotifiedFIFO(&gNB->resp_L1); +#ifndef USE_MSGQ notifiedFIFO_elt_t *msg = newNotifiedFIFO_elt(sizeof(processingData_L1_t), 0, &gNB->resp_L1, rx_func); pushNotifiedFIFO(&gNB->resp_L1, msg); // to unblock the process in the beginning - +#endif // L1 TX result FIFO initNotifiedFIFO(&gNB->L1_tx_free); initNotifiedFIFO(&gNB->L1_tx_filled); initNotifiedFIFO(&gNB->L1_tx_out); - - // we create 2 threads for L1 tx processing - for (int i=0; i < 2; i++) { + +#ifdef USE_MSGQ + threadCreate(&gNB->L1_rx_thread, L1_rx_thread, (void *)gNB, "L1_rx_thread", + gNB->L1_rx_thread_core, OAI_PRIORITY_RT_MAX); +// threadCreate(&gNB->L1_tx_thread, L1_tx_thread, (void *)gNB, "L1_tx_thread", +// gNB->L1_tx_thread_core, OAI_PRIORITY_RT_MAX); +#endif + if (get_softmodem_params()->reorder_thread_disable) { notifiedFIFO_elt_t *msgL1Tx = newNotifiedFIFO_elt(sizeof(processingData_L1tx_t), 0, &gNB->L1_tx_out, tx_func); processingData_L1tx_t *msgDataTx = (processingData_L1tx_t *)NotifiedFifoData(msgL1Tx); memset(msgDataTx, 0, sizeof(processingData_L1tx_t)); init_DLSCH_struct(gNB, msgDataTx); memset(msgDataTx->ssb, 0, 64*sizeof(NR_gNB_SSB_t)); - pushNotifiedFIFO(&gNB->L1_tx_free, msgL1Tx); // to unblock the process in the beginning +#ifndef USE_MSGQ + pushNotifiedFIFO(&gNB->L1_tx_out, msgL1Tx); // to unblock the process in the beginning +#else + gNB->msgDataTx = msgDataTx; +#endif + } else { + // we create 2 threads for L1 tx processing + for (int i=0; i < 2; i++) { + notifiedFIFO_elt_t *msgL1Tx = newNotifiedFIFO_elt(sizeof(processingData_L1tx_t), 0, &gNB->L1_tx_out, tx_func); + processingData_L1tx_t *msgDataTx = (processingData_L1tx_t *)NotifiedFifoData(msgL1Tx); + memset(msgDataTx, 0, sizeof(processingData_L1tx_t)); + init_DLSCH_struct(gNB, msgDataTx); + memset(msgDataTx->ssb, 0, 64*sizeof(NR_gNB_SSB_t)); + pushNotifiedFIFO(&gNB->L1_tx_free, msgL1Tx); // to unblock the process in the beginning + } + + LOG_I(PHY,"Creating thread for TX reordering and dispatching to RU\n"); + threadCreate(&proc->pthread_tx_reorder, tx_reorder_thread, (void *)gNB, "thread_tx_reorder", + gNB->RU_list[0] ? gNB->RU_list[0]->tpcores[1] : -1, OAI_PRIORITY_RT_MAX); } if ((!get_softmodem_params()->emulate_l1) && (!IS_SOFTMODEM_NOSTATS_BIT) && (NFAPI_MODE!=NFAPI_MODE_VNF)) threadCreate(&proc->L1_stats_thread,nrL1_stats_thread,(void*)gNB,"L1_stats",-1,OAI_PRIORITY_RT_LOW); - LOG_I(PHY,"Creating thread for TX reordering and dispatching to RU\n"); - threadCreate(&proc->pthread_tx_reorder, tx_reorder_thread, (void *)gNB, "thread_tx_reorder", - gNB->RU_list[0] ? gNB->RU_list[0]->tpcores[1] : -1, OAI_PRIORITY_RT_MAX); - } @@ -538,6 +617,7 @@ void init_eNB_afterRU(void) { gNB = RC.gNB[inst]; gNB->ldpc_offload_flag = ldpc_offload_flag; + gNB->reorder_thread_disable = get_softmodem_params()->reorder_thread_disable; phy_init_nr_gNB(gNB); @@ -616,6 +696,7 @@ void init_gNB(int single_thread_flag,int wait_for_sync) { gNB->prach_energy_counter = 0; gNB->chest_time = get_softmodem_params()->chest_time; gNB->chest_freq = get_softmodem_params()->chest_freq; + } diff --git a/executables/nr-ru.c b/executables/nr-ru.c index e6b6eeac26..7edaaf98a8 100644 --- a/executables/nr-ru.c +++ b/executables/nr-ru.c @@ -60,6 +60,8 @@ unsigned short config_frames[4] = {2,9,11,13}; #endif +#define USE_MSGQ 1 + /* these variables have to be defined before including ENB_APP/enb_paramdef.h and GNB_APP/gnb_paramdef.h */ static int DEFBANDS[] = {7}; static int DEFENBS[] = {0}; @@ -83,6 +85,8 @@ static int DEFRUTPCORES[] = {-1,-1,-1,-1}; extern int oai_exit; +uint16_t sl_ahead; + extern struct timespec timespec_sub(struct timespec lhs, struct timespec rhs); extern struct timespec timespec_add(struct timespec lhs, struct timespec rhs); extern void nr_phy_free_RU(RU_t *); @@ -1083,7 +1087,7 @@ void *ru_thread( void *param ) { ru_thread_status = 0; // set default return value sprintf(threadname,"ru_thread %u",ru->idx); - LOG_I(PHY,"Starting RU %d (%s,%s) on core %d\n",ru->idx,NB_functions[ru->function],NB_timing[ru->if_timing],sched_getcpu()); + LOG_I(PHY,"Starting RU %d (%s,%s) on cpu %d\n",ru->idx,NB_functions[ru->function],NB_timing[ru->if_timing],sched_getcpu()); memcpy((void *)&ru->config,(void *)&RC.gNB[0]->gNB_config,sizeof(ru->config)); if(emulate_rf) { @@ -1279,9 +1283,13 @@ void *ru_thread( void *param ) { } // end if (slot_type == NR_UPLINK_SLOT || slot_type == NR_MIXED_SLOT) { // At this point, all information for subframe has been received on FH interface +#ifndef USE_MSGQ res = pullTpool(&gNB->resp_L1, &gNB->threadPool); if (res == NULL) break; // Tpool has been stopped +#else + res=newNotifiedFIFO_elt(sizeof(processingData_L1_t),0, &gNB->resp_L1,NULL); +#endif syncMsg = (processingData_L1_t *)NotifiedFifoData(res); syncMsg->gNB = gNB; syncMsg->frame_rx = proc->frame_rx; @@ -1290,7 +1298,11 @@ void *ru_thread( void *param ) { syncMsg->slot_tx = proc->tti_tx; syncMsg->timestamp_tx = proc->timestamp_tx; res->key = proc->tti_rx; +#ifndef USE_MSGQ pushTpool(&gNB->threadPool, res); +#else + pushNotifiedFIFO(&gNB->resp_L1, res); +#endif } printf( "Exiting ru_thread \n"); @@ -1305,7 +1317,8 @@ int start_streaming(RU_t *ru) { } int nr_start_if(struct RU_t_s *ru, struct PHY_VARS_gNB_s *gNB) { - for (int i=0;inb_rx;i++) ru->openair0_cfg.rxbase[i] = ru->common.rxdata[i]; + if (ru->if_south <= REMOTE_IF5) + for (int i=0;inb_rx;i++) ru->openair0_cfg.rxbase[i] = ru->common.rxdata[i]; ru->openair0_cfg.rxsize = ru->nr_frame_parms->samples_per_subframe*10; reset_meas(&ru->ifdevice.tx_fhaul); return(ru->ifdevice.trx_start_func(&ru->ifdevice)); @@ -1342,7 +1355,7 @@ void init_RU_proc(RU_t *ru) { pthread_mutex_init( &proc->mutex_emulateRF,NULL); pthread_cond_init( &proc->cond_emulateRF, NULL); - threadCreate( &proc->pthread_FH, ru_thread, (void *)ru, "ru_thread", -1 /*ru->tpcores[0]*/, OAI_PRIORITY_RT_MAX ); + threadCreate( &proc->pthread_FH, ru_thread, (void *)ru, "ru_thread", ru->ru_thread_core, OAI_PRIORITY_RT_MAX ); if(emulate_rf) threadCreate( &proc->pthread_emulateRF, emulatedRF_thread, (void *)proc, "emulateRF", -1, OAI_PRIORITY_RT ); @@ -1668,7 +1681,7 @@ void set_function_spec_param(RU_t *ru) { case REMOTE_IF4p5: ru->do_prach = 0; ru->feprx = NULL; // DFTs - ru->feptx_prec = NULL; // Precoding operation + ru->feptx_prec = nr_feptx_prec; // Precoding operation ru->feptx_ofdm = NULL; // no OFDM mod ru->fh_south_in = fh_if4p5_south_in; // synchronous IF4p5 reception ru->fh_south_out = fh_if4p5_south_out; // synchronous IF4p5 transmission @@ -1780,23 +1793,25 @@ void init_NR_RU(char *rf_config_file) { set_function_spec_param(ru); LOG_I(PHY,"Starting ru_thread %d\n",ru_id); init_RU_proc(ru); - int threadCnt = ru->num_tpcores; - if (threadCnt < 2) LOG_E(PHY,"Number of threads for gNB should be more than 1. Allocated only %d\n",threadCnt); - else LOG_I(PHY,"RU Thread pool size %d\n",threadCnt); - char pool[80]; - int s_offset = sprintf(pool,"%d",ru->tpcores[0]); - for (int icpu=1; icputpcores[icpu]); + if (ru->if_south != REMOTE_IF4p5) { + int threadCnt = ru->num_tpcores; + if (threadCnt < 2) LOG_E(PHY,"Number of threads for gNB should be more than 1. Allocated only %d\n",threadCnt); + else LOG_I(PHY,"RU Thread pool size %d\n",threadCnt); + char pool[80]; + int s_offset = sprintf(pool,"%d",ru->tpcores[0]); + for (int icpu=1; icputpcores[icpu]); + } + LOG_I(PHY,"RU thread-pool core string %s\n",pool); + ru->threadPool = (tpool_t*)malloc(sizeof(tpool_t)); + initTpool(pool, ru->threadPool, cpumeas(CPUMEAS_GETSTATE)); + // FEP RX result FIFO + ru->respfeprx = (notifiedFIFO_t*) malloc(sizeof(notifiedFIFO_t)); + initNotifiedFIFO(ru->respfeprx); + // FEP TX result FIFO + ru->respfeptx = (notifiedFIFO_t*) malloc(sizeof(notifiedFIFO_t)); + initNotifiedFIFO(ru->respfeptx); } - LOG_I(PHY,"RU thread-pool core string %s\n",pool); - ru->threadPool = (tpool_t*)malloc(sizeof(tpool_t)); - initTpool(pool, ru->threadPool, cpumeas(CPUMEAS_GETSTATE)); - // FEP RX result FIFO - ru->respfeprx = (notifiedFIFO_t*) malloc(sizeof(notifiedFIFO_t)); - initNotifiedFIFO(ru->respfeprx); - // FEP TX result FIFO - ru->respfeptx = (notifiedFIFO_t*) malloc(sizeof(notifiedFIFO_t)); - initNotifiedFIFO(ru->respfeptx); } // for ru_id // sleep(1); @@ -1975,6 +1990,7 @@ static void NRRCconfig_RU(void) { RC.ru[j]->openair0_cfg.txfh_cores[0] = *(RUParamList.paramarray[j][RU_TXFH_CORE_ID].iptr); RC.ru[j]->num_tpcores = *(RUParamList.paramarray[j][RU_NUM_TP_CORES].iptr); RC.ru[j]->half_slot_parallelization = *(RUParamList.paramarray[j][RU_HALF_SLOT_PARALLELIZATION].iptr); + RC.ru[j]->ru_thread_core = *(RUParamList.paramarray[j][RU_RU_THREAD_CORE].iptr); printf("[RU %d] Setting half-slot parallelization to %d\n",j,RC.ru[j]->half_slot_parallelization); AssertFatal(RC.ru[j]->num_tpcores <= RUParamList.paramarray[j][RU_TP_CORES].numelt, "Number of TP cores should be <=16\n"); for (i=0; inum_tpcores; i++) RC.ru[j]->tpcores[i] = RUParamList.paramarray[j][RU_TP_CORES].iptr[i]; diff --git a/executables/softmodem-common.c b/executables/softmodem-common.c index 2b289b5352..d420b3ac1c 100644 --- a/executables/softmodem-common.c +++ b/executables/softmodem-common.c @@ -46,6 +46,7 @@ static softmodem_params_t softmodem_params; char *parallel_config=NULL; char *worker_config=NULL; int usrp_tx_thread = 0; +char *nfapi_str=NULL; int ldpc_offload_flag=0; uint8_t nfapi_mode=0; diff --git a/executables/softmodem-common.h b/executables/softmodem-common.h index 09eaea1c92..2b430ee8fb 100644 --- a/executables/softmodem-common.h +++ b/executables/softmodem-common.h @@ -43,6 +43,7 @@ extern "C" default no pool (runs in calling thread),\n\ list of cores, comma separated (negative value is no core affinity)\n\ example: -1,3 launches two working threads one floating, the second set on core 3" +#define CONFIG_HLP_REORDER "Disable reorder thread\n" #define CONFIG_HLP_ULMAXE "set the eNodeB max ULSCH erros\n" #define CONFIG_HLP_CALUER "set UE RX calibration\n" #define CONFIG_HLP_CALUERM "" @@ -109,6 +110,7 @@ extern "C" /*-----------------------------------------------------------------------------------------------------------------------------------------------------*/ #define RF_CONFIG_FILE softmodem_params.rf_config_file #define TP_CONFIG softmodem_params.threadPoolConfig +#define CONTINUOUS_TX softmodem_params.continuous_tx #define PHY_TEST softmodem_params.phy_test #define DO_RA softmodem_params.do_ra #define SA softmodem_params.sa @@ -131,13 +133,16 @@ extern "C" #define NON_STOP softmodem_params.non_stop #define EMULATE_L1 softmodem_params.emulate_l1 #define CONTINUOUS_TX softmodem_params.continuous_tx - +#define REORDER_THREAD_DISABLE softmodem_params.reorder_thread_disable #define DEFAULT_RFCONFIG_FILE "/usr/local/etc/syriq/ue.band7.tm1.PRB100.NR40.dat"; extern int usrp_tx_thread; +extern char *nfapi_str; + #define CMDLINE_PARAMS_DESC { \ {"rf-config-file", CONFIG_HLP_RFCFGF, 0, strptr:&RF_CONFIG_FILE, defstrval:NULL, TYPE_STRING, 0},\ {"thread-pool", CONFIG_HLP_TPOOL, 0, strptr:&TP_CONFIG, defstrval:"-1,-1,-1,-1,-1,-1,-1,-1", TYPE_STRING, 0}, \ + {"reorder-thread-disable",CONFIG_HLP_REORDER, PARAMFLAG_BOOL, iptr:&REORDER_THREAD_DISABLE, defintval:0, TYPE_INT, 0}, \ {"phy-test", CONFIG_HLP_PHYTST, PARAMFLAG_BOOL, iptr:&PHY_TEST, defintval:0, TYPE_INT, 0}, \ {"do-ra", CONFIG_HLP_DORA, PARAMFLAG_BOOL, iptr:&DO_RA, defintval:0, TYPE_INT, 0}, \ {"sa", CONFIG_HLP_SA, PARAMFLAG_BOOL, iptr:&SA, defintval:0, TYPE_INT, 0}, \ @@ -166,8 +171,8 @@ extern int usrp_tx_thread; {"chest-time", CONFIG_HLP_CHESTTIME, 0, iptr:&CHEST_TIME, defintval:0, TYPE_INT, 0}, \ {"nsa", CONFIG_HLP_NSA, PARAMFLAG_BOOL, iptr:&NSA, defintval:0, TYPE_INT, 0}, \ {"node-number", NULL, 0, u16ptr:&NODE_NUMBER, defuintval:0, TYPE_UINT16, 0}, \ - {"usrp-tx-thread-config", CONFIG_HLP_USRP_THREAD, 0, iptr:&usrp_tx_thread, defstrval:0, TYPE_INT, 0}, \ - {"nfapi", CONFIG_HLP_NFAPI, 0, strptr:NULL, defstrval:"MONOLITHIC", TYPE_STRING, 0}, \ + {"usrp-tx-thread-config", CONFIG_HLP_USRP_THREAD, 0, iptr:&usrp_tx_thread, defintval:0, TYPE_INT, 0}, \ + {"nfapi", CONFIG_HLP_NFAPI, 0, strptr:&nfapi_str, defstrval:"MONOLITHIC", TYPE_STRING, 0}, \ {"non-stop", CONFIG_HLP_NONSTOP, PARAMFLAG_BOOL, iptr:&NON_STOP, defintval:0, TYPE_INT, 0}, \ {"emulate-l1", CONFIG_L1_EMULATOR, PARAMFLAG_BOOL, iptr:&EMULATE_L1, defintval:0, TYPE_INT, 0}, \ {"continuous-tx", CONFIG_HLP_CONTINUOUS_TX,PARAMFLAG_BOOL, iptr:&CONTINUOUS_TX, defintval:0, TYPE_INT, 0}, \ @@ -208,6 +213,7 @@ extern int usrp_tx_thread; { .s5 = { NULL } }, \ { .s5 = { NULL } }, \ { .s5 = { NULL } }, \ + { .s5 = { NULL } }, \ { .s3a = { config_checkstr_assign_integer, \ {"MONOLITHIC", "PNF", "VNF","UE_STUB_PNF","UE_STUB_OFFNET","STANDALONE_PNF"}, \ {NFAPI_MONOLITHIC, NFAPI_MODE_PNF, NFAPI_MODE_VNF,NFAPI_UE_STUB_PNF,NFAPI_UE_STUB_OFFNET,NFAPI_MODE_STANDALONE_PNF}, \ @@ -290,6 +296,7 @@ typedef struct { //THREAD_STRUCT thread_struct; char *rf_config_file; char *threadPoolConfig; + int reorder_thread_disable; int phy_test; int do_ra; int sa; diff --git a/openair1/PHY/defs_RU.h b/openair1/PHY/defs_RU.h index 0ee7562346..20f59a1aa8 100644 --- a/openair1/PHY/defs_RU.h +++ b/openair1/PHY/defs_RU.h @@ -655,6 +655,8 @@ typedef struct RU_t_s { int txfh_core_id; /// number of RU interfaces int num_fd; + /// Core id of ru_thread + int ru_thread_core; /// list of cores for RU ThreadPool int tpcores[16]; /// number of cores for RU ThreadPool diff --git a/openair1/PHY/defs_gNB.h b/openair1/PHY/defs_gNB.h index 60bf49796b..c3dd0caa0b 100644 --- a/openair1/PHY/defs_gNB.h +++ b/openair1/PHY/defs_gNB.h @@ -657,6 +657,8 @@ typedef struct PHY_VARS_gNB_s { int ldpc_offload_flag; + int reorder_thread_disable; + int max_ldpc_iterations; /// indicate the channel estimation technique in time domain int chest_time; @@ -739,6 +741,11 @@ typedef struct PHY_VARS_gNB_s { notifiedFIFO_t L1_tx_out; notifiedFIFO_t resp_RU_tx; tpool_t threadPool; + pthread_t L1_rx_thread; + int L1_rx_thread_core; + pthread_t L1_tx_thread; + int L1_tx_thread_core; + struct processingData_L1tx *msgDataTx; int nbDecode; void *scopeData; /// structure for analyzing high-level RT measurements diff --git a/openair1/SCHED_NR/fapi_nr_l1.c b/openair1/SCHED_NR/fapi_nr_l1.c index b9f0d8e681..4617189805 100644 --- a/openair1/SCHED_NR/fapi_nr_l1.c +++ b/openair1/SCHED_NR/fapi_nr_l1.c @@ -36,6 +36,8 @@ #include "PHY/NR_TRANSPORT/nr_dci.h" #include "nfapi/oai_integration/vendor_ext.h" +#define USE_MSGQ 1 + extern int oai_nfapi_dl_tti_req(nfapi_nr_dl_tti_request_t *dl_config_req); extern int oai_nfapi_tx_data_req(nfapi_nr_tx_data_request_t *tx_data_req); extern int oai_nfapi_ul_dci_req(nfapi_nr_ul_dci_request_t *ul_dci_req); @@ -163,13 +165,20 @@ void nr_schedule_response(NR_Sched_Rsp_t *Sched_INFO){ if (slot_type == NR_DOWNLINK_SLOT || slot_type == NR_MIXED_SLOT) { notifiedFIFO_elt_t *res; - res = pullTpool(&gNB->L1_tx_free, &gNB->threadPool); +#ifndef USE_MSGQ + if (gNB->reorder_thread_disable) + res = pullTpool(&gNB->L1_tx_out, &gNB->threadPool); + else + res = pullTpool(&gNB->L1_tx_free, &gNB->threadPool); if (res == NULL) return; // Tpool has been stopped, nothing to process processingData_L1tx_t *msgTx = (processingData_L1tx_t *)NotifiedFifoData(res); - const time_stats_t ts = exec_time_stats_NotifiedFIFO(res); +#else + processingData_L1tx_t *msgTx = gNB->msgDataTx; //newNotifiedFIFO_elt(sizeof(processingData_L1tx_t),0, &gNB->L1_tx_out,NULL); +#endif + /*const time_stats_t ts = exec_time_stats_NotifiedFIFO(res); merge_meas(&gNB->phy_proc_tx, &ts); - +*/ msgTx->num_pdsch_slot=0; msgTx->num_dl_pdcch=0; msgTx->num_ul_pdcch=number_ul_dci_pdu; @@ -212,7 +221,14 @@ void nr_schedule_response(NR_Sched_Rsp_t *Sched_INFO){ for (int i=0; iul_pdcch_pdu[i] = UL_dci_req->ul_dci_pdu_list[i]; - pushNotifiedFIFO(&gNB->L1_tx_filled,res); +#ifndef USE_MSGQ + if (gNB->reorder_thread_disable) + pushNotifiedFIFO(&gNB->L1_tx_out,res); + else + pushNotifiedFIFO(&gNB->L1_tx_filled,res); +#else + //pushNotifiedFIFO(&gNB->L1_tx_out, res); +#endif } for (int i = 0; i < number_ul_tti_pdu; i++) { diff --git a/openair1/SCHED_NR/phy_procedures_nr_gNB.c b/openair1/SCHED_NR/phy_procedures_nr_gNB.c index 6a1a8f7b36..f5d2cc47a6 100644 --- a/openair1/SCHED_NR/phy_procedures_nr_gNB.c +++ b/openair1/SCHED_NR/phy_procedures_nr_gNB.c @@ -390,11 +390,13 @@ void nr_ulsch_procedures(PHY_VARS_gNB *gNB, int frame_rx, int slot_rx, int ULSCH stop_meas(&gNB->ulsch_decoding_stats); } +#define USE_MSGQ 1 void nr_fill_indication(PHY_VARS_gNB *gNB, int frame, int slot_rx, int ULSCH_id, uint8_t harq_pid, uint8_t crc_flag, int dtx_flag) { +#ifndef USE_MSGQ pthread_mutex_lock(&gNB->UL_INFO_mutex); - +#endif NR_gNB_ULSCH_t *ulsch = gNB->ulsch[ULSCH_id]; NR_UL_gNB_HARQ_t *harq_process = ulsch->harq_processes[harq_pid]; NR_gNB_SCH_STATS_t *stats=get_ulsch_stats(gNB,ulsch); @@ -507,8 +509,9 @@ void nr_fill_indication(PHY_VARS_gNB *gNB, int frame, int slot_rx, int ULSCH_id, gNB->UL_INFO.rx_ind.number_of_pdus++; +#ifndef USE_MSGQ pthread_mutex_unlock(&gNB->UL_INFO_mutex); - +#endif } // Function to fill UL RB mask to be used for N0 measurements diff --git a/openair2/ENB_APP/enb_paramdef.h b/openair2/ENB_APP/enb_paramdef.h index 647ac96c18..2070e9e2a1 100644 --- a/openair2/ENB_APP/enb_paramdef.h +++ b/openair2/ENB_APP/enb_paramdef.h @@ -111,6 +111,7 @@ typedef enum { #define CONFIG_STRING_RU_NUM_TP_CORES "num_tp_cores" #define CONFIG_STRING_RU_NUM_INTERFACES "num_interfaces" #define CONFIG_STRING_RU_HALF_SLOT_PARALLELIZATION "half_slot_parallelization" +#define CONFIG_STRING_RU_RU_THREAD_CORE "ru_thread_core" #define HLP_RU_SF_AHEAD "LTE TX processing advance" #define HLP_RU_SL_AHEAD "NR TX processing advance" @@ -122,6 +123,7 @@ typedef enum { #define HLP_RU_NUM_TP_CORES "Number of cores for RU ThreadPool" #define HLP_RU_NUM_INTERFACES "Number of network interfaces for RU" #define HLP_RU_HALF_SLOT_PARALLELIZATION "run half slots in parallel in RU FEP" +#define HLP_RU_RU_THREAD_CORE "id of core to pin ru_thread, -1 is default" #define RU_LOCAL_IF_NAME_IDX 0 #define RU_LOCAL_ADDRESS_IDX 1 @@ -164,6 +166,7 @@ typedef enum { #define RU_NUM_TP_CORES 38 #define RU_NUM_INTERFACES 39 #define RU_HALF_SLOT_PARALLELIZATION 40 +#define RU_RU_THREAD_CORE 41 /*-----------------------------------------------------------------------------------------------------------------------------------------*/ /* RU configuration parameters */ /* optname helpstr paramflags XXXptr defXXXval type numelt */ @@ -210,6 +213,7 @@ typedef enum { {CONFIG_STRING_RU_NUM_TP_CORES, HLP_RU_NUM_TP_CORES, 0, uptr:NULL, defintval:2, TYPE_UINT, 0}, \ {CONFIG_STRING_RU_NUM_INTERFACES, HLP_RU_NUM_INTERFACES, 0, uptr:NULL, defintval:1, TYPE_UINT, 0}, \ {CONFIG_STRING_RU_HALF_SLOT_PARALLELIZATION, HLP_RU_HALF_SLOT_PARALLELIZATION, 0, uptr:NULL, defintval:1, TYPE_UINT, 0}, \ + {CONFIG_STRING_RU_RU_THREAD_CORE, HLP_RU_RU_THREAD_CORE, 0, uptr:NULL, defintval:-1, TYPE_INT, 0}, \ } /*---------------------------------------------------------------------------------------------------------------------------------------*/ diff --git a/openair2/GNB_APP/L1_nr_paramdef.h b/openair2/GNB_APP/L1_nr_paramdef.h index 62f2f43926..efe378820d 100644 --- a/openair2/GNB_APP/L1_nr_paramdef.h +++ b/openair2/GNB_APP/L1_nr_paramdef.h @@ -53,6 +53,9 @@ #define CONFIG_STRING_L1_PUSCH_DTX_THRESHOLD "pusch_dtx_threshold" #define CONFIG_STRING_L1_SRS_DTX_THRESHOLD "srs_dtx_threshold" #define CONFIG_STRING_L1_MAX_LDPC_ITERATIONS "max_ldpc_iterations" +#define CONFIG_STRING_L1_RX_THREAD_CORE "L1_rx_thread_core" +#define CONFIG_STRING_L1_TX_THREAD_CORE "L1_tx_thread_core" + /*----------------------------------------------------------------------------------------------------------------------------------------------------*/ /* L1 configuration parameters */ /* optname helpstr paramflags XXXptr defXXXval type numelt */ @@ -74,6 +77,8 @@ {CONFIG_STRING_L1_PUSCH_DTX_THRESHOLD, NULL, 0, uptr:NULL, defintval:50, TYPE_UINT, 0}, \ {CONFIG_STRING_L1_SRS_DTX_THRESHOLD, NULL, 0, uptr:NULL, defintval:50, TYPE_UINT, 0}, \ {CONFIG_STRING_L1_MAX_LDPC_ITERATIONS, NULL, 0, uptr:NULL, defintval:5, TYPE_UINT, 0}, \ +{CONFIG_STRING_L1_RX_THREAD_CORE, NULL, 0, uptr:NULL, defintval:-1, TYPE_UINT, 0}, \ +{CONFIG_STRING_L1_TX_THREAD_CORE, NULL, 0, uptr:NULL, defintval:-1, TYPE_UINT, 0}, \ } #define L1_CC_IDX 0 #define L1_TRANSPORT_N_PREFERENCE_IDX 1 @@ -91,6 +96,8 @@ #define L1_PUSCH_DTX_THRESHOLD 13 #define L1_SRS_DTX_THRESHOLD 14 #define L1_MAX_LDPC_ITERATIONS 15 +#define L1_RX_THREAD_CORE 16 +#define L1_TX_THREAD_CORE 17 /*----------------------------------------------------------------------------------------------------------------------------------------------------*/ #endif diff --git a/openair2/GNB_APP/gnb_config.c b/openair2/GNB_APP/gnb_config.c index c1fa1d49b9..4a378033dd 100644 --- a/openair2/GNB_APP/gnb_config.c +++ b/openair2/GNB_APP/gnb_config.c @@ -769,6 +769,9 @@ void RCconfig_NR_L1(void) RC.gNB[j]->pusch_thres = *(L1_ParamList.paramarray[j][L1_PUSCH_DTX_THRESHOLD].uptr); RC.gNB[j]->srs_thres = *(L1_ParamList.paramarray[j][L1_SRS_DTX_THRESHOLD].uptr); RC.gNB[j]->max_ldpc_iterations = *(L1_ParamList.paramarray[j][L1_MAX_LDPC_ITERATIONS].uptr); + RC.gNB[j]->L1_rx_thread_core = *(L1_ParamList.paramarray[j][L1_RX_THREAD_CORE].iptr); + RC.gNB[j]->L1_tx_thread_core = *(L1_ParamList.paramarray[j][L1_TX_THREAD_CORE].iptr); + LOG_I(PHY,"L1_RX_THREAD_CORE %d (%d)\n",*(L1_ParamList.paramarray[j][L1_RX_THREAD_CORE].iptr),L1_RX_THREAD_CORE); if (strcmp(*(L1_ParamList.paramarray[j][L1_TRANSPORT_N_PREFERENCE_IDX].strptr), "local_mac") == 0) { // sf_ahead = 2; // Need 4 subframe gap between RX and TX } else if (strcmp(*(L1_ParamList.paramarray[j][L1_TRANSPORT_N_PREFERENCE_IDX].strptr), "nfapi") == 0) { From e4b3757e1e616d841c6f4d42489fa73713759a18 Mon Sep 17 00:00:00 2001 From: Raymond Knopp Date: Wed, 8 Mar 2023 21:43:44 +0100 Subject: [PATCH 3/6] comment L1_tx_thread for now --- executables/nr-gnb.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/executables/nr-gnb.c b/executables/nr-gnb.c index e797b685df..105465df02 100644 --- a/executables/nr-gnb.c +++ b/executables/nr-gnb.c @@ -153,7 +153,7 @@ void *L1_rx_thread(void *arg) { } return NULL; } - +/* to be added for URLLC, requires MAC scheduling to be split from UL indication void *L1_tx_thread(void *arg) { PHY_VARS_gNB *gNB = (PHY_VARS_gNB*)arg; @@ -165,6 +165,7 @@ void *L1_tx_thread(void *arg) { } return NULL; } +*/ void rx_func(void *param) { processingData_L1_t *info = (processingData_L1_t *) param; PHY_VARS_gNB *gNB = info->gNB; From 2b143370ee7905e78050ed80cc26314233dc9379 Mon Sep 17 00:00:00 2001 From: Raymond Knopp Date: Sun, 12 Mar 2023 22:51:09 +0100 Subject: [PATCH 4/6] small changes in eth_udp.c for AW2S --- common/utils/system.c | 4 +++- openair1/PHY/NR_TRANSPORT/nr_dlsch.c | 2 +- radio/ETHERNET/USERSPACE/LIB/eth_udp.c | 26 ++++++++++++++++---------- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/common/utils/system.c b/common/utils/system.c index 9d52639b2f..5b30b84f53 100644 --- a/common/utils/system.c +++ b/common/utils/system.c @@ -231,7 +231,9 @@ void threadCreate(pthread_t* t, void * (*func)(void*), void * param, char* name, int settingPriority = 1; ret=pthread_attr_init(&attr); AssertFatal(ret==0,"ret: %d, errno: %d\n",ret, errno); - + + LOG_I(UTIL,"Creating thread %s with affinity %d and priority %d\n",name,affinity,priority); + if (checkIfFedoraDistribution()) if (checkIfGenericKernelOnFedora()) if (checkIfInsideContainer()) diff --git a/openair1/PHY/NR_TRANSPORT/nr_dlsch.c b/openair1/PHY/NR_TRANSPORT/nr_dlsch.c index 9b6adb8702..b98263bbf7 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_dlsch.c +++ b/openair1/PHY/NR_TRANSPORT/nr_dlsch.c @@ -466,7 +466,7 @@ void nr_generate_pdsch(processingData_L1tx_t *msgTx, for (int rb=0; rbrbSize; rb++) { //get pmi info uint8_t pmi; - if (rel15->precodingAndBeamforming.prg_size > 0) + if (0 /*rel15->precodingAndBeamforming.prg_size > 0*/) pmi = rel15->precodingAndBeamforming.prgs_list[(int)rb/rel15->precodingAndBeamforming.prg_size].pm_idx; else pmi = 0;//no precoding diff --git a/radio/ETHERNET/USERSPACE/LIB/eth_udp.c b/radio/ETHERNET/USERSPACE/LIB/eth_udp.c index 31971f1d1a..0d46d793ba 100644 --- a/radio/ETHERNET/USERSPACE/LIB/eth_udp.c +++ b/radio/ETHERNET/USERSPACE/LIB/eth_udp.c @@ -314,9 +314,15 @@ void *trx_eth_write_udp_cmd(udpTXelem_t *udpTXelem) { uint64_t last_rxTS = fhstate->TS[0]-fhstate->TS0; uint64_t TS_advance=0; if (timestamp > last_rxTS) TS_advance = timestamp - last_rxTS; - - if (TS_advance < (nsamps/2)) LOG_W(PHY,"Starting TX FH for TS %llu absslot %llu(%llu) last_rxTS %llu TS_advance %llu samples\n",(unsigned long long)timestamp,(unsigned long long)timestamp/nsamps,((unsigned long long)timestamp/nsamps)%20,(unsigned long long)last_rxTS,(unsigned long long)TS_advance); - void *buff2; + else { + LOG_W(PHY,"TS_advance is < 0 TS %llu absslot %llu(%llu) last_rxTS %llu TS_advance %llu samples\n", (unsigned long long)timestamp,(unsigned long long)timestamp/nsamps,((unsigned long long)timestamp/nsamps)%20,(unsigned long long)last_rxTS,(unsigned long long)TS_advance); + free(buff); + return(NULL); + } + if (TS_advance < (nsamps/2)) { + LOG_W(PHY,"Starting TX FH for TS %llu absslot %llu(%llu) last_rxTS %llu TS_advance %llu samples\n",(unsigned long long)timestamp,(unsigned long long)timestamp/nsamps,((unsigned long long)timestamp/nsamps)%20,(unsigned long long)last_rxTS,(unsigned long long)TS_advance); + } + void *buff2; #if defined(__x86_64) || defined(__i386__) int nsamps2 = 256>>3; __m256i buff_tx[nsamps2+1]; @@ -445,7 +451,7 @@ void *udp_read_thread(void *arg) { char buffer[UDP_PACKET_SIZE_BYTES(256)]; int first_read=0; while (oai_exit == 0) { - LOG_I(PHY,"UDP read thread %d, waiting for start sampling_rate_d %d, sampling_rate_n %d\n",u->thread_id,device->sampling_rate_ratio_n,device->sampling_rate_ratio_d); + LOG_I(PHY,"UDP read thread %d on core %d, waiting for start sampling_rate_d %d, sampling_rate_n %d\n",u->thread_id,sched_getcpu(),device->sampling_rate_ratio_n,device->sampling_rate_ratio_d); while (fhstate->active > 0) { ssize_t count = recvfrom(((eth_state_t*)device->priv)->sockfdd[0], buffer,sizeof(buffer),0, @@ -500,23 +506,23 @@ void *udp_read_thread(void *arg) { int trx_eth_read_udp(openair0_device *device, openair0_timestamp *timestamp, uint32_t **buff, int nsamps) { fhstate_t *fhstate = &device->fhstate; - openair0_timestamp prev_read_TS= fhstate->TS_read, min_TS; + openair0_timestamp prev_read_TS= fhstate->TS_read; + volatile openair0_timestamp min_TS; // block until FH is ready while (fhstate->r[0] == 0 || fhstate->r[1] == 0 || fhstate->r[2] == 0 || fhstate->r[3] == 0 || fhstate->r[4] == 0 || fhstate->r[5] == 0 || fhstate->r[6] == 0 || fhstate->r[7] == 0) usleep(100); // get minimum TS over all antennas - min_TS = fhstate->TS[0]; + min_TS = (volatile openair0_timestamp)fhstate->TS[0]; for (int i=1;iopenair0_cfg->rx_num_channels;i++) min_TS = min(min_TS,fhstate->TS[i]); // poll/sleep until we accumulated enough samples on each antenna port int count=0; while (fhstate->first_read == 1 && min_TS < (fhstate->TS0+prev_read_TS + nsamps)) { - usleep(50); - min_TS = fhstate->TS[0]; - for (int i=1;iopenair0_cfg->rx_num_channels;i++) min_TS = min(min_TS,fhstate->TS[i]); + usleep(10); + min_TS = (volatile openair0_timestamp)fhstate->TS[0]; + for (int i=1;iopenair0_cfg->rx_num_channels;i++) min_TS = min(min_TS,(volatile openair0_timestamp)fhstate->TS[i]); count++; } - if (fhstate->first_read == 0) { *timestamp = 0; fhstate->TS_read = *timestamp+nsamps; From 80d84d3271b272e1632315fa7eaea0c3e1e8373e Mon Sep 17 00:00:00 2001 From: Raymond Knopp Date: Mon, 27 Mar 2023 16:15:47 +0200 Subject: [PATCH 5/6] compilation after merge conflicts --- executables/softmodem-common.h | 2 +- openair2/ENB_APP/enb_paramdef.h | 2 +- openair2/GNB_APP/L1_nr_paramdef.h | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/executables/softmodem-common.h b/executables/softmodem-common.h index 3044acf851..2499a31ac1 100644 --- a/executables/softmodem-common.h +++ b/executables/softmodem-common.h @@ -141,7 +141,7 @@ extern int usrp_tx_thread; #define CMDLINE_PARAMS_DESC { \ {"rf-config-file", CONFIG_HLP_RFCFGF, 0, .strptr=&RF_CONFIG_FILE, .defstrval=NULL, TYPE_STRING, 0}, \ {"thread-pool", CONFIG_HLP_TPOOL, 0, .strptr=&TP_CONFIG, .defstrval="-1,-1,-1,-1,-1,-1,-1,-1", TYPE_STRING, 0}, \ - {"reorder-thread-disable",CONFIG_HLP_REORDER, PARAMFLAG_BOOL, .iptr:&REORDER_THREAD_DISABLE, .defintval:0, TYPE_INT, 0}, \ + {"reorder-thread-disable",CONFIG_HLP_REORDER, PARAMFLAG_BOOL, .iptr=&REORDER_THREAD_DISABLE, .defintval=0, TYPE_INT, 0}, \ {"phy-test", CONFIG_HLP_PHYTST, PARAMFLAG_BOOL, .iptr=&PHY_TEST, .defintval=0, TYPE_INT, 0}, \ {"do-ra", CONFIG_HLP_DORA, PARAMFLAG_BOOL, .iptr=&DO_RA, .defintval=0, TYPE_INT, 0}, \ {"sa", CONFIG_HLP_SA, PARAMFLAG_BOOL, .iptr=&SA, .defintval=0, TYPE_INT, 0}, \ diff --git a/openair2/ENB_APP/enb_paramdef.h b/openair2/ENB_APP/enb_paramdef.h index 2f6bf04f58..e37c5388d0 100644 --- a/openair2/ENB_APP/enb_paramdef.h +++ b/openair2/ENB_APP/enb_paramdef.h @@ -214,7 +214,7 @@ typedef enum { {CONFIG_STRING_RU_NUM_TP_CORES, HLP_RU_NUM_TP_CORES, 0, .uptr=NULL, .defintval=2, TYPE_UINT, 0}, \ {CONFIG_STRING_RU_NUM_INTERFACES, HLP_RU_NUM_INTERFACES, 0, .uptr=NULL, .defintval=1, TYPE_UINT, 0}, \ {CONFIG_STRING_RU_HALF_SLOT_PARALLELIZATION, HLP_RU_HALF_SLOT_PARALLELIZATION, 0, .uptr=NULL, .defintval=1, TYPE_UINT, 0}, \ - {CONFIG_STRING_RU_RU_THREAD_CORE, HLP_RU_RU_THREAD_CORE, 0, .uptr:NULL, .defintval:-1, TYPE_UINT, 0}, \ + {CONFIG_STRING_RU_RU_THREAD_CORE, HLP_RU_RU_THREAD_CORE, 0, .uptr=NULL, .defintval=-1, TYPE_UINT, 0}, \ } // clang-format on diff --git a/openair2/GNB_APP/L1_nr_paramdef.h b/openair2/GNB_APP/L1_nr_paramdef.h index dec87bf985..8c4381d8bd 100644 --- a/openair2/GNB_APP/L1_nr_paramdef.h +++ b/openair2/GNB_APP/L1_nr_paramdef.h @@ -78,8 +78,8 @@ {CONFIG_STRING_L1_PUSCH_DTX_THRESHOLD, NULL, 0, .uptr=NULL, .defintval=50, TYPE_UINT, 0}, \ {CONFIG_STRING_L1_SRS_DTX_THRESHOLD, NULL, 0, .uptr=NULL, .defintval=50, TYPE_UINT, 0}, \ {CONFIG_STRING_L1_MAX_LDPC_ITERATIONS, NULL, 0, .uptr=NULL, .defintval=5, TYPE_UINT, 0}, \ - {CONFIG_STRING_L1_RX_THREAD_CORE, NULL, 0, .uptr:NULL, .defintval:-1, TYPE_UINT, 0}, \ - {CONFIG_STRING_L1_TX_THREAD_CORE, NULL, 0, .uptr:NULL, .defintval:-1, TYPE_UINT, 0}, \ + {CONFIG_STRING_L1_RX_THREAD_CORE, NULL, 0, .uptr=NULL, .defintval=-1, TYPE_UINT, 0}, \ + {CONFIG_STRING_L1_TX_THREAD_CORE, NULL, 0, .uptr=NULL, .defintval=-1, TYPE_UINT, 0}, \ } // clang-format on #define L1_CC_IDX 0 From 2a6b18863504393340362c92718971604ce169c1 Mon Sep 17 00:00:00 2001 From: Raymond Knopp Date: Thu, 27 Apr 2023 15:17:07 +0200 Subject: [PATCH 6/6] removed temporary "USE_MSGQ" definition. Now, with reorder_thread_disable we use the native message queue and L1_rx_thread instead of the thread-pool for the main L1 thread. There is a single TX thread (no slot parallelization) --- executables/nr-gnb.c | 59 +++++++++++------------ executables/nr-ru.c | 24 +++++---- openair1/SCHED_NR/fapi_nr_l1.c | 27 ++++------- openair1/SCHED_NR/phy_procedures_nr_gNB.c | 11 ++--- 4 files changed, 52 insertions(+), 69 deletions(-) diff --git a/executables/nr-gnb.c b/executables/nr-gnb.c index d2cd85be59..7161125f89 100644 --- a/executables/nr-gnb.c +++ b/executables/nr-gnb.c @@ -88,7 +88,6 @@ #include #include -#define USE_MSGQ 1 //#define USRP_DEBUG 1 // Fix per CC openair rf/if device update // extern openair0_device openair0; @@ -264,24 +263,22 @@ void rx_func(void *param) processingData_L1tx_t *syncMsg; // Its a FIFO so it maitains the order in which the MAC fills the messages // so no need for checking for right slot - #ifndef USE_MSGQ - if (get_softmodem_params()->reorder_thread_disable) - res = pullTpool(&gNB->L1_tx_out, &gNB->threadPool); - else + if (get_softmodem_params()->reorder_thread_disable) { + // call the TX function directly from this thread + syncMsg = gNB->msgDataTx; + syncMsg->gNB = gNB; + syncMsg->timestamp_tx = info->timestamp_tx; + tx_func(syncMsg); + } else { res = pullTpool(&gNB->L1_tx_filled, &gNB->threadPool); - if (res == NULL) - return; // Tpool has been stopped - syncMsg = (processingData_L1tx_t *)NotifiedFifoData(res); - syncMsg->gNB = gNB; - syncMsg->timestamp_tx = info->timestamp_tx; - res->key = slot_tx; - pushTpool(&gNB->threadPool, res); -#else - syncMsg = gNB->msgDataTx; - syncMsg->gNB = gNB; - syncMsg->timestamp_tx = info->timestamp_tx; - tx_func(syncMsg); -#endif + if (res == NULL) + return; // Tpool has been stopped + syncMsg = (processingData_L1tx_t *)NotifiedFifoData(res); + syncMsg->gNB = gNB; + syncMsg->timestamp_tx = info->timestamp_tx; + res->key = slot_tx; + pushTpool(&gNB->threadPool, res); + } } else if (get_softmodem_params()->continuous_tx) { notifiedFIFO_elt_t *res = pullTpool(&gNB->L1_tx_free, &gNB->threadPool); if (res == NULL) @@ -476,32 +473,30 @@ void init_gNB_Tpool(int inst) { // L1 RX result FIFO initNotifiedFIFO(&gNB->resp_L1); -#ifndef USE_MSGQ - notifiedFIFO_elt_t *msg = newNotifiedFIFO_elt(sizeof(processingData_L1_t), 0, &gNB->resp_L1, rx_func); - pushNotifiedFIFO(&gNB->resp_L1, msg); // to unblock the process in the beginning -#endif + if (!get_softmodem_params()->reorder_thread_disable) { + notifiedFIFO_elt_t *msg = newNotifiedFIFO_elt(sizeof(processingData_L1_t), 0, &gNB->resp_L1, rx_func); + pushNotifiedFIFO(&gNB->resp_L1, msg); // to unblock the process in the beginning + } // L1 TX result FIFO initNotifiedFIFO(&gNB->L1_tx_free); initNotifiedFIFO(&gNB->L1_tx_filled); initNotifiedFIFO(&gNB->L1_tx_out); -#ifdef USE_MSGQ - threadCreate(&gNB->L1_rx_thread, L1_rx_thread, (void *)gNB, "L1_rx_thread", - gNB->L1_rx_thread_core, OAI_PRIORITY_RT_MAX); -// threadCreate(&gNB->L1_tx_thread, L1_tx_thread, (void *)gNB, "L1_tx_thread", -// gNB->L1_tx_thread_core, OAI_PRIORITY_RT_MAX); -#endif if (get_softmodem_params()->reorder_thread_disable) { + // create the RX thread responsible for triggering RX processing and then TX processing if a single thread is used + threadCreate(&gNB->L1_rx_thread, L1_rx_thread, (void *)gNB, "L1_rx_thread", + gNB->L1_rx_thread_core, OAI_PRIORITY_RT_MAX); + // if separate threads are used for RX and TX, create the TX thread + // threadCreate(&gNB->L1_tx_thread, L1_tx_thread, (void *)gNB, "L1_tx_thread", + // gNB->L1_tx_thread_core, OAI_PRIORITY_RT_MAX); + notifiedFIFO_elt_t *msgL1Tx = newNotifiedFIFO_elt(sizeof(processingData_L1tx_t), 0, &gNB->L1_tx_out, tx_func); processingData_L1tx_t *msgDataTx = (processingData_L1tx_t *)NotifiedFifoData(msgL1Tx); memset(msgDataTx, 0, sizeof(processingData_L1tx_t)); init_DLSCH_struct(gNB, msgDataTx); memset(msgDataTx->ssb, 0, 64*sizeof(NR_gNB_SSB_t)); -#ifndef USE_MSGQ - pushNotifiedFIFO(&gNB->L1_tx_out, msgL1Tx); // to unblock the process in the beginning -#else + // this will be removed when the msgDataTx is not necessary anymore gNB->msgDataTx = msgDataTx; -#endif } else { // we create 2 threads for L1 tx processing for (int i=0; i < 2; i++) { diff --git a/executables/nr-ru.c b/executables/nr-ru.c index 4017c8e9ac..01a5e7fea8 100644 --- a/executables/nr-ru.c +++ b/executables/nr-ru.c @@ -60,7 +60,6 @@ unsigned short config_frames[4] = {2,9,11,13}; #endif -#define USE_MSGQ 1 /* these variables have to be defined before including ENB_APP/enb_paramdef.h and GNB_APP/gnb_paramdef.h */ static int DEFBANDS[] = {7}; @@ -1285,13 +1284,13 @@ void *ru_thread( void *param ) { } // end if (slot_type == NR_UPLINK_SLOT || slot_type == NR_MIXED_SLOT) { // At this point, all information for subframe has been received on FH interface -#ifndef USE_MSGQ - res = pullTpool(&gNB->resp_L1, &gNB->threadPool); - if (res == NULL) - break; // Tpool has been stopped -#else - res=newNotifiedFIFO_elt(sizeof(processingData_L1_t),0, &gNB->resp_L1,NULL); -#endif + if (!get_softmodem_params()->reorder_thread_disable) { + res = pullTpool(&gNB->resp_L1, &gNB->threadPool); + if (res == NULL) + break; // Tpool has been stopped + } else { + res=newNotifiedFIFO_elt(sizeof(processingData_L1_t),0, &gNB->resp_L1,NULL); + } syncMsg = (processingData_L1_t *)NotifiedFifoData(res); syncMsg->gNB = gNB; syncMsg->frame_rx = proc->frame_rx; @@ -1300,11 +1299,10 @@ void *ru_thread( void *param ) { syncMsg->slot_tx = proc->tti_tx; syncMsg->timestamp_tx = proc->timestamp_tx; res->key = proc->tti_rx; -#ifndef USE_MSGQ - pushTpool(&gNB->threadPool, res); -#else - pushNotifiedFIFO(&gNB->resp_L1, res); -#endif + if (!get_softmodem_params()->reorder_thread_disable) + pushTpool(&gNB->threadPool, res); + else + pushNotifiedFIFO(&gNB->resp_L1, res); } printf( "Exiting ru_thread \n"); diff --git a/openair1/SCHED_NR/fapi_nr_l1.c b/openair1/SCHED_NR/fapi_nr_l1.c index d72eb208c3..9662780737 100644 --- a/openair1/SCHED_NR/fapi_nr_l1.c +++ b/openair1/SCHED_NR/fapi_nr_l1.c @@ -37,7 +37,6 @@ #include "nfapi/oai_integration/vendor_ext.h" #include "openair2/NR_PHY_INTERFACE/nr_sched_response.h" -#define USE_MSGQ 1 extern int oai_nfapi_dl_tti_req(nfapi_nr_dl_tti_request_t *dl_config_req); extern int oai_nfapi_tx_data_req(nfapi_nr_tx_data_request_t *tx_data_req); @@ -159,18 +158,16 @@ void nr_schedule_response(NR_Sched_Rsp_t *Sched_INFO) if (NFAPI_MODE == NFAPI_MONOLITHIC){ if (slot_type == NR_DOWNLINK_SLOT || slot_type == NR_MIXED_SLOT) { - notifiedFIFO_elt_t *res; -#ifndef USE_MSGQ - if (gNB->reorder_thread_disable) - res = pullTpool(&gNB->L1_tx_out, &gNB->threadPool); - else + notifiedFIFO_elt_t *res=NULL; + processingData_L1tx_t *msgTx=NULL; + if (!gNB->reorder_thread_disable) { res = pullTpool(&gNB->L1_tx_free, &gNB->threadPool); - if (res == NULL) - return; // Tpool has been stopped, nothing to process - processingData_L1tx_t *msgTx = (processingData_L1tx_t *)NotifiedFifoData(res); -#else - processingData_L1tx_t *msgTx = gNB->msgDataTx; //newNotifiedFIFO_elt(sizeof(processingData_L1tx_t),0, &gNB->L1_tx_out,NULL); -#endif + if (res == NULL) + return; // Tpool has been stopped, nothing to process + msgTx = (processingData_L1tx_t *)NotifiedFifoData(res); + } else { + msgTx = gNB->msgDataTx; //newNotifiedFIFO_elt(sizeof(processingData_L1tx_t),0, &gNB->L1_tx_out,NULL); + } /*const time_stats_t ts = exec_time_stats_NotifiedFIFO(res); merge_meas(&gNB->phy_proc_tx, &ts); */ @@ -225,12 +222,8 @@ void nr_schedule_response(NR_Sched_Rsp_t *Sched_INFO) * released only when both threads are done with it. */ inc_ref_sched_response(Sched_INFO->sched_response_id); -#ifndef USE_MSGQ - if (gNB->reorder_thread_disable) - pushNotifiedFIFO(&gNB->L1_tx_out,res); - else + if (!gNB->reorder_thread_disable) pushNotifiedFIFO(&gNB->L1_tx_filled,res); -#endif } for (int i = 0; i < number_ul_tti_pdu; i++) { diff --git a/openair1/SCHED_NR/phy_procedures_nr_gNB.c b/openair1/SCHED_NR/phy_procedures_nr_gNB.c index 566da85fb8..e24e17920b 100644 --- a/openair1/SCHED_NR/phy_procedures_nr_gNB.c +++ b/openair1/SCHED_NR/phy_procedures_nr_gNB.c @@ -395,13 +395,11 @@ void nr_ulsch_procedures(PHY_VARS_gNB *gNB, int frame_rx, int slot_rx, int ULSCH stop_meas(&gNB->ulsch_decoding_stats); } -#define USE_MSGQ 1 void nr_fill_indication(PHY_VARS_gNB *gNB, int frame, int slot_rx, int ULSCH_id, uint8_t harq_pid, uint8_t crc_flag, int dtx_flag) { -#ifndef USE_MSGQ - pthread_mutex_lock(&gNB->UL_INFO_mutex); -#endif + if (!get_softmodem_params()->reorder_thread_disable) + pthread_mutex_lock(&gNB->UL_INFO_mutex); NR_gNB_ULSCH_t *ulsch = &gNB->ulsch[ULSCH_id]; NR_UL_gNB_HARQ_t *harq_process = ulsch->harq_process; @@ -560,9 +558,8 @@ void nr_fill_indication(PHY_VARS_gNB *gNB, int frame, int slot_rx, int ULSCH_id, gNB->UL_INFO.rx_ind.number_of_pdus++; -#ifndef USE_MSGQ - pthread_mutex_unlock(&gNB->UL_INFO_mutex); -#endif + if (!get_softmodem_params()->reorder_thread_disable) + pthread_mutex_unlock(&gNB->UL_INFO_mutex); } // Function to fill UL RB mask to be used for N0 measurements