diff --git a/common/utils/system.c b/common/utils/system.c index 9d52639b2f..5b30b84f53 100644 --- a/common/utils/system.c +++ b/common/utils/system.c @@ -231,7 +231,9 @@ void threadCreate(pthread_t* t, void * (*func)(void*), void * param, char* name, int settingPriority = 1; ret=pthread_attr_init(&attr); AssertFatal(ret==0,"ret: %d, errno: %d\n",ret, errno); - + + LOG_I(UTIL,"Creating thread %s with affinity %d and priority %d\n",name,affinity,priority); + if (checkIfFedoraDistribution()) if (checkIfGenericKernelOnFedora()) if (checkIfInsideContainer()) diff --git a/executables/nr-gnb.c b/executables/nr-gnb.c index 773f89416c..7161125f89 100644 --- a/executables/nr-gnb.c +++ b/executables/nr-gnb.c @@ -110,7 +110,8 @@ time_stats_t softmodem_stats_rx_sf; // total rx time #define L1STATSSTRLEN 16384 -void tx_func(void *param) { +void tx_func(void *param) +{ processingData_L1tx_t *info = (processingData_L1tx_t *) param; int frame_tx = info->frame; @@ -119,6 +120,7 @@ void tx_func(void *param) { int absslot_tx = info->timestamp_tx/info->gNB->frame_parms.get_samples_per_slot(slot_tx,&info->gNB->frame_parms); int absslot_rx = absslot_tx-info->gNB->RU_list[0]->sl_ahead; int rt_prof_idx = absslot_rx % RT_PROF_DEPTH; + start_meas(&info->gNB->phy_proc_tx); clock_gettime(CLOCK_MONOTONIC,&info->gNB->rt_L1_profiling.start_L1_TX[rt_prof_idx]); phy_procedures_gNB_TX(info, @@ -127,10 +129,47 @@ void tx_func(void *param) { 1); clock_gettime(CLOCK_MONOTONIC,&info->gNB->rt_L1_profiling.return_L1_TX[rt_prof_idx]); + if (get_softmodem_params()->reorder_thread_disable) { + PHY_VARS_gNB *gNB = info->gNB; + processingData_RU_t syncMsgRU; + syncMsgRU.frame_tx = frame_tx; + syncMsgRU.slot_tx = slot_tx; + syncMsgRU.ru = gNB->RU_list[0]; + syncMsgRU.timestamp_tx = info->timestamp_tx; + LOG_D(PHY,"gNB: %d.%d : calling RU TX function\n",syncMsgRU.frame_tx,syncMsgRU.slot_tx); + ru_tx_func((void*)&syncMsgRU); + } /* this thread is done with the sched_info, decrease the reference counter */ deref_sched_response(info->sched_response_id); + stop_meas(&info->gNB->phy_proc_tx); } + +void *L1_rx_thread(void *arg) +{ + PHY_VARS_gNB *gNB = (PHY_VARS_gNB*)arg; + + while (oai_exit == 0) { + notifiedFIFO_elt_t *res = pullNotifiedFIFO(&gNB->resp_L1); + processingData_L1_t *info = (processingData_L1_t *)NotifiedFifoData(res); + rx_func(info); + delNotifiedFIFO_elt(res); + } + return NULL; +} +/* to be added for URLLC, requires MAC scheduling to be split from UL indication +void *L1_tx_thread(void *arg) { + PHY_VARS_gNB *gNB = (PHY_VARS_gNB*)arg; + + while (oai_exit == 0) { + notifiedFIFO_elt_t *res = pullNotifiedFIFO(&gNB->L1_tx_out); + processingData_L1tx_t *info = (processingData_L1tx_t *)NotifiedFifoData(res); + tx_func(info); + delNotifiedFIFO_elt(res); + } + return NULL; +} +*/ void rx_func(void *param) { processingData_L1_t *info = (processingData_L1_t *) param; @@ -189,13 +228,16 @@ void rx_func(void *param) // Do PRACH RU processing L1_nr_prach_procedures(gNB,frame_rx,slot_rx); - //apply the rx signal rotation here - for (int aa = 0; aa < gNB->frame_parms.nb_antennas_rx; aa++) { - apply_nr_rotation_ul(&gNB->frame_parms, - gNB->common_vars.rxdataF[aa], - slot_rx, - 0, - gNB->frame_parms.Ncp==EXTENDED?12:14); + //WA: comment rotation in tx/rx + if((gNB->num_RU == 1) && (gNB->RU_list[0]->if_south != REMOTE_IF4p5)) { + //apply the rx signal rotation here + for (int aa = 0; aa < gNB->frame_parms.nb_antennas_rx; aa++) { + apply_nr_rotation_ul(&gNB->frame_parms, + gNB->common_vars.rxdataF[aa], + slot_rx, + 0, + gNB->frame_parms.Ncp==EXTENDED?12:14); + } } phy_procedures_gNB_uespec_RX(gNB, frame_rx, slot_rx); } @@ -206,29 +248,37 @@ void rx_func(void *param) // Call the scheduler start_meas(&gNB->ul_indication_stats); - pthread_mutex_lock(&gNB->UL_INFO_mutex); +// pthread_mutex_lock(&gNB->UL_INFO_mutex); gNB->UL_INFO.frame = frame_rx; gNB->UL_INFO.slot = slot_rx; gNB->UL_INFO.module_id = gNB->Mod_id; gNB->UL_INFO.CC_id = gNB->CC_id; gNB->if_inst->NR_UL_indication(&gNB->UL_INFO); - pthread_mutex_unlock(&gNB->UL_INFO_mutex); +// pthread_mutex_unlock(&gNB->UL_INFO_mutex); stop_meas(&gNB->ul_indication_stats); - int tx_slot_type = nr_slot_select(cfg,frame_rx,slot_tx); + int tx_slot_type = nr_slot_select(cfg,frame_tx,slot_tx); if ((tx_slot_type == NR_DOWNLINK_SLOT || tx_slot_type == NR_MIXED_SLOT) && NFAPI_MODE != NFAPI_MODE_PNF) { notifiedFIFO_elt_t *res; processingData_L1tx_t *syncMsg; // Its a FIFO so it maitains the order in which the MAC fills the messages // so no need for checking for right slot - res = pullTpool(&gNB->L1_tx_filled, &gNB->threadPool); - if (res == NULL) - return; // Tpool has been stopped - syncMsg = (processingData_L1tx_t *)NotifiedFifoData(res); - syncMsg->gNB = gNB; - syncMsg->timestamp_tx = info->timestamp_tx; - res->key = slot_tx; - pushTpool(&gNB->threadPool, res); + if (get_softmodem_params()->reorder_thread_disable) { + // call the TX function directly from this thread + syncMsg = gNB->msgDataTx; + syncMsg->gNB = gNB; + syncMsg->timestamp_tx = info->timestamp_tx; + tx_func(syncMsg); + } else { + res = pullTpool(&gNB->L1_tx_filled, &gNB->threadPool); + if (res == NULL) + return; // Tpool has been stopped + syncMsg = (processingData_L1tx_t *)NotifiedFifoData(res); + syncMsg->gNB = gNB; + syncMsg->timestamp_tx = info->timestamp_tx; + res->key = slot_tx; + pushTpool(&gNB->threadPool, res); + } } else if (get_softmodem_params()->continuous_tx) { notifiedFIFO_elt_t *res = pullTpool(&gNB->L1_tx_free, &gNB->threadPool); if (res == NULL) @@ -287,6 +337,11 @@ static size_t dump_L1_meas_stats(PHY_VARS_gNB *gNB, RU_t *ru, char *output, size const char *end = output + outputlen; output += print_meas_log(&gNB->phy_proc_tx, "L1 Tx processing", NULL, NULL, output, end - output); output += print_meas_log(&gNB->dlsch_encoding_stats, "DLSCH encoding", NULL, NULL, output, end - output); + output += print_meas_log(&gNB->dlsch_scrambling_stats, "DLSCH scrambling", NULL, NULL, output, end-output); + output += print_meas_log(&gNB->dlsch_modulation_stats, "DLSCH modulation", NULL, NULL, output, end-output); + output += print_meas_log(&gNB->dlsch_layer_mapping_stats, "DLSCH layer mapping", NULL, NULL, output,end-output); + output += print_meas_log(&gNB->dlsch_resource_mapping_stats, "DLSCH resource mapping", NULL, NULL, output,end-output); + output += print_meas_log(&gNB->dlsch_precoding_stats, "DLSCH precoding", NULL, NULL, output,end-output); output += print_meas_log(&gNB->phy_proc_rx, "L1 Rx processing", NULL, NULL, output, end - output); output += print_meas_log(&gNB->ul_indication_stats, "UL Indication", NULL, NULL, output, end - output); output += print_meas_log(&gNB->rx_pusch_stats, "PUSCH inner-receiver", NULL, NULL, output, end - output); @@ -331,7 +386,11 @@ void *nrL1_stats_thread(void *param) { reset_meas(&gNB->rx_pusch_stats); reset_meas(&gNB->ulsch_decoding_stats); reset_meas(&gNB->schedule_response_stats); - + reset_meas(&gNB->dlsch_scrambling_stats); + reset_meas(&gNB->dlsch_modulation_stats); + reset_meas(&gNB->dlsch_layer_mapping_stats); + reset_meas(&gNB->dlsch_resource_mapping_stats); + reset_meas(&gNB->dlsch_precoding_stats); while (!oai_exit) { sleep(1); dump_nr_I0_stats(fd,gNB); @@ -414,31 +473,49 @@ void init_gNB_Tpool(int inst) { // L1 RX result FIFO initNotifiedFIFO(&gNB->resp_L1); - notifiedFIFO_elt_t *msg = newNotifiedFIFO_elt(sizeof(processingData_L1_t), 0, &gNB->resp_L1, rx_func); - pushNotifiedFIFO(&gNB->resp_L1, msg); // to unblock the process in the beginning - + if (!get_softmodem_params()->reorder_thread_disable) { + notifiedFIFO_elt_t *msg = newNotifiedFIFO_elt(sizeof(processingData_L1_t), 0, &gNB->resp_L1, rx_func); + pushNotifiedFIFO(&gNB->resp_L1, msg); // to unblock the process in the beginning + } // L1 TX result FIFO initNotifiedFIFO(&gNB->L1_tx_free); initNotifiedFIFO(&gNB->L1_tx_filled); initNotifiedFIFO(&gNB->L1_tx_out); - - // we create 2 threads for L1 tx processing - for (int i=0; i < 2; i++) { + + if (get_softmodem_params()->reorder_thread_disable) { + // create the RX thread responsible for triggering RX processing and then TX processing if a single thread is used + threadCreate(&gNB->L1_rx_thread, L1_rx_thread, (void *)gNB, "L1_rx_thread", + gNB->L1_rx_thread_core, OAI_PRIORITY_RT_MAX); + // if separate threads are used for RX and TX, create the TX thread + // threadCreate(&gNB->L1_tx_thread, L1_tx_thread, (void *)gNB, "L1_tx_thread", + // gNB->L1_tx_thread_core, OAI_PRIORITY_RT_MAX); + notifiedFIFO_elt_t *msgL1Tx = newNotifiedFIFO_elt(sizeof(processingData_L1tx_t), 0, &gNB->L1_tx_out, tx_func); processingData_L1tx_t *msgDataTx = (processingData_L1tx_t *)NotifiedFifoData(msgL1Tx); memset(msgDataTx, 0, sizeof(processingData_L1tx_t)); init_DLSCH_struct(gNB, msgDataTx); memset(msgDataTx->ssb, 0, 64*sizeof(NR_gNB_SSB_t)); - pushNotifiedFIFO(&gNB->L1_tx_free, msgL1Tx); // to unblock the process in the beginning + // this will be removed when the msgDataTx is not necessary anymore + gNB->msgDataTx = msgDataTx; + } else { + // we create 2 threads for L1 tx processing + for (int i=0; i < 2; i++) { + notifiedFIFO_elt_t *msgL1Tx = newNotifiedFIFO_elt(sizeof(processingData_L1tx_t), 0, &gNB->L1_tx_out, tx_func); + processingData_L1tx_t *msgDataTx = (processingData_L1tx_t *)NotifiedFifoData(msgL1Tx); + memset(msgDataTx, 0, sizeof(processingData_L1tx_t)); + init_DLSCH_struct(gNB, msgDataTx); + memset(msgDataTx->ssb, 0, 64*sizeof(NR_gNB_SSB_t)); + pushNotifiedFIFO(&gNB->L1_tx_free, msgL1Tx); // to unblock the process in the beginning + } + + LOG_I(PHY,"Creating thread for TX reordering and dispatching to RU\n"); + threadCreate(&proc->pthread_tx_reorder, tx_reorder_thread, (void *)gNB, "thread_tx_reorder", + gNB->RU_list[0] ? gNB->RU_list[0]->tpcores[1] : -1, OAI_PRIORITY_RT_MAX); } if ((!get_softmodem_params()->emulate_l1) && (!IS_SOFTMODEM_NOSTATS_BIT) && (NFAPI_MODE!=NFAPI_MODE_VNF)) threadCreate(&proc->L1_stats_thread,nrL1_stats_thread,(void*)gNB,"L1_stats",-1,OAI_PRIORITY_RT_LOW); - LOG_I(PHY,"Creating thread for TX reordering and dispatching to RU\n"); - threadCreate(&proc->pthread_tx_reorder, tx_reorder_thread, (void *)gNB, "thread_tx_reorder", - gNB->RU_list[0] ? gNB->RU_list[0]->tpcores[1] : -1, OAI_PRIORITY_RT_MAX); - } @@ -507,6 +584,7 @@ void init_eNB_afterRU(void) { gNB = RC.gNB[inst]; gNB->ldpc_offload_flag = ldpc_offload_flag; + gNB->reorder_thread_disable = get_softmodem_params()->reorder_thread_disable; phy_init_nr_gNB(gNB); @@ -585,6 +663,7 @@ void init_gNB(int single_thread_flag,int wait_for_sync) { gNB->prach_energy_counter = 0; gNB->chest_time = get_softmodem_params()->chest_time; gNB->chest_freq = get_softmodem_params()->chest_freq; + } diff --git a/executables/nr-ru.c b/executables/nr-ru.c index 6a2b6e7e9f..01a5e7fea8 100644 --- a/executables/nr-ru.c +++ b/executables/nr-ru.c @@ -60,6 +60,7 @@ unsigned short config_frames[4] = {2,9,11,13}; #endif + /* these variables have to be defined before including ENB_APP/enb_paramdef.h and GNB_APP/gnb_paramdef.h */ static int DEFBANDS[] = {7}; static int DEFENBS[] = {0}; @@ -83,6 +84,8 @@ static int DEFRUTPCORES[] = {-1,-1,-1,-1}; extern int oai_exit; +uint16_t sl_ahead; + extern struct timespec timespec_sub(struct timespec lhs, struct timespec rhs); extern struct timespec timespec_add(struct timespec lhs, struct timespec rhs); extern void nr_phy_free_RU(RU_t *); @@ -1085,7 +1088,7 @@ void *ru_thread( void *param ) { ru_thread_status = 0; // set default return value sprintf(threadname,"ru_thread %u",ru->idx); - LOG_I(PHY,"Starting RU %d (%s,%s),\n",ru->idx,NB_functions[ru->function],NB_timing[ru->if_timing]); + LOG_I(PHY,"Starting RU %d (%s,%s) on cpu %d\n",ru->idx,NB_functions[ru->function],NB_timing[ru->if_timing],sched_getcpu()); memcpy((void *)&ru->config,(void *)&RC.gNB[0]->gNB_config,sizeof(ru->config)); if(emulate_rf) { @@ -1281,9 +1284,13 @@ void *ru_thread( void *param ) { } // end if (slot_type == NR_UPLINK_SLOT || slot_type == NR_MIXED_SLOT) { // At this point, all information for subframe has been received on FH interface - res = pullTpool(&gNB->resp_L1, &gNB->threadPool); - if (res == NULL) - break; // Tpool has been stopped + if (!get_softmodem_params()->reorder_thread_disable) { + res = pullTpool(&gNB->resp_L1, &gNB->threadPool); + if (res == NULL) + break; // Tpool has been stopped + } else { + res=newNotifiedFIFO_elt(sizeof(processingData_L1_t),0, &gNB->resp_L1,NULL); + } syncMsg = (processingData_L1_t *)NotifiedFifoData(res); syncMsg->gNB = gNB; syncMsg->frame_rx = proc->frame_rx; @@ -1292,7 +1299,10 @@ void *ru_thread( void *param ) { syncMsg->slot_tx = proc->tti_tx; syncMsg->timestamp_tx = proc->timestamp_tx; res->key = proc->tti_rx; - pushTpool(&gNB->threadPool, res); + if (!get_softmodem_params()->reorder_thread_disable) + pushTpool(&gNB->threadPool, res); + else + pushNotifiedFIFO(&gNB->resp_L1, res); } printf( "Exiting ru_thread \n"); @@ -1307,7 +1317,8 @@ int start_streaming(RU_t *ru) { } int nr_start_if(struct RU_t_s *ru, struct PHY_VARS_gNB_s *gNB) { - for (int i=0;inb_rx;i++) ru->openair0_cfg.rxbase[i] = ru->common.rxdata[i]; + if (ru->if_south <= REMOTE_IF5) + for (int i=0;inb_rx;i++) ru->openair0_cfg.rxbase[i] = ru->common.rxdata[i]; ru->openair0_cfg.rxsize = ru->nr_frame_parms->samples_per_subframe*10; reset_meas(&ru->ifdevice.tx_fhaul); return(ru->ifdevice.trx_start_func(&ru->ifdevice)); @@ -1344,7 +1355,7 @@ void init_RU_proc(RU_t *ru) { pthread_mutex_init( &proc->mutex_emulateRF,NULL); pthread_cond_init( &proc->cond_emulateRF, NULL); - threadCreate( &proc->pthread_FH, ru_thread, (void *)ru, "ru_thread", ru->tpcores[0], OAI_PRIORITY_RT_MAX ); + threadCreate( &proc->pthread_FH, ru_thread, (void *)ru, "ru_thread", ru->ru_thread_core, OAI_PRIORITY_RT_MAX ); if(emulate_rf) threadCreate( &proc->pthread_emulateRF, emulatedRF_thread, (void *)proc, "emulateRF", -1, OAI_PRIORITY_RT ); @@ -1670,7 +1681,7 @@ void set_function_spec_param(RU_t *ru) { case REMOTE_IF4p5: ru->do_prach = 0; ru->feprx = NULL; // DFTs - ru->feptx_prec = NULL; // Precoding operation + ru->feptx_prec = nr_feptx_prec; // Precoding operation ru->feptx_ofdm = NULL; // no OFDM mod ru->fh_south_in = fh_if4p5_south_in; // synchronous IF4p5 reception ru->fh_south_out = fh_if4p5_south_out; // synchronous IF4p5 transmission @@ -1782,23 +1793,25 @@ void init_NR_RU(char *rf_config_file) { set_function_spec_param(ru); LOG_I(PHY,"Starting ru_thread %d\n",ru_id); init_RU_proc(ru); - int threadCnt = ru->num_tpcores; - if (threadCnt < 2) LOG_E(PHY,"Number of threads for gNB should be more than 1. Allocated only %d\n",threadCnt); - else LOG_I(PHY,"RU Thread pool size %d\n",threadCnt); - char pool[80]; - int s_offset = sprintf(pool,"%d",ru->tpcores[0]); - for (int icpu=1; icputpcores[icpu]); + if (ru->if_south != REMOTE_IF4p5) { + int threadCnt = ru->num_tpcores; + if (threadCnt < 2) LOG_E(PHY,"Number of threads for gNB should be more than 1. Allocated only %d\n",threadCnt); + else LOG_I(PHY,"RU Thread pool size %d\n",threadCnt); + char pool[80]; + int s_offset = sprintf(pool,"%d",ru->tpcores[0]); + for (int icpu=1; icputpcores[icpu]); + } + LOG_I(PHY,"RU thread-pool core string %s\n",pool); + ru->threadPool = (tpool_t*)malloc(sizeof(tpool_t)); + initTpool(pool, ru->threadPool, cpumeas(CPUMEAS_GETSTATE)); + // FEP RX result FIFO + ru->respfeprx = (notifiedFIFO_t*) malloc(sizeof(notifiedFIFO_t)); + initNotifiedFIFO(ru->respfeprx); + // FEP TX result FIFO + ru->respfeptx = (notifiedFIFO_t*) malloc(sizeof(notifiedFIFO_t)); + initNotifiedFIFO(ru->respfeptx); } - LOG_I(PHY,"RU thread-pool core string %s\n",pool); - ru->threadPool = (tpool_t*)malloc(sizeof(tpool_t)); - initTpool(pool, ru->threadPool, cpumeas(CPUMEAS_GETSTATE)); - // FEP RX result FIFO - ru->respfeprx = (notifiedFIFO_t*) malloc(sizeof(notifiedFIFO_t)); - initNotifiedFIFO(ru->respfeprx); - // FEP TX result FIFO - ru->respfeptx = (notifiedFIFO_t*) malloc(sizeof(notifiedFIFO_t)); - initNotifiedFIFO(ru->respfeptx); } // for ru_id // sleep(1); @@ -1987,6 +2000,7 @@ static void NRRCconfig_RU(void) { RC.ru[j]->openair0_cfg.txfh_cores[0] = *(RUParamList.paramarray[j][RU_TXFH_CORE_ID].iptr); RC.ru[j]->num_tpcores = *(RUParamList.paramarray[j][RU_NUM_TP_CORES].iptr); RC.ru[j]->half_slot_parallelization = *(RUParamList.paramarray[j][RU_HALF_SLOT_PARALLELIZATION].iptr); + RC.ru[j]->ru_thread_core = *(RUParamList.paramarray[j][RU_RU_THREAD_CORE].iptr); printf("[RU %d] Setting half-slot parallelization to %d\n",j,RC.ru[j]->half_slot_parallelization); AssertFatal(RC.ru[j]->num_tpcores <= RUParamList.paramarray[j][RU_TP_CORES].numelt, "Number of TP cores should be <=16\n"); for (i=0; inum_tpcores; i++) RC.ru[j]->tpcores[i] = RUParamList.paramarray[j][RU_TP_CORES].iptr[i]; diff --git a/executables/softmodem-common.c b/executables/softmodem-common.c index f1d61d9032..25672f2280 100644 --- a/executables/softmodem-common.c +++ b/executables/softmodem-common.c @@ -46,6 +46,7 @@ static softmodem_params_t softmodem_params; char *parallel_config=NULL; char *worker_config=NULL; int usrp_tx_thread = 0; +char *nfapi_str=NULL; int ldpc_offload_flag=0; uint8_t nfapi_mode=0; diff --git a/executables/softmodem-common.h b/executables/softmodem-common.h index cf6ef56ea4..b8f7e6ff62 100644 --- a/executables/softmodem-common.h +++ b/executables/softmodem-common.h @@ -44,6 +44,7 @@ extern "C" example: -1,3 launches two working threads one floating, the second set on core 3\n\ default 8 floating threads\n\ use N for no pool (runs in calling thread) recommended with rfsim.\n" +#define CONFIG_HLP_REORDER "Disable reorder thread\n" #define CONFIG_HLP_ULMAXE "set the eNodeB max ULSCH erros\n" #define CONFIG_HLP_CALUER "set UE RX calibration\n" #define CONFIG_HLP_CALUERM "" @@ -110,6 +111,7 @@ extern "C" /*-----------------------------------------------------------------------------------------------------------------------------------------------------*/ #define RF_CONFIG_FILE softmodem_params.rf_config_file #define TP_CONFIG softmodem_params.threadPoolConfig +#define CONTINUOUS_TX softmodem_params.continuous_tx #define PHY_TEST softmodem_params.phy_test #define DO_RA softmodem_params.do_ra #define SA softmodem_params.sa @@ -132,7 +134,7 @@ extern "C" #define NON_STOP softmodem_params.non_stop #define EMULATE_L1 softmodem_params.emulate_l1 #define CONTINUOUS_TX softmodem_params.continuous_tx - +#define REORDER_THREAD_DISABLE softmodem_params.reorder_thread_disable #define DEFAULT_RFCONFIG_FILE "/usr/local/etc/syriq/ue.band7.tm1.PRB100.NR40.dat"; extern int usrp_tx_thread; @@ -140,6 +142,7 @@ extern int usrp_tx_thread; #define CMDLINE_PARAMS_DESC { \ {"rf-config-file", CONFIG_HLP_RFCFGF, 0, .strptr=&RF_CONFIG_FILE, .defstrval=NULL, TYPE_STRING, 0}, \ {"thread-pool", CONFIG_HLP_TPOOL, 0, .strptr=&TP_CONFIG, .defstrval="-1,-1,-1,-1,-1,-1,-1,-1", TYPE_STRING, 0}, \ + {"reorder-thread-disable",CONFIG_HLP_REORDER, PARAMFLAG_BOOL, .iptr=&REORDER_THREAD_DISABLE, .defintval=0, TYPE_INT, 0}, \ {"phy-test", CONFIG_HLP_PHYTST, PARAMFLAG_BOOL, .iptr=&PHY_TEST, .defintval=0, TYPE_INT, 0}, \ {"do-ra", CONFIG_HLP_DORA, PARAMFLAG_BOOL, .iptr=&DO_RA, .defintval=0, TYPE_INT, 0}, \ {"sa", CONFIG_HLP_SA, PARAMFLAG_BOOL, .iptr=&SA, .defintval=0, TYPE_INT, 0}, \ @@ -210,6 +213,7 @@ extern int usrp_tx_thread; { .s5 = { NULL } }, \ { .s5 = { NULL } }, \ { .s5 = { NULL } }, \ + { .s5 = { NULL } }, \ { .s3a = { config_checkstr_assign_integer, \ {"MONOLITHIC", "PNF", "VNF","UE_STUB_PNF","UE_STUB_OFFNET","STANDALONE_PNF"}, \ {NFAPI_MONOLITHIC, NFAPI_MODE_PNF, NFAPI_MODE_VNF,NFAPI_UE_STUB_PNF,NFAPI_UE_STUB_OFFNET,NFAPI_MODE_STANDALONE_PNF}, \ @@ -300,6 +304,7 @@ typedef struct { //THREAD_STRUCT thread_struct; char *rf_config_file; char *threadPoolConfig; + int reorder_thread_disable; int phy_test; int do_ra; int sa; diff --git a/openair1/PHY/NR_TRANSPORT/nr_dlsch.c b/openair1/PHY/NR_TRANSPORT/nr_dlsch.c index 93dad56823..ea3c84be57 100644 --- a/openair1/PHY/NR_TRANSPORT/nr_dlsch.c +++ b/openair1/PHY/NR_TRANSPORT/nr_dlsch.c @@ -491,7 +491,7 @@ void nr_generate_pdsch(processingData_L1tx_t *msgTx, for (int rb=0; rbrbSize; rb++) { //get pmi info uint8_t pmi; - if (rel15->precodingAndBeamforming.prg_size > 0) + if (0 /*rel15->precodingAndBeamforming.prg_size > 0*/) pmi = rel15->precodingAndBeamforming.prgs_list[(int)rb/rel15->precodingAndBeamforming.prg_size].pm_idx; else pmi = 0;//no precoding diff --git a/openair1/PHY/defs_RU.h b/openair1/PHY/defs_RU.h index 0ee7562346..20f59a1aa8 100644 --- a/openair1/PHY/defs_RU.h +++ b/openair1/PHY/defs_RU.h @@ -655,6 +655,8 @@ typedef struct RU_t_s { int txfh_core_id; /// number of RU interfaces int num_fd; + /// Core id of ru_thread + int ru_thread_core; /// list of cores for RU ThreadPool int tpcores[16]; /// number of cores for RU ThreadPool diff --git a/openair1/PHY/defs_gNB.h b/openair1/PHY/defs_gNB.h index d9c4ad69e0..0b5c7ba564 100644 --- a/openair1/PHY/defs_gNB.h +++ b/openair1/PHY/defs_gNB.h @@ -683,6 +683,8 @@ typedef struct PHY_VARS_gNB_s { int ldpc_offload_flag; + int reorder_thread_disable; + int max_ldpc_iterations; /// indicate the channel estimation technique in time domain int chest_time; @@ -765,6 +767,11 @@ typedef struct PHY_VARS_gNB_s { notifiedFIFO_t L1_tx_out; notifiedFIFO_t resp_RU_tx; tpool_t threadPool; + pthread_t L1_rx_thread; + int L1_rx_thread_core; + pthread_t L1_tx_thread; + int L1_tx_thread_core; + struct processingData_L1tx *msgDataTx; int nbDecode; void *scopeData; /// structure for analyzing high-level RT measurements diff --git a/openair1/SCHED_NR/fapi_nr_l1.c b/openair1/SCHED_NR/fapi_nr_l1.c index 6f644c4ba5..9662780737 100644 --- a/openair1/SCHED_NR/fapi_nr_l1.c +++ b/openair1/SCHED_NR/fapi_nr_l1.c @@ -37,6 +37,7 @@ #include "nfapi/oai_integration/vendor_ext.h" #include "openair2/NR_PHY_INTERFACE/nr_sched_response.h" + extern int oai_nfapi_dl_tti_req(nfapi_nr_dl_tti_request_t *dl_config_req); extern int oai_nfapi_tx_data_req(nfapi_nr_tx_data_request_t *tx_data_req); extern int oai_nfapi_ul_dci_req(nfapi_nr_ul_dci_request_t *ul_dci_req); @@ -157,14 +158,19 @@ void nr_schedule_response(NR_Sched_Rsp_t *Sched_INFO) if (NFAPI_MODE == NFAPI_MONOLITHIC){ if (slot_type == NR_DOWNLINK_SLOT || slot_type == NR_MIXED_SLOT) { - notifiedFIFO_elt_t *res; - res = pullTpool(&gNB->L1_tx_free, &gNB->threadPool); - if (res == NULL) - return; // Tpool has been stopped, nothing to process - processingData_L1tx_t *msgTx = (processingData_L1tx_t *)NotifiedFifoData(res); - const time_stats_t ts = exec_time_stats_NotifiedFIFO(res); + notifiedFIFO_elt_t *res=NULL; + processingData_L1tx_t *msgTx=NULL; + if (!gNB->reorder_thread_disable) { + res = pullTpool(&gNB->L1_tx_free, &gNB->threadPool); + if (res == NULL) + return; // Tpool has been stopped, nothing to process + msgTx = (processingData_L1tx_t *)NotifiedFifoData(res); + } else { + msgTx = gNB->msgDataTx; //newNotifiedFIFO_elt(sizeof(processingData_L1tx_t),0, &gNB->L1_tx_out,NULL); + } + /*const time_stats_t ts = exec_time_stats_NotifiedFIFO(res); merge_meas(&gNB->phy_proc_tx, &ts); - +*/ msgTx->num_pdsch_slot = 0; msgTx->num_dl_pdcch = 0; msgTx->num_ul_pdcch = number_ul_dci_pdu; @@ -216,7 +222,8 @@ void nr_schedule_response(NR_Sched_Rsp_t *Sched_INFO) * released only when both threads are done with it. */ inc_ref_sched_response(Sched_INFO->sched_response_id); - pushNotifiedFIFO(&gNB->L1_tx_filled,res); + if (!gNB->reorder_thread_disable) + pushNotifiedFIFO(&gNB->L1_tx_filled,res); } for (int i = 0; i < number_ul_tti_pdu; i++) { diff --git a/openair1/SCHED_NR/phy_procedures_nr_gNB.c b/openair1/SCHED_NR/phy_procedures_nr_gNB.c index 3da7971f8b..3b5592753a 100644 --- a/openair1/SCHED_NR/phy_procedures_nr_gNB.c +++ b/openair1/SCHED_NR/phy_procedures_nr_gNB.c @@ -396,9 +396,11 @@ void nr_ulsch_procedures(PHY_VARS_gNB *gNB, int frame_rx, int slot_rx, int ULSCH stop_meas(&gNB->ulsch_decoding_stats); } + void nr_fill_indication(PHY_VARS_gNB *gNB, int frame, int slot_rx, int ULSCH_id, uint8_t harq_pid, uint8_t crc_flag, int dtx_flag) { - pthread_mutex_lock(&gNB->UL_INFO_mutex); + if (!get_softmodem_params()->reorder_thread_disable) + pthread_mutex_lock(&gNB->UL_INFO_mutex); NR_gNB_ULSCH_t *ulsch = &gNB->ulsch[ULSCH_id]; NR_UL_gNB_HARQ_t *harq_process = ulsch->harq_process; @@ -557,7 +559,8 @@ void nr_fill_indication(PHY_VARS_gNB *gNB, int frame, int slot_rx, int ULSCH_id, gNB->UL_INFO.rx_ind.number_of_pdus++; - pthread_mutex_unlock(&gNB->UL_INFO_mutex); + if (!get_softmodem_params()->reorder_thread_disable) + pthread_mutex_unlock(&gNB->UL_INFO_mutex); } // Function to fill UL RB mask to be used for N0 measurements diff --git a/openair2/ENB_APP/enb_paramdef.h b/openair2/ENB_APP/enb_paramdef.h index 2b9eef3033..e37c5388d0 100644 --- a/openair2/ENB_APP/enb_paramdef.h +++ b/openair2/ENB_APP/enb_paramdef.h @@ -111,6 +111,7 @@ typedef enum { #define CONFIG_STRING_RU_NUM_TP_CORES "num_tp_cores" #define CONFIG_STRING_RU_NUM_INTERFACES "num_interfaces" #define CONFIG_STRING_RU_HALF_SLOT_PARALLELIZATION "half_slot_parallelization" +#define CONFIG_STRING_RU_RU_THREAD_CORE "ru_thread_core" #define HLP_RU_SF_AHEAD "LTE TX processing advance" #define HLP_RU_SL_AHEAD "NR TX processing advance" @@ -122,6 +123,7 @@ typedef enum { #define HLP_RU_NUM_TP_CORES "Number of cores for RU ThreadPool" #define HLP_RU_NUM_INTERFACES "Number of network interfaces for RU" #define HLP_RU_HALF_SLOT_PARALLELIZATION "run half slots in parallel in RU FEP" +#define HLP_RU_RU_THREAD_CORE "id of core to pin ru_thread, -1 is default" #define RU_LOCAL_IF_NAME_IDX 0 #define RU_LOCAL_ADDRESS_IDX 1 @@ -164,6 +166,7 @@ typedef enum { #define RU_NUM_TP_CORES 38 #define RU_NUM_INTERFACES 39 #define RU_HALF_SLOT_PARALLELIZATION 40 +#define RU_RU_THREAD_CORE 41 /*-----------------------------------------------------------------------------------------------------------------------------------------*/ /* RU configuration parameters */ /* optname helpstr paramflags XXXptr defXXXval type numelt */ @@ -211,6 +214,7 @@ typedef enum { {CONFIG_STRING_RU_NUM_TP_CORES, HLP_RU_NUM_TP_CORES, 0, .uptr=NULL, .defintval=2, TYPE_UINT, 0}, \ {CONFIG_STRING_RU_NUM_INTERFACES, HLP_RU_NUM_INTERFACES, 0, .uptr=NULL, .defintval=1, TYPE_UINT, 0}, \ {CONFIG_STRING_RU_HALF_SLOT_PARALLELIZATION, HLP_RU_HALF_SLOT_PARALLELIZATION, 0, .uptr=NULL, .defintval=1, TYPE_UINT, 0}, \ + {CONFIG_STRING_RU_RU_THREAD_CORE, HLP_RU_RU_THREAD_CORE, 0, .uptr=NULL, .defintval=-1, TYPE_UINT, 0}, \ } // clang-format on diff --git a/openair2/GNB_APP/L1_nr_paramdef.h b/openair2/GNB_APP/L1_nr_paramdef.h index 7791adfff3..8c4381d8bd 100644 --- a/openair2/GNB_APP/L1_nr_paramdef.h +++ b/openair2/GNB_APP/L1_nr_paramdef.h @@ -53,6 +53,8 @@ #define CONFIG_STRING_L1_PUSCH_DTX_THRESHOLD "pusch_dtx_threshold" #define CONFIG_STRING_L1_SRS_DTX_THRESHOLD "srs_dtx_threshold" #define CONFIG_STRING_L1_MAX_LDPC_ITERATIONS "max_ldpc_iterations" +#define CONFIG_STRING_L1_RX_THREAD_CORE "L1_rx_thread_core" +#define CONFIG_STRING_L1_TX_THREAD_CORE "L1_tx_thread_core" #define HLP_TP_SIZ "thread_pool_size paramter removed, please use --thread-pool" /*----------------------------------------------------------------------------------------------------------------------------------------------------*/ /* L1 configuration parameters */ @@ -76,6 +78,8 @@ {CONFIG_STRING_L1_PUSCH_DTX_THRESHOLD, NULL, 0, .uptr=NULL, .defintval=50, TYPE_UINT, 0}, \ {CONFIG_STRING_L1_SRS_DTX_THRESHOLD, NULL, 0, .uptr=NULL, .defintval=50, TYPE_UINT, 0}, \ {CONFIG_STRING_L1_MAX_LDPC_ITERATIONS, NULL, 0, .uptr=NULL, .defintval=5, TYPE_UINT, 0}, \ + {CONFIG_STRING_L1_RX_THREAD_CORE, NULL, 0, .uptr=NULL, .defintval=-1, TYPE_UINT, 0}, \ + {CONFIG_STRING_L1_TX_THREAD_CORE, NULL, 0, .uptr=NULL, .defintval=-1, TYPE_UINT, 0}, \ } // clang-format on #define L1_CC_IDX 0 @@ -94,6 +98,8 @@ #define L1_PUSCH_DTX_THRESHOLD 13 #define L1_SRS_DTX_THRESHOLD 14 #define L1_MAX_LDPC_ITERATIONS 15 +#define L1_RX_THREAD_CORE 16 +#define L1_TX_THREAD_CORE 17 /*----------------------------------------------------------------------------------------------------------------------------------------------------*/ #endif diff --git a/openair2/GNB_APP/gnb_config.c b/openair2/GNB_APP/gnb_config.c index 3705f62840..a9ce850d9b 100644 --- a/openair2/GNB_APP/gnb_config.c +++ b/openair2/GNB_APP/gnb_config.c @@ -770,6 +770,9 @@ void RCconfig_NR_L1(void) RC.gNB[j]->pusch_thres = *(L1_ParamList.paramarray[j][L1_PUSCH_DTX_THRESHOLD].uptr); RC.gNB[j]->srs_thres = *(L1_ParamList.paramarray[j][L1_SRS_DTX_THRESHOLD].uptr); RC.gNB[j]->max_ldpc_iterations = *(L1_ParamList.paramarray[j][L1_MAX_LDPC_ITERATIONS].uptr); + RC.gNB[j]->L1_rx_thread_core = *(L1_ParamList.paramarray[j][L1_RX_THREAD_CORE].iptr); + RC.gNB[j]->L1_tx_thread_core = *(L1_ParamList.paramarray[j][L1_TX_THREAD_CORE].iptr); + LOG_I(PHY,"L1_RX_THREAD_CORE %d (%d)\n",*(L1_ParamList.paramarray[j][L1_RX_THREAD_CORE].iptr),L1_RX_THREAD_CORE); if (strcmp(*(L1_ParamList.paramarray[j][L1_TRANSPORT_N_PREFERENCE_IDX].strptr), "local_mac") == 0) { // sf_ahead = 2; // Need 4 subframe gap between RX and TX } else if (strcmp(*(L1_ParamList.paramarray[j][L1_TRANSPORT_N_PREFERENCE_IDX].strptr), "nfapi") == 0) { diff --git a/radio/AW2SORI/oaiori.c b/radio/AW2SORI/oaiori.c index 43d24b40c5..b9368bb22b 100644 --- a/radio/AW2SORI/oaiori.c +++ b/radio/AW2SORI/oaiori.c @@ -22,7 +22,6 @@ #include #include #include -#include #include #include "common_lib.h" diff --git a/radio/ETHERNET/USERSPACE/LIB/eth_udp.c b/radio/ETHERNET/USERSPACE/LIB/eth_udp.c index 31971f1d1a..0d46d793ba 100644 --- a/radio/ETHERNET/USERSPACE/LIB/eth_udp.c +++ b/radio/ETHERNET/USERSPACE/LIB/eth_udp.c @@ -314,9 +314,15 @@ void *trx_eth_write_udp_cmd(udpTXelem_t *udpTXelem) { uint64_t last_rxTS = fhstate->TS[0]-fhstate->TS0; uint64_t TS_advance=0; if (timestamp > last_rxTS) TS_advance = timestamp - last_rxTS; - - if (TS_advance < (nsamps/2)) LOG_W(PHY,"Starting TX FH for TS %llu absslot %llu(%llu) last_rxTS %llu TS_advance %llu samples\n",(unsigned long long)timestamp,(unsigned long long)timestamp/nsamps,((unsigned long long)timestamp/nsamps)%20,(unsigned long long)last_rxTS,(unsigned long long)TS_advance); - void *buff2; + else { + LOG_W(PHY,"TS_advance is < 0 TS %llu absslot %llu(%llu) last_rxTS %llu TS_advance %llu samples\n", (unsigned long long)timestamp,(unsigned long long)timestamp/nsamps,((unsigned long long)timestamp/nsamps)%20,(unsigned long long)last_rxTS,(unsigned long long)TS_advance); + free(buff); + return(NULL); + } + if (TS_advance < (nsamps/2)) { + LOG_W(PHY,"Starting TX FH for TS %llu absslot %llu(%llu) last_rxTS %llu TS_advance %llu samples\n",(unsigned long long)timestamp,(unsigned long long)timestamp/nsamps,((unsigned long long)timestamp/nsamps)%20,(unsigned long long)last_rxTS,(unsigned long long)TS_advance); + } + void *buff2; #if defined(__x86_64) || defined(__i386__) int nsamps2 = 256>>3; __m256i buff_tx[nsamps2+1]; @@ -445,7 +451,7 @@ void *udp_read_thread(void *arg) { char buffer[UDP_PACKET_SIZE_BYTES(256)]; int first_read=0; while (oai_exit == 0) { - LOG_I(PHY,"UDP read thread %d, waiting for start sampling_rate_d %d, sampling_rate_n %d\n",u->thread_id,device->sampling_rate_ratio_n,device->sampling_rate_ratio_d); + LOG_I(PHY,"UDP read thread %d on core %d, waiting for start sampling_rate_d %d, sampling_rate_n %d\n",u->thread_id,sched_getcpu(),device->sampling_rate_ratio_n,device->sampling_rate_ratio_d); while (fhstate->active > 0) { ssize_t count = recvfrom(((eth_state_t*)device->priv)->sockfdd[0], buffer,sizeof(buffer),0, @@ -500,23 +506,23 @@ void *udp_read_thread(void *arg) { int trx_eth_read_udp(openair0_device *device, openair0_timestamp *timestamp, uint32_t **buff, int nsamps) { fhstate_t *fhstate = &device->fhstate; - openair0_timestamp prev_read_TS= fhstate->TS_read, min_TS; + openair0_timestamp prev_read_TS= fhstate->TS_read; + volatile openair0_timestamp min_TS; // block until FH is ready while (fhstate->r[0] == 0 || fhstate->r[1] == 0 || fhstate->r[2] == 0 || fhstate->r[3] == 0 || fhstate->r[4] == 0 || fhstate->r[5] == 0 || fhstate->r[6] == 0 || fhstate->r[7] == 0) usleep(100); // get minimum TS over all antennas - min_TS = fhstate->TS[0]; + min_TS = (volatile openair0_timestamp)fhstate->TS[0]; for (int i=1;iopenair0_cfg->rx_num_channels;i++) min_TS = min(min_TS,fhstate->TS[i]); // poll/sleep until we accumulated enough samples on each antenna port int count=0; while (fhstate->first_read == 1 && min_TS < (fhstate->TS0+prev_read_TS + nsamps)) { - usleep(50); - min_TS = fhstate->TS[0]; - for (int i=1;iopenair0_cfg->rx_num_channels;i++) min_TS = min(min_TS,fhstate->TS[i]); + usleep(10); + min_TS = (volatile openair0_timestamp)fhstate->TS[0]; + for (int i=1;iopenair0_cfg->rx_num_channels;i++) min_TS = min(min_TS,(volatile openair0_timestamp)fhstate->TS[i]); count++; } - if (fhstate->first_read == 0) { *timestamp = 0; fhstate->TS_read = *timestamp+nsamps; diff --git a/radio/USRP/USERSPACE/LIB/usrp_lib.cpp b/radio/USRP/USERSPACE/LIB/usrp_lib.cpp index 802604b2af..8053411516 100644 --- a/radio/USRP/USERSPACE/LIB/usrp_lib.cpp +++ b/radio/USRP/USERSPACE/LIB/usrp_lib.cpp @@ -556,6 +556,7 @@ void *trx_usrp_write_thread(void * arg){ signed char last_packet; int flags_gpio; + printf("trx_usrp_write_thread started on cpu %d\n",sched_getcpu()); while(1){ pthread_mutex_lock(&write_thread->mutex_write); while (write_thread->count_write == 0) { @@ -664,7 +665,6 @@ int trx_usrp_write_init(openair0_device *device){ (char*)"trx_usrp_write_thread", -1, OAI_PRIORITY_RT_MAX); - return(0); }