diff --git a/executables/nr-gnb.c b/executables/nr-gnb.c index 8ea7f16712..cbd59b59ec 100644 --- a/executables/nr-gnb.c +++ b/executables/nr-gnb.c @@ -375,6 +375,8 @@ void *tx_reorder_thread(void* param) { resL1Reserve=pullTpool(gNB->L1_tx_out, gNB->threadPool); int next_tx_slot=((processingData_L1tx_t *)NotifiedFifoData(resL1Reserve))->slot; + LOG_I(PHY,"tx_reorder_thread started\n"); + while (!oai_exit) { notifiedFIFO_elt_t *resL1; if (resL1Reserve) { @@ -433,6 +435,7 @@ void init_gNB_Tpool(int inst) { s_offset += 3; } if (getenv("noThreads")) strcpy(pool, "n"); + LOG_I(PHY,"Initializing gNB ThreadPool with %s\n",pool); initTpool(pool, gNB->threadPool, cpumeas(CPUMEAS_GETSTATE)); // ULSCH decoder result FIFO gNB->respDecode = (notifiedFIFO_t*) malloc(sizeof(notifiedFIFO_t)); @@ -465,7 +468,8 @@ void init_gNB_Tpool(int inst) { if (!get_softmodem_params()->emulate_l1) threadCreate(&proc->L1_stats_thread,nrL1_stats_thread,(void*)gNB,"L1_stats",-1,OAI_PRIORITY_RT_LOW); - threadCreate(&proc->pthread_tx_reorder, tx_reorder_thread, (void *)gNB, "thread_tx_reorder", -1, OAI_PRIORITY_RT_MAX); + LOG_I(PHY,"Creating thread for TX reordering and dispatching to RU\n"); + threadCreate(&proc->pthread_tx_reorder, tx_reorder_thread, (void *)gNB, "thread_tx_reorder", gNB->RU_list[0]->tpcores[1], OAI_PRIORITY_RT_MAX); } diff --git a/executables/nr-ru.c b/executables/nr-ru.c index d73e7cade8..9449d96650 100644 --- a/executables/nr-ru.c +++ b/executables/nr-ru.c @@ -64,7 +64,7 @@ unsigned short config_frames[4] = {2,9,11,13}; static int DEFBANDS[] = {7}; static int DEFENBS[] = {0}; static int DEFBFW[] = {0x00007fff}; - +static int DEFRUTPCORES[] = {2,4,6,8}; //static int DEFNRBANDS[] = {7}; //static int DEFGNBS[] = {0}; @@ -279,6 +279,7 @@ void fh_if5_south_out(RU_t *ru, int frame, int slot, uint64_t timestamp) { if (ru == RC.ru[0]) VCD_SIGNAL_DUMPER_DUMP_VARIABLE_BY_NAME( VCD_SIGNAL_DUMPER_VARIABLES_TRX_TST, ru->proc.timestamp_tx&0xffffffff ); int offset = ru->nr_frame_parms->get_samples_slot_timestamp(slot,ru->nr_frame_parms,0); start_meas(&ru->tx_fhaul); + /* for (int aid=0;aidnb_tx;aid++) ru->ifdevice.trx_write_func2(&ru->ifdevice, timestamp, @@ -286,6 +287,7 @@ void fh_if5_south_out(RU_t *ru, int frame, int slot, uint64_t timestamp) { aid, ru->nr_frame_parms->get_samples_per_slot(slot,ru->nr_frame_parms), 0); + */ stop_meas(&ru->tx_fhaul); LOG_D(PHY,"IF5 TX %d.%d\n",frame,slot); } @@ -338,7 +340,8 @@ void fh_if5_south_in(RU_t *ru, } stop_meas(&ru->rx_fhaul); - LOG_D(PHY,"IF5 %d.%d => RX %d.%d first_rx %d: time %f\n",*frame,*tti,proc->frame_rx,proc->tti_rx,proc->first_rx,ru->rx_fhaul.p_time/(cpu_freq_GHz*1000.0)); + double fhtime = ru->rx_fhaul.p_time/(cpu_freq_GHz*1000.0); + if (fhtime > 800) LOG_W(PHY,"IF5 %d.%d => RX %d.%d first_rx %d: time %f\n",*frame,*tti,proc->frame_rx,proc->tti_rx,proc->first_rx,ru->rx_fhaul.p_time/(cpu_freq_GHz*1000.0)); VCD_SIGNAL_DUMPER_DUMP_VARIABLE_BY_NAME( VCD_SIGNAL_DUMPER_VARIABLES_TRX_TS, proc->timestamp_rx&0xffffffff ); VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_RECV_IF5, 0 ); @@ -1555,7 +1558,7 @@ void init_RU_proc(RU_t *ru) { pthread_mutex_init( &proc->mutex_emulateRF,NULL); pthread_cond_init( &proc->cond_emulateRF, NULL); - threadCreate( &proc->pthread_FH, ru_thread, (void *)ru, "thread_FH", -1, OAI_PRIORITY_RT_MAX ); + threadCreate( &proc->pthread_FH, ru_thread, (void *)ru, "ru_thread", ru->tpcores[0], OAI_PRIORITY_RT_MAX ); if(emulate_rf) threadCreate( &proc->pthread_emulateRF, emulatedRF_thread, (void *)proc, "emulateRF", -1, OAI_PRIORITY_RT ); @@ -1872,7 +1875,7 @@ void set_function_spec_param(RU_t *ru) { case REMOTE_IF5: // the remote unit is IF5 RRU ru->do_prach = 0; - ru->txfh_in_fep = 1; + ru->txfh_in_fep = 0; #ifdef USE_TPOOL ru->feprx = (get_thread_worker_conf() == WORKER_ENABLE) ? nr_fep_tp : nr_fep_full; // this is frequency-shift + DFTs ru->feptx_prec = (get_thread_worker_conf() == WORKER_ENABLE) ? NULL : nr_feptx_prec; // need to do transmit Precoding + IDFTs @@ -2020,16 +2023,16 @@ void init_NR_RU(char *rf_config_file) { set_function_spec_param(ru); LOG_I(PHY,"Starting ru_thread %d\n",ru_id); init_RU_proc(ru); - int threadCnt = ru->nb_rx+ru->nb_tx; + int threadCnt = ru->num_tpcores; if (threadCnt < 2) LOG_E(PHY,"Number of threads for gNB should be more than 1. Allocated only %d\n",threadCnt); + else LOG_I(PHY,"RU Thread pool size %d\n",threadCnt); char pool[80]; - sprintf(pool,"-1"); - int s_offset = 0; + int s_offset = sprintf(pool,"%d",ru->tpcores[0]); for (int icpu=1; icputpcores[icpu]); } #ifdef USE_TPOOL + LOG_I(PHY,"RU thread-pool core string %s\n",pool); ru->threadPool = (tpool_t*)malloc(sizeof(tpool_t)); initTpool(pool, ru->threadPool, cpumeas(CPUMEAS_GETSTATE)); // FEP RX result FIFO @@ -2211,6 +2214,10 @@ static void NRRCconfig_RU(void) { RC.ru[j]->openair0_cfg.nr_band = RC.ru[j]->band[0]; RC.ru[j]->openair0_cfg.nr_scs_for_raster = *(RUParamList.paramarray[j][RU_NR_SCS_FOR_RASTER].iptr); printf("[RU %d] Setting nr_flag %d, nr_band %d, nr_scs_for_raster %d\n",j,RC.ru[j]->openair0_cfg.nr_flag,RC.ru[j]->openair0_cfg.nr_band,RC.ru[j]->openair0_cfg.nr_scs_for_raster); + RC.ru[j]->openair0_cfg.fh_cores[0] = *(RUParamList.paramarray[j][RU_RXFH_CORE_ID].iptr); + RC.ru[j]->num_tpcores = *(RUParamList.paramarray[j][RU_NUM_TP_CORES].iptr); + AssertFatal(RC.ru[j]->num_tpcores <= RUParamList.paramarray[j][RU_TP_CORES].numelt, "Number of TP cores should be <=16\n"); + for (i=0; inum_tpcores; i++) RC.ru[j]->tpcores[i] = RUParamList.paramarray[j][RU_TP_CORES].iptr[i]; if (config_isparamset(RUParamList.paramarray[j], RU_BF_WEIGHTS_LIST_IDX)) { RC.ru[j]->nb_bfw = RUParamList.paramarray[j][RU_BF_WEIGHTS_LIST_IDX].numelt; diff --git a/openair1/PHY/defs_RU.h b/openair1/PHY/defs_RU.h index 20505ca6f4..688ebf3a88 100644 --- a/openair1/PHY/defs_RU.h +++ b/openair1/PHY/defs_RU.h @@ -668,7 +668,12 @@ typedef struct RU_t_s { notifiedFIFO_t *respfeprx; /// to signal end of feptx notifiedFIFO_t *respfeptx; - + /// core id for RX fhaul (IF5 ECPRI) + int rxfh_core_id; + /// list of cores for RU ThreadPool + int tpcores[16]; + /// number of cores for RU ThreadPool + int num_tpcores; } RU_t; diff --git a/openair2/ENB_APP/enb_paramdef.h b/openair2/ENB_APP/enb_paramdef.h index e6e97069fa..b972a5b120 100644 --- a/openair2/ENB_APP/enb_paramdef.h +++ b/openair2/ENB_APP/enb_paramdef.h @@ -103,11 +103,17 @@ typedef enum { #define CONFIG_STRING_RU_SL_AHEAD "sl_ahead" #define CONFIG_STRING_RU_NR_FLAG "nr_flag" #define CONFIG_STRING_RU_NR_SCS_FOR_RASTER "nr_scs_for_raster" +#define CONFIG_STRING_RU_RXFH_CORE_ID "rxfh_core_id" +#define CONFIG_STRING_RU_TP_CORES "tp_cores" +#define CONFIG_STRING_RU_NUM_TP_CORES "num_tp_cores" #define HLP_RU_SF_AHEAD "LTE TX processing advance" #define HLP_RU_SL_AHEAD "NR TX processing advance" #define HLP_RU_NR_FLAG "Use NR numerology (for AW2SORI)" #define HLP_RU_NR_SCS_FOR_RASTER "NR SCS for raster (for AW2SORI)" +#define HLP_RU_RXFH_CORE_ID "Core ID for RX Fronthaul thread (ECPRI IF5)" +#define HLP_RU_TP_CORES "List of cores for RU ThreadPool" +#define HLP_RU_NUM_TP_CORES "Number of cores for RU ThreadPool" #define RU_LOCAL_IF_NAME_IDX 0 #define RU_LOCAL_ADDRESS_IDX 1 @@ -142,6 +148,9 @@ typedef enum { #define RU_SL_AHEAD 30 #define RU_NR_FLAG 31 #define RU_NR_SCS_FOR_RASTER 32 +#define RU_RXFH_CORE_ID 33 +#define RU_TP_CORES 34 +#define RU_NUM_TP_CORES 35 /*-----------------------------------------------------------------------------------------------------------------------------------------*/ /* RU configuration parameters */ /* optname helpstr paramflags XXXptr defXXXval type numelt */ @@ -180,6 +189,9 @@ typedef enum { {CONFIG_STRING_RU_SL_AHEAD, HLP_RU_SL_AHEAD, 0, iptr:NULL, defintval:6, TYPE_INT, 0}, \ {CONFIG_STRING_RU_NR_FLAG, HLP_RU_NR_FLAG, 0, iptr:NULL, defintval:0, TYPE_INT, 0}, \ {CONFIG_STRING_RU_NR_SCS_FOR_RASTER, HLP_RU_NR_SCS_FOR_RASTER, 0, iptr:NULL, defintval:1, TYPE_INT, 0}, \ + {CONFIG_STRING_RU_RXFH_CORE_ID, HLP_RU_RXFH_CORE_ID, 0, uptr:NULL, defintval:0, TYPE_UINT, 0}, \ + {CONFIG_STRING_RU_TP_CORES, HLP_RU_TP_CORES, 0, uptr:NULL, defintarrayval:DEFRUTPCORES, TYPE_INTARRAY, 8}, \ + {CONFIG_STRING_RU_NUM_TP_CORES, HLP_RU_NUM_TP_CORES, 0, uptr:NULL, defintval:4, TYPE_UINT, 0}, \ } /*---------------------------------------------------------------------------------------------------------------------------------------*/ diff --git a/targets/ARCH/COMMON/common_lib.h b/targets/ARCH/COMMON/common_lib.h index d8a9d11c8a..b2a0517d12 100644 --- a/targets/ARCH/COMMON/common_lib.h +++ b/targets/ARCH/COMMON/common_lib.h @@ -254,6 +254,8 @@ typedef struct { int nr_band; //! NR scs for raster int nr_scs_for_raster; + //! Core IDs for RX FH + int fh_cores[4]; } openair0_config_t; /*! \brief RF mapping */ diff --git a/targets/ARCH/ETHERNET/USERSPACE/LIB/eth_udp.c b/targets/ARCH/ETHERNET/USERSPACE/LIB/eth_udp.c index 662b19ba45..ad910e1e3e 100644 --- a/targets/ARCH/ETHERNET/USERSPACE/LIB/eth_udp.c +++ b/targets/ARCH/ETHERNET/USERSPACE/LIB/eth_udp.c @@ -43,6 +43,7 @@ #include #include #include "common/utils/LOG/vcd_signal_dumper.h" +#include #include "common_lib.h" #include "ethernet_lib.h" @@ -148,11 +149,11 @@ int eth_socket_init_udp(openair0_device *device) { printf("ETHERNET: Cannot set SO_NO_CHECK option on socket (user %d)",i); exit(0); } +#if 0 /*def SO_ATTACH_REUSEPORT_EBPF*/ if (setsockopt(eth->sockfdd[i], SOL_SOCKET, SO_REUSEPORT, &enable, sizeof(int))) { printf("ETHERNET: Cannot set SO_REUSEPORT option on socket (user %d)",i); exit(0); } -#ifdef SO_ATTACH_REUSEPORT_EBPF*/ struct sock_filter code[]={ { BPF_LD | BPF_W | BPF_ABS, 0, 0, SKF_AD_OFF + SKF_AD_CPU }, // A = #cpu { BPF_RET | BPF_A, 0, 0, 0 }, // return A @@ -451,8 +452,8 @@ void *udp_read_thread(void *arg) { // zero-copy and corrected the header component. memcpy((void*)(device->openair0_cfg->rxbase[aid]+offset), (void*)&buffer[APP_HEADER_SIZE_BYTES], - count-APP_HEADER_SIZE_BYTES); - LOG_D(PHY,"thread_id %d, aid %d, TS %llu, TS0 %llu, offset %d\n",u->thread_id,aid,(unsigned long long)TS,fhstate->TS0,offset); + count-APP_HEADER_SIZE_BYTES); + LOG_D(PHY,"thread_id %d (%d), aid %d, TS %llu, TS0 %llu, offset %d\n",u->thread_id,sched_getcpu(),aid,(unsigned long long)TS,fhstate->TS0,offset); } sleep(1); } @@ -473,7 +474,7 @@ int trx_eth_read_udp(openair0_device *device, openair0_timestamp *timestamp, uin // poll/sleep until we accumulated enough samples on each antenna port int count=0; while (fhstate->first_read == 1 && min_TS < (fhstate->TS0+prev_read_TS + nsamps)) { - usleep(100); + usleep(50); min_TS = fhstate->TS[0]; for (int i=1;iopenair0_cfg->rx_num_channels;i++) min_TS = min(min_TS,fhstate->TS[i]); count++; diff --git a/targets/ARCH/ETHERNET/USERSPACE/LIB/ethernet_lib.c b/targets/ARCH/ETHERNET/USERSPACE/LIB/ethernet_lib.c index fbe9aed34b..4170cb1ac3 100644 --- a/targets/ARCH/ETHERNET/USERSPACE/LIB/ethernet_lib.c +++ b/targets/ARCH/ETHERNET/USERSPACE/LIB/ethernet_lib.c @@ -65,13 +65,13 @@ int trx_eth_start(openair0_device *device) AssertFatal(device->thirdparty_init(device) == 0, "third-party init failed\n"); device->openair0_cfg->samples_per_packet = 256; eth->num_fd = 1; //max(device->openair0_cfg->rx_num_channels,device->openair0_cfg->tx_num_channels); - printf("Creating %d UDP threads ...\n",device->openair0_cfg->rx_num_channels); - udp_read_t *u[device->openair0_cfg->rx_num_channels]; - for (int i=0;i<(device->openair0_cfg->rx_num_channels);i++) { + udp_read_t *u[2]; + for (int i=0;inum_fd;i++) { u[i] = malloc(sizeof(udp_read_t)); u[i]->thread_id=i; u[i]->device = device; - threadCreate(&u[i]->pthread,udp_read_thread,u[i],"udp read thread",-1,OAI_PRIORITY_RT_MAX); + printf("UDP Read Thread %d on core %d\n",i,device->openair0_cfg->fh_cores[i]); + threadCreate(&u[i]->pthread,udp_read_thread,u[i],"udp read thread",device->openair0_cfg->fh_cores[i],OAI_PRIORITY_RT_MAX); } device->sampling_rate_ratio_n=1; device->sampling_rate_ratio_d=1; @@ -112,6 +112,7 @@ int trx_eth_start(openair0_device *device) } else AssertFatal(1==0,"num_rb_dl %d not ok with ECPRI\n",device->openair0_cfg->num_rb_dl); } +#ifdef USE_TX_TPOOL int threadCnt = device->openair0_cfg->tx_num_channels; if (threadCnt < 2) LOG_E(PHY,"Number of threads for gNB should be more than 1. Allocated only %d\n",threadCnt); char pool[80]; @@ -126,6 +127,7 @@ int trx_eth_start(openair0_device *device) // ULSCH decoder result FIFO device->respudpTX = (notifiedFIFO_t*) malloc(sizeof(notifiedFIFO_t)); initNotifiedFIFO(device->respudpTX); +#endif } /* initialize socket */ if (eth->flags == ETH_RAW_MODE) { @@ -211,6 +213,7 @@ int trx_eth_start(openair0_device *device) if(ethernet_tune (device, RCV_BUF_SIZE,67108864)!=0) return -1; if(ethernet_tune (device, KERNEL_SND_BUF_MAX_SIZE, 67108864)!=0) return -1; if(ethernet_tune (device, KERNEL_RCV_BUF_MAX_SIZE, 67108864)!=0) return -1; + if(ethernet_tune (device, TX_Q_LEN, 10000)!=0) return -1; return 0; @@ -445,7 +448,7 @@ int ethernet_tune(openair0_device *device, } break; case KERNEL_SND_BUF_MAX_SIZE: - ret=snprintf(system_cmd,sizeof(system_cmd),"sysctl -w net.core.wmem_max=%d;sysctl -w net.core.wmem_default=%d;sysctl -w net.core.netdev_max_backlog=416384;sysctl -w net.core.optmem_max=524288;",value,value); + ret=snprintf(system_cmd,sizeof(system_cmd),"sysctl -w net.core.wmem_max=%d;sysctl -w net.core.wmem_default=%d;sysctl -w net.core.netdev_max_backlog=5000;sysctl -w net.core.optmem_max=524288;",value,value); if (ret > 0) { ret=system(system_cmd); if (ret == -1) { @@ -487,7 +490,7 @@ int transport_init(openair0_device *device, eth->compression = ALAW_COMPRESS; } - eth->num_fd = max(openair0_cfg->rx_num_channels,openair0_cfg->tx_num_channels); + eth->num_fd = 1; printf("[ETHERNET]: Initializing openair0_device for %s ...\n", ((device->host_type == RAU_HOST) ? "RAU": "RRU")); printf("[ETHERNET]: num_fd %d\n",eth->num_fd); diff --git a/targets/RT/USER/lte-ru.c b/targets/RT/USER/lte-ru.c index 571551cee9..30919ec6a4 100644 --- a/targets/RT/USER/lte-ru.c +++ b/targets/RT/USER/lte-ru.c @@ -66,6 +66,8 @@ static int DEFBANDS[] = {7}; static int DEFENBS[] = {0}; static int DEFBFW[] = {0x00007fff}; +static int DEFRUTPCORES[] = {2,4,6,8}; + #include "ENB_APP/enb_paramdef.h" #include "common/config/config_userapi.h"